From 00343a48d39d9ff74ceb662c5140048295f2610a Mon Sep 17 00:00:00 2001 From: TTrapper Date: Mon, 2 Oct 2017 17:51:09 -0300 Subject: [PATCH 0001/1924] sampled version of sparse_softmax_cross_entropy_with_logits --- tensorflow/python/ops/nn.py | 1 + tensorflow/python/ops/nn_impl.py | 98 ++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index a80662c8b5..f7edace5b1 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -90,6 +90,7 @@ See the @{$python/nn} guide. @@in_top_k @@nce_loss @@sampled_softmax_loss +@@sampled_sparse_softmax_loss @@uniform_candidate_sampler @@log_uniform_candidate_sampler @@learned_unigram_candidate_sampler diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index db8e92831e..b2b57a055f 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -1258,3 +1258,101 @@ def sampled_softmax_loss(weights, labels=labels, logits=logits) # sampled_losses is a [batch_size] tensor. return sampled_losses + + +def sampled_sparse_softmax_loss(weights, + biases, + labels, + inputs, + num_sampled, + num_classes, + sampled_values=None, + remove_accidental_hits=True, + partition_strategy="mod", + name="sampled_sparse_softmax_loss"): + """Computes and returns the sampled sparse softmax training loss. + + This is a faster way to train a softmax classifier over a huge number of + classes. + + This operation is for training only. It is generally an underestimate of + the full softmax loss. + + A common use case is to use this method for training, and calculate the full + softmax loss for evaluation or inference. In this case, you must set + `partition_strategy="div"` for the two losses to be consistent, as in the + following example: + + ```python + if mode == "train": + loss = tf.nn.sampled_sparse_softmax_loss( + weights=weights, + biases=biases, + labels=labels, + inputs=inputs, + ..., + partition_strategy="div") + elif mode == "eval": + logits = tf.matmul(inputs, tf.transpose(weights)) + logits = tf.nn.bias_add(logits, biases) + labels_one_hot = tf.one_hot(labels, n_classes) + loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=labels_one_hot, + logits=logits) + ``` + + See our [Candidate Sampling Algorithms Reference] + (https://www.tensorflow.org/extras/candidate_sampling.pdf) + + Also see Section 3 of [Jean et al., 2014](http://arxiv.org/abs/1412.2007) + ([pdf](http://arxiv.org/pdf/1412.2007.pdf)) for the math. + + Args: + weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor` + objects whose concatenation along dimension 0 has shape + [num_classes, dim]. The (possibly-sharded) class embeddings. + biases: A `Tensor` of shape `[num_classes]`. The class biases. + labels: A `Tensor` of type `int64` and shape `[batch_size, 1]`. + The index of the single target class for each row of logits. Note that + this format differs from the `labels` argument of + `nn.sparse_softmax_cross_entropy_with_logits`. + inputs: A `Tensor` of shape `[batch_size, dim]`. The forward + activations of the input network. + num_sampled: An `int`. The number of classes to randomly sample per batch. + num_classes: An `int`. The number of possible classes. + sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`, + `sampled_expected_count`) returned by a `*_candidate_sampler` function. + (if None, we default to `log_uniform_candidate_sampler`) + remove_accidental_hits: A `bool`. whether to remove "accidental hits" + where a sampled class equals one of the target classes. Default is + True. + partition_strategy: A string specifying the partitioning strategy, relevant + if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. + Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. + name: A name for the operation (optional). + + Returns: + A `batch_size` 1-D tensor of per-example sampled softmax losses. + + """ + logits, labels = _compute_sampled_logits( + weights=weights, + biases=biases, + labels=labels, + inputs=inputs, + num_sampled=num_sampled, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_values, + subtract_log_q=True, + remove_accidental_hits=remove_accidental_hits, + partition_strategy=partition_strategy, + name=name) + + # labels returned by _compute_sampled_logits are one_hot. Convert to indices. + labels = array_ops.reshape(math_ops.argmax(labels, axis=1), [-1]) + + sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( + labels=labels, logits=logits) + # sampled_losses is a [batch_size] tensor. + return sampled_losses -- GitLab From 499376eb38b6b5b991e330d87c91d879a6f7bbbe Mon Sep 17 00:00:00 2001 From: Daniyar Date: Mon, 2 Oct 2017 20:58:00 +0100 Subject: [PATCH 0002/1924] unpack for int64 tensors on gpu --- tensorflow/core/kernels/unpack_op.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow/core/kernels/unpack_op.cc b/tensorflow/core/kernels/unpack_op.cc index 7fd1def1fe..7ece912557 100644 --- a/tensorflow/core/kernels/unpack_op.cc +++ b/tensorflow/core/kernels/unpack_op.cc @@ -153,6 +153,12 @@ REGISTER_KERNEL_BUILDER(Name("Unpack") .HostMemory("output") .TypeConstraint("T"), UnpackOp); +REGISTER_KERNEL_BUILDER(Name("Unpack") + .Device(DEVICE_GPU) + .HostMemory("value") + .HostMemory("output") + .TypeConstraint("T"), + UnpackOp); #endif // GOOGLE_CUDA @@ -170,6 +176,12 @@ REGISTER_KERNEL_BUILDER(Name("Unpack") .HostMemory("output") .TypeConstraint("T"), UnpackOp); +REGISTER_KERNEL_BUILDER(Name("Unpack") + .Device(DEVICE_SYCL) + .HostMemory("value") + .HostMemory("output") + .TypeConstraint("T"), + UnpackOp); #undef REGISTER_SYCL #endif // TENSORFLOW_USE_SYCL -- GitLab From 7fe8a6decd3b1c077de5a3cdedff198195b16ee1 Mon Sep 17 00:00:00 2001 From: Daniyar Date: Thu, 5 Oct 2017 14:34:12 +0100 Subject: [PATCH 0003/1924] unstack op tests for dtypes --- .../python/kernel_tests/unstack_op_test.py | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/kernel_tests/unstack_op_test.py b/tensorflow/python/kernel_tests/unstack_op_test.py index c2dcff978a..d937108599 100644 --- a/tensorflow/python/kernel_tests/unstack_op_test.py +++ b/tensorflow/python/kernel_tests/unstack_op_test.py @@ -22,6 +22,7 @@ import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import constant_op +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker from tensorflow.python.platform import test @@ -42,15 +43,33 @@ class UnstackOpTest(test.TestCase): np.random.seed(7) with self.test_session(use_gpu=True): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): - data = np.random.randn(*shape) - # Convert data to a single tensorflow tensor - x = constant_op.constant(data) - # Unpack into a list of tensors - cs = array_ops.unstack(x, num=shape[0]) - self.assertEqual(type(cs), list) - self.assertEqual(len(cs), shape[0]) - cs = [c.eval() for c in cs] - self.assertAllEqual(cs, data) + for dtype in [np.bool, np.float16, np.float32, np.float64, np.int32, np.int64]: + data = np.random.randn(*shape).astype(dtype) + # Convert data to a single tensorflow tensor + x = constant_op.constant(data) + # Unpack into a list of tensors + cs = array_ops.unstack(x, num=shape[0]) + self.assertEqual(type(cs), list) + self.assertEqual(len(cs), shape[0]) + cs = [c.eval() for c in cs] + self.assertAllEqual(cs, data) + + def testSimpleGpu(self): + if not test_util.is_gpu_available(): + self.skipTest("No GPU available") + np.random.seed(7) + with self.test_session(use_gpu=True, force_gpu=True): + for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): + for dtype in [np.float16, np.float32, np.float64, np.int32, np.int64]: + data = np.random.randn(*shape).astype(dtype) + # Convert data to a single tensorflow tensor + x = constant_op.constant(data) + # Unpack into a list of tensors + cs = array_ops.unstack(x, num=shape[0]) + self.assertEqual(type(cs), list) + self.assertEqual(len(cs), shape[0]) + cs = [c.eval() for c in cs] + self.assertAllEqual(cs, data) def testGradientsAxis0(self): for shape in (2,), (3,), (2, 3), (3, 2), (4, 3, 2): -- GitLab From 03233a04cf07d639d8d2b5f3fbcab479b267ac4e Mon Sep 17 00:00:00 2001 From: TTrapper Date: Fri, 6 Oct 2017 00:21:08 -0300 Subject: [PATCH 0004/1924] Adressed reviewer comments: moved to contrib, fixed erroneous doc, modified _compute_sampled_logits to optionally return target indices --- tensorflow/contrib/nn/__init__.py | 1 + .../contrib/nn/python/ops/sampling_ops.py | 97 +++++++++++++ tensorflow/python/ops/nn.py | 1 - tensorflow/python/ops/nn_impl.py | 128 +++--------------- 4 files changed, 120 insertions(+), 107 deletions(-) diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py index be0957f473..89b70ddfc2 100644 --- a/tensorflow/contrib/nn/__init__.py +++ b/tensorflow/contrib/nn/__init__.py @@ -19,6 +19,7 @@ @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits @@deprecated_flipped_sigmoid_cross_entropy_with_logits @@rank_sampled_softmax_loss +@@sampled_sparse_softmax_loss """ from __future__ import absolute_import diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops.py b/tensorflow/contrib/nn/python/ops/sampling_ops.py index 2ae529e015..b26da52f01 100644 --- a/tensorflow/contrib/nn/python/ops/sampling_ops.py +++ b/tensorflow/contrib/nn/python/ops/sampling_ops.py @@ -24,6 +24,8 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import embedding_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn +from tensorflow.python.ops import nn_impl +from tensorflow.python.ops import nn_ops def _rank_resample(weights, biases, inputs, sampled_values, num_resampled, @@ -240,3 +242,98 @@ def rank_sampled_softmax_loss(weights, remove_accidental_hits=remove_accidental_hits, partition_strategy=partition_strategy, name=name) + + +def sampled_sparse_softmax_loss(weights, + biases, + labels, + inputs, + num_sampled, + num_classes, + sampled_values=None, + remove_accidental_hits=True, + partition_strategy="mod", + name="sampled_sparse_softmax_loss"): + """Computes and returns the sampled sparse softmax training loss. + + This is a faster way to train a softmax classifier over a huge number of + classes. + + This operation is for training only. It is generally an underestimate of + the full softmax loss. + + A common use case is to use this method for training, and calculate the full + softmax loss for evaluation or inference. In this case, you must set + `partition_strategy="div"` for the two losses to be consistent, as in the + following example: + + ```python + if mode == "train": + loss = tf.nn.sampled_sparse_softmax_loss( + weights=weights, + biases=biases, + labels=labels, + inputs=inputs, + ..., + partition_strategy="div") + elif mode == "eval": + logits = tf.matmul(inputs, tf.transpose(weights)) + logits = tf.nn.bias_add(logits, biases) + loss = tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=tf.squeeze(labels), + logits=logits) + ``` + + See our [Candidate Sampling Algorithms Reference] + (https://www.tensorflow.org/extras/candidate_sampling.pdf) + + Also see Section 3 of [Jean et al., 2014](http://arxiv.org/abs/1412.2007) + ([pdf](http://arxiv.org/pdf/1412.2007.pdf)) for the math. + + Args: + weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor` + objects whose concatenation along dimension 0 has shape + [num_classes, dim]. The (possibly-sharded) class embeddings. + biases: A `Tensor` of shape `[num_classes]`. The class biases. + labels: A `Tensor` of type `int64` and shape `[batch_size, 1]`. + The index of the single target class for each row of logits. Note that + this format differs from the `labels` argument of + `nn.sparse_softmax_cross_entropy_with_logits`. + inputs: A `Tensor` of shape `[batch_size, dim]`. The forward + activations of the input network. + num_sampled: An `int`. The number of classes to randomly sample per batch. + num_classes: An `int`. The number of possible classes. + sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`, + `sampled_expected_count`) returned by a `*_candidate_sampler` function. + (if None, we default to `log_uniform_candidate_sampler`) + remove_accidental_hits: A `bool`. whether to remove "accidental hits" + where a sampled class equals one of the target classes. Default is + True. + partition_strategy: A string specifying the partitioning strategy, relevant + if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. + Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. + name: A name for the operation (optional). + + Returns: + A `batch_size` 1-D tensor of per-example sampled softmax losses. + + """ + logits, labels = nn_impl._compute_sampled_logits( + weights=weights, + biases=biases, + labels=labels, + inputs=inputs, + num_sampled=num_sampled, + num_classes=num_classes, + num_true=1, + sampled_values=sampled_values, + subtract_log_q=True, + remove_accidental_hits=remove_accidental_hits, + partition_strategy=partition_strategy, + labels_as_indices=True, + name=name) + + sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( + labels=array_ops.squeeze(labels), logits=logits) + # sampled_losses is a [batch_size] tensor. + return sampled_losses diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index f7edace5b1..a80662c8b5 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -90,7 +90,6 @@ See the @{$python/nn} guide. @@in_top_k @@nce_loss @@sampled_softmax_loss -@@sampled_sparse_softmax_loss @@uniform_candidate_sampler @@log_uniform_candidate_sampler @@learned_unigram_candidate_sampler diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index b2b57a055f..ad18eedfb0 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import candidate_sampling_ops from tensorflow.python.ops import embedding_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_nn_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_ops @@ -893,6 +894,7 @@ def _compute_sampled_logits(weights, subtract_log_q=True, remove_accidental_hits=False, partition_strategy="mod", + labels_as_indices=False, name=None): """Helper function for nce_loss and sampled_softmax_loss functions. @@ -930,12 +932,18 @@ def _compute_sampled_logits(weights, partition_strategy: A string specifying the partitioning strategy, relevant if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. + labels_as_indices: A `bool`. Whether the returned labels represent the + indices of the true classes. Default is `False`. name: A name for the operation (optional). Returns: - out_logits, out_labels: `Tensor` objects each with shape + out_logits: `Tensor` object with shape `[batch_size, num_true + num_sampled]`, for passing to either `nn.sigmoid_cross_entropy_with_logits` (NCE) or `nn.softmax_cross_entropy_with_logits` (sampled softmax). + out_labels: If `labels_as_indices` is `False`, a Tensor object with the same + shape as `out_logits`. Otherwise a `Tensor` of shape + `[batch_size, num_true]` with the indices of the target classes for each + row of `out_logits`. """ if isinstance(weights, variables.PartitionedVariable): @@ -1046,13 +1054,19 @@ def _compute_sampled_logits(weights, # Construct output logits and labels. The true labels/logits start at col 0. out_logits = array_ops.concat([true_logits, sampled_logits], 1) - # true_logits is a float tensor, ones_like(true_logits) is a float tensor - # of ones. We then divide by num_true to ensure the per-example labels sum - # to 1.0, i.e. form a proper probability distribution. - out_labels = array_ops.concat([ - array_ops.ones_like(true_logits) / num_true, - array_ops.zeros_like(sampled_logits) - ], 1) + if labels_as_indices: + # We want each row of labels to be the indices of the targets, which + # start at col 0 and end at col num_true-1. + out_labels = gen_array_ops.tile( + [math_ops.range(num_true)], [array_ops.shape(true_logits)[0], 1]) + else: + # true_logits is a float tensor, ones_like(true_logits) is a float + # tensor of ones. We then divide by num_true to ensure the per-example + # labels sum to 1.0, i.e. form a proper probability distribution. + out_labels = array_ops.concat([ + array_ops.ones_like(true_logits) / num_true, + array_ops.zeros_like(sampled_logits) + ], 1) return out_logits, out_labels @@ -1258,101 +1272,3 @@ def sampled_softmax_loss(weights, labels=labels, logits=logits) # sampled_losses is a [batch_size] tensor. return sampled_losses - - -def sampled_sparse_softmax_loss(weights, - biases, - labels, - inputs, - num_sampled, - num_classes, - sampled_values=None, - remove_accidental_hits=True, - partition_strategy="mod", - name="sampled_sparse_softmax_loss"): - """Computes and returns the sampled sparse softmax training loss. - - This is a faster way to train a softmax classifier over a huge number of - classes. - - This operation is for training only. It is generally an underestimate of - the full softmax loss. - - A common use case is to use this method for training, and calculate the full - softmax loss for evaluation or inference. In this case, you must set - `partition_strategy="div"` for the two losses to be consistent, as in the - following example: - - ```python - if mode == "train": - loss = tf.nn.sampled_sparse_softmax_loss( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - ..., - partition_strategy="div") - elif mode == "eval": - logits = tf.matmul(inputs, tf.transpose(weights)) - logits = tf.nn.bias_add(logits, biases) - labels_one_hot = tf.one_hot(labels, n_classes) - loss = tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels_one_hot, - logits=logits) - ``` - - See our [Candidate Sampling Algorithms Reference] - (https://www.tensorflow.org/extras/candidate_sampling.pdf) - - Also see Section 3 of [Jean et al., 2014](http://arxiv.org/abs/1412.2007) - ([pdf](http://arxiv.org/pdf/1412.2007.pdf)) for the math. - - Args: - weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor` - objects whose concatenation along dimension 0 has shape - [num_classes, dim]. The (possibly-sharded) class embeddings. - biases: A `Tensor` of shape `[num_classes]`. The class biases. - labels: A `Tensor` of type `int64` and shape `[batch_size, 1]`. - The index of the single target class for each row of logits. Note that - this format differs from the `labels` argument of - `nn.sparse_softmax_cross_entropy_with_logits`. - inputs: A `Tensor` of shape `[batch_size, dim]`. The forward - activations of the input network. - num_sampled: An `int`. The number of classes to randomly sample per batch. - num_classes: An `int`. The number of possible classes. - sampled_values: a tuple of (`sampled_candidates`, `true_expected_count`, - `sampled_expected_count`) returned by a `*_candidate_sampler` function. - (if None, we default to `log_uniform_candidate_sampler`) - remove_accidental_hits: A `bool`. whether to remove "accidental hits" - where a sampled class equals one of the target classes. Default is - True. - partition_strategy: A string specifying the partitioning strategy, relevant - if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. - Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. - name: A name for the operation (optional). - - Returns: - A `batch_size` 1-D tensor of per-example sampled softmax losses. - - """ - logits, labels = _compute_sampled_logits( - weights=weights, - biases=biases, - labels=labels, - inputs=inputs, - num_sampled=num_sampled, - num_classes=num_classes, - num_true=1, - sampled_values=sampled_values, - subtract_log_q=True, - remove_accidental_hits=remove_accidental_hits, - partition_strategy=partition_strategy, - name=name) - - # labels returned by _compute_sampled_logits are one_hot. Convert to indices. - labels = array_ops.reshape(math_ops.argmax(labels, axis=1), [-1]) - - sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits) - # sampled_losses is a [batch_size] tensor. - return sampled_losses -- GitLab From 7680d8d00dec8897b64ea864da71537b7be957de Mon Sep 17 00:00:00 2001 From: TTrapper Date: Fri, 6 Oct 2017 00:47:54 -0300 Subject: [PATCH 0005/1924] checkstyle fix --- tensorflow/python/ops/nn_impl.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index ad18eedfb0..8e64259143 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -1055,18 +1055,18 @@ def _compute_sampled_logits(weights, # Construct output logits and labels. The true labels/logits start at col 0. out_logits = array_ops.concat([true_logits, sampled_logits], 1) if labels_as_indices: - # We want each row of labels to be the indices of the targets, which - # start at col 0 and end at col num_true-1. - out_labels = gen_array_ops.tile( - [math_ops.range(num_true)], [array_ops.shape(true_logits)[0], 1]) + # We want each row of labels to be the indices of the targets, which + # start at col 0 and end at col num_true-1. + out_labels = gen_array_ops.tile( + [math_ops.range(num_true)], [array_ops.shape(true_logits)[0], 1]) else: - # true_logits is a float tensor, ones_like(true_logits) is a float - # tensor of ones. We then divide by num_true to ensure the per-example - # labels sum to 1.0, i.e. form a proper probability distribution. - out_labels = array_ops.concat([ - array_ops.ones_like(true_logits) / num_true, - array_ops.zeros_like(sampled_logits) - ], 1) + # true_logits is a float tensor, ones_like(true_logits) is a float + # tensor of ones. We then divide by num_true to ensure the per-example + # labels sum to 1.0, i.e. form a proper probability distribution. + out_labels = array_ops.concat([ + array_ops.ones_like(true_logits) / num_true, + array_ops.zeros_like(sampled_logits) + ], 1) return out_logits, out_labels -- GitLab From dfe2abae81f331389891e38de5193566e2716b9b Mon Sep 17 00:00:00 2001 From: Shreyash sharma Date: Sat, 21 Oct 2017 12:56:24 +0530 Subject: [PATCH 0006/1924] Update layers_test.py --- .../contrib/layers/python/layers/layers_test.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py index 2d08a0cb91..d93c1552e4 100644 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ b/tensorflow/contrib/layers/python/layers/layers_test.py @@ -3688,7 +3688,17 @@ class LegacyFullyConnectedTest(test.TestCase): 'rank of x must be at least 2 not: 1'): x = constant_op.constant([[]], shape=[0]) _layers.legacy_fully_connected(x, 2, activation_fn=nn_ops.softmax) - - + +class zero_debias_moving_mean(test.TestCase): + + def Error_in_tf.contrib.layers.batch_norm_when(self): + import tensorflow as tf + a = tf.placeholder(tf.float32, shape=(10, 10, 10, 10)) + b = tf.contrib.layers.batch_norm(a, center=False, data_format='NCHW', + zero_debias_moving_mean=True) + sess = tf.Session() + sess.run(tf.global_variables_initializer()) + + if __name__ == '__main__': test.main() -- GitLab From f300bcbb3419e7ad7130a84d5375ae53d92e1568 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sun, 22 Oct 2017 21:36:25 +0800 Subject: [PATCH 0007/1924] Propagate -DPCRE_STATIC from pcre.BUILD to swig.BUILD To fix a build error on Windows: ERROR: C:/os/t/external/swig/BUILD.bazel:5:1: Linking of rule '@swig//:swig' failed (Exit 1120): link.exe failed: error executing command misc.o : error LNK2019: unresolved external symbol __imp_pcre_compile referenced in function Swig_string_regex ... --- third_party/pcre.BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/pcre.BUILD b/third_party/pcre.BUILD index 68aadd1d40..e2cdec4029 100644 --- a/third_party/pcre.BUILD +++ b/third_party/pcre.BUILD @@ -50,12 +50,12 @@ cc_library( "-DNEWLINE=10", "-DNO_RECURSE", "-DPARENS_NEST_LIMIT=50", - "-DPCRE_STATIC=1", "-DPOSIX_MALLOC_THRESHOLD=10", "-DSTDC_HEADERS=1", "-DSUPPORT_UCP", "-DSUPPORT_UTF", ], + defines = ["PCRE_STATIC=1"], includes = ["."], visibility = ["@swig//:__pkg__"], # Please use RE2 alwayslink = 1, -- GitLab From 52e3b184556920c2ca7b58d7632b988ec229af13 Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Wed, 25 Oct 2017 13:11:29 -0400 Subject: [PATCH 0008/1924] initial tf sru implementation --- .../python/kernel_tests/core_rnn_cell_test.py | 27 ++++++++ tensorflow/python/ops/rnn_cell.py | 1 + tensorflow/python/ops/rnn_cell_impl.py | 68 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index 8349188f6f..0c030f9169 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -140,6 +140,33 @@ class RNNCellTest(test.TestCase): # Smoke test self.assertAllClose(res[0], [[0.156736, 0.156736]]) + def testSRUCell(self): + with self.test_session() as sess: + with variable_scope.variable_scope( + "root", initializer=init_ops.constant_initializer(0.5)): + x = array_ops.zeros([1, 2]) + m = array_ops.zeros([1, 2]) + g, _ = rnn_cell_impl.SRUCell(2)(x, m) + sess.run([variables_lib.global_variables_initializer()]) + res = sess.run( + [g], {x.name: np.array([[1., 1.]]), + m.name: np.array([[0.1, 0.1]])}) + # Smoke test + self.assertAllClose(res[0], [[0.39352643, 0.39352643]]) + with variable_scope.variable_scope( + "other", initializer=init_ops.constant_initializer(0.5)): + x = array_ops.zeros( + [1, 3]) # Test SRUCell with input_size != num_units. + m = array_ops.zeros([1, 2]) + g, _ = rnn_cell_impl.SRUCell(2)(x, m) + sess.run([variables_lib.global_variables_initializer()]) + res = sess.run( + [g], + {x.name: np.array([[1., 1., 1.]]), + m.name: np.array([[0.1, 0.1]])}) + # Smoke test + self.assertAllClose(res[0], [[0.40844864, 0.40844864]]) + def testBasicLSTMCell(self): for dtype in [dtypes.float16, dtypes.float32]: np_dtype = dtype.as_numpy_dtype diff --git a/tensorflow/python/ops/rnn_cell.py b/tensorflow/python/ops/rnn_cell.py index c0dac8fb01..35dbbf34c4 100644 --- a/tensorflow/python/ops/rnn_cell.py +++ b/tensorflow/python/ops/rnn_cell.py @@ -24,6 +24,7 @@ @@BasicLSTMCell @@GRUCell @@LSTMCell +@@SRUCell ## Classes storing split `RNNCell` state diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 4056eade81..b5ac4af32b 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -268,6 +268,74 @@ class BasicRNNCell(RNNCell): output = self._activation(self._linear([inputs, state])) return output, output +class SRUCell(RNNCell): + """Training RNNs as Fast as CNNs (cf. https://arxiv.org/abs/1709.02755). + + Args: + num_units: int, The number of units in the SRU cell. + activation: Nonlinearity to use. Default: `tanh`. + reuse: (optional) Python boolean describing whether to reuse variables + in an existing scope. If not `True`, and the existing scope already has + the given variables, an error is raised. + kernel_initializer: (optional) The initializer to use for the weight and + projection matrices. + bias_initializer: (optional) The initializer to use for the bias. + """ + + def __init__(self, + num_units, + activation=None, + reuse=None, + kernel_initializer=None, + bias_initializer=None): + super(SRUCell, self).__init__(_reuse=reuse) + self._num_units = num_units + self._activation = activation or math_ops.tanh + self._kernel_initializer = kernel_initializer + self._bias_initializer = bias_initializer + self._gate_linear = None + + @property + def state_size(self): + return self._num_units + + @property + def output_size(self): + return self._num_units + + def call(self, inputs, state): + """Gated recurrent unit (SRU) with nunits cells.""" + if self._gate_linear is None: + self._gate_linear = _Linear( + [inputs], + 3 * self._num_units, + False, + kernel_initializer=self._kernel_initializer) + + value = self._gate_linear([inputs]) + x_bar, f_intermediate, r_intermediate = \ + array_ops.split(value=value, num_or_size_splits=3, axis=1) + + if self._bias_initializer is None: + self._bias_initializer = init_ops.constant_initializer(0.0, \ + dtype=inputs.dtype) + _biases = vs.get_variable( + _BIAS_VARIABLE_NAME, [2 * self._num_units], + dtype=inputs.dtype, + initializer=self._bias_initializer) + + f_r = math_ops.sigmoid(nn_ops.bias_add(array_ops.concat([f_intermediate, \ + r_intermediate], 1), _biases)) + f, r = array_ops.split(value=f_r, num_or_size_splits=2, axis=1) + + c = f * state + (1.0 - f) * x_bar + # For the following equaiton, in the paper, we have: + # h = r * self._activation(c) + (1.0 - r) * inputs + # However, this causes a dimension mismatch when num_units + # is not input_size. + h = r * self._activation(c) + (1.0 - r) * x_bar + + return h, c class GRUCell(RNNCell): """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). -- GitLab From 836d9fe7d3de3057db30bbb475bfb82ea258a880 Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Wed, 25 Oct 2017 14:31:39 -0400 Subject: [PATCH 0009/1924] fix equation --- .../python/kernel_tests/core_rnn_cell_test.py | 16 ++-------------- tensorflow/python/ops/rnn_cell_impl.py | 6 +----- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index 0c030f9169..3e99b9af85 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -146,26 +146,14 @@ class RNNCellTest(test.TestCase): "root", initializer=init_ops.constant_initializer(0.5)): x = array_ops.zeros([1, 2]) m = array_ops.zeros([1, 2]) - g, _ = rnn_cell_impl.SRUCell(2)(x, m) + g, _ = rnn_cell_impl.SRUCell(2, \ + bias_initializer=init_ops.constant_initializer(0.5))(x, m) sess.run([variables_lib.global_variables_initializer()]) res = sess.run( [g], {x.name: np.array([[1., 1.]]), m.name: np.array([[0.1, 0.1]])}) # Smoke test self.assertAllClose(res[0], [[0.39352643, 0.39352643]]) - with variable_scope.variable_scope( - "other", initializer=init_ops.constant_initializer(0.5)): - x = array_ops.zeros( - [1, 3]) # Test SRUCell with input_size != num_units. - m = array_ops.zeros([1, 2]) - g, _ = rnn_cell_impl.SRUCell(2)(x, m) - sess.run([variables_lib.global_variables_initializer()]) - res = sess.run( - [g], - {x.name: np.array([[1., 1., 1.]]), - m.name: np.array([[0.1, 0.1]])}) - # Smoke test - self.assertAllClose(res[0], [[0.40844864, 0.40844864]]) def testBasicLSTMCell(self): for dtype in [dtypes.float16, dtypes.float32]: diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index b5ac4af32b..e5204b925d 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -329,11 +329,7 @@ class SRUCell(RNNCell): f, r = array_ops.split(value=f_r, num_or_size_splits=2, axis=1) c = f * state + (1.0 - f) * x_bar - # For the following equaiton, in the paper, we have: - # h = r * self._activation(c) + (1.0 - r) * inputs - # However, this causes a dimension mismatch when num_units - # is not input_size. - h = r * self._activation(c) + (1.0 - r) * x_bar + h = r * self._activation(c) + (1.0 - r) * inputs return h, c -- GitLab From 391b4a0ec1a2a7a3ef33acbf3aa41590ddc62999 Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Wed, 1 Nov 2017 16:05:16 -0400 Subject: [PATCH 0010/1924] Refactor to BasicLSTM like --- tensorflow/python/ops/rnn_cell_impl.py | 123 +++++++++++++------------ 1 file changed, 65 insertions(+), 58 deletions(-) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 62fd2f4d87..030d190840 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -284,7 +284,46 @@ class BasicRNNCell(RNNCell): output = self._activation(self._linear([inputs, state])) return output, output -class SRUCell(RNNCell): +class _LayerRNNCell(RNNCell): + """Subclass of RNNCells that act like proper `tf.Layer` objects. + + For backwards compatibility purposes, most `RNNCell` instances allow their + `call` methods to instantiate variables via `tf.get_variable`. The underlying + variable scope thus keeps track of any variables, and returning cached + versions. This is atypical of `tf.layer` objects, which separate this + part of layer building into a `build` method that is only called once. + + Here we provide a subclass for `RNNCell` objects that act exactly as + `Layer` objects do. They must provide a `build` method and their + `call` methods do not access Variables `tf.get_variable`. + """ + + def __call__(self, inputs, state, scope=None): + """Run this RNN cell on inputs, starting from the given state. + + Args: + inputs: `2-D` tensor with shape `[batch_size, input_size]`. + state: if `self.state_size` is an integer, this should be a `2-D Tensor` + with shape `[batch_size, self.state_size]`. Otherwise, if + `self.state_size` is a tuple of integers, this should be a tuple + with shapes `[batch_size, s] for s in self.state_size`. + scope: `VariableScope` for the created subgraph; if not provided, + defaults to standard `tf.layers.Layer` behavior. + + Returns: + A pair containing: + + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. + - New state: Either a single `2-D` tensor, or a tuple of tensors matching + the arity and shapes of `state`. + """ + # Bypass RNNCell's variable capturing semantics for LayerRNNCell. + # Instead, it is up to subclasses to provide a proper build + # method. See the class docstring for more details. + return base_layer.Layer.__call__(self, inputs, state, scope=scope) + + +class SRUCell(_LayerRNNCell): """Training RNNs as Fast as CNNs (cf. https://arxiv.org/abs/1709.02755). Args: @@ -319,29 +358,36 @@ class SRUCell(RNNCell): def output_size(self): return self._num_units + def build(self, inputs_shape): + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth, 3 * self._num_units], + initializer=self._kernel_initializer) + + bias_initializer = self._bias_initializer or \ + init_ops.constant_initializer(0.0, dtype=self.dtype) + + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[2 * self._num_units], + initializer=bias_initializer) + + self._built = True + def call(self, inputs, state): - """Gated recurrent unit (SRU) with nunits cells.""" - if self._gate_linear is None: - self._gate_linear = _Linear( - [inputs], - 3 * self._num_units, - False, - kernel_initializer=self._kernel_initializer) + """Simple recurrent unit (SRU) with nunits cells.""" - value = self._gate_linear([inputs]) + U = math_ops.matmul(inputs, self._kernel) x_bar, f_intermediate, r_intermediate = \ - array_ops.split(value=value, num_or_size_splits=3, axis=1) - - if self._bias_initializer is None: - self._bias_initializer = init_ops.constant_initializer(0.0, \ - dtype=inputs.dtype) - _biases = vs.get_variable( - _BIAS_VARIABLE_NAME, [2 * self._num_units], - dtype=inputs.dtype, - initializer=self._bias_initializer) + array_ops.split(value=U, num_or_size_splits=3, axis=1) f_r = math_ops.sigmoid(nn_ops.bias_add(array_ops.concat([f_intermediate, \ - r_intermediate], 1), _biases)) + r_intermediate], 1), self._bias)) f, r = array_ops.split(value=f_r, num_or_size_splits=2, axis=1) c = f * state + (1.0 - f) * x_bar @@ -349,45 +395,6 @@ class SRUCell(RNNCell): return h, c -class _LayerRNNCell(RNNCell): - """Subclass of RNNCells that act like proper `tf.Layer` objects. - - For backwards compatibility purposes, most `RNNCell` instances allow their - `call` methods to instantiate variables via `tf.get_variable`. The underlying - variable scope thus keeps track of any variables, and returning cached - versions. This is atypical of `tf.layer` objects, which separate this - part of layer building into a `build` method that is only called once. - - Here we provide a subclass for `RNNCell` objects that act exactly as - `Layer` objects do. They must provide a `build` method and their - `call` methods do not access Variables `tf.get_variable`. - """ - - def __call__(self, inputs, state, scope=None): - """Run this RNN cell on inputs, starting from the given state. - - Args: - inputs: `2-D` tensor with shape `[batch_size, input_size]`. - state: if `self.state_size` is an integer, this should be a `2-D Tensor` - with shape `[batch_size, self.state_size]`. Otherwise, if - `self.state_size` is a tuple of integers, this should be a tuple - with shapes `[batch_size, s] for s in self.state_size`. - scope: `VariableScope` for the created subgraph; if not provided, - defaults to standard `tf.layers.Layer` behavior. - - Returns: - A pair containing: - - - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. - - New state: Either a single `2-D` tensor, or a tuple of tensors matching - the arity and shapes of `state`. - """ - # Bypass RNNCell's variable capturing semantics for LayerRNNCell. - # Instead, it is up to subclasses to provide a proper build - # method. See the class docstring for more details. - return base_layer.Layer.__call__(self, inputs, state, scope=scope) - - class GRUCell(RNNCell): """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). -- GitLab From 5049da67cd28ef3d78a1734532bd3b4708f1fa56 Mon Sep 17 00:00:00 2001 From: Tian Jin Date: Wed, 1 Nov 2017 16:08:40 -0400 Subject: [PATCH 0011/1924] Refactor to BasicLSTM like --- tensorflow/python/ops/rnn_cell_impl.py | 122 +++++++++++++------------ 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index 62fd2f4d87..fa23ac54c8 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -284,7 +284,45 @@ class BasicRNNCell(RNNCell): output = self._activation(self._linear([inputs, state])) return output, output -class SRUCell(RNNCell): +class _LayerRNNCell(RNNCell): + """Subclass of RNNCells that act like proper `tf.Layer` objects. + + For backwards compatibility purposes, most `RNNCell` instances allow their + `call` methods to instantiate variables via `tf.get_variable`. The underlying + variable scope thus keeps track of any variables, and returning cached + versions. This is atypical of `tf.layer` objects, which separate this + part of layer building into a `build` method that is only called once. + + Here we provide a subclass for `RNNCell` objects that act exactly as + `Layer` objects do. They must provide a `build` method and their + `call` methods do not access Variables `tf.get_variable`. + """ + + def __call__(self, inputs, state, scope=None): + """Run this RNN cell on inputs, starting from the given state. + + Args: + inputs: `2-D` tensor with shape `[batch_size, input_size]`. + state: if `self.state_size` is an integer, this should be a `2-D Tensor` + with shape `[batch_size, self.state_size]`. Otherwise, if + `self.state_size` is a tuple of integers, this should be a tuple + with shapes `[batch_size, s] for s in self.state_size`. + scope: `VariableScope` for the created subgraph; if not provided, + defaults to standard `tf.layers.Layer` behavior. + + Returns: + A pair containing: + + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. + - New state: Either a single `2-D` tensor, or a tuple of tensors matching + the arity and shapes of `state`. + """ + # Bypass RNNCell's variable capturing semantics for LayerRNNCell. + # Instead, it is up to subclasses to provide a proper build + # method. See the class docstring for more details. + return base_layer.Layer.__call__(self, inputs, state, scope=scope) + +class SRUCell(_LayerRNNCell): """Training RNNs as Fast as CNNs (cf. https://arxiv.org/abs/1709.02755). Args: @@ -319,29 +357,36 @@ class SRUCell(RNNCell): def output_size(self): return self._num_units + def build(self, inputs_shape): + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth, 3 * self._num_units], + initializer=self._kernel_initializer) + + bias_initializer = self._bias_initializer or \ + init_ops.constant_initializer(0.0, dtype=self.dtype) + + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[2 * self._num_units], + initializer=bias_initializer) + + self._built = True + def call(self, inputs, state): - """Gated recurrent unit (SRU) with nunits cells.""" - if self._gate_linear is None: - self._gate_linear = _Linear( - [inputs], - 3 * self._num_units, - False, - kernel_initializer=self._kernel_initializer) + """Simple recurrent unit (SRU) with nunits cells.""" - value = self._gate_linear([inputs]) + U = math_ops.matmul(inputs, self._kernel) x_bar, f_intermediate, r_intermediate = \ - array_ops.split(value=value, num_or_size_splits=3, axis=1) - - if self._bias_initializer is None: - self._bias_initializer = init_ops.constant_initializer(0.0, \ - dtype=inputs.dtype) - _biases = vs.get_variable( - _BIAS_VARIABLE_NAME, [2 * self._num_units], - dtype=inputs.dtype, - initializer=self._bias_initializer) + array_ops.split(value=U, num_or_size_splits=3, axis=1) f_r = math_ops.sigmoid(nn_ops.bias_add(array_ops.concat([f_intermediate, \ - r_intermediate], 1), _biases)) + r_intermediate], 1), self._bias)) f, r = array_ops.split(value=f_r, num_or_size_splits=2, axis=1) c = f * state + (1.0 - f) * x_bar @@ -349,45 +394,6 @@ class SRUCell(RNNCell): return h, c -class _LayerRNNCell(RNNCell): - """Subclass of RNNCells that act like proper `tf.Layer` objects. - - For backwards compatibility purposes, most `RNNCell` instances allow their - `call` methods to instantiate variables via `tf.get_variable`. The underlying - variable scope thus keeps track of any variables, and returning cached - versions. This is atypical of `tf.layer` objects, which separate this - part of layer building into a `build` method that is only called once. - - Here we provide a subclass for `RNNCell` objects that act exactly as - `Layer` objects do. They must provide a `build` method and their - `call` methods do not access Variables `tf.get_variable`. - """ - - def __call__(self, inputs, state, scope=None): - """Run this RNN cell on inputs, starting from the given state. - - Args: - inputs: `2-D` tensor with shape `[batch_size, input_size]`. - state: if `self.state_size` is an integer, this should be a `2-D Tensor` - with shape `[batch_size, self.state_size]`. Otherwise, if - `self.state_size` is a tuple of integers, this should be a tuple - with shapes `[batch_size, s] for s in self.state_size`. - scope: `VariableScope` for the created subgraph; if not provided, - defaults to standard `tf.layers.Layer` behavior. - - Returns: - A pair containing: - - - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. - - New state: Either a single `2-D` tensor, or a tuple of tensors matching - the arity and shapes of `state`. - """ - # Bypass RNNCell's variable capturing semantics for LayerRNNCell. - # Instead, it is up to subclasses to provide a proper build - # method. See the class docstring for more details. - return base_layer.Layer.__call__(self, inputs, state, scope=scope) - - class GRUCell(RNNCell): """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). -- GitLab From 40fc0cb0258352b5d00f25bab55a6991b06b959b Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sun, 5 Nov 2017 14:52:42 +0000 Subject: [PATCH 0012/1924] Fix issue in the `Defun` docs This fix fixes a couple of typos in the `Defun` docs: `tf.Constant` -> `tf.constant` Signed-off-by: Yong Tang --- tensorflow/python/framework/function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index cef3f8d4c4..f55ee5b1e1 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -82,8 +82,8 @@ class Defun(object): return x + y, x - y # Building the graph. - a = tf.Constant([1.0]) - b = tf.Constant([2.0]) + a = tf.constant([1.0]) + b = tf.constant([2.0]) c, d = MyFunc(a, b, name='mycall') ``` """ -- GitLab From b739ee1734baf437f1925ea09d07647ec7602caf Mon Sep 17 00:00:00 2001 From: Viraj Date: Fri, 27 Oct 2017 20:38:45 -0700 Subject: [PATCH 0013/1924] svd backpropagation when full_matrices is False --- tensorflow/python/kernel_tests/svd_op_test.py | 32 +++++---- tensorflow/python/ops/linalg_grad.py | 69 +++++++++++-------- 2 files changed, 60 insertions(+), 41 deletions(-) diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py index 9871eacb03..d20567bf0e 100644 --- a/tensorflow/python/kernel_tests/svd_op_test.py +++ b/tensorflow/python/kernel_tests/svd_op_test.py @@ -190,10 +190,11 @@ class SvdGradOpTest(test.TestCase): pass # Filled in below -def _GetSvdGradOpTest(dtype_, shape_, compute_uv_): +def _GetSvdGradOpTest(dtype_, shape_, compute_uv_, full_matrices_): def _NormalizingSvd(tf_a): - tf_s, tf_u, tf_v = linalg_ops.svd(tf_a, compute_uv=True, full_matrices=True) + tf_s, tf_u, tf_v = linalg_ops.svd( + tf_a, compute_uv=True, full_matrices=full_matrices_) # Singular vectors are only unique up to an arbitrary phase. We normalize # the vectors such that the first component of u (if m >=n) or v (if n > m) # have phase 0. @@ -270,17 +271,20 @@ if __name__ == "__main__": _GetSvdOpTest(dtype, shape, use_static_shape, compute_uv, full_matrices)) for compute_uv in False, True: - dtypes = ([np.float32, np.float64] + [np.complex64, np.complex128] * - (not compute_uv)) - for dtype in dtypes: - mat_shapes = ([(10, 11), (11, 10), - (11, 11)] + [(5, 11), (11, 5)] * (not compute_uv)) - for mat_shape in mat_shapes: - for batch_dims in [(), (3,)]: - shape = batch_dims + mat_shape - name = "%s_%s_compute_uv_%s" % (dtype.__name__, - "_".join(map(str, shape)), compute_uv) - _AddTest(SvdGradOpTest, "SvdGrad", name, - _GetSvdGradOpTest(dtype, shape, compute_uv)) + for full_matrices in False, True: + dtypes = ([np.float32, np.float64] + + [np.complex64, np.complex128] * (not compute_uv)) + for dtype in dtypes: + mat_shapes = [(10, 11), (11, 10), (11, 11)] + if not full_matrices or not compute_uv: + mat_shapes += [(5, 11), (11, 5)] + for mat_shape in mat_shapes: + for batch_dims in [(), (3,)]: + shape = batch_dims + mat_shape + name = "%s_%s_compute_uv_%s_full_%s" % ( + dtype.__name__, "_".join(map(str, shape)), compute_uv, + full_matrices) + _AddTest(SvdGradOpTest, "SvdGrad", name, + _GetSvdGradOpTest(dtype, shape, compute_uv, full_matrices)) test.main() diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py index 8a76fe3ce5..1316ed5b54 100644 --- a/tensorflow/python/ops/linalg_grad.py +++ b/tensorflow/python/ops/linalg_grad.py @@ -268,13 +268,14 @@ def _SelfAdjointEigV2Grad(op, grad_e, grad_v): @ops.RegisterGradient("Svd") def _SvdGrad(op, grad_s, grad_u, grad_v): - """Gradient for Svd based on Giles' algorithm. Reference at top of file.""" - - if op.get_attr("compute_uv") and not op.get_attr("full_matrices"): - raise NotImplementedError( - "SVD gradient is not implemented for compute_uv=True and " - "full_matrices=False.") - + """Gradient for the singular value decomposition + + The derivation for the compute_uv=False case, and most of + the derivation for the full_matrices=True case, are in + Giles' paper (see reference at top of file). A derivation for + the full_matrices=False case is available at + https://j-towns.github.io/papers/svd-derivative.pdf + """ a = op.inputs[0] a_shape = a.get_shape().with_rank_at_least(2) @@ -300,7 +301,7 @@ def _SvdGrad(op, grad_s, grad_u, grad_v): "SVD gradient has not been implemented for input with unknown " "inner matrix shape.") - if not op.get_attr("full_matrices") or not op.get_attr("compute_uv"): + if not op.get_attr("compute_uv"): s, u, v = linalg_ops.svd(a, compute_uv=True, full_matrices=True) else: s = op.outputs[0] @@ -334,9 +335,10 @@ def _SvdGrad(op, grad_s, grad_u, grad_v): # multiple singular values with value zero. I am not sure if this is a true # instability or if it simply throws off the finite difference gradient # checker. - if abs(m - n) > 1: + if op.get_attr("full_matrices") and abs(m - n) > 1: raise NotImplementedError( - "svd gradient is not implemented for abs(m - n) > 1") + "svd gradient is not implemented for abs(m - n) > 1 " + "when full_matrices is True") s_mat = array_ops.matrix_diag(s) s2 = math_ops.square(s) @@ -352,32 +354,45 @@ def _SvdGrad(op, grad_s, grad_u, grad_v): array_ops.expand_dims(s2, -2) - array_ops.expand_dims(s2, -1)), array_ops.zeros_like(s)) s_inv_mat = array_ops.matrix_diag(math_ops.reciprocal(s)) + + v1 = v[..., :, :m] + grad_v1 = grad_v[..., :, :m] + u_gu = math_ops.matmul(u, grad_u, adjoint_a=True) - v_gv = math_ops.matmul(v, grad_v, adjoint_a=True) + v_gv = math_ops.matmul(v1, grad_v1, adjoint_a=True) - if m == n: - f_u = f * u_gu - f_v = f * v_gv - else: - dv2 = array_ops.matrix_transpose(v_gv[..., m:n, :m]) - v_gv[..., :m, m:n] - f_u = f * u_gu - f_v = f * v_gv[..., :m, :m] + f_u = f * u_gu + f_v = f * v_gv - grad_a_nouv = ( + term1_nouv = ( grad_s_mat + math_ops.matmul(f_u + _linalg.adjoint(f_u), s_mat) + math_ops.matmul(s_mat, f_v + _linalg.adjoint(f_v))) - if m != n: - grad_a_nouv = array_ops.concat( - [grad_a_nouv, math_ops.matmul(s_inv_mat, dv2)], -1) + term1 = math_ops.matmul(u, math_ops.matmul(term1_nouv, v1, adjoint_b=True)) + + if m == n: + grad_a_before_transpose = term1 + else: + proj_v1_perp = (linalg_ops.eye(n, dtype=v.dtype) + - math_ops.matmul(v1, v1, adjoint_b=True)) + term2_nous = math_ops.matmul(grad_v1, proj_v1_perp, adjoint_a=True) + + if op.get_attr("full_matrices"): + v2 = v[..., :, m:n] + grad_v2 = grad_v[..., :, m:n] + + v1t_gv2 = math_ops.matmul(v1, grad_v2, adjoint_a=True) + term2_nous -= math_ops.matmul(v1t_gv2, v2, adjoint_b=True) + + u_s_inv = math_ops.matmul(u, s_inv_mat) + term2 = math_ops.matmul(u_s_inv, term2_nous) + + grad_a_before_transpose = term1 + term2 if use_adjoint: - # Use (U X V^H)^H = V (U X)^H. - grad_a = math_ops.matmul( - v, math_ops.matmul(u, grad_a_nouv), adjoint_b=True) + grad_a = array_ops.matrix_transpose(grad_a_before_transpose) else: - grad_a = math_ops.matmul(u, - math_ops.matmul(grad_a_nouv, v, adjoint_b=True)) + grad_a = grad_a_before_transpose grad_a.set_shape(a_shape) return grad_a -- GitLab From f552fb90e94ccfb72475327553c968412282eb26 Mon Sep 17 00:00:00 2001 From: Alex Rothberg Date: Tue, 7 Nov 2017 22:04:16 -0500 Subject: [PATCH 0014/1924] update create_train_op to use get_global_step --- tensorflow/contrib/training/python/training/training.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py index 6a4d79796d..59f02fa38f 100644 --- a/tensorflow/contrib/training/python/training/training.py +++ b/tensorflow/contrib/training/python/training/training.py @@ -255,6 +255,7 @@ from tensorflow.python.platform import tf_logging as logging from tensorflow.python.summary import summary from tensorflow.python.training import monitored_session from tensorflow.python.training import optimizer as tf_optimizer +from tensorflow.python.training import training_util # TODO(nsilberman): move add_gradients_summaries, clip_gradient_norms and # multiply_gradients into contrib/summaries and contrib/optimizers.py @@ -409,7 +410,7 @@ def create_train_op(total_loss, loss value. """ if global_step is _USE_GLOBAL_STEP: - global_step = variables.get_or_create_global_step() + global_step = training_util.get_global_step() # Update ops use GraphKeys.UPDATE_OPS collection if update_ops is None. global_update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS)) -- GitLab From d97dcf04a889e5f75c5651ca994f499655564416 Mon Sep 17 00:00:00 2001 From: jfaath Date: Wed, 8 Nov 2017 11:38:59 -0700 Subject: [PATCH 0015/1924] fix indenting on cache check --- tensorflow/python/layers/base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index db608aa79a..220450214c 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -2087,17 +2087,17 @@ class Network(Layer): # Store in cache. self._output_shape_cache[cache_key] = output_shapes - else: - # Cache hit. - output_shapes = self._output_shape_cache[cache_key] + else: + # Cache hit. + output_shapes = self._output_shape_cache[cache_key] - if isinstance(output_shapes, list): - if len(output_shapes) == 1: - return tensor_shape.TensorShape(output_shapes[0]) - else: - return [tensor_shape.TensorShape(shape) for shape in output_shapes] + if isinstance(output_shapes, list): + if len(output_shapes) == 1: + return tensor_shape.TensorShape(output_shapes[0]) else: - return tensor_shape.TensorShape(output_shapes) + return [tensor_shape.TensorShape(shape) for shape in output_shapes] + else: + return tensor_shape.TensorShape(output_shapes) def _run_internal_graph(self, inputs, masks=None): """Computes output tensors for new inputs. -- GitLab From 5de6f68848b8bc431e18a53fa03700820bcee57f Mon Sep 17 00:00:00 2001 From: Cameron Thomas Date: Thu, 9 Nov 2017 01:19:51 +0000 Subject: [PATCH 0016/1924] Forward declare condition_variable Necessary to enable friendship with mutex --- tensorflow/core/platform/default/mutex.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/platform/default/mutex.h b/tensorflow/core/platform/default/mutex.h index c3e44c42d9..044c754e80 100644 --- a/tensorflow/core/platform/default/mutex.h +++ b/tensorflow/core/platform/default/mutex.h @@ -31,6 +31,8 @@ namespace tensorflow { enum LinkerInitialized { LINKER_INITIALIZED }; +class condition_variable; + // Mimic std::mutex + C++17's shared_mutex, adding a LinkerInitialized // constructor interface. This type is as fast as mutex, but is also a shared // lock. -- GitLab From b58ee215e631b9c2a0400cbd5b52ea7a3a8bfca0 Mon Sep 17 00:00:00 2001 From: PW486 Date: Thu, 9 Nov 2017 19:12:41 +0900 Subject: [PATCH 0017/1924] Fixed typos, comments --- tensorflow/core/public/session.h | 2 +- tensorflow/core/util/saved_tensor_slice.proto | 2 +- tensorflow/core/util/strided_slice_op.cc | 4 ++-- tensorflow/core/util/tensor_slice_reader.h | 1 - tensorflow/core/util/tensor_slice_reader_cache.h | 1 - tensorflow/core/util/tensor_slice_writer.h | 1 - 6 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/public/session.h b/tensorflow/core/public/session.h index bca384e59f..75ad50f6f2 100644 --- a/tensorflow/core/public/session.h +++ b/tensorflow/core/public/session.h @@ -186,7 +186,7 @@ class Session { /// the `SessionOptions::target` field). virtual Status Close() = 0; - // NOTE(ashankar): As of July 2017, this method was added to faciliate some + // NOTE(ashankar): As of July 2017, this method was added to facilitate some // experimentation. Reconsider/re-evaluate after September 2017. // // Sets `*output` to the `DeviceMgr` that owns accessible devices in the diff --git a/tensorflow/core/util/saved_tensor_slice.proto b/tensorflow/core/util/saved_tensor_slice.proto index 6278685957..8a6dd7bdb7 100644 --- a/tensorflow/core/util/saved_tensor_slice.proto +++ b/tensorflow/core/util/saved_tensor_slice.proto @@ -1,7 +1,7 @@ // Protocol buffers for saved tensor slices. It's used for the brain tensor // ops checkpoints and the V3 checkpoints in dist_belief. -// A checkpoint file is an sstable. The value for each record is a serialized +// A checkpoint file is a stable. The value for each record is a serialized // SavedTensorSlices message (defined below). // // Each checkpoint file has a record with the empty key (""), which corresponds diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc index cfe9275a09..d5bc676a9a 100644 --- a/tensorflow/core/util/strided_slice_op.cc +++ b/tensorflow/core/util/strided_slice_op.cc @@ -218,8 +218,8 @@ Status ValidateStridedSliceOp( // Step 2: Make a sparse spec into a full index spec // - // The sparse spec does not corresopnds to the number of dimensions - // Make a dense spec that corresponds to thte number of dimensions + // The sparse spec does not corresponds to the number of dimensions + // Make a dense spec that corresponds to the number of dimensions // // For example suppose foo[...,3:] on foo.shape=(2,2,3) then // we need to produce the missing begin_mask for the first two diff --git a/tensorflow/core/util/tensor_slice_reader.h b/tensorflow/core/util/tensor_slice_reader.h index 4bb2b24615..263f56c7fc 100644 --- a/tensorflow/core/util/tensor_slice_reader.h +++ b/tensorflow/core/util/tensor_slice_reader.h @@ -15,7 +15,6 @@ limitations under the License. // The utility to read checkpoints for google brain tensor ops and v3 // checkpoints for dist_belief. -// #ifndef TENSORFLOW_UTIL_TENSOR_SLICE_READER_H_ #define TENSORFLOW_UTIL_TENSOR_SLICE_READER_H_ diff --git a/tensorflow/core/util/tensor_slice_reader_cache.h b/tensorflow/core/util/tensor_slice_reader_cache.h index bdd36a2791..63a8d0b068 100644 --- a/tensorflow/core/util/tensor_slice_reader_cache.h +++ b/tensorflow/core/util/tensor_slice_reader_cache.h @@ -15,7 +15,6 @@ limitations under the License. // The utility to read checkpoints for google brain tensor ops and v3 // checkpoints for dist_belief. -// #ifndef TENSORFLOW_UTIL_TENSOR_SLICE_READER_CACHE_H_ #define TENSORFLOW_UTIL_TENSOR_SLICE_READER_CACHE_H_ diff --git a/tensorflow/core/util/tensor_slice_writer.h b/tensorflow/core/util/tensor_slice_writer.h index 95d6384afe..bdb4921e1b 100644 --- a/tensorflow/core/util/tensor_slice_writer.h +++ b/tensorflow/core/util/tensor_slice_writer.h @@ -15,7 +15,6 @@ limitations under the License. // The utility to write checkpoints for google brain tensor ops and v3 // checkpoints for dist_belief. -// #ifndef TENSORFLOW_UTIL_TENSOR_SLICE_WRITER_H_ #define TENSORFLOW_UTIL_TENSOR_SLICE_WRITER_H_ -- GitLab From c25cd200ddb2728aec1302f655ff220b08d60007 Mon Sep 17 00:00:00 2001 From: MyungJoo Ham Date: Thu, 9 Nov 2017 19:23:07 +0900 Subject: [PATCH 0018/1924] CMake: configure default string values of options properly Because cmake configures defaults values as ON or OFF only, string values as default doesn't work. Thus, when it is set "OFF", we need to re-set the values. Fixes #14400 Signed-off-by: MyungJoo Ham --- tensorflow/contrib/cmake/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt index 77a3fc0c83..846daf3213 100644 --- a/tensorflow/contrib/cmake/CMakeLists.txt +++ b/tensorflow/contrib/cmake/CMakeLists.txt @@ -53,7 +53,15 @@ if (NOT WIN32) set(tensorflow_CUDNN_INCLUDE /usr/include) endif (NOT tensorflow_CUDNN_INCLUDE) option(tensorflow_PATH_CUDNN_STATIC_LIB "Override PATH_STATIC_LIB for libcudnn_static.a" ${tensorflow_PATH_STATIC_LIB}) + if (NOT tensorflow_PATH_CUDNN_STATIC_LIB) + # option's default value is OFF. Fill it with real default values + set (tensorflow_PATH_CUDNN_STATIC_LIB ${tensorflow_PATH_STATIC_LIB}) + endif (NOT tensorflow_PATH_CUDNN_STATIC_LIB) option(tensorflow_PATH_NCCL_STATIC_LIB "Override PATH_STATIC_LIB for libnccl_static.a" ${tensorflow_PATH_STATIC_LIB}) + if (NOT tensorflow_PATH_NCCL_STATIC_LIB) + # option's default value is OFF. Fill it with real default values + set (tensorflow_PATH_NCCL_STATIC_LIB ${tensorflow_PATH_STATIC_LIB}) + endif (NOT tensorflow_PATH_NCCL_STATIC_LIB) option(tensorflow_CUDA_LIBRARY_PATH "Designate the default CUDA library paths" /usr/local/cuda/lib64) if (NOT tensorflow_CUDA_LIBRARY_PATH) # option's default value is OFF. Fill it with real default values -- GitLab From 7a5041df41c5c98b175e57729a4976e78887d6fd Mon Sep 17 00:00:00 2001 From: Zhengsheng Wei Date: Thu, 9 Nov 2017 19:02:46 +0800 Subject: [PATCH 0019/1924] modified document --- tensorflow/python/layers/convolutional.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 8c327d7e27..6ad18a4e25 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -64,8 +64,8 @@ class _Conv(base.Layer): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. -- GitLab From d45f27d4586ef2d2dcc405eaac97b1515dad9671 Mon Sep 17 00:00:00 2001 From: PW486 Date: Thu, 9 Nov 2017 23:05:38 +0900 Subject: [PATCH 0020/1924] Fixed typos, comments --- tensorflow/core/util/saved_tensor_slice.proto | 2 +- tensorflow/core/util/strided_slice_op.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/util/saved_tensor_slice.proto b/tensorflow/core/util/saved_tensor_slice.proto index 8a6dd7bdb7..6278685957 100644 --- a/tensorflow/core/util/saved_tensor_slice.proto +++ b/tensorflow/core/util/saved_tensor_slice.proto @@ -1,7 +1,7 @@ // Protocol buffers for saved tensor slices. It's used for the brain tensor // ops checkpoints and the V3 checkpoints in dist_belief. -// A checkpoint file is a stable. The value for each record is a serialized +// A checkpoint file is an sstable. The value for each record is a serialized // SavedTensorSlices message (defined below). // // Each checkpoint file has a record with the empty key (""), which corresponds diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc index d5bc676a9a..f0264c0a9d 100644 --- a/tensorflow/core/util/strided_slice_op.cc +++ b/tensorflow/core/util/strided_slice_op.cc @@ -218,8 +218,8 @@ Status ValidateStridedSliceOp( // Step 2: Make a sparse spec into a full index spec // - // The sparse spec does not corresponds to the number of dimensions - // Make a dense spec that corresponds to the number of dimensions + // The sparse spec does not correspond to the number of dimensions + // Make a dense spec that correspond to the number of dimensions // // For example suppose foo[...,3:] on foo.shape=(2,2,3) then // we need to produce the missing begin_mask for the first two -- GitLab From 6f6eb52a89ec6e360d8604fa68516cf2d819207f Mon Sep 17 00:00:00 2001 From: PW486 Date: Thu, 9 Nov 2017 23:10:41 +0900 Subject: [PATCH 0021/1924] Fixed typos, comments --- tensorflow/core/util/strided_slice_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/util/strided_slice_op.cc b/tensorflow/core/util/strided_slice_op.cc index f0264c0a9d..aca60b942d 100644 --- a/tensorflow/core/util/strided_slice_op.cc +++ b/tensorflow/core/util/strided_slice_op.cc @@ -219,7 +219,7 @@ Status ValidateStridedSliceOp( // Step 2: Make a sparse spec into a full index spec // // The sparse spec does not correspond to the number of dimensions - // Make a dense spec that correspond to the number of dimensions + // Make a dense spec that corresponds to the number of dimensions // // For example suppose foo[...,3:] on foo.shape=(2,2,3) then // we need to produce the missing begin_mask for the first two -- GitLab From 2c3f70192850a4eee3d190f796948f2324d66821 Mon Sep 17 00:00:00 2001 From: amilioto Date: Thu, 9 Nov 2017 15:28:45 +0100 Subject: [PATCH 0022/1924] added patch needed for jetson tx2/1 --- tensorflow/stream_executor/cuda/cuda_gpu_executor.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index 6c522264e1..a8759376e4 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -866,6 +866,9 @@ static int TryToReadNumaNode(const string &pci_bus_id, int device_ordinal) { #elif defined(PLATFORM_WINDOWS) // Windows support for NUMA is not currently implemented. Return node 0. return 0; +#elif defined(__aarch64__) + LOG(INFO) << "ARM64 does not support NUMA - returning NUMA node zero"; + return 0; #else VLOG(2) << "trying to read NUMA node for device ordinal: " << device_ordinal; static const int kUnknownNumaNode = -1; -- GitLab From c83f5353267104225a938b21317eed3c32bcc6a7 Mon Sep 17 00:00:00 2001 From: Florian Courtial Date: Thu, 9 Nov 2017 22:52:58 +0100 Subject: [PATCH 0023/1924] Add the C++ gradient of the Prod operation. --- tensorflow/cc/gradients/math_grad.cc | 172 ++++++++++++++++++++++ tensorflow/cc/gradients/math_grad_test.cc | 9 ++ 2 files changed, 181 insertions(+) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index d7446b9560..92f451beea 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -794,6 +794,178 @@ Status MinOrMaxGrad(const Scope& scope, const Operation& op, REGISTER_GRADIENT_OP("Min", MinOrMaxGrad); REGISTER_GRADIENT_OP("Max", MinOrMaxGrad); +Status ProdGrad(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { +auto zero = Const(scope, 0); +auto one = Const(scope, 1); + +// The gradient can be expressed by dividing the product by each entry of +// the input tensor. If our input is +// [ +// [3, 4], +// [5, 6], +// [7, 8] +// ] +// and we do a Prod operation on the axis 1, we will obtain [[105, 192]]. +// The gradient will have the same shape as the input +// [ +// [105/3, 192/4], +// dz * [105/5, 192/6], +// [105/7, 192/6] +// ] +// If the input contains a zero, the division is impossible but +// if we take the calculation that gave the first gradient +// (3 * 5 * 6)/3 is equal to 5 * 6 +// the trick will be to cumprod the elements on the axis without +// the element at the current position (3 in the example above). +// We will take as example: +// [ +// [ +// [3.0, 4.0], +// [5.0, 6.0], +// [7.0, 8.0] +// ], +// [ +// [3.0, 5.0], +// [0.0, 6.0], +// [5.0, 6.0] +// ] +// ] + +// [2, 3, 2] +auto input_shape = Shape(scope, op.input(0)); + +// [1] +auto reduction_indices = op.input(1); + +// [2, 1, 2] +auto output_shape_kept_dims = + ReducedShapeHelper(scope, input_shape, reduction_indices); + +// [1, 3, 1] +auto tile_scaling = SafeDivHelper(scope, input_shape, output_shape_kept_dims); + +// [[[105, 192]], [[0, 180]]] +auto grad = Reshape(scope, grad_inputs[0], output_shape_kept_dims); + +// [[[105, 192], [105, 192], [105, 192]], [[0, 180], [0, 180], [0, 180]]] +auto grad_tiled = Tile(scope, grad, tile_scaling); + +Scope cpu_scope = scope.WithDevice("/cpu:0"); + +// [3] +auto rank = Rank(cpu_scope, op.input(0)); + +// [1] +auto reduced = Cast(cpu_scope, reduction_indices, DataType::DT_INT32); + +// [0, 1, 2] +auto idx = Range(cpu_scope, zero, rank, one); + +// [0, 2] +auto other = SetDiff1D(cpu_scope, idx, reduced).out; + +// [1, 0, 2] +auto perm = + Concat(cpu_scope, std::initializer_list{reduced, other}, 0); + +// 3 => [3] +auto reduced_num = Prod(cpu_scope, Gather(scope, input_shape, reduced), 0); + +// 2 * 2 => [2] +auto other_num = Prod(cpu_scope, Gather(scope, input_shape, other), 0); + +// [ +// [ +// [ 3., 4.], +// [ 3., 5.] +// ], +// [ +// [ 5., 6.], +// [ 0., 6.] +// ], +// [ +// [ 7., 8.], +// [ 5., 6.] +// ] +// ] +auto permuted = Transpose(scope, op.input(0), perm); + +// [3, 2, 2] +auto permuted_shape = Shape(scope, permuted); + +// [ +// [ 3., 4., 3., 5.], +// [ 5., 6., 0., 6.], +// [ 7., 8., 5., 6.] +// ] +auto reshaped = Reshape( + scope, permuted, + Stack(scope, std::initializer_list{reduced_num, other_num})); + +// [ +// [ 1., 1., 1., 1.], +// [ 3., 4., 3., 5.], +// [ 15., 24., 0., 30.] +// ] +auto left = Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true)); + +// [ +// [ 35., 48., 0., 36.], +// [ 7., 8., 5., 6.], +// [ 1., 1., 1., 1.] +// ] +auto right = + Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true).Reverse(true)); + +// left * right = +// [ +// [ 35., 48., 0., 36.], +// [ 21., 32., 15., 30.], +// [ 15., 24., 0., 30.] +// ] +// y = +// [ +// [ +// [ 35., 48.], +// [ 0., 36.] +// ], +// [ +// [ 21., 32.], +// [ 15., 30.] +// ], +// [ +// [ 15., 24.], +// [ 0., 30.] +// ] +// ] +auto y = Reshape(scope, Mul(scope, left, right), permuted_shape); + +// out = +// [ +// [ +// [ 35., 48.], +// [ 21., 32.], +// [ 15., 24.] +// ], +// [ +// [ 0., 36.], +// [ 15., 30.], +// [ 0., 30.] +// ] +// ] +auto out = + Mul(scope, grad, Transpose(scope, y, InvertPermutation(scope, perm))); + +grad_outputs->push_back(Reshape(scope, out, input_shape)); + +// stop propagation along reduction_indices +grad_outputs->push_back(NoGradient()); +return scope.status(); +} +REGISTER_GRADIENT_OP("Prod", ProdGrad); + // MatMulGrad helper function used to compute two MatMul operations // based on input matrix transposition combinations. Status MatMulGradHelper(const Scope& scope, const bool is_batch, diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc index 6313f41da5..512a29660f 100644 --- a/tensorflow/cc/gradients/math_grad_test.cc +++ b/tensorflow/cc/gradients/math_grad_test.cc @@ -865,5 +865,14 @@ TEST_F(NaryGradTest, Minimum) { RunTest(x, x_init_value, y, shape); } +TEST_F(NaryGradTest, Prod) { + TensorShape x_shape({2, 3, 2}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto y = Prod(scope_, x, {1}); + // y's shape is the result of reducing x along axes 1 + TensorShape y_shape({2, 1, 2}); + RunTest({x}, {x_shape}, {y}, {y_shape}); +} + } // namespace } // namespace tensorflow -- GitLab From d7860eb053e4f91de010e923ebad23883fdd832f Mon Sep 17 00:00:00 2001 From: Florian Courtial Date: Fri, 10 Nov 2017 00:00:02 +0100 Subject: [PATCH 0024/1924] Use of grad_tiled. --- tensorflow/cc/gradients/math_grad.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index 92f451beea..d73ff95761 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -956,7 +956,7 @@ auto y = Reshape(scope, Mul(scope, left, right), permuted_shape); // ] // ] auto out = - Mul(scope, grad, Transpose(scope, y, InvertPermutation(scope, perm))); + Mul(scope, grad_tiled, Transpose(scope, y, InvertPermutation(scope, perm))); grad_outputs->push_back(Reshape(scope, out, input_shape)); -- GitLab From 589279dab6259e578a94d5eb7b2e7498d459c8ec Mon Sep 17 00:00:00 2001 From: Zhengsheng Wei Date: Fri, 10 Nov 2017 09:54:12 +0800 Subject: [PATCH 0025/1924] modify relevant docstring(s) of convolution. Include con2d, con3d and their relevant. --- .../python/layers/core_layers.py | 8 ++-- tensorflow/python/layers/convolutional.py | 48 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tensorflow/contrib/model_pruning/python/layers/core_layers.py b/tensorflow/contrib/model_pruning/python/layers/core_layers.py index ae60d8b1e1..95dfd8f421 100644 --- a/tensorflow/contrib/model_pruning/python/layers/core_layers.py +++ b/tensorflow/contrib/model_pruning/python/layers/core_layers.py @@ -72,8 +72,8 @@ class _MaskedConv(base.Layer): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Regularizer function for the output. @@ -279,8 +279,8 @@ class MaskedConv2D(_MaskedConv): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Regularizer function for the output. diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 6ad18a4e25..ea3c0de5e1 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -254,8 +254,8 @@ class Conv1D(_Conv): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -362,8 +362,8 @@ def conv1d(inputs, linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -450,8 +450,8 @@ class Conv2D(_Conv): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -565,8 +565,8 @@ def conv2d(inputs, linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -654,8 +654,8 @@ class Conv3D(_Conv): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -770,8 +770,8 @@ def conv3d(inputs, linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -860,8 +860,8 @@ class SeparableConv2D(Conv2D): use_bias: Boolean, whether the layer uses a bias. depthwise_initializer: An initializer for the depthwise convolution kernel. pointwise_initializer: An initializer for the pointwise convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. depthwise_regularizer: Optional regularizer for the depthwise convolution kernel. pointwise_regularizer: Optional regularizer for the pointwise @@ -1088,8 +1088,8 @@ def separable_conv2d(inputs, use_bias: Boolean, whether the layer uses a bias. depthwise_initializer: An initializer for the depthwise convolution kernel. pointwise_initializer: An initializer for the pointwise convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. depthwise_regularizer: Optional regularizer for the depthwise convolution kernel. pointwise_regularizer: Optional regularizer for the pointwise @@ -1174,8 +1174,8 @@ class Conv2DTranspose(Conv2D): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -1391,8 +1391,8 @@ def conv2d_transpose(inputs, linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If `None`, then no - bias will be applied. + bias_initializer: An initializer for the bias vector. If `None`, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -1464,8 +1464,8 @@ class Conv3DTranspose(Conv3D): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If `None`, then no - bias will be applied. + bias_initializer: An initializer for the bias vector. If `None`, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. @@ -1705,8 +1705,8 @@ def conv3d_transpose(inputs, linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, no bias will - be applied. + bias_initializer: An initializer for the bias vector. If None, the default + initializer will be used. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Optional regularizer function for the output. -- GitLab From 70c79c257c9f4a15252c678a402445bdaeceef2c Mon Sep 17 00:00:00 2001 From: Zhengsheng Wei Date: Fri, 10 Nov 2017 10:12:29 +0800 Subject: [PATCH 0026/1924] restore docstring of core_layers --- .../contrib/model_pruning/python/layers/core_layers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/contrib/model_pruning/python/layers/core_layers.py b/tensorflow/contrib/model_pruning/python/layers/core_layers.py index 95dfd8f421..ae60d8b1e1 100644 --- a/tensorflow/contrib/model_pruning/python/layers/core_layers.py +++ b/tensorflow/contrib/model_pruning/python/layers/core_layers.py @@ -72,8 +72,8 @@ class _MaskedConv(base.Layer): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. + bias_initializer: An initializer for the bias vector. If None, no bias will + be applied. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Regularizer function for the output. @@ -279,8 +279,8 @@ class MaskedConv2D(_MaskedConv): linear activation. use_bias: Boolean, whether the layer uses a bias. kernel_initializer: An initializer for the convolution kernel. - bias_initializer: An initializer for the bias vector. If None, the default - initializer will be used. + bias_initializer: An initializer for the bias vector. If None, no bias will + be applied. kernel_regularizer: Optional regularizer for the convolution kernel. bias_regularizer: Optional regularizer for the bias vector. activity_regularizer: Regularizer function for the output. -- GitLab From 17626168cb05e9edc6cbbd57d04c1da8a43ecfb2 Mon Sep 17 00:00:00 2001 From: PW486 Date: Fri, 10 Nov 2017 12:33:44 +0900 Subject: [PATCH 0027/1924] Fixed typos, comments --- tensorflow/contrib/batching/shared_batch_scheduler.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/batching/shared_batch_scheduler.h b/tensorflow/contrib/batching/shared_batch_scheduler.h index 41a3f99137..1853827dc0 100644 --- a/tensorflow/contrib/batching/shared_batch_scheduler.h +++ b/tensorflow/contrib/batching/shared_batch_scheduler.h @@ -63,7 +63,7 @@ namespace serving { // instead of N independent ones, with their sharing deliberately coordinated. // // SharedBatchScheduler does not implement the BatchScheduler API; rather, it -// presents an abstraction of "queues", where each queue coresponds to one type +// presents an abstraction of "queues", where each queue corresponds to one type // of task. Tasks submitted to a given queue are placed in their own batches, // and cannot be mixed with other tasks. Queues can be added and deleted // dynamically, to accommodate e.g. versions of a model being brought up and -- GitLab From e058a030f88f19a60e3a4d5ed6b5cbcf85b1a5d6 Mon Sep 17 00:00:00 2001 From: PW486 Date: Fri, 10 Nov 2017 14:19:31 +0900 Subject: [PATCH 0028/1924] Fixed typos --- tensorflow/c/c_test_util.h | 2 +- tensorflow/compiler/xla/client/computation_builder.h | 2 +- tensorflow/contrib/boosted_trees/lib/utils/batch_features.h | 2 +- tensorflow/core/grappler/costs/virtual_placer.h | 2 +- tensorflow/python/util/util.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/c/c_test_util.h b/tensorflow/c/c_test_util.h index d547337492..bc44a7b840 100644 --- a/tensorflow/c/c_test_util.h +++ b/tensorflow/c/c_test_util.h @@ -74,7 +74,7 @@ TF_Operation* Neg(TF_Operation* n, TF_Graph* graph, TF_Status* s, TF_Operation* LessThan(TF_Output l, TF_Output r, TF_Graph* graph, TF_Status* s); -// Split `input` along the first dimention into 3 tensors +// Split `input` along the first dimension into 3 tensors TF_Operation* Split3(TF_Operation* input, TF_Graph* graph, TF_Status* s, const char* name = "split3"); diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index 8e1b4be1f3..9159b26614 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -806,7 +806,7 @@ class ComputationBuilder { // The operand must represent a constant value, which in this case // means that it must not statically depend on any parameter of the // computation that is being built other then the ones specified on the - // paramtere list. The parameters in the list will be indexed by their + // parameter list. The parameters in the list will be indexed by their // parameter id property so the number of parameters specified should be at // least as many as the largest used parameter index. // diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h index 7a550d6f73..badc629a11 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h @@ -56,7 +56,7 @@ class BatchFeatures { *num_sparse_int_features = sparse_int_feature_columns_.size(); if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 && *num_sparse_int_features == 0) { - return errors::FailedPrecondition("Not intialized yet."); + return errors::FailedPrecondition("Not initialized yet."); } return Status::OK(); } diff --git a/tensorflow/core/grappler/costs/virtual_placer.h b/tensorflow/core/grappler/costs/virtual_placer.h index 7ccb1ebb99..fee5ce0f51 100644 --- a/tensorflow/core/grappler/costs/virtual_placer.h +++ b/tensorflow/core/grappler/costs/virtual_placer.h @@ -41,7 +41,7 @@ class VirtualPlacer { private: // Converts given device name to Lowercase Fully-Qualified Name (LFQN) string. // This helps us disambiguate device names internally and simplify matching. - // If device_name couldn't be parsed succesfully, returns empty string. + // If device_name couldn't be parsed successfully, returns empty string. string to_lfqn_or_empty(const string& device_name) const; // Map based on the cluster info: cluster device name -> device properties. diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc index c3d7611ad4..a41fa7df25 100644 --- a/tensorflow/python/util/util.cc +++ b/tensorflow/python/util/util.cc @@ -29,7 +29,7 @@ bool WarnedThatSetIsNotSequence = false; // Returns 1 if `o` is considered a sequence for the purposes of Flatten(). // Returns 0 otherwise. -// Returns -1 if an error occured. +// Returns -1 if an error occurred. int IsSequenceHelper(PyObject* o) { if (PyDict_Check(o)) return true; if (PySet_Check(o) && !WarnedThatSetIsNotSequence) { -- GitLab From 736718d7acdae6f66b3726561048eaaa4eefb458 Mon Sep 17 00:00:00 2001 From: CSJY Date: Sat, 11 Nov 2017 12:32:18 +0800 Subject: [PATCH 0029/1924] remove redundant code --- tensorflow/core/kernels/slice_op.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/kernels/slice_op.cc b/tensorflow/core/kernels/slice_op.cc index 28a379774b..2faa1a24b1 100644 --- a/tensorflow/core/kernels/slice_op.cc +++ b/tensorflow/core/kernels/slice_op.cc @@ -262,7 +262,6 @@ class MklSliceOp : public OpKernel { HANDLE_DIM(1); HANDLE_DIM(2); HANDLE_DIM(3); - HANDLE_DIM(4); HANDLE_DIM(5); HANDLE_DIM(6); -- GitLab From 74265fefa38897e015e25cc3f4b9d77e6e430823 Mon Sep 17 00:00:00 2001 From: Florian Courtial Date: Sun, 12 Nov 2017 10:33:30 +0100 Subject: [PATCH 0030/1924] Add reshape -1 and converts negative reduction indices. --- tensorflow/cc/gradients/math_grad.cc | 337 ++++++++++++++------------- 1 file changed, 171 insertions(+), 166 deletions(-) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index d73ff95761..3d8e4d66aa 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -797,172 +797,177 @@ REGISTER_GRADIENT_OP("Max", MinOrMaxGrad); Status ProdGrad(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { -auto zero = Const(scope, 0); -auto one = Const(scope, 1); - -// The gradient can be expressed by dividing the product by each entry of -// the input tensor. If our input is -// [ -// [3, 4], -// [5, 6], -// [7, 8] -// ] -// and we do a Prod operation on the axis 1, we will obtain [[105, 192]]. -// The gradient will have the same shape as the input -// [ -// [105/3, 192/4], -// dz * [105/5, 192/6], -// [105/7, 192/6] -// ] -// If the input contains a zero, the division is impossible but -// if we take the calculation that gave the first gradient -// (3 * 5 * 6)/3 is equal to 5 * 6 -// the trick will be to cumprod the elements on the axis without -// the element at the current position (3 in the example above). -// We will take as example: -// [ -// [ -// [3.0, 4.0], -// [5.0, 6.0], -// [7.0, 8.0] -// ], -// [ -// [3.0, 5.0], -// [0.0, 6.0], -// [5.0, 6.0] -// ] -// ] - -// [2, 3, 2] -auto input_shape = Shape(scope, op.input(0)); - -// [1] -auto reduction_indices = op.input(1); - -// [2, 1, 2] -auto output_shape_kept_dims = - ReducedShapeHelper(scope, input_shape, reduction_indices); - -// [1, 3, 1] -auto tile_scaling = SafeDivHelper(scope, input_shape, output_shape_kept_dims); - -// [[[105, 192]], [[0, 180]]] -auto grad = Reshape(scope, grad_inputs[0], output_shape_kept_dims); - -// [[[105, 192], [105, 192], [105, 192]], [[0, 180], [0, 180], [0, 180]]] -auto grad_tiled = Tile(scope, grad, tile_scaling); - -Scope cpu_scope = scope.WithDevice("/cpu:0"); - -// [3] -auto rank = Rank(cpu_scope, op.input(0)); - -// [1] -auto reduced = Cast(cpu_scope, reduction_indices, DataType::DT_INT32); - -// [0, 1, 2] -auto idx = Range(cpu_scope, zero, rank, one); - -// [0, 2] -auto other = SetDiff1D(cpu_scope, idx, reduced).out; - -// [1, 0, 2] -auto perm = - Concat(cpu_scope, std::initializer_list{reduced, other}, 0); - -// 3 => [3] -auto reduced_num = Prod(cpu_scope, Gather(scope, input_shape, reduced), 0); - -// 2 * 2 => [2] -auto other_num = Prod(cpu_scope, Gather(scope, input_shape, other), 0); - -// [ -// [ -// [ 3., 4.], -// [ 3., 5.] -// ], -// [ -// [ 5., 6.], -// [ 0., 6.] -// ], -// [ -// [ 7., 8.], -// [ 5., 6.] -// ] -// ] -auto permuted = Transpose(scope, op.input(0), perm); - -// [3, 2, 2] -auto permuted_shape = Shape(scope, permuted); - -// [ -// [ 3., 4., 3., 5.], -// [ 5., 6., 0., 6.], -// [ 7., 8., 5., 6.] -// ] -auto reshaped = Reshape( - scope, permuted, - Stack(scope, std::initializer_list{reduced_num, other_num})); - -// [ -// [ 1., 1., 1., 1.], -// [ 3., 4., 3., 5.], -// [ 15., 24., 0., 30.] -// ] -auto left = Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true)); - -// [ -// [ 35., 48., 0., 36.], -// [ 7., 8., 5., 6.], -// [ 1., 1., 1., 1.] -// ] -auto right = - Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true).Reverse(true)); - -// left * right = -// [ -// [ 35., 48., 0., 36.], -// [ 21., 32., 15., 30.], -// [ 15., 24., 0., 30.] -// ] -// y = -// [ -// [ -// [ 35., 48.], -// [ 0., 36.] -// ], -// [ -// [ 21., 32.], -// [ 15., 30.] -// ], -// [ -// [ 15., 24.], -// [ 0., 30.] -// ] -// ] -auto y = Reshape(scope, Mul(scope, left, right), permuted_shape); - -// out = -// [ -// [ -// [ 35., 48.], -// [ 21., 32.], -// [ 15., 24.] -// ], -// [ -// [ 0., 36.], -// [ 15., 30.], -// [ 0., 30.] -// ] -// ] -auto out = - Mul(scope, grad_tiled, Transpose(scope, y, InvertPermutation(scope, perm))); - -grad_outputs->push_back(Reshape(scope, out, input_shape)); - -// stop propagation along reduction_indices -grad_outputs->push_back(NoGradient()); -return scope.status(); + auto zero = Const(scope, 0); + auto one = Const(scope, 1); + + // The gradient can be expressed by dividing the product by each entry of + // the input tensor. If our input is + // [ + // [3, 4], + // [5, 6], + // [7, 8] + // ] + // and we do a Prod operation on the axis 1, we will obtain [[105, 192]]. + // The gradient will have the same shape as the input + // [ + // [105/3, 192/4], + // dz * [105/5, 192/6], + // [105/7, 192/6] + // ] + // If the input contains a zero, the division is impossible but + // if we take the calculation that gave the first gradient + // (3 * 5 * 6)/3 is equal to 5 * 6 + // the trick will be to cumprod the elements on the axis without + // the element at the current position (3 in the example above). + // We will take as example: + // [ + // [ + // [3.0, 4.0], + // [5.0, 6.0], + // [7.0, 8.0] + // ], + // [ + // [3.0, 5.0], + // [0.0, 6.0], + // [5.0, 6.0] + // ] + // ] + + // [2, 3, 2] + auto input_shape = Shape(scope, op.input(0)); + + // The Reshape with -1 flattens the reduction indices. + // [1] + auto reduction_indices = Reshape(scope, op.input(1), {-1}); + + // [2, 1, 2] + auto output_shape_kept_dims = + ReducedShapeHelper(scope, input_shape, reduction_indices); + + // [1, 3, 1] + auto tile_scaling = SafeDivHelper(scope, input_shape, output_shape_kept_dims); + + // [[[105, 192]], [[0, 180]]] + auto grad = Reshape(scope, grad_inputs[0], output_shape_kept_dims); + + // [[[105, 192], [105, 192], [105, 192]], [[0, 180], [0, 180], [0, 180]]] + auto grad_tiled = Tile(scope, grad, tile_scaling); + + Scope cpu_scope = scope.WithDevice("/cpu:0"); + + // [3] + auto rank = Rank(cpu_scope, op.input(0)); + + + // Normalize any negative indices in the reduction_axes to positive values. + auto reduction_indices_pos = Mod(scope, Add(scope, reduction_indices, rank), rank); + + // [1] + auto reduced = Cast(cpu_scope, reduction_indices_pos, DataType::DT_INT32); + + // [0, 1, 2] + auto idx = Range(cpu_scope, zero, rank, one); + + // [0, 2] + auto other = SetDiff1D(cpu_scope, idx, reduced).out; + + // [1, 0, 2] + auto perm = + Concat(cpu_scope, std::initializer_list{reduced, other}, 0); + + // 3 => [3] + auto reduced_num = Prod(cpu_scope, Gather(scope, input_shape, reduced), 0); + + // 2 * 2 => [2] + auto other_num = Prod(cpu_scope, Gather(scope, input_shape, other), 0); + + // [ + // [ + // [ 3., 4.], + // [ 3., 5.] + // ], + // [ + // [ 5., 6.], + // [ 0., 6.] + // ], + // [ + // [ 7., 8.], + // [ 5., 6.] + // ] + // ] + auto permuted = Transpose(scope, op.input(0), perm); + + // [3, 2, 2] + auto permuted_shape = Shape(scope, permuted); + + // [ + // [ 3., 4., 3., 5.], + // [ 5., 6., 0., 6.], + // [ 7., 8., 5., 6.] + // ] + auto reshaped = Reshape( + scope, permuted, + Stack(scope, std::initializer_list{reduced_num, other_num})); + + // [ + // [ 1., 1., 1., 1.], + // [ 3., 4., 3., 5.], + // [ 15., 24., 0., 30.] + // ] + auto left = Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true)); + + // [ + // [ 35., 48., 0., 36.], + // [ 7., 8., 5., 6.], + // [ 1., 1., 1., 1.] + // ] + auto right = + Cumprod(scope, reshaped, zero, Cumprod::Exclusive(true).Reverse(true)); + + // left * right = + // [ + // [ 35., 48., 0., 36.], + // [ 21., 32., 15., 30.], + // [ 15., 24., 0., 30.] + // ] + // y = + // [ + // [ + // [ 35., 48.], + // [ 0., 36.] + // ], + // [ + // [ 21., 32.], + // [ 15., 30.] + // ], + // [ + // [ 15., 24.], + // [ 0., 30.] + // ] + // ] + auto y = Reshape(scope, Mul(scope, left, right), permuted_shape); + + // out = + // [ + // [ + // [ 35., 48.], + // [ 21., 32.], + // [ 15., 24.] + // ], + // [ + // [ 0., 36.], + // [ 15., 30.], + // [ 0., 30.] + // ] + // ] + auto out = + Mul(scope, grad_tiled, Transpose(scope, y, InvertPermutation(scope, perm))); + + grad_outputs->push_back(Reshape(scope, out, input_shape)); + + // stop propagation along reduction_indices + grad_outputs->push_back(NoGradient()); + return scope.status(); } REGISTER_GRADIENT_OP("Prod", ProdGrad); -- GitLab From cbdae0cf40b067db45c7d7fadb44bd1e126aee55 Mon Sep 17 00:00:00 2001 From: Florian Courtial Date: Sun, 12 Nov 2017 10:36:32 +0100 Subject: [PATCH 0031/1924] Formatting. --- tensorflow/cc/gradients/math_grad.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index 3d8e4d66aa..da8bae596f 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -795,8 +795,8 @@ REGISTER_GRADIENT_OP("Min", MinOrMaxGrad); REGISTER_GRADIENT_OP("Max", MinOrMaxGrad); Status ProdGrad(const Scope& scope, const Operation& op, - const std::vector& grad_inputs, - std::vector* grad_outputs) { + const std::vector& grad_inputs, + std::vector* grad_outputs) { auto zero = Const(scope, 0); auto one = Const(scope, 1); -- GitLab From 95ed2e833abd80727164270fdc299e99ab86ffaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Mon, 13 Nov 2017 13:35:14 +0800 Subject: [PATCH 0032/1924] TST: add test case --- .../python/kernel_tests/lookup_ops_test.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py index 76c790a0a2..11778d8ddb 100644 --- a/tensorflow/python/kernel_tests/lookup_ops_test.py +++ b/tensorflow/python/kernel_tests/lookup_ops_test.py @@ -573,15 +573,19 @@ class IndexToStringTableFromFileTest(test.TestCase): return vocabulary_file def test_index_to_string_table(self): - vocabulary_file = self._createVocabFile("i2f_vocab1.txt") - with self.test_session(): - table = lookup_ops.index_to_string_table_from_file( - vocabulary_file=vocabulary_file) - features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64)) - self.assertRaises(errors_impl.OpError, features.eval) - lookup_ops.tables_initializer().run() - self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), - features.eval()) + vocabulary_path = self._createVocabFile("i2f_vocab1.txt") + # vocabulary_file supports string and tensor + type_funcs = [str, constant_op.constant] + for type_func in type_funcs: + vocabulary_file = type_func(vocabulary_path) + with self.test_session(): + table = lookup_ops.index_to_string_table_from_file( + vocabulary_file=vocabulary_file) + features = table.lookup(constant_op.constant([0, 1, 2, 3], dtypes.int64)) + self.assertRaises(errors_impl.OpError, features.eval) + lookup_ops.tables_initializer().run() + self.assertAllEqual((b"brain", b"salad", b"surgery", b"UNK"), + features.eval()) def test_index_to_string_table_with_default_value(self): default_value = b"NONE" -- GitLab From 603a2f3db38753cb4281f367f413e8c1975835f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Mon, 13 Nov 2017 13:37:00 +0800 Subject: [PATCH 0033/1924] BUG: don't check tensor --- tensorflow/python/ops/lookup_ops.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index fa58ffc37e..10b7cd7001 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -1123,8 +1123,10 @@ def index_to_string_table_from_file(vocabulary_file, ValueError: when `vocabulary_file` is empty. ValueError: when `vocab_size` is invalid. """ - if not vocabulary_file: - raise ValueError("vocabulary_file must be specified.") + if vocabulary_file is None or ( + isinstance(vocabulary_file, str) and not vocabulary_file): + raise ValueError("vocabulary_file must be specified and must not be empty.") + if vocab_size is not None and vocab_size < 1: raise ValueError("vocab_size must be greater than 0, got %d." % vocab_size) -- GitLab From bd1074ab5d2bc87d4fc37e9f6941dc138a3fb961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Mon, 13 Nov 2017 13:39:33 +0800 Subject: [PATCH 0034/1924] DOC: add docment --- tensorflow/python/ops/lookup_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index 10b7cd7001..cb5e9d08c0 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -1110,7 +1110,7 @@ def index_to_string_table_from_file(vocabulary_file, ``` Args: - vocabulary_file: The vocabulary filename. + vocabulary_file: The vocabulary filename, may be a constant scalar `Tensor`. vocab_size: Number of the elements in the vocabulary, if known. default_value: The value to use for out-of-vocabulary indices. name: A name for this op (optional). -- GitLab From 9e966e9e540d245950dcfccdb982304dac740294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Mon, 13 Nov 2017 13:45:21 +0800 Subject: [PATCH 0035/1924] ENH: use six.string_types --- tensorflow/python/ops/lookup_ops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index cb5e9d08c0..c489a8ab6b 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -20,6 +20,7 @@ from __future__ import print_function import collections import functools +import six from tensorflow.python.eager import context from tensorflow.python.framework import constant_op @@ -922,7 +923,7 @@ def index_table_from_file(vocabulary_file=None, than zero. """ if vocabulary_file is None or ( - isinstance(vocabulary_file, str) and not vocabulary_file): + isinstance(vocabulary_file, six.string_types) and not vocabulary_file): raise ValueError("vocabulary_file must be specified and must not be empty.") if num_oov_buckets < 0: raise ValueError("num_oov_buckets must be greater or equal than 0, got %d." @@ -1124,7 +1125,7 @@ def index_to_string_table_from_file(vocabulary_file, ValueError: when `vocab_size` is invalid. """ if vocabulary_file is None or ( - isinstance(vocabulary_file, str) and not vocabulary_file): + isinstance(vocabulary_file, six.string_types) and not vocabulary_file): raise ValueError("vocabulary_file must be specified and must not be empty.") if vocab_size is not None and vocab_size < 1: -- GitLab From 6c1ab6d34213057f5d70d194094ff48137815ae3 Mon Sep 17 00:00:00 2001 From: Daniel Zhang Date: Mon, 13 Nov 2017 14:43:54 +0800 Subject: [PATCH 0036/1924] Fix control input name for quantize node --- tensorflow/tools/graph_transforms/quantize_nodes.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/graph_transforms/quantize_nodes.cc b/tensorflow/tools/graph_transforms/quantize_nodes.cc index 97e8f77616..5ccd88cfa1 100644 --- a/tensorflow/tools/graph_transforms/quantize_nodes.cc +++ b/tensorflow/tools/graph_transforms/quantize_nodes.cc @@ -759,7 +759,7 @@ Status QuantizeNodes(const GraphDef& input_graph_def, NodeDef reshape_dims; reshape_dims.set_op("Const"); reshape_dims.set_name(unique_input_name + "/reshape_dims"); - AddNodeInput("^" + input_name, &reshape_dims); + AddNodeInput("^" + NodeNameFromInput(input_name), &reshape_dims); SetNodeAttr("dtype", DT_INT32, &reshape_dims); Tensor reshape_dims_tensor(DT_INT32, {1}); reshape_dims_tensor.flat()(0) = -1; @@ -769,7 +769,7 @@ Status QuantizeNodes(const GraphDef& input_graph_def, NodeDef reduction_dims; reduction_dims.set_op("Const"); reduction_dims.set_name(unique_input_name + "/reduction_dims"); - AddNodeInput("^" + input_name, &reduction_dims); + AddNodeInput("^" + NodeNameFromInput(input_name), &reduction_dims); SetNodeAttr("dtype", DT_INT32, &reduction_dims); Tensor reduction_dims_tensor(DT_INT32, {1}); reduction_dims_tensor.flat()(0) = 0; -- GitLab From cedb85f2cbda30b9dada94930af9ba40bbbdcf86 Mon Sep 17 00:00:00 2001 From: TTrapper Date: Tue, 14 Nov 2017 12:41:15 -0400 Subject: [PATCH 0037/1924] Removing labels_as_indices logic from _compute_sampled_logits. Now computing 0-index labels in sampled_sparse_softmax_loss. --- .../contrib/nn/python/ops/sampling_ops.py | 7 ++-- tensorflow/python/ops/nn_impl.py | 33 +++++++------------ 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops.py b/tensorflow/contrib/nn/python/ops/sampling_ops.py index b26da52f01..02aa1efc5a 100644 --- a/tensorflow/contrib/nn/python/ops/sampling_ops.py +++ b/tensorflow/contrib/nn/python/ops/sampling_ops.py @@ -318,7 +318,7 @@ def sampled_sparse_softmax_loss(weights, A `batch_size` 1-D tensor of per-example sampled softmax losses. """ - logits, labels = nn_impl._compute_sampled_logits( + logits, _ = nn_impl._compute_sampled_logits( weights=weights, biases=biases, labels=labels, @@ -330,9 +330,12 @@ def sampled_sparse_softmax_loss(weights, subtract_log_q=True, remove_accidental_hits=remove_accidental_hits, partition_strategy=partition_strategy, - labels_as_indices=True, name=name) + # There is only one true label. _compute_sampled_logits puts the true logit + # at index 0. + labels = tf.zeros([array_ops.shape(logits)[0], 1], dtype=dtypes.int64) + sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( labels=array_ops.squeeze(labels), logits=logits) # sampled_losses is a [batch_size] tensor. diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index 8e64259143..2bf5514c64 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -894,7 +894,6 @@ def _compute_sampled_logits(weights, subtract_log_q=True, remove_accidental_hits=False, partition_strategy="mod", - labels_as_indices=False, name=None): """Helper function for nce_loss and sampled_softmax_loss functions. @@ -932,18 +931,13 @@ def _compute_sampled_logits(weights, partition_strategy: A string specifying the partitioning strategy, relevant if `len(weights) > 1`. Currently `"div"` and `"mod"` are supported. Default is `"mod"`. See `tf.nn.embedding_lookup` for more details. - labels_as_indices: A `bool`. Whether the returned labels represent the - indices of the true classes. Default is `False`. name: A name for the operation (optional). Returns: out_logits: `Tensor` object with shape `[batch_size, num_true + num_sampled]`, for passing to either `nn.sigmoid_cross_entropy_with_logits` (NCE) or `nn.softmax_cross_entropy_with_logits` (sampled softmax). - out_labels: If `labels_as_indices` is `False`, a Tensor object with the same - shape as `out_logits`. Otherwise a `Tensor` of shape - `[batch_size, num_true]` with the indices of the target classes for each - row of `out_logits`. + out_labels: A Tensor object with the same shape as `out_logits`. """ if isinstance(weights, variables.PartitionedVariable): @@ -1054,21 +1048,16 @@ def _compute_sampled_logits(weights, # Construct output logits and labels. The true labels/logits start at col 0. out_logits = array_ops.concat([true_logits, sampled_logits], 1) - if labels_as_indices: - # We want each row of labels to be the indices of the targets, which - # start at col 0 and end at col num_true-1. - out_labels = gen_array_ops.tile( - [math_ops.range(num_true)], [array_ops.shape(true_logits)[0], 1]) - else: - # true_logits is a float tensor, ones_like(true_logits) is a float - # tensor of ones. We then divide by num_true to ensure the per-example - # labels sum to 1.0, i.e. form a proper probability distribution. - out_labels = array_ops.concat([ - array_ops.ones_like(true_logits) / num_true, - array_ops.zeros_like(sampled_logits) - ], 1) - - return out_logits, out_labels + + # true_logits is a float tensor, ones_like(true_logits) is a float + # tensor of ones. We then divide by num_true to ensure the per-example + # labels sum to 1.0, i.e. form a proper probability distribution. + out_labels = array_ops.concat([ + array_ops.ones_like(true_logits) / num_true, + array_ops.zeros_like(sampled_logits) + ], 1) + + return out_logits, out_labels def nce_loss(weights, -- GitLab From 7ba5810c105640f218993d989142d7e91da6703e Mon Sep 17 00:00:00 2001 From: TTrapper Date: Tue, 14 Nov 2017 13:48:29 -0400 Subject: [PATCH 0038/1924] calling array_ops instead of erroneus tf --- tensorflow/contrib/nn/python/ops/sampling_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops.py b/tensorflow/contrib/nn/python/ops/sampling_ops.py index 02aa1efc5a..ca719ccaf3 100644 --- a/tensorflow/contrib/nn/python/ops/sampling_ops.py +++ b/tensorflow/contrib/nn/python/ops/sampling_ops.py @@ -334,7 +334,7 @@ def sampled_sparse_softmax_loss(weights, # There is only one true label. _compute_sampled_logits puts the true logit # at index 0. - labels = tf.zeros([array_ops.shape(logits)[0], 1], dtype=dtypes.int64) + labels = array_ops.zeros([array_ops.shape(logits)[0], 1], dtype=dtypes.int64) sampled_losses = nn_ops.sparse_softmax_cross_entropy_with_logits( labels=array_ops.squeeze(labels), logits=logits) -- GitLab From ec4cac9b0cc81a8a47f5ae25d53a0320e3c274b1 Mon Sep 17 00:00:00 2001 From: Florian Courtial Date: Thu, 16 Nov 2017 17:41:34 +0100 Subject: [PATCH 0039/1924] Use CPU scope when normalizing any negative indices in the reduction_axes. --- tensorflow/cc/gradients/math_grad.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index da8bae596f..5cfd8472ec 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -860,7 +860,7 @@ Status ProdGrad(const Scope& scope, const Operation& op, // Normalize any negative indices in the reduction_axes to positive values. - auto reduction_indices_pos = Mod(scope, Add(scope, reduction_indices, rank), rank); + auto reduction_indices_pos = Mod(cpu_scope, Add(cpu_scope, reduction_indices, rank), rank); // [1] auto reduced = Cast(cpu_scope, reduction_indices_pos, DataType::DT_INT32); -- GitLab From 1a63168ff0196f1579a1f6b4cfae2d65f1e7c04e Mon Sep 17 00:00:00 2001 From: Dave MacLachlan Date: Thu, 16 Nov 2017 15:05:58 -0800 Subject: [PATCH 0040/1924] Add LICENSES to gitignore Update gitignore file for ios to cover the license files that get installed following the install instructions. --- tensorflow/examples/ios/.gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/examples/ios/.gitignore b/tensorflow/examples/ios/.gitignore index e572b3012c..dbabfb33bf 100644 --- a/tensorflow/examples/ios/.gitignore +++ b/tensorflow/examples/ios/.gitignore @@ -2,3 +2,6 @@ project.xcworkspace xcuserdata imagenet_comp_graph_label_strings.txt tensorflow_inception_graph.pb +simple/data/LICENSE +camera/data/LICENSE +benchmark/data/LICENSE -- GitLab From 0f9a9c854f7dfee904c4e88130cc496ec9f2611e Mon Sep 17 00:00:00 2001 From: Alex Rothberg Date: Thu, 16 Nov 2017 18:53:42 -0500 Subject: [PATCH 0041/1924] Use get_or_create_global_step --- tensorflow/contrib/training/python/training/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/training/python/training/training.py b/tensorflow/contrib/training/python/training/training.py index 59f02fa38f..8e0139bdd6 100644 --- a/tensorflow/contrib/training/python/training/training.py +++ b/tensorflow/contrib/training/python/training/training.py @@ -410,7 +410,7 @@ def create_train_op(total_loss, loss value. """ if global_step is _USE_GLOBAL_STEP: - global_step = training_util.get_global_step() + global_step = training_util.get_or_create_global_step() # Update ops use GraphKeys.UPDATE_OPS collection if update_ops is None. global_update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS)) -- GitLab From 204e88b8387b9bc95a72981548bd4b14c1bb17d7 Mon Sep 17 00:00:00 2001 From: Gregg Helt Date: Fri, 17 Nov 2017 14:06:25 -0800 Subject: [PATCH 0042/1924] Fixed bug in code within programmer's guide markdown docs for Variables. Had a call to {var}.run() method, but Variable instances have no run() method, switched to eval() instead. --- tensorflow/docs_src/programmers_guide/variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/variables.md b/tensorflow/docs_src/programmers_guide/variables.md index f310b89380..bda39cc28e 100644 --- a/tensorflow/docs_src/programmers_guide/variables.md +++ b/tensorflow/docs_src/programmers_guide/variables.md @@ -205,7 +205,7 @@ methods: v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) assignment = v.assign_add(1) tf.global_variables_initializer().run() -assignment.run() +assignment.eval() ``` Most TensorFlow optimizers have specialized ops that efficiently update the -- GitLab From 1b4ad65bc5830513d10ecde6d3e96e96117f7bad Mon Sep 17 00:00:00 2001 From: Daniyar Date: Mon, 20 Nov 2017 19:36:58 +0000 Subject: [PATCH 0043/1924] restart tests --- tensorflow/core/kernels/unpack_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/unpack_op.cc b/tensorflow/core/kernels/unpack_op.cc index 7ece912557..e4963f9d4c 100644 --- a/tensorflow/core/kernels/unpack_op.cc +++ b/tensorflow/core/kernels/unpack_op.cc @@ -176,6 +176,7 @@ REGISTER_KERNEL_BUILDER(Name("Unpack") .HostMemory("output") .TypeConstraint("T"), UnpackOp); + REGISTER_KERNEL_BUILDER(Name("Unpack") .Device(DEVICE_SYCL) .HostMemory("value") -- GitLab From 3953afc536c362574a454c1bd6e7047f161ee79f Mon Sep 17 00:00:00 2001 From: Shreyash sharma Date: Wed, 22 Nov 2017 12:16:40 +0530 Subject: [PATCH 0044/1924] Update layers_test The Changes have been made as suggested in my previous pull request. https://github.com/tensorflow/tensorflow/pull/13829 The unit test case has been updated along with the update in layers.py with reference to the issue https://github.com/tensorflow/tensorflow/issues/11673. Please verify and get back. --- .../layers/python/layers/layers_test.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py index d93c1552e4..b11de4008e 100644 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ b/tensorflow/contrib/layers/python/layers/layers_test.py @@ -507,13 +507,13 @@ class ConvolutionTest(test.TestCase): self.assertEqual( len(ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)), 1) - def testConvWithBatchNorm(self): + def testConvWithNorm(self): height, width = 7, 9 with self.test_session(): images = random_ops.random_uniform((5, height, width, 32), seed=1) with arg_scope( [layers_lib.convolution2d], - normalizer_fn=_layers.batch_norm, + normalizer_fn=_layers._norm, normalizer_params={'decay': 0.9}): net = layers_lib.convolution2d(images, 32, [3, 3]) net = layers_lib.convolution2d(net, 32, [3, 3]) @@ -1739,6 +1739,11 @@ class BatchNormTest(test.TestCase): expected_var *= correction_factor return expected_var, correction_factor + def testBatchNormCenterFalse(self) + a = array_ops.placeholder(dtype.float32, shape=(10, 10, 10, 10)) + b = _layers.batch_norm(a, center=False, data_format='NCHW', + zero_debias_moving_mean=True) + def testUnknownShape(self): with ops.Graph().as_default() as g, self.test_session(g): inputs = array_ops.placeholder(dtype=dtypes.float32) @@ -3688,16 +3693,6 @@ class LegacyFullyConnectedTest(test.TestCase): 'rank of x must be at least 2 not: 1'): x = constant_op.constant([[]], shape=[0]) _layers.legacy_fully_connected(x, 2, activation_fn=nn_ops.softmax) - -class zero_debias_moving_mean(test.TestCase): - - def Error_in_tf.contrib.layers.batch_norm_when(self): - import tensorflow as tf - a = tf.placeholder(tf.float32, shape=(10, 10, 10, 10)) - b = tf.contrib.layers.batch_norm(a, center=False, data_format='NCHW', - zero_debias_moving_mean=True) - sess = tf.Session() - sess.run(tf.global_variables_initializer()) if __name__ == '__main__': -- GitLab From b7606e872e6bc37e2356ce7725e0f42356b795ec Mon Sep 17 00:00:00 2001 From: Shreyash sharma Date: Wed, 22 Nov 2017 12:44:31 +0530 Subject: [PATCH 0045/1924] Update layers.py Updated layers.py --- tensorflow/contrib/layers/python/layers/layers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 8b3ccea995..a416aa5fb6 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -805,7 +805,8 @@ def batch_norm(inputs, if data_format == DATA_FORMAT_NCHW: mean = array_ops.reshape(mean, params_shape_broadcast) variance = array_ops.reshape(variance, params_shape_broadcast) - beta = array_ops.reshape(beta, params_shape_broadcast) + if beta is not None: + beta = array_ops.reshape(beta, params_shape_broadcast) if gamma is not None: gamma = array_ops.reshape(gamma, params_shape_broadcast) -- GitLab From 758ac9cb907fdd7d9c295ea076e985c9f545667f Mon Sep 17 00:00:00 2001 From: FredZhang <654496915@qq.com> Date: Sat, 25 Nov 2017 22:13:08 +0800 Subject: [PATCH 0046/1924] Some PATH typo : no `train_dir` in tutorial This file uses `train_dir`. But in code file, there is no `train_dir` anymore, it should be replaced with `log_dir` `input_data_dir` and `checkpoint_file` respectively --- tensorflow/docs_src/get_started/mnist/mechanics.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/docs_src/get_started/mnist/mechanics.md b/tensorflow/docs_src/get_started/mnist/mechanics.md index 27fae45b5b..a5c784b30d 100644 --- a/tensorflow/docs_src/get_started/mnist/mechanics.md +++ b/tensorflow/docs_src/get_started/mnist/mechanics.md @@ -47,7 +47,7 @@ training folder and then unpack that data to return a dictionary of `DataSet` instances. ```python -data_sets = input_data.read_data_sets(FLAGS.train_dir, FLAGS.fake_data) +data_sets = input_data.read_data_sets(FLAGS.input_data_dir, FLAGS.fake_data) ``` **NOTE**: The `fake_data` flag is used for unit-testing purposes and may be @@ -369,7 +369,7 @@ may be instantiated to write the events files, which contain both the graph itself and the values of the summaries. ```python -summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph) +summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph) ``` Lastly, the events file will be updated with new summary values every time the @@ -403,7 +403,7 @@ method will periodically be called to write a checkpoint file to the training directory with the current values of all the trainable variables. ```python -saver.save(sess, FLAGS.train_dir, global_step=step) +saver.save(sess, checkpoint_file, global_step=step) ``` At some later point in the future, training might be resumed by using the @@ -411,7 +411,7 @@ At some later point in the future, training might be resumed by using the method to reload the model parameters. ```python -saver.restore(sess, FLAGS.train_dir) +saver.restore(sess, checkpoint_file) ``` ## Evaluate the Model -- GitLab From 70bf5374524bc2f19b9196eeb066b883ee504db5 Mon Sep 17 00:00:00 2001 From: concerttttt Date: Mon, 27 Nov 2017 11:58:38 +0800 Subject: [PATCH 0047/1924] Update tf_core_framework.cmake gpu_tracer.cc replaced by device_tracer.cc --- tensorflow/contrib/cmake/tf_core_framework.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index c607546f4a..5ec1a8d04f 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -211,7 +211,7 @@ if (NOT tensorflow_ENABLE_GPU) list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_gpu_srcs}) else() file(GLOB tf_core_platform_srcs_exclude - "${tensorflow_source_dir}/tensorflow/core/platform/default/gpu_tracer.cc") + "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc") list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_srcs_exclude}) endif() -- GitLab From 58970731ba6d899c827ab1ce5c853d9ac8ae1414 Mon Sep 17 00:00:00 2001 From: lanhin Date: Mon, 27 Nov 2017 20:43:30 +0800 Subject: [PATCH 0048/1924] Comment typo fix. --- tensorflow/core/common_runtime/function.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc index 93bd3a6adb..6fb0dc252e 100644 --- a/tensorflow/core/common_runtime/function.cc +++ b/tensorflow/core/common_runtime/function.cc @@ -318,7 +318,7 @@ Status FunctionLibraryRuntimeImpl::CreateKernel(const NodeDef& ndef, kernel); } - // Try to instantiate this function for the func/attr. Maybe its + // Try to instantiate this function for the func/attr. Maybe it's // cached already. Handle handle; TF_RETURN_IF_ERROR(Instantiate(ndef.op(), AttrSlice(&ndef.attr()), &handle)); -- GitLab From b924e9f4c380dc85df433106df5f3c6a875318ac Mon Sep 17 00:00:00 2001 From: scott Date: Mon, 27 Nov 2017 23:14:19 +0800 Subject: [PATCH 0049/1924] add extra document to parameter:num_epochs --- .../contrib/slim/python/slim/data/dataset_data_provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py b/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py index 82c6b5a619..41426a6508 100644 --- a/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py +++ b/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py @@ -62,7 +62,9 @@ class DatasetDataProvider(data_provider.DataProvider): seed=None, scope=None): """Creates a DatasetDataProvider. - + Note: if `num_epochs` is not `None`, local counter `epochs` will be created + by relevant function. Use `local_variables_initializer()` to initialize + local variables. Args: dataset: An instance of the Dataset class. num_readers: The number of parallel readers to use. -- GitLab From 6ec7e7680a8a1c5eaf1054a9eb81c8f608aadb90 Mon Sep 17 00:00:00 2001 From: Daniel Ylitalo Date: Tue, 28 Nov 2017 15:30:47 +0100 Subject: [PATCH 0050/1924] Add FreeBSD compatibility --- tensorflow/contrib/lite/kernels/internal/BUILD | 9 +++++++++ tensorflow/core/platform/env.cc | 14 ++++++++++++++ third_party/flatbuffers/flatbuffers.BUILD | 14 ++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) mode change 100644 => 100755 tensorflow/contrib/lite/kernels/internal/BUILD mode change 100644 => 100755 tensorflow/core/platform/env.cc mode change 100644 => 100755 third_party/flatbuffers/flatbuffers.BUILD diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD old mode 100644 new mode 100755 index 288534099b..a3ecb2ebf6 --- a/tensorflow/contrib/lite/kernels/internal/BUILD +++ b/tensorflow/contrib/lite/kernels/internal/BUILD @@ -124,6 +124,13 @@ config_setting( }, ) +config_setting( + name = "freebsd", + values = { + "cpu": "freebsd", + }, +) + cc_library( name = "optimized_base", srcs = [], @@ -147,6 +154,7 @@ cc_library( ":x86": tflite_deps_intel, ":x86_64": tflite_deps_intel, ":darwin": tflite_deps_intel, + ":freebsd": tflite_deps_intel, "//conditions:default": [], }), ) @@ -224,6 +232,7 @@ cc_library( ":x86": tflite_deps_intel, ":x86_64": tflite_deps_intel, ":darwin": tflite_deps_intel, + ":freebsd": tflite_deps_intel, "//conditions:default": [], }), ) diff --git a/tensorflow/core/platform/env.cc b/tensorflow/core/platform/env.cc old mode 100644 new mode 100755 index 12ef55ec26..5118c4cb59 --- a/tensorflow/core/platform/env.cc +++ b/tensorflow/core/platform/env.cc @@ -20,6 +20,10 @@ limitations under the License. #if defined(__APPLE__) #include #endif +#if defined(__FreeBSD__) +#include +#include +#endif #if defined(PLATFORM_WINDOWS) #include #include "tensorflow/core/platform/windows/windows_file_system.h" @@ -266,6 +270,13 @@ string Env::GetExecutablePath() { char unresolved_path[buffer_size]; _NSGetExecutablePath(unresolved_path, &buffer_size); CHECK(realpath(unresolved_path, exe_path)); +#elif defined(__FreeBSD__) + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + size_t exe_path_size = PATH_MAX; + + if (sysctl(mib, 4, exe_path, &exe_path_size, NULL, 0) != 0) { + // Not sure what to do if it fails? + } #elif defined(PLATFORM_WINDOWS) HMODULE hModule = GetModuleHandleW(NULL); WCHAR wc_file_path[MAX_PATH] = {0}; @@ -293,6 +304,9 @@ bool Env::LocalTempFilename(string* filename) { pthread_threadid_np(nullptr, &tid64); int32 tid = static_cast(tid64); int32 pid = static_cast(getpid()); +#elif defined(__FreeBSD__) + int32 tid = static_cast((long) pthread_self()); + int32 pid = static_cast(getpid()); #elif defined(PLATFORM_WINDOWS) int32 tid = static_cast(GetCurrentThreadId()); int32 pid = static_cast(GetCurrentProcessId()); diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD old mode 100644 new mode 100755 index 0a76adcf91..c06c269bb2 --- a/third_party/flatbuffers/flatbuffers.BUILD +++ b/third_party/flatbuffers/flatbuffers.BUILD @@ -4,6 +4,12 @@ package( licenses(["notice"]) # Apache 2.0 +config_setting( + name = "freebsd", + values = {"cpu": "freebsd"}, + visibility = ["//visibility:public"], +) + FLATBUFFERS_COPTS = [ "-fexceptions", ] + select({ @@ -107,10 +113,14 @@ cc_binary( "grpc/", "include/", ], - linkopts = [ + linkopts = select({ + ":freebsd": [ "-lm", - "-ldl", ], + "//conditions:default": [ + "-lm", + "-ldl", + ]}), deps = [ ":flatc_library", ], -- GitLab From 1a53e4a82f1d077859214dab4d4fb84479ae70e6 Mon Sep 17 00:00:00 2001 From: Daniel Ylitalo Date: Tue, 28 Nov 2017 15:33:34 +0100 Subject: [PATCH 0051/1924] change back file permissions --- tensorflow/contrib/lite/kernels/internal/BUILD | 0 tensorflow/core/platform/env.cc | 0 third_party/flatbuffers/flatbuffers.BUILD | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tensorflow/contrib/lite/kernels/internal/BUILD mode change 100755 => 100644 tensorflow/core/platform/env.cc mode change 100755 => 100644 third_party/flatbuffers/flatbuffers.BUILD diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD old mode 100755 new mode 100644 diff --git a/tensorflow/core/platform/env.cc b/tensorflow/core/platform/env.cc old mode 100755 new mode 100644 diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD old mode 100755 new mode 100644 -- GitLab From 7ea0fd6ccadc2922560ee66e3b11ae45324cc946 Mon Sep 17 00:00:00 2001 From: Guenther Schmuelling Date: Tue, 28 Nov 2017 10:46:48 -0800 Subject: [PATCH 0052/1924] add support for quantized ops on windows --- tensorflow/contrib/cmake/README.md | 17 ------ .../contrib/cmake/external/gemmlowp.cmake | 4 +- .../contrib/cmake/tf_core_kernels.cmake | 3 - tensorflow/contrib/cmake/tf_tests.cmake | 2 + tensorflow/core/kernels/quantized_conv_ops.cc | 7 +++ .../python/ops/quantized_conv_ops_test.py | 2 +- tensorflow/python/ops/quantized_ops_test.py | 57 +++++++++++++++++++ 7 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 tensorflow/python/ops/quantized_ops_test.py diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md index 4ddfec5960..4be733a280 100644 --- a/tensorflow/contrib/cmake/README.md +++ b/tensorflow/contrib/cmake/README.md @@ -19,23 +19,6 @@ for instructions on how to install a pre-built TensorFlow package on Windows. ### Current known limitations * It is not possible to load a custom Op library. * GCS file system is not supported. -* The following Ops are not currently implemented: - - Dequantize - - QuantizeAndDequantize - - QuantizedAvgPool - - QuantizedBatchNomWithGlobalNormalization - - QuantizedBiasAdd - - QuantizedConcat - - QuantizedConv2D - - QuantizedMatmul - - QuantizedMaxPoo - - QuantizeDownAndShrinkRange - - QuantizedRelu - - QuantizedRelu6 - - QuantizedReshape - - QuantizeV2 - - RequantizationRange - - Requantize ## Building with CMake diff --git a/tensorflow/contrib/cmake/external/gemmlowp.cmake b/tensorflow/contrib/cmake/external/gemmlowp.cmake index 3b146657bf..a235442dc5 100644 --- a/tensorflow/contrib/cmake/external/gemmlowp.cmake +++ b/tensorflow/contrib/cmake/external/gemmlowp.cmake @@ -14,8 +14,8 @@ # ============================================================================== include (ExternalProject) -set(gemmlowp_URL https://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip) -set(gemmlowp_HASH SHA256=dd2557072bde12141419cb8320a9c25e6ec41a8ae53c2ac78c076a347bb46d9d) +set(gemmlowp_URL https://github.com/google/gemmlowp/archive/6a2a90822e8546fc2bfa7044de0faf1c1cb4862f.zip) +set(gemmlowp_HASH SHA256=3447948d219f3270383766bbe08942888c0eb4e0ca6663c0e0548502ec5bb77d) set(gemmlowp_BUILD ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) set(gemmlowp_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/gemmlowp/src/gemmlowp) diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake index 2d015908a8..eb6bf567aa 100644 --- a/tensorflow/contrib/cmake/tf_core_kernels.cmake +++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake @@ -150,9 +150,6 @@ list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_exclude_srcs}) if(WIN32) file(GLOB_RECURSE tf_core_kernels_windows_exclude_srcs # not working on windows yet - "${tensorflow_source_dir}/tensorflow/core/kernels/meta_support.*" - "${tensorflow_source_dir}/tensorflow/core/kernels/*quantiz*.h" - "${tensorflow_source_dir}/tensorflow/core/kernels/*quantiz*.cc" "${tensorflow_source_dir}/tensorflow/core/kernels/neon/*" # not in core - those are loaded dynamically as dll "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/kernels/hyperplane_lsh_probes.cc" diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 18b71d1f9a..46134f4455 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -145,6 +145,8 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/contrib/estimator/python/estimator/*_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/*.py" "${tensorflow_source_dir}/tensorflow/python/meta_graph_transform/*_test.py" + "${tensorflow_source_dir}/tensorflow/python/ops/quantized_conv_ops_test.py" + "${tensorflow_source_dir}/tensorflow/python/ops/quantized_ops_test.py" "${tensorflow_source_dir}/tensorflow/python/platform/build_info_test.py" "${tensorflow_source_dir}/tensorflow/python/profiler/*_test.py" "${tensorflow_source_dir}/tensorflow/python/profiler/internal/*_test.py" diff --git a/tensorflow/core/kernels/quantized_conv_ops.cc b/tensorflow/core/kernels/quantized_conv_ops.cc index 3b0764bb9b..54090bac7e 100644 --- a/tensorflow/core/kernels/quantized_conv_ops.cc +++ b/tensorflow/core/kernels/quantized_conv_ops.cc @@ -268,6 +268,13 @@ class Im2ColConvFunctor { Im2ColBufferResource* im2col_buffer_resource; std::function**)> creator = [](Im2ColBufferResource** resource) { +#ifdef _MSC_VER + // MSVC complains about the capture of chunk_value_count which oddly + // works fine in conv_ops_using_gemm.cc for example. + // Define chunk_value_count inside the lambda for now. + const int64 chunk_value_count = + (kMaxChunkSize + (sizeof(T1) - 1)) / sizeof(T1); +#endif *resource = new Im2ColBufferResource(); return Status::OK(); }; diff --git a/tensorflow/python/ops/quantized_conv_ops_test.py b/tensorflow/python/ops/quantized_conv_ops_test.py index 5ea47ea40e..5e9e710027 100644 --- a/tensorflow/python/ops/quantized_conv_ops_test.py +++ b/tensorflow/python/ops/quantized_conv_ops_test.py @@ -93,7 +93,7 @@ class Conv2DTest(test.TestCase): quantized_range = ((quantized_max - quantized_min) * range_adjust) range_scale = (quantized_range / number_of_steps) lowest_quantized = -(1 << (number_of_bits - 1)) - result = np.array([(quantized_min + ((x - lowest_quantized) * range_scale)) + result = np.array([(quantized_min + ((float(x) - lowest_quantized) * range_scale)) for x in quantized.flatten()]) return result diff --git a/tensorflow/python/ops/quantized_ops_test.py b/tensorflow/python/ops/quantized_ops_test.py new file mode 100644 index 0000000000..4bf3b35e13 --- /dev/null +++ b/tensorflow/python/ops/quantized_ops_test.py @@ -0,0 +1,57 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for quantized operations.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class QuantizedOpsTest(test.TestCase): + + def __init__(self, method_name="runTest"): + super(QuantizedOpsTest, self).__init__(method_name) + + def testQuantizeOp(self): + expected_output = [1, 1, 2, 127, 255, 255] + with self.test_session(use_gpu=False) as sess: + x = constant_op.constant([1.0, 1.25, 1.75, 127.0, 255.0, 500.0], shape=[6], dtype=dtypes.float32) + x_min = 0.0 + x_max = 255.0 + op = array_ops.quantize(x, x_min, x_max, dtypes.quint8, mode="MIN_FIRST") + value = sess.run(op) + self.assertArrayNear(expected_output, value.output, 0.1) + + def testDequantizeOp(self): + expected_output = [1.0, 2.0, 4.0, 8.0, 16.0, 255.0] + inp = np.array([1, 2, 4, 8, 16, 255]).astype(np.uint8) + with self.test_session(use_gpu=False) as sess: + x = constant_op.constant(inp, shape=[6], dtype=dtypes.quint8) + x_min = 0.0 + x_max = 255.0 + op = array_ops.dequantize(x, x_min, x_max, mode="MIN_FIRST") + value = sess.run(op) + self.assertArrayNear(expected_output, value, 0.1) + + +if __name__ == "__main__": + test.main() -- GitLab From bc8718b090c565c6562dce098d16cdadffc6a213 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 11:30:15 -0800 Subject: [PATCH 0053/1924] Removed unused variables from curl_http_request_test. PiperOrigin-RevId: 177191730 --- tensorflow/core/platform/cloud/curl_http_request_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/core/platform/cloud/curl_http_request_test.cc b/tensorflow/core/platform/cloud/curl_http_request_test.cc index 6c0f081852..d476a1a4db 100644 --- a/tensorflow/core/platform/cloud/curl_http_request_test.cc +++ b/tensorflow/core/platform/cloud/curl_http_request_test.cc @@ -263,7 +263,6 @@ TEST(CurlHttpRequestTest, GetRequest) { std::vector scratch; scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end()); - StringPiece result; scratch.reserve(100); TF_EXPECT_OK(http_request.SetUri("http://www.testuri.com")); @@ -594,7 +593,6 @@ TEST(CurlHttpRequestTest, ErrorReturnsNoResponse) { std::vector scratch; scratch.insert(scratch.begin(), kTestContent.begin(), kTestContent.end()); - StringPiece result; scratch.reserve(100); TF_EXPECT_OK(http_request.SetUri("http://www.testuri.com")); -- GitLab From cbd31dd4d30663344d0d15d8897d8ce652cf6294 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 28 Nov 2017 11:47:47 -0800 Subject: [PATCH 0054/1924] Eager function definition no longer adds control dependencies after constructing nodes. PiperOrigin-RevId: 177194441 --- tensorflow/python/eager/function.py | 59 ++++++++++++++--------- tensorflow/python/eager/graph_callable.py | 1 + 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 9bcd9c23c7..2f4b59e938 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -30,7 +30,7 @@ from tensorflow.python.eager import execute from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes +from tensorflow.python.framework import dtypes as dtypes_module from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.ops import gradients_impl @@ -48,23 +48,7 @@ _scoped_captures.tensors = None def make_function_def(graph, operations, inputs, outputs): - """Makes function def where accesses to resources are serialized.""" - last_op_using_resource_tensor = {} - - # TODO(apassos) probably control flow has to be handled delicately here as in - # if a resource is accessed inside a control flow context we need the control - # dependency to point to something outside the context which is guaranteed to - # happen after the access. - # - # TODO(apassos) this should do some form of alias analysis as ops which - # forward the resources such as Identity and Switch can cause serialization to - # fail. - for op in operations: - for t in op.inputs: - if t.dtype == dtypes.resource: - if t.name in last_op_using_resource_tensor: - op._add_control_input(last_op_using_resource_tensor[t.name]) # pylint: disable=protected-access - last_op_using_resource_tensor[t.name] = op + """Makes function def from the given graph with the operations.""" return graph_to_function_def.graph_to_function_def( graph, operations, inputs, outputs) @@ -85,7 +69,7 @@ def capture_value(tensor_map, value, dtype, name): if captured_value is None: captured_value = graph_placeholder( dtype=dtype or value.dtype, shape=value.shape, name=name) - if captured_value.dtype == dtypes.resource: + if captured_value.dtype == dtypes_module.resource: captured_value._handle_data = value._handle_data # pylint: disable=protected-access tensor_map[ops.tensor_id(value)] = (value, captured_value) else: @@ -120,11 +104,19 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False): class CapturingGraph(ops.Graph): + """Graph used when constructing eager functions.""" def __init__(self, captures): super(CapturingGraph, self).__init__() self._building_function = True self.captures = captures + # Map from resource tensor name to last op (in program order) which uses + # this tensor. Used to enforce that execution order matches program order + # for resource tensors. + self._last_op_using_resource_tensor = {} + + def clear_resource_control_flow_state(self): + self._last_op_using_resource_tensor = {} def create_op( self, @@ -137,12 +129,31 @@ class CapturingGraph(ops.Graph): op_def=None, compute_shapes=True, compute_device=True): + # TODO(apassos) probably control flow has to be handled delicately here as + # in if a resource is accessed inside a control flow context we need the + # control dependency to point to something outside the context which is + # guaranteed to happen after the access. + # + # TODO(apassos) this should do some form of alias analysis as ops which + # forward the resources such as Identity and Switch can cause serialization + # to fail. + resource_inputs = set() + control_inputs = set() for i, inp in enumerate(inputs): if inp.graph is not self: inputs[i] = capture_value(self.captures, inp, inp.dtype, inp.op.name) - return super(CapturingGraph, self).create_op( - op_type, inputs, dtypes, input_types, name, attrs, op_def, - compute_shapes, compute_device) + inp = inputs[i] + if inp.dtype == dtypes_module.resource: + if inp.name in self._last_op_using_resource_tensor: + control_inputs.add(self._last_op_using_resource_tensor[inp.name]) + resource_inputs.add(inp.name) + with self.control_dependencies(list(control_inputs)): + op = super(CapturingGraph, self).create_op( + op_type, inputs, dtypes, input_types, name, attrs, op_def, + compute_shapes, compute_device) + for name in resource_inputs: + self._last_op_using_resource_tensor[name] = op + return op # TODO(apassos): it'd be really nice if we could scope this registration. @@ -314,7 +325,7 @@ class GraphModeFunction(object): return ops.internal_convert_to_tensor(x, ctx=ctx) op = g.create_op( signature.name, [make_tensor(x) for x in all_args], - [dtypes.DType(x.type) for x in signature.output_arg], + [dtypes_module.DType(x.type) for x in signature.output_arg], op_def=signature, name="FunctionCall", compute_shapes=False) @@ -373,7 +384,7 @@ class GraphModeFunction(object): args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.convert_to_tensor(x) for x in args], - [dtypes.DType(x.type) for x in signature.output_arg], + [dtypes_module.DType(x.type) for x in signature.output_arg], op_def=signature, name="FunctionCall", compute_shapes=False) diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 837a75c808..faf0ac88bc 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -296,6 +296,7 @@ def _graph_callable_internal(func, shape_and_dtypes): # Call the function again, now replacing usages of variables with # placeholders. This assumes the variable capturing scope created above # knows about all variables. + tmp_graph.clear_resource_control_flow_state() with variable_captures.capturing_scope(), function.capture_tensors( captures): captured_outputs = func(*func_inputs) -- GitLab From 570d2772796bd642dc4808bd869e293f74553620 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 12:09:49 -0800 Subject: [PATCH 0055/1924] Bump LLVM snapshot to r319150. PiperOrigin-RevId: 177197719 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index dd5dc37a87..cb77f96be5 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -578,11 +578,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "llvm", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/8d26b8bee4d8e7230870a600bc968c7ee8cf6f67.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/8d26b8bee4d8e7230870a600bc968c7ee8cf6f67.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9ab4c272cb604a7f947865428c4ef2169fee2100.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/9ab4c272cb604a7f947865428c4ef2169fee2100.tar.gz", ], - sha256 = "ff5ddbe5af5e264426c8d489e7fddfc5ad7e0975f19cefe9db8c0a5d0faeb23e", - strip_prefix = "llvm-8d26b8bee4d8e7230870a600bc968c7ee8cf6f67", + sha256 = "1b1b7d3800a94ca2302e3dd670dbe84238749583027883784b55297059d83da8", + strip_prefix = "llvm-9ab4c272cb604a7f947865428c4ef2169fee2100", build_file = str(Label("//third_party/llvm:llvm.BUILD")), repository = tf_repo_name, ) -- GitLab From ba87a8030aa30f24c354cf705e79734658bb0a8b Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 28 Nov 2017 12:17:52 -0800 Subject: [PATCH 0056/1924] Eager: Better errors for invalid options to enable_eager_execution. Fixes #14739 PiperOrigin-RevId: 177198861 --- tensorflow/python/framework/ops.py | 11 +++++++++++ tensorflow/python/framework/ops_test.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 60df8f82f0..cfef5e35f4 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -35,6 +35,7 @@ from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 from tensorflow.core.framework import op_def_pb2 from tensorflow.core.framework import versions_pb2 +from tensorflow.core.protobuf import config_pb2 from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.eager import context from tensorflow.python.eager import core @@ -4794,6 +4795,16 @@ def enable_eager_execution(config=None, device_policy=None): or if trying to create a context with nontrivial options which differ from those of the existing context. """ + if config is not None and not isinstance(config, config_pb2.ConfigProto): + raise TypeError( + "config must be a tf.ConfigProto, but got %s" % type(config)) + if device_policy not in (None, context.DEVICE_PLACEMENT_EXPLICIT, + context.DEVICE_PLACEMENT_WARN, + context.DEVICE_PLACEMENT_SILENT): + raise ValueError( + "device_policy must be one of None, tfe.DEVICE_PLACEMENT_EXPLICIT, " + "tfe.DEVICE_PLACEMENT_WARN, tfe.DEVICE_PLACEMENT_SILENT" + ) # pylint: disable=protected-access if context._default_mode == context.GRAPH_MODE: graph_mode_has_been_used = ( diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index cd296ccdc5..e929cc8abf 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -2395,6 +2395,13 @@ class InputTypesTest(test_util.TensorFlowTestCase): self.assertEqual([dtypes.double, dtypes.double], z.op._input_dtypes) # pylint: enable=protected-access + def testBadArgumentsToEnableEagerExecution(self): + with self.assertRaisesRegexp(TypeError, "config must be a tf.ConfigProto"): + ops.enable_eager_execution(context.DEVICE_PLACEMENT_SILENT) + with self.assertRaisesRegexp(ValueError, "device_policy must be one of"): + c = config_pb2.ConfigProto() + ops.enable_eager_execution(c, c) + if __name__ == "__main__": googletest.main() -- GitLab From b911049edfbb4a4eb07b3b46ed144da6cd33f9c1 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Tue, 28 Nov 2017 12:26:12 -0800 Subject: [PATCH 0057/1924] Continue to allow old argument names specified in tf.flags.DEFINE functions. There are more DEFINE functions in absl.flags, they only accept the absl names. PiperOrigin-RevId: 177199982 --- tensorflow/python/platform/flags.py | 48 ++++++++++++++++++++++++ tensorflow/python/platform/flags_test.py | 41 +++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/platform/flags.py b/tensorflow/python/platform/flags.py index e9a36ae75d..abd6f3d855 100644 --- a/tensorflow/python/platform/flags.py +++ b/tensorflow/python/platform/flags.py @@ -18,5 +18,53 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import logging as _logging + # go/tf-wildcard-import from absl.flags import * # pylint: disable=wildcard-import +import six as _six + +from tensorflow.python.util import tf_decorator + + +# Since we wrap absl.flags DEFINE functions, we need to declare this module +# does not affect key flags. +disclaim_key_flags() # pylint: disable=undefined-variable + + +_RENAMED_ARGUMENTS = { + 'flag_name': 'name', + 'default_value': 'default', + 'docstring': 'help', +} + + +def _wrap_define_function(original_function): + """Wraps absl.flags's define functions so tf.flags accepts old names.""" + + def wrapper(*args, **kwargs): + """Wrapper function that turns old keyword names to new ones.""" + has_old_names = False + for old_name, new_name in _six.iteritems(_RENAMED_ARGUMENTS): + if old_name in kwargs: + has_old_names = True + value = kwargs.pop(old_name) + kwargs[new_name] = value + if has_old_names: + _logging.warning( + 'Use of the keyword argument names (flag_name, default_value, ' + 'docstring) is deprecated, please use (name, default, help) instead.') + return original_function(*args, **kwargs) + + return tf_decorator.make_decorator(original_function, wrapper) + + +# pylint: disable=invalid-name,used-before-assignment +# absl.flags APIs use `default` as the name of the default value argument. +# Allow the following functions continue to accept `default_value`. +DEFINE_string = _wrap_define_function(DEFINE_string) +DEFINE_boolean = _wrap_define_function(DEFINE_boolean) +DEFINE_bool = DEFINE_boolean +DEFINE_float = _wrap_define_function(DEFINE_float) +DEFINE_integer = _wrap_define_function(DEFINE_integer) +# pylint: enable=invalid-name,used-before-assignment diff --git a/tensorflow/python/platform/flags_test.py b/tensorflow/python/platform/flags_test.py index 23060e17d2..e8200142dd 100644 --- a/tensorflow/python/platform/flags_test.py +++ b/tensorflow/python/platform/flags_test.py @@ -24,11 +24,50 @@ from absl import flags as absl_flags from tensorflow.python.platform import flags +flags.DEFINE_string( + flag_name='old_string', default_value='default', docstring='docstring') +flags.DEFINE_string( + name='new_string', default='default', help='docstring') +flags.DEFINE_integer( + flag_name='old_integer', default_value=1, docstring='docstring') +flags.DEFINE_integer( + name='new_integer', default=1, help='docstring') +flags.DEFINE_float( + flag_name='old_float', default_value=1.5, docstring='docstring') +flags.DEFINE_float( + name='new_float', default=1.5, help='docstring') +flags.DEFINE_bool( + flag_name='old_bool', default_value=True, docstring='docstring') +flags.DEFINE_bool( + name='new_bool', default=True, help='docstring') +flags.DEFINE_boolean( + flag_name='old_boolean', default_value=False, docstring='docstring') +flags.DEFINE_boolean( + name='new_boolean', default=False, help='docstring') + + class FlagsTest(unittest.TestCase): def test_global_flags_object(self): self.assertIs(flags.FLAGS, absl_flags.FLAGS) + def test_keyword_arguments(self): + test_cases = ( + ('old_string', 'default'), + ('new_string', 'default'), + ('old_integer', 1), + ('new_integer', 1), + ('old_float', 1.5), + ('new_float', 1.5), + ('old_bool', True), + ('new_bool', True), + ('old_boolean', False), + ('new_boolean', False), + ) + for flag_name, default_value in test_cases: + self.assertEqual(default_value, absl_flags.FLAGS[flag_name].default) + self.assertEqual('docstring', absl_flags.FLAGS[flag_name].help) + -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() -- GitLab From a86f2d2c1af7ac0e5c36eedb18d74c022737fc25 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Tue, 28 Nov 2017 12:49:38 -0800 Subject: [PATCH 0058/1924] Added an option to assume that the shape of fed nodes in unknown since any shape can be actually used. PiperOrigin-RevId: 177203023 --- tensorflow/core/grappler/costs/BUILD | 1 + .../core/grappler/costs/graph_properties.cc | 74 ++++++++++++++--- .../core/grappler/costs/graph_properties.h | 37 ++++++--- .../grappler/costs/graph_properties_test.cc | 82 +++++++++++++++---- .../core/grappler/costs/virtual_scheduler.cc | 2 +- .../optimizers/arithmetic_optimizer.cc | 2 +- .../grappler/optimizers/constant_folding.cc | 2 +- .../grappler/optimizers/layout_optimizer.cc | 2 +- .../grappler/optimizers/memory_optimizer.cc | 2 +- .../grappler/optimizers/static_schedule.cc | 4 +- tensorflow/python/grappler/item.i | 2 +- tensorflow/python/grappler/model_analyzer.cc | 2 +- 12 files changed, 164 insertions(+), 48 deletions(-) diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD index f02cb51038..f1edbbb602 100644 --- a/tensorflow/core/grappler/costs/BUILD +++ b/tensorflow/core/grappler/costs/BUILD @@ -50,6 +50,7 @@ cc_library( "//tensorflow/core:framework", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", + "//tensorflow/core/grappler:utils", "//tensorflow/core/grappler/clusters:cluster", ], ) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index dd389de636..fb7e20fca0 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/grappler/costs/utils.h" +#include "tensorflow/core/grappler/utils.h" namespace tensorflow { namespace grappler { @@ -316,7 +317,11 @@ class SymbolicShapeRefiner { shape_inference::ShapeHandle shape) { return shape_refiner_->SetShape(node, output_port, shape); } - + Status SetUnknownShape(const Node* node, int output_port) { + shape_inference::ShapeHandle shape = + GetUnknownOutputShape(node, output_port); + return shape_refiner_->SetShape(node, output_port, shape); + } struct ShapeId { const Node* node; int port_id; @@ -646,6 +651,23 @@ Status GraphProperties::UpdateMergeNode(SymbolicShapeRefiner* shape_refiner, return Status::OK(); } +Status GraphProperties::OverwriteFedPorts( + SymbolicShapeRefiner* shape_refiner, + const std::unordered_map>& fed_ports, + const Node* node, TopoQueue* new_shapes) const { + auto it = fed_ports.find(node->name()); + Status status; + if (it != fed_ports.end()) { + // It is possible to feed node output ports with tensors of any shape: as a + // result, the shape of a fed port is completely unknown. + for (const int output_port : it->second) { + status.Update(shape_refiner->SetUnknownShape(node, output_port)); + } + new_shapes->push(node); + } + return status; +} + // Manually propagate the input shape for Enter nodes and update any Merge node // outputs. Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner, @@ -673,9 +695,10 @@ Status GraphProperties::UpdateEnter(SymbolicShapeRefiner* shape_refiner, return Status::OK(); } -Status GraphProperties::UpdateShapes(SymbolicShapeRefiner* shape_refiner, - bool relax, const Node* n, - TopoQueue* new_shapes) { +Status GraphProperties::UpdateShapes( + SymbolicShapeRefiner* shape_refiner, bool relax, + const std::unordered_map>& fed_ports, + const Node* n, TopoQueue* new_shapes) const { if (n->IsEnter()) { // The Enter shape function always forwards an UnknownShape, so do the right // thing here. @@ -695,7 +718,9 @@ Status GraphProperties::UpdateShapes(SymbolicShapeRefiner* shape_refiner, } } } - return Status::OK(); + // Nodes can be fed with any shape. The TensorFlow shape inference code can't + // handle this properly, so overwrite its behavior here. + return OverwriteFedPorts(shape_refiner, fed_ports, n, new_shapes); } // Propagates the shapes in the transitive fan-out of . @@ -703,6 +728,7 @@ Status GraphProperties::PropagateShapes( SymbolicShapeRefiner* shape_refiner, bool relax, TopoQueue* new_shapes, const std::unordered_map>& resources, + const std::unordered_map>& fed_ports, int num_loops) const { // Limit the number of iterations to prevent infinite loops in the presence of // incorrect shape functions. The algoritm should converge in at most @@ -728,8 +754,8 @@ Status GraphProperties::PropagateShapes( for (const Edge* e : n->out_edges()) { if (!e->IsControlEdge()) { const Node* fanout = e->dst(); - TF_RETURN_IF_ERROR( - UpdateShapes(shape_refiner, relax, fanout, new_shapes)); + TF_RETURN_IF_ERROR(UpdateShapes(shape_refiner, relax, fed_ports, + fanout, new_shapes)); } } } @@ -803,7 +829,7 @@ Status GraphProperties::UpdateResource( return Status::OK(); } -Status GraphProperties::InferStatically() { +Status GraphProperties::InferStatically(bool assume_valid_feeds) { Graph graph(OpRegistry::Global()); FunctionLibraryDefinition function_library(graph.op_registry(), item_.graph.library()); @@ -820,11 +846,21 @@ Status GraphProperties::InferStatically() { Status s = ImportGraphDef(options, item_.graph, &graph, &shape_refiner); TF_RETURN_IF_ERROR(s); + std::unordered_map> fed_ports; + if (!assume_valid_feeds) { + for (const auto& feed : item_.feed) { + int port_index = 0; + string node_name = ParseNodeName(feed.first, &port_index); + fed_ports[node_name].insert(port_index); + } + } + // List the resources and the nodes using them. Also collect the Enter and // Merge nodes. std::unordered_map> resources; std::unordered_set enter_nodes; std::unordered_set merge_nodes; + std::unordered_set fed_nodes; int num_loops = 0; for (const Node* const node : graph.nodes()) { for (int i = 0; i < node->num_inputs(); ++i) { @@ -841,6 +877,9 @@ Status GraphProperties::InferStatically() { } else if (node->IsNextIteration()) { ++num_loops; } + if (fed_ports.find(node->name()) != fed_ports.end()) { + fed_nodes.insert(node); + } } SymbolicShapeRefiner refiner(&shape_refiner); @@ -855,15 +894,22 @@ Status GraphProperties::InferStatically() { // Force the propagation of shapes of Enter nodes manually (the Enter shape // function always forwards an UnknownShape). for (const Node* node : enter_nodes) { - TF_RETURN_IF_ERROR(UpdateShapes(&refiner, relax, node, &new_shapes)); + TF_RETURN_IF_ERROR( + UpdateShapes(&refiner, relax, fed_ports, node, &new_shapes)); } // Seed the propagation of shapes through merge nodes. for (const Node* node : merge_nodes) { - TF_RETURN_IF_ERROR(UpdateShapes(&refiner, relax, node, &new_shapes)); + TF_RETURN_IF_ERROR( + UpdateShapes(&refiner, relax, fed_ports, node, &new_shapes)); + } + // Also seed the propagation of shapes in the fanout of fed nodes. + for (const Node* node : fed_nodes) { + TF_RETURN_IF_ERROR( + OverwriteFedPorts(&refiner, fed_ports, node, &new_shapes)); } // Propagate shapes normally. - TF_RETURN_IF_ERROR( - PropagateShapes(&refiner, relax, &new_shapes, resources, num_loops)); + TF_RETURN_IF_ERROR(PropagateShapes(&refiner, relax, &new_shapes, resources, + fed_ports, num_loops)); } // Track shapes globally across the graph. @@ -874,6 +920,10 @@ Status GraphProperties::InferStatically() { if (!node_ctx) { continue; } + // Skip any information that comes from fed nodes. + if (fed_ports.find(node->name()) != fed_ports.end()) { + continue; + } for (const auto& merged_shapes : node_ctx->MergedShapes()) { if (!shape_manager.Merge(merged_shapes.first, merged_shapes.second) .ok()) { diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h index 95bc5044d0..6fc53a7f2e 100644 --- a/tensorflow/core/grappler/costs/graph_properties.h +++ b/tensorflow/core/grappler/costs/graph_properties.h @@ -34,12 +34,19 @@ class TopoQueue; // nodes, and potentially a set of nodes to feed. class GraphProperties { public: - // Factory method for creating a GrapplerShapes from a MetaGraphDef. - // Returns nullptr if the given meta_graph cannot be converted. explicit GraphProperties(const GrapplerItem& item) : item_(item) {} - Status InferStatically(); + // Infer the shapes through abstract interpretation. Feed information can be + // incorrect so it should be discarded to ensure correctness of the analysis. + // However, it can help infer shapes in the fanout of fed nodes (even though + // the correctness of these shapes can't be guaranteed), so in some cases + // (such as simulation or scheduling) it makes sense of keep these shapes. + Status InferStatically(bool assume_valid_feeds); + // Infer the shape by running the graph on the specified cluster and recording + // the shapes of the processed tensors. Status InferDynamically(Cluster* cluster); + // Extract the properties from a cost graph. For testing only since there is + // no way to ensure that the cost graph match the item. Status InferFromCostGraph(const CostGraphDef& cost_graph); // Stores `item_.graph` with the inferred output shapes to `output_graph_def`. @@ -65,12 +72,6 @@ class GraphProperties { OpInfo::TensorProperties*); private: - // Inputs - GrapplerItem item_; - std::map> input_properties_; - std::map> output_properties_; - const std::vector missing_properties_; - // Merges shapes , determined from an EnqueueV2 node, into // <*queue_shapes_and_types>. static Status MergeEnqueueShapesAndTypes( @@ -99,17 +100,31 @@ class GraphProperties { static Status UpdateEnter(SymbolicShapeRefiner* shape_refiner, const Node* node, bool relax, TopoQueue* new_shapes); + // Process a node that is used to feed the model. + Status OverwriteFedPorts( + SymbolicShapeRefiner* shape_refiner, + const std::unordered_map>& fed_ports, + const Node* node, TopoQueue* new_shapes) const; // Update the shapes for node 'n'. If output shapes for n have changed, // enqueue its fanout in 'new_shapes'. - static Status UpdateShapes(SymbolicShapeRefiner* shape_refiner, bool relax, - const Node* n, TopoQueue* new_shapes); + Status UpdateShapes( + SymbolicShapeRefiner* shape_refiner, bool relax, + const std::unordered_map>& fed_ports, + const Node* n, TopoQueue* new_shapes) const; // Propagate the shapes for the nodes enqueued in new_shapes and their // transitive fanout until a fixed point is reached. Status PropagateShapes( SymbolicShapeRefiner* shape_refiner, bool relax, TopoQueue* new_shapes, const std::unordered_map>& resources, + const std::unordered_map>& fed_ports, int num_loops) const; + + // Data members + GrapplerItem item_; + std::map> input_properties_; + std::map> output_properties_; + const std::vector missing_properties_; }; } // end namespace grappler diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index c11af5777a..ad8e768f1f 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -73,7 +73,7 @@ TEST_F(GraphPropertiesTest, StaticProperties) { CHECK(fake_input.NextItem(&item)); GraphProperties properties(item); - Status s = properties.InferStatically(); + Status s = properties.InferStatically(true); TF_CHECK_OK(s); for (const auto& node : item.graph.node()) { @@ -179,7 +179,7 @@ TEST_F(GraphPropertiesTest, Variables) { { GraphProperties static_properties(item); - TF_CHECK_OK(static_properties.InferStatically()); + TF_CHECK_OK(static_properties.InferStatically(false)); const auto props = static_properties.GetOutputProperties("Var"); EXPECT_EQ(1, props.size()); @@ -219,7 +219,7 @@ TEST_F(GraphPropertiesTest, VarHandles) { .Finalize(item.graph.add_node())); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto props = properties.GetOutputProperties("VarRead"); EXPECT_EQ(1, props.size()); @@ -286,7 +286,7 @@ TEST_F(GraphPropertiesTest, Queues) { TF_CHECK_OK(root.ToGraphDef(&item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto props1 = properties.GetOutputProperties("Dequeue1"); ASSERT_EQ(1, props1.size()); @@ -335,7 +335,7 @@ TEST_F(GraphPropertiesTest, MergeWithoutLoops) { "merge_without_loops.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector nodes{"cond/Merge", "cond/concat", "cond/concat_1"}; std::vector expected_outputs{"float: [-1,-1,1]", "float: [2,1,1]", @@ -377,7 +377,7 @@ TEST_F(GraphPropertiesTest, WhileLoop) { "while_loop.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector nodes{"while/Merge_1", "while/NextIteration_1", "while/Exit_1"}; @@ -435,7 +435,7 @@ TEST_F(GraphPropertiesTest, NestedLoop) { "nested_loop.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector outer_nodes{"while/Merge_1", "while/NextIteration_1", "while/Exit_1"}; @@ -498,7 +498,7 @@ TEST_F(GraphPropertiesTest, LoopsAndQueues) { "loops_and_queues.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector outer_nodes{"while/Merge_1", "while/NextIteration_1", "while/Exit_1"}; @@ -556,7 +556,7 @@ TEST_F(GraphPropertiesTest, LoopsAndResourceVars) { "loops_and_resource_vars.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector outer_nodes{"while/Merge_1", "while/NextIteration_1", "while/Exit_1"}; @@ -608,7 +608,7 @@ TEST_F(GraphPropertiesTest, QueuesAndLoops) { "queues_and_loops.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); std::vector nodes{"while/Merge_1", "while/NextIteration_1", "while/Exit_1"}; @@ -657,7 +657,7 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape) { item.fetch.push_back("init_restore"); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto restore_props = properties.GetOutputProperties("restore"); const OpInfo::TensorProperties& restore_prop = restore_props[0]; @@ -704,7 +704,7 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape_WithTwoNodesShareSameOutput) { item.fetch.push_back("init2"); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto props = properties.GetOutputProperties("restore"); const OpInfo::TensorProperties& prop = props[0]; @@ -732,7 +732,7 @@ TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) { "simple_function.pbtxt"); TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto props = properties.GetOutputProperties("MyAdd_55e046a8_1"); const OpInfo::TensorProperties& prop = props[0]; EXPECT_EQ(DT_FLOAT, prop.dtype()); @@ -766,7 +766,7 @@ TEST_F(GraphPropertiesTest, SymbolicShapes) { TF_CHECK_OK(s.ToGraphDef(&item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto shape_a = properties.GetOutputProperties("a").at(0).shape(); const auto shape_c = properties.GetOutputProperties("c").at(0).shape(); EXPECT_EQ(2, shape_a.dim_size()); @@ -822,7 +822,7 @@ TEST_F(GraphPropertiesTest, DoNotValidateColocationConstraints) { GraphProperties properties(item); // This function should return OK, since it doesn't validate the colocation // constraints internally. - TF_EXPECT_OK(properties.InferStatically()); + TF_EXPECT_OK(properties.InferStatically(false)); } TEST_F(GraphPropertiesTest, ShapeTracking) { @@ -842,7 +842,7 @@ TEST_F(GraphPropertiesTest, ShapeTracking) { TF_CHECK_OK(s.ToGraphDef(&item.graph)); GraphProperties properties(item); - TF_CHECK_OK(properties.InferStatically()); + TF_CHECK_OK(properties.InferStatically(false)); const auto shape_a = properties.GetOutputProperties("a").at(0).shape(); const auto shape_b = properties.GetOutputProperties("b").at(0).shape(); const auto shape_o1 = properties.GetOutputProperties("o1").at(0).shape(); @@ -851,6 +851,56 @@ TEST_F(GraphPropertiesTest, ShapeTracking) { EXPECT_EQ(shape_b.DebugString(), shape_o2.DebugString()); } +TEST_F(GraphPropertiesTest, FedNodes) { + TrivialTestGraphInputYielder fake_input(4, 1, 10, false, + cluster_->GetDeviceNames()); + GrapplerItem item; + CHECK(fake_input.NextItem(&item)); + item.feed.emplace_back("AddN", Tensor()); + + { + // Conservative shape analysis: the shape of fed ports should be unknown + GraphProperties properties(item); + Status s = properties.InferStatically(false); + TF_CHECK_OK(s); + for (const auto& node : item.graph.node()) { + if (node.name() == "AddN") { + const auto in_props = properties.GetInputProperties(node.name()); + EXPECT_EQ(1, in_props.size()); + const OpInfo::TensorProperties& in_prop = in_props[0]; + EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + EXPECT_FALSE(in_prop.shape().unknown_rank()); + EXPECT_EQ(2, in_prop.shape().dim_size()); + const auto out_props = properties.GetOutputProperties(node.name()); + EXPECT_EQ(1, out_props.size()); + EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + EXPECT_TRUE(in_prop.shape().unknown_rank()); + } + } + } + { + // Optimistic shape analysis: the shape of fed ports should be derived from + // the shape of the fanin. + GraphProperties properties(item); + Status s = properties.InferStatically(true); + TF_CHECK_OK(s); + for (const auto& node : item.graph.node()) { + if (node.name() == "AddN") { + const auto in_props = properties.GetInputProperties(node.name()); + EXPECT_EQ(1, in_props.size()); + const OpInfo::TensorProperties& in_prop = in_props[0]; + EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + EXPECT_FALSE(in_prop.shape().unknown_rank()); + EXPECT_EQ(2, in_prop.shape().dim_size()); + const auto out_props = properties.GetOutputProperties(node.name()); + EXPECT_EQ(1, out_props.size()); + const OpInfo::TensorProperties& out_prop = out_props[0]; + EXPECT_EQ(in_prop.DebugString(), out_prop.DebugString()); + } + } + } +} + } // namespace } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc index e5e1ee3292..6640de668d 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc @@ -122,7 +122,7 @@ Status VirtualScheduler::Init() { // Construct graph properties. Status status; if (use_static_shapes_) { - status = graph_properties_.InferStatically(); + status = graph_properties_.InferStatically(true); } else { status = graph_properties_.InferDynamically(cluster_); } diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 1e39c610a4..930d122234 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -1067,7 +1067,7 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/, if (opt_level_ == RewriterConfig::AGGRESSIVE) { graph_properties_.reset(new GraphProperties(item)); // Shapes are only needed in aggressive mode. - TF_RETURN_IF_ERROR(graph_properties_->InferStatically()); + TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false)); TF_RETURN_IF_ERROR( graph_properties_->AnnotateOutputShapes(optimized_graph_)); } diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index c77b2badf4..33a9dddba7 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1163,7 +1163,7 @@ Status ConstantFolding::RunOptimizationPass(Cluster* cluster, Status s = errors::Unknown( "The graph properties are needed but were not initialized"); if (needs_shapes) { - s = properties.InferStatically(); + s = properties.InferStatically(false); } if (!has_feed && s.ok()) { diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index d5563e9d4c..1b8046b787 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -1620,7 +1620,7 @@ Status LayoutOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, virtual_placer_.reset(new VirtualPlacer(cluster)); nodes_to_preserve_ = item.NodesToPreserve(); GraphProperties graph_properties(item); - auto status = graph_properties.InferStatically(); + auto status = graph_properties.InferStatically(false); if (!status.ok()) { *output = item.graph; return status; diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer.cc b/tensorflow/core/grappler/optimizers/memory_optimizer.cc index 7c44ce15c6..a2a2680c4f 100644 --- a/tensorflow/core/grappler/optimizers/memory_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/memory_optimizer.cc @@ -716,7 +716,7 @@ Status MemoryOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, { // Estimate the size of the data to swap for each node. GraphProperties properties(item); - TF_RETURN_IF_ERROR(properties.InferStatically()); + TF_RETURN_IF_ERROR(properties.InferStatically(true)); for (auto& swap : nodes_to_swap) { const NodeDef* node = swap.first; std::vector props = diff --git a/tensorflow/core/grappler/optimizers/static_schedule.cc b/tensorflow/core/grappler/optimizers/static_schedule.cc index 6ce6deef2c..450e853407 100644 --- a/tensorflow/core/grappler/optimizers/static_schedule.cc +++ b/tensorflow/core/grappler/optimizers/static_schedule.cc @@ -86,7 +86,7 @@ Status EstimateEarliestExecutionTimes( name_map.clear(); GraphProperties properties(item); - TF_RETURN_IF_ERROR(properties.InferStatically()); + TF_RETURN_IF_ERROR(properties.InferStatically(true)); OpLevelCostEstimator estimator; VirtualPlacer placer(cluster); @@ -154,7 +154,7 @@ Status EstimateRequiredTimes( } } GraphProperties properties(item); - TF_RETURN_IF_ERROR(properties.InferStatically()); + TF_RETURN_IF_ERROR(properties.InferStatically(true)); OpLevelCostEstimator estimator; VirtualPlacer placer(cluster); diff --git a/tensorflow/python/grappler/item.i b/tensorflow/python/grappler/item.i index 7dd79f7c82..8f72a425c3 100644 --- a/tensorflow/python/grappler/item.i +++ b/tensorflow/python/grappler/item.i @@ -120,7 +120,7 @@ static PyObject* TF_GetOpProperties(GItem item) { Py_RETURN_NONE; } tensorflow::grappler::GraphProperties properties(*item); - tensorflow::Status status = properties.InferStatically(); + tensorflow::Status status = properties.InferStatically(false); if (!status.ok()) { Py_RETURN_NONE; } diff --git a/tensorflow/python/grappler/model_analyzer.cc b/tensorflow/python/grappler/model_analyzer.cc index 7d365c3be9..da5b03234e 100644 --- a/tensorflow/python/grappler/model_analyzer.cc +++ b/tensorflow/python/grappler/model_analyzer.cc @@ -27,7 +27,7 @@ ModelAnalyzer::ModelAnalyzer(const GrapplerItem& item) : item_(item) {} Status ModelAnalyzer::GenerateReport(std::ostream& os) { GraphProperties properties(item_); - TF_RETURN_IF_ERROR(properties.InferStatically()); + TF_RETURN_IF_ERROR(properties.InferStatically(false)); for (const auto& node : item_.MainOpsFanin()) { PrintNodeInfo(node, properties, os); -- GitLab From d96e936fffb8ccd5761c4bf59a8f8ce185f4d50c Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 28 Nov 2017 12:50:21 -0800 Subject: [PATCH 0059/1924] Add custom_estimators PiperOrigin-RevId: 177203120 --- .../docs_src/get_started/custom_estimators.md | 576 ++++++++++++++++++ 1 file changed, 576 insertions(+) create mode 100644 tensorflow/docs_src/get_started/custom_estimators.md diff --git a/tensorflow/docs_src/get_started/custom_estimators.md b/tensorflow/docs_src/get_started/custom_estimators.md new file mode 100644 index 0000000000..e347aa6bd0 --- /dev/null +++ b/tensorflow/docs_src/get_started/custom_estimators.md @@ -0,0 +1,576 @@ + +# Creating Custom Estimators +This document introduces custom Estimators. In particular, this document +demonstrates how to create a custom @{tf.estimator.Estimator$Estimator} that +mimics the behavior of the pre-made Estimator +@{tf.estimator.DNNClassifier$`DNNClassifier`} in solving the Iris problem. See +the @{$get_started/estimator$Pre-Made Estimators chapter} for details. + +If you are feeling impatient, feel free to compare and contrast the following +full programs: + +* Iris implemented with the [pre-made DNNClassifier Estimator](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). +* Iris implemented with a [custom Estimator](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py). + +## Pre-made vs. custom + +As the following figure shows, pre-made Estimators are subclasses of the +@{tf.estimator.Estimator} base class, while custom Estimators are an instance +of tf.estimator.Estimator: + +
+Premade estimators are sub-classes of `Estimator`. Custom Estimators are usually (direct) instances of `Estimator` +
+
+Pre-made and custom Estimators are all Estimators. +
+ +Pre-made Estimators are fully baked. Sometimes though, you need more control +over an Estimator's behavior. That's where custom Estimators come in. You can +create a custom Estimator to do just about anything. If you want hidden layers +connected in some unusual fashion, write a custom Estimator. If you want to +calculate a unique +[metric](https://developers.google.com/machine-learning/glossary/#metric) +for your model, write a custom Estimator. Basically, if you want an Estimator +optimized for your specific problem, write a custom Estimator. + +A model function (or `model_fn`) implements the ML algorithm. The +only difference between working with pre-made Estimators and custom Estimators +is: + +* With pre-made Estimators, someone already wrote the model function for you. +* With custom Estimators, you must write the model function. + +Your model function could implement a wide range of algorithms, defining all +sorts of hidden layers and metrics. Like input functions, all model functions +must accept a standard group of input parameters and return a standard group of +output values. Just as input functions can leverage the Dataset API, model +functions can leverage the Layers API and the Metrics API. + +Let's see how to solve the Iris problem with a custom Estimator. A quick +reminder--here's the organization of the Iris model that we're trying to mimic: + +
+A diagram of the network architecture: Inputs, 2 hidden layers, and outputs +
+
+Our implementation of Iris contains four features, two hidden layers, +and a logits output layer. +
+ +## Write an Input function + +In our custom Estimator implementation, we'll reuse the input function we used +in the pre-made Estimator implementation. Namely: + +```python +def train_input_fn(features, labels, batch_size): + """An input function for training""" + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((features, labels)) + + # Shuffle, repeat, and batch the examples. + dataset = dataset.shuffle(1000).repeat().batch(batch_size) + + # Return the read end of the pipeline. + return dataset.make_one_shot_iterator().get_next() +``` + +This input function builds an input pipeline that yields batches of +`(features, labels)` pairs, where `features` is a dictionary features. + +## Create feature columns + + +As detailed in @{$get_started/estimator$Premade Estimators}, you must define +your model's feature columns to specify how the model should use each feature. +Whether working with pre-made Estimators or custom Estimators, you define +feature columns in the same fashion. + +The following code creates a simple `numeric_column` for each input feature, +indicating that the value of the input feature should be used directly as an +input to the model: + +```python +# Feature columns describe how to use the input. +my_feature_columns = [] +for key in train_x.keys(): + my_feature_columns.append(tf.feature_column.numeric_column(key=key)) +``` + +## Write a model function + +The model function we'll use has the following call signature: + +```python +def my_model_fn( + features, # This is batch_features from input_fn + labels, # This is batch_labels from input_fn + mode, # An instance of tf.estimator.ModeKeys + params): # Additional configuration +``` + +The first two arguments are the batches of features and labels returned from +the input function; that is, `features` and `labels` are the handles to the +data your model will use. The `mode` argument indicates whether the caller is +requesting training, predicting, or evaluation. + +The caller may pass `params` to an Estimator's constructor. The `params` passed +to the constructor become the `params` passed to `model_fn`. + +```python + # Build 2 hidden layer DNN with 10, 10 units respectively. + classifier = tf.estimator.Estimator( + model_fn=my_model, + params={ + 'feature_columns': my_feature_columns, + # Two hidden layers of 10 nodes each. + 'hidden_units': [10, 10], + # The model must choose between 3 classes. + 'n_classes': 3, + }) +``` + +To implement a typical model function, you must do the following: + +* (Define the model)[#define_the_model]. +* Specify additional calculations for each of + the [three different modes](#modes): + * [Predict](#predict) + * [Evaluate](#evaluate) + * [Train](#train) + +## Define the model + +The basic deep neural network model must define the following three sections: + +* An [input layer](https://developers.google.com/machine-learning/glossary/#input_layer) +* One or more [hidden layers](https://developers.google.com/machine-learning/glossary/#hidden_layer) +* An [output layer](https://developers.google.com/machine-learning/glossary/#output_layer) + +### Define the input layer + +Call @{tf.feature_column.input_layer} to convert your feature dictionary and +feature columns into input for your model. For example: + +```python + # Use `input_layer` to apply the feature columns. + net = tf.feature_column.input_layer(features, params['feature_columns']) +``` + +The preceding line applies the transformations defined by your feature columns, +creating the input layer of our model. + +
+A diagram of the input layer, in this case a 1:1 mapping from raw-inputs to features. +
+ + +### Hidden Layers + +If you are creating a deep neural network, you must define one or more hidden +layers. The Layers API provides a rich set of functions to define all types of +hidden layers, including convolutional, pooling, and dropout layers. For Iris, +we're simply going to call @{tf.layers.dense} to create hidden layers, with +dimensions defined by `params['hidden_layers']`. In a `dense` layer each node +is connected to every node in the preceding layer. Here's the relevant code: + +``` python + # Build the hidden layers, sized according to the 'hidden_units' param. + for units in params['hidden_units']: + net = tf.layers.dense(net, units=units, activation=tf.nn.relu) +``` +* The `units` parameter defines the number of output neurons in a given layer. +* The `activation` parameter defines the [activation function](https://developers.google.com/machine-learning/glossary/#a) — + [Relu](https://developers.google.com/machine-learning/glossary/#ReLU) in this + case. + +The variable `net` here signifies the current top layer of the network. During +the first iteration, `net` signifies the input layer. On each loop iteration +`tf.layers.dense` creates a new layer, which takes the previous layer as its +input. So, the loop uses `net` to pass the previously created layer as input +to the layer being created. + +After creating two hidden layers, our network looks as follows. For +simplicity, the figure only shows four hidden units in each layer. + +
+The input layer with two hidden layers added. +
+ +Note that @{tf.layers.dense} provides many additional capabilities, including +the ability to set a multitude of regularization parameters. For the sake of +simplicity, though, we're going to simply accept the default values of the +other parameters. + +### Output Layer + +We'll define the output layer by calling @{tf.layers.dense} yet again, this +time without an activation function: + +```python + # Compute logits (1 per class). + logits = tf.layers.dense(net, params['n_classes'], activation=None) +``` + +Here, `net` signifies the final hidden layer. Therefore, the full set of layers +is now connected as follows: + +
+A logit output layer connected to the top hidden layer +
+
+The final hidden layer feeds into the output layer. +
+ +When defining an output layer, the `units` parameter specifies the number of +outputs. So, by setting `units` to `params['n_classes']`, the model produces +one output value per class. Each element of the output vector will contains the +score, or "logit", calculated to the associated class of Iris: Setosa, +Versicolor, or Virginica, respectively. + +Later on, these logits will be transformed into probabilities by the +@{tf.nn.softmax} function. + +## Implement training, evaluation, and prediction {modes} + +The final step in creating a model function is to write branching code that +implements prediction, evaluation, and training. + +The model function gets invoked whenever someone calls the Estimator's `train`, +`evaluate`, or `predict` methods. Recall that the signature for the model +function looks like this: + +``` python +def my_model_fn( + features, # This is batch_features from input_fn + labels, # This is batch_labels from input_fn + mode): # An instance of tf.estimator.ModeKeys, see below +``` + +Focus on that third argument, mode. As the following table shows, when someone +calls train, evaluate, or predict, the Estimator framework invokes your model +function with the mode parameter set as follows: + +| Estimator method | Estimator Mode | +|:---------------------------------|:------------------| +|@{tf.estimator.Estimator.train$`train()`} |@{tf.estimator.ModeKeys.TRAIN$`ModeKeys.TRAIN`} | +|@{tf.estimator.Estimator.evaluate$`evaluate()`} |@{tf.estimator.ModeKeys.EVAL$`ModeKeys.EVAL`} | +|@{tf.estimator.Estimator.predict$`predict()`}|@{tf.estimator.ModeKeys.PREDICT$`ModeKeys.PREDICT`} | + +For example, suppose you instantiate a custom Estimator to generate an object +named `classifier`. Then, you make the following call: + +``` python +classifier = tf.estimator.Estimator(...) +classifier.train(input_fn=lambda: my_input_fn(FILE_TRAIN, True, 500)) +``` +The Estimator framework then calls your model function with mode set to +`ModeKeys.TRAIN`. + +Your model function must provide code to handle all three of the mode values. +For each mode value, your code must return an instance of +`tf.estimator.EstimatorSpec`, which contains the information the caller +requires. Let's examine each mode. + +### Predict + +When the Estimator's `predict` method is called, the `model_fn` receives +`mode = ModeKeys.PREDICT`. In this case, the model function must return a +`tf.estimator.EstimatorSpec` containing the prediction. + +The model must have been trained prior to making a prediction. The trained model +is stored on disk in the `model_dir` directory established when you +instantiated the Estimator. + +The code to generate the prediction for this model looks as follows: + +```python +# Compute predictions. +predicted_classes = tf.argmax(logits, 1) +if mode == tf.estimator.ModeKeys.PREDICT: + predictions = { + 'class_ids': predicted_classes[:, tf.newaxis], + 'probabilities': tf.nn.softmax(logits), + 'logits': logits, + } + return tf.estimator.EstimatorSpec(mode, predictions=predictions) +``` +The prediction dictionary contains everything that your model returns when run +in prediction mode. + +
+Additional outputs added to the output layer. +
+ +The `predictions` holds the following three key/value pairs: + +* `class_ids` holds the class id (0, 1, or 2) representing the model's + prediction of the most likely species for this example. +* `probabilities` holds the three probabilities (in this example, 0.02, 0.95, + and 0.03) +* `logit` holds the raw logit values (in this example, -1.3, 2.6, and -0.9) + +We return that dictionary to the caller via the `predictions` parameter of the +@{tf.estimator.EstimatorSpec}. The Estimator's +@{tf.estimator.Estimator.predict$`predict`} method will yield these +dictionaries. + +### Calculate the loss + +For both [training](#train) and [evaluation](#evaluate) we need to calculate the +model's loss. This is the +[objective](https://developers.google.com/machine-learning/glossary/#objective) +that will be optimized. + +Before we calculate loss, we we must first convert the labels from a list of +indexes `(0, 1, 2)` to a +[one-hot representation](https://developers.google.com/machine-learning/glossary/#one-hot_encoding) +by calling @{tf.one_hot}. Then, we can calculate the loss by calling +@{tf.losses.softmax_cross_entropy}. Here's the complete code: + + +```python + # Convert the labels to a one-hot tensor of shape (length of features, 3) + # and with a on-value of 1 for each one-hot vector of length 3. + onehot_labels = tf.one_hot(labels, 3, 1, 0) + + # Compute loss. + loss = tf.losses.softmax_cross_entropy( + onehot_labels=onehot_labels, logits=logits) +``` + +### Evaluate + +When the Estimator's `evaluate` method is called, the `model_fn` receives +`mode = ModeKeys.EVAL`. In this case, the model function must return a +`tf.estimator.EstimatorSpec` containing the model's loss and optionally one +or more metrics. + +Although returning metrics is optional, most custom Estimators do return at +least one metric. TensorFlow provides a Metrics module @{tf.metrics} to +calculate common metrics. For brevity's sake, we'll only return accuracy. The +@{tf.metrics.accuracy} function compares our predictions against the +true values, that is, against the labels provided by the input function. The +@{tf.metrics.accuracy} function requires the labels and predictions to have the +same shape. Here's the call to @{tf.metrics.accuracy}: + +``` python + # Compute evaluation metrics. + accuracy = tf.metrics.accuracy(labels=labels, + predictions=predicted_classes, + name='acc_op') +``` + +The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for evaluation +typically contains the following information: + +* `loss`, which is the model's loss +* `eval_metric_ops`, which is an optional dictionary of metrics. + +So, we'll create a dictionary containing our sole metric. If we had calculated +other metrics, we would have added them as additional key/value pairs to that +same dictionary. Then, we'll pass that dictionary in the `eval_metric_ops` +argument of `tf.estimator.EstimatorSpec`. Here's the code: + +```python + metrics = {'accuracy': accuracy} + tf.summary.scalar('accuracy', accuracy[1]) + + if mode == tf.estimator.ModeKeys.EVAL: + return tf.estimator.EstimatorSpec( + mode, loss=loss, eval_metric_ops=metrics) +``` + +The @{tf.summary.scalar} will make accuracy available to TensorBoard (more on +this later). + +### Train + +When the Estimator's `train` method is called, the `model_fn` is called +with `mode = ModeKeys.TRAIN`. In this case, the model function must return an +`EstimatorSpec` that contains the loss and a training operation. + +Building the training operation will require an optimizer. We will use +@{tf.train.AdagradOptimizer} because we're mimicking the `DNNClassifier`, which +also uses `Adagrad` by default. The `tf.train` package provides many other +optimizers—feel free to experiment with them. + +Here is the code that builds the optimizer: + +``` python + # Instantiate an optimizer. + optimizer = tf.train.AdagradOptimizer(learning_rate=0.1) +``` + +Next, we train the model using the optimizer's +@{tf.train.Optimizer.minimize$`minimize`} method on the loss we calculated +earlier. + +The `minimize` method also takes a `global_step` parameter. TensorFlow uses this +parameter to count the number of training steps that have been processed +(to know when to end a training run). Furthermore, the `global_step` is +essential for TensorBoard graphs to work correctly. Simply call +@{tf.train.get_global_step} and pass the result to the `global_step` +argument of `minimize`. + +Here's the code to train the model: + +``` python + # Train the model by establishing an objective, which is to + # minimize loss using that optimizer. + train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) +``` + +The @{tf.estimator.EstimatorSpec$`EstimatorSpec`} returned for training +must have the following fields set: + +* `loss`, which contains the value of the loss function. +* `train_op`, which executes a training step. + +Here's our code to call `EstimatorSpec`: + +```python + # Return training information. + return tf.estimator.EstimatorSpec( + mode=tf.estimator.ModeKeys.TRAIN, + loss=loss, + train_op=train_op) +``` + +The model function is now complete. + +## The custom Estimator + +Instantiate the custom Estimator through the Estimator base class as follows: + +```python + # Build 2 hidden layer DNN with 10, 10 units respectively. + classifier = tf.estimator.Estimator( + model_fn=my_model, + params={ + 'feature_columns': my_feature_columns, + # Two hidden layers of 10 nodes each. + 'hidden_units': [10, 10], + # The model must choose between 3 classes. + 'n_classes': 3, + }) +``` +Here the `params` dictionary serves the same purpose as the key-word +arguments of `DNNClassifier`; that is, the `params` dictionary lets you +configure your Estimator without modifying the code in the `model_fn`. + +The rest of the code to train, evaluate, and generate predictions using our +Estimator is the same as for the pre-made `DNNClassifier`. For example, the +following line will train the model: + +```python + # Train the Model. + classifier.train( + input_fn=lambda:train_input_fn(train_x, train_y, args.batch_size), + steps=args.train_steps) +``` + +## TensorBoard + +You can view training results for your custom Estimator in TensorBoard. To see +this reporting, start TensorBoard from your command line as follows: + +```bsh +# Replace PATH with the actual path passed as model_dir +tensorboard --logdir=PATH +``` + +Then, open TensorBoard by browsing to: [http://localhost:6006](http://localhost:6006) + +All the pre-made Estimators automatically log a lot of information to +TensorBoard. With custom Estimators, however, TensorBoard only provides one +default log (a graph of the loss) plus the information you explicitly tell +TensorBoard to log. For the custom Estimator you just created, TensorBoard +generates the following: + +
+Accuracy, steps/second, and loss 'scalar' graphs from tensorboard +
+
+TensorBoard displays three graphs. +
+ +In brief, here's what the three graphs tell you: + +* global_step/sec: A performance indicator showing how many batches (gradient + updates) we processed per second as the model trains. + +* loss: The loss reported. + +* accuracy: The accuracy is recorded by the following two lines: + + * `eval_metric_ops={'my_accuracy': accuracy})`, during evaluation. + * `tf.summary.scalar('accuracy', accuracy[1])`, during training. + +These tensorboard graphs are one of the main reasons it's important to pass a +`global_step` to your optimizer's `minimize` method. The model can't record +the x-coordinate for these graphs without it. + +Note the following in the `my_accuracy` and `loss` graphs: + +* The orange line represents training. +* The blue dot represents evaluation. + +During training, summaries (the orange line) are recorded periodically as +batches are processed, which is why it becomes a graph spanning x-axis range. + +By contrast, evaluation produces only a single point on the graph for each call +to `evaluate`. This point contains the average over the entire evaluation call. +This has no width on the graph as it is evaluated entirely from the model state +at a particular training step (from a single checkpoint). + +As suggested in the following figure, you may see and also selectively +disable/enable the reporting using the controls on the left side. + +
+Check-boxes allowing the user to select which runs are shown. +
+
+Enable or disable reporting. +
+ + +## Summary + +Although pre-made Estimators can be an effective way to quickly create new +models, you will often need the additional flexibility that custom Estimators +provide. Fortunately, pre-made and custom Estimators follow the same +programming model. The only practical difference is that you must write a model +function for custom Estimators; everything else is the same. + +For more details, be sure to check out: + +* The +[official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/master/official/mnist), +which uses a custom estimator. + +* The TensorFlow +[official models repository](https://github.com/tensorflow/models/tree/master/official), +which contains more curated examples using custom estimators. + +* This [TensorBoard video](https://youtu.be/eBbEDRsCmv4), which introduces +TensorBoard. + + -- GitLab From c68d35becf59396f86b5d90d236405eafef3349e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 12:57:42 -0800 Subject: [PATCH 0060/1924] Change HLO verifier semantics for bitcasts to: Bitcasts that are not the root of a computation can be any shape byte size. Bitcasts that are the root of a computation must have the same shape byte size as their operand. PiperOrigin-RevId: 177204171 --- tensorflow/compiler/xla/service/hlo_verifier.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 15188c4057..2c09d2defb 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -143,9 +143,13 @@ class ShapeVerifier : public DfsHloVisitor { } Status HandleBitcast(HloInstruction* bitcast) override { - // Bitcasts can be any shape, as long as the size matches the operand size. - TF_RET_CHECK(shape_size_fn_(bitcast->shape()) == - shape_size_fn_(bitcast->operand(0)->shape())); + // Bitcasts that are not the root of a computation can be any shape. + // Bitcasts that are the root of a computation must have the same shape + // byte size as their operand. + if (bitcast->parent()->root_instruction() == bitcast) { + TF_RET_CHECK(shape_size_fn_(bitcast->shape()) == + shape_size_fn_(bitcast->operand(0)->shape())); + } return tensorflow::Status::OK(); } -- GitLab From 9c7fd28542b37e7980f2b0a155996cc1703bd0d7 Mon Sep 17 00:00:00 2001 From: James Keeling Date: Tue, 28 Nov 2017 12:59:35 -0800 Subject: [PATCH 0061/1924] Allow unsorted_segment_sum and unsorted_segment_max to take int64 num_segments Previously this argument could only be an int32. It can now be int32 or int64, defaulting to int32. This fixes bugs that can occur when calling _IndexedSlicesToTensor with int64 arguments. PiperOrigin-RevId: 177204464 --- tensorflow/core/ops/math_ops.cc | 6 ++++-- .../segment_reduction_ops_test.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index d30b847696..d7afd02df6 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -1811,10 +1811,11 @@ output: Has same shape as data, except for dimension 0 which REGISTER_OP("UnsortedSegmentSum") .Input("data: T") .Input("segment_ids: Tindices") - .Input("num_segments: int32") + .Input("num_segments: Tnumsegments") .Output("output: T") .Attr("T: numbertype") .Attr("Tindices: {int32,int64}") + .Attr("Tnumsegments: {int32,int64} = DT_INT32") .SetShapeFn(UnsortedSegmentReductionShapeFn) .Doc(R"doc( Computes the sum along segments of a tensor. @@ -1849,10 +1850,11 @@ output: Has same shape as data, except for the first `segment_ids.rank` REGISTER_OP("UnsortedSegmentMax") .Input("data: T") .Input("segment_ids: Tindices") - .Input("num_segments: int32") + .Input("num_segments: Tnumsegments") .Output("output: T") .Attr("T: realnumbertype") .Attr("Tindices: {int32,int64}") + .Attr("Tnumsegments: {int32,int64} = DT_INT32") .SetShapeFn(UnsortedSegmentReductionShapeFn) .Doc(R"doc( Computes the Max along segments of a tensor. diff --git a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py index 99f9f09690..fd58cdb170 100644 --- a/tensorflow/python/kernel_tests/segment_reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/segment_reduction_ops_test.py @@ -266,6 +266,27 @@ class UnsortedSegmentSumTest(SegmentReductionHelper): self.assertAllClose(np_ans, tf_ans) self.assertShapeEqual(np_ans, s) + def testNumSegmentsTypes(self): + dtypes = [dtypes_lib.int32, dtypes_lib.int64] + indices_flat = np.array([0, 4, 0, 8, 3, 8, 4, 7, 7, 3]) + num_segments = 12 + for indices in indices_flat, indices_flat.reshape(5, 2): + shape = indices.shape + (2,) + for dtype in dtypes: + with self.test_session(use_gpu=True): + tf_x, np_x = self._input(shape) + num_segments_constant = constant_op.constant( + num_segments, dtype=dtype) + np_ans = self._segmentReduce( + indices, np_x, np.add, op2=None, num_out_rows=num_segments) + s = math_ops.unsorted_segment_sum( + data=tf_x, + segment_ids=indices, + num_segments=num_segments_constant) + tf_ans = s.eval() + self.assertAllClose(np_ans, tf_ans) + self.assertShapeEqual(np_ans, s) + def testGradientSegmentSum(self): num_cols = 2 indices_flat = np.array([0, 4, 0, 8, 3, 8, 4, 7, 7, 3]) -- GitLab From f93c8a72154fd22fe1578bf448df156acd54fddf Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Tue, 28 Nov 2017 13:40:05 -0800 Subject: [PATCH 0062/1924] [XLA:CPU] Avoid using the untiled lowering for dot when possible We still need the "make rhs column major" layout assignment optimization since we see significant regressions without it. I did not port the logic around single_threaded_eigen from ProfitableToImplementDotInUntiledLlvmIr. That logic became stale after we changed dot-matrix products to always be single threaded, even when calling into the Eigen implementation. PiperOrigin-RevId: 177210146 --- .../xla/service/cpu/dot_op_emitter.cc | 53 +++---------------- .../compiler/xla/service/cpu/dot_op_emitter.h | 16 ++---- .../xla/service/cpu/layout_assignment.cc | 6 +-- 3 files changed, 12 insertions(+), 63 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc index 4c40dae512..8f7b478cee 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc @@ -518,9 +518,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot, bool transpose_lhs, bool DotOpEmitter::ShapesAreLegalForRuntimeDot() const { return true; } bool DotOpEmitter::EmitLlvmIrDotIfProfitable() { - if (dot_.shape().dimensions_size() != 2 || - ProfitableToImplementDotInUntiledLlvmIr(dot_) == - DotInLlvmIrProfitable::kYes) { + if (dot_.shape().dimensions_size() != 2) { return false; } @@ -977,9 +975,7 @@ bool PotentiallyImplementedAsEigenDot(const HloInstruction& hlo) { return false; } - if (ProfitableToImplementDotInUntiledLlvmIr(hlo) == - DotInLlvmIrProfitable::kYes || - ProfitableToImplementDotInTiledLlvmIr(hlo)) { + if (ProfitableToImplementDotInTiledLlvmIr(hlo)) { return false; } @@ -1010,46 +1006,11 @@ bool PotentiallyImplementedAsEigenDot(const HloInstruction& hlo) { return false; } -DotInLlvmIrProfitable ProfitableToImplementDotInUntiledLlvmIr( - const HloInstruction& dot) { - if (dot.opcode() == HloOpcode::kDot && dot.shape().dimensions_size() == 2) { - const Shape& result_shape = dot.shape(); - // kReductionDimensionThresholdBytes was chosen to be 1/4 of a typical L1 - // cache line size, so that we can have the reduction dimension of both the - // LHS and RHS matrices and still have some space "left over". This needs - // to be tuned further. - const int64 kReductionDimensionThresholdBytes = 8 * 1024; - const bool single_threaded_eigen = - !dot.GetModule()->config().debug_options().xla_cpu_multi_thread_eigen(); - - // This is the point at which it is better to call into Eigen and shard the - // dot across multiple worker threads. This is a rough estimate by running - // a matmult benchmark on my local machine, and it can be tuned further. - const int64 kMaxSingleThreadedFlops = 16 * 1024; - - const int64 M = result_shape.dimensions(0); - const int64 N = result_shape.dimensions(1); - const int64 K = dot.operand(1)->shape().dimensions(0); - const int64 primitive_type_size = - ShapeUtil::ByteSizeOfPrimitiveType(result_shape.element_type()); - if (M == 1 && - K * primitive_type_size <= kReductionDimensionThresholdBytes && - (single_threaded_eigen || M * K * N <= kMaxSingleThreadedFlops)) { - // Heuristics: - // - // - Look for a configuration where we will likely be able to keep LHS in - // L1 and do a cache-optimal traversal of RHS. - // - // - Bail out on matrices that are large enough that Eigen can profitably - // shard the computation across multiple cores. This only applies when - // multi-threading is enabled. - return LayoutUtil::IsMonotonicWithDim0Major( - dot.operand(1)->shape().layout()) - ? DotInLlvmIrProfitable::kWithColumnMajorRhs - : DotInLlvmIrProfitable::kYes; - } - } - return DotInLlvmIrProfitable::kNo; +// For vector-matrix dot products, it is always profitable to make the Rhs +// column major. +bool ProfitableToMakeDotRhsColumnMajor(const HloInstruction& hlo) { + return hlo.opcode() == HloOpcode::kDot && + hlo.shape().dimensions_size() == 2 && hlo.shape().dimensions(0) == 1; } bool ProfitableToImplementDotInTiledLlvmIr(const HloInstruction& dot) { diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h index c9168ccc0f..2badb26f90 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.h @@ -32,19 +32,9 @@ namespace cpu { bool PotentiallyImplementedAsEigenDot(const HloInstruction& hlo); -enum class DotInLlvmIrProfitable { kYes, kNo, kWithColumnMajorRhs }; - -// Returns a value to indicate if (and under what conditions) will lowering -// |dot| as a untiled LLVM IR dot operation be profitable over calling into -// Eigen or emitting a tiled LLVM IR implementation. Possible return values -// are: -// -// * DotInLlvmIrProfitable::kYes - always profitable. -// * DotInLlvmIrProfitable::kNo - never profitable. -// * DotInLlvmIrProfitable::kWithColumnMajorRhs - only if we can manage to make -// the Rhs layout column major. -DotInLlvmIrProfitable ProfitableToImplementDotInUntiledLlvmIr( - const HloInstruction& dot); +// Returns true to indicate that |hlo| is a dot, and that it is profitable to +// switch the layout of the |hlo|'s RHS operand to column major. +bool ProfitableToMakeDotRhsColumnMajor(const HloInstruction& hlo); // Returns true to indicate that we can generate a tiled LLVM IR implementation // for |dot|. diff --git a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc index 3f2d101959..69466fd32e 100644 --- a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc @@ -52,8 +52,7 @@ Status CpuLayoutAssignment::AddBackendConstraints( tensorflow::gtl::FlatMap should_make_rhs_col_major_cache; auto should_make_rhs_col_major = [&](const HloInstruction& instruction) { - if (ProfitableToImplementDotInUntiledLlvmIr(instruction) != - DotInLlvmIrProfitable::kWithColumnMajorRhs) { + if (!ProfitableToMakeDotRhsColumnMajor(instruction)) { return false; } @@ -69,8 +68,7 @@ Status CpuLayoutAssignment::AddBackendConstraints( bool result = std::all_of( rhs->users().begin(), rhs->users().end(), [&](HloInstruction* user) { - return ProfitableToImplementDotInUntiledLlvmIr(*user) == - DotInLlvmIrProfitable::kWithColumnMajorRhs && + return ProfitableToMakeDotRhsColumnMajor(*user) && user->operand(0) != rhs; }); -- GitLab From c294fcfd85c03a801d3aad83cfd08055dadbad1a Mon Sep 17 00:00:00 2001 From: Mustafa Ispir Date: Tue, 28 Nov 2017 14:10:24 -0800 Subject: [PATCH 0063/1924] Dataset support within Estimator. With this cl Input_fn can return a Dataset. PiperOrigin-RevId: 177215252 --- tensorflow/python/estimator/BUILD | 1 + tensorflow/python/estimator/estimator.py | 49 +++++++++--- tensorflow/python/estimator/estimator_test.py | 74 +++++++++++++++++++ 3 files changed, 114 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index 03f386e9cf..8e6945b0f3 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -433,6 +433,7 @@ py_library( "//tensorflow/python:summary", "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python/data", "//tensorflow/python/saved_model:builder", "//tensorflow/python/saved_model:tag_constants", "//third_party/py/numpy", diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index f267f4a54e..63103ef4c1 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -30,6 +30,7 @@ from google.protobuf import message from tensorflow.core.framework import summary_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as tf_session +from tensorflow.python.data.ops import dataset_ops from tensorflow.python.eager import context from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import run_config @@ -416,7 +417,7 @@ class Estimator(object): with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) self._create_and_assert_global_step(g) - features = self._get_features_from_input_fn( + features, input_hooks = self._get_features_from_input_fn( input_fn, model_fn_lib.ModeKeys.PREDICT) estimator_spec = self._call_model_fn( features, None, model_fn_lib.ModeKeys.PREDICT, self.config) @@ -426,7 +427,7 @@ class Estimator(object): checkpoint_filename_with_path=checkpoint_path, scaffold=estimator_spec.scaffold, config=self._session_config), - hooks=hooks) as mon_sess: + hooks=input_hooks + hooks) as mon_sess: while not mon_sess.should_stop(): preds_evaluated = mon_sess.run(predictions) if not isinstance(predictions, dict): @@ -582,6 +583,11 @@ class Estimator(object): def _get_features_from_input_fn(self, input_fn, mode): """Extracts the `features` from return values of `input_fn`.""" result = self._call_input_fn(input_fn, mode) + input_hooks = [] + if isinstance(result, dataset_ops.Dataset): + iterator = result.make_initializable_iterator() + input_hooks.append(_DatasetInitializerHook(iterator)) + result = iterator.get_next() if isinstance(result, (list, tuple)): # Unconditionally drop the label (the second element of result). result = result[0] @@ -590,16 +596,22 @@ class Estimator(object): logging.warning('Input graph does not use tf.data.Dataset or contain a ' 'QueueRunner. That means predict yields forever. ' 'This is probably a mistake.') - return result + return result, input_hooks def _get_features_and_labels_from_input_fn(self, input_fn, mode): + """Extracts the `features` and labels from return values of `input_fn`.""" result = self._call_input_fn(input_fn, mode) + input_hooks = [] + if isinstance(result, dataset_ops.Dataset): + iterator = result.make_initializable_iterator() + input_hooks.append(_DatasetInitializerHook(iterator)) + result = iterator.get_next() if isinstance(result, (list, tuple)): if len(result) != 2: raise ValueError( 'input_fn should return (feautures, labels) as a len 2 tuple.') - return result - return result, None + return result[0], result[1], input_hooks + return result, None, input_hooks def _extract_batch_length(self, preds_evaluated): """Extracts batch length of predictions.""" @@ -723,8 +735,10 @@ class Estimator(object): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) training_util._get_or_create_global_step_read() # pylint: disable=protected-access - features, labels = self._get_features_and_labels_from_input_fn( - input_fn, model_fn_lib.ModeKeys.TRAIN) + features, labels, input_hooks = ( + self._get_features_and_labels_from_input_fn( + input_fn, model_fn_lib.ModeKeys.TRAIN)) + worker_hooks.extend(input_hooks) estimator_spec = self._call_model_fn( features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. @@ -822,8 +836,9 @@ class Estimator(object): with ops.Graph().as_default() as g: random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) - features, labels = self._get_features_and_labels_from_input_fn( - input_fn, model_fn_lib.ModeKeys.EVAL) + features, labels, input_hooks = ( + self._get_features_and_labels_from_input_fn( + input_fn, model_fn_lib.ModeKeys.EVAL)) estimator_spec = self._call_model_fn( features, labels, model_fn_lib.ModeKeys.EVAL, self.config) @@ -844,7 +859,8 @@ class Estimator(object): 'already defines a default metric with the same name.') eval_dict[ops.GraphKeys.GLOBAL_STEP] = global_step_tensor - all_hooks = list(hooks or []) + all_hooks = list(input_hooks) + all_hooks.extend(hooks) all_hooks.extend(list(estimator_spec.evaluation_hooks or [])) eval_results = evaluation._evaluate_once( # pylint: disable=protected-access @@ -1039,3 +1055,16 @@ def _has_dataset_or_queue_runner(maybe_tensor): # Now, check queue. return ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS) + + +class _DatasetInitializerHook(training.SessionRunHook): + + def __init__(self, iterator): + self._iterator = iterator + + def begin(self): + self._initializer = self._iterator.initializer + + def after_create_session(self, session, coord): + del coord + session.run(self._initializer) diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py index c1b773b8c4..db64fbc9cc 100644 --- a/tensorflow/python/estimator/estimator_test.py +++ b/tensorflow/python/estimator/estimator_test.py @@ -913,6 +913,80 @@ class EstimatorGetVariablesTest(test.TestCase): self.assertEqual(3., est.get_variable_value('three')) +class EstimatorDatasetIntegrationTest(test.TestCase): + """Tests dataset integration.""" + + def test_returned_by_input_fn(self): + + def _input_fn(): + return dataset_ops.Dataset.from_tensors(([1.], [2.])) + + def _model_fn(features, labels, mode): + return model_fn_lib.EstimatorSpec( + mode, + loss=features + labels, # 1 + 2 + train_op=state_ops.assign_add(training.get_global_step(), 1)) + + est = estimator.Estimator(model_fn=_model_fn) + est.train(_input_fn, steps=1) + scores = est.evaluate(_input_fn, steps=1) + self.assertEqual(3., scores[model_fn_lib.LOSS_METRIC_KEY]) + + def test_with_none_labels(self): + + def _input_fn(): + return dataset_ops.Dataset.from_tensors([7.]) + + def _model_fn(features, labels, mode): + self.assertIsNone(labels) + return model_fn_lib.EstimatorSpec( + mode, + loss=features, # 7 + train_op=state_ops.assign_add(training.get_global_step(), 1)) + + est = estimator.Estimator(model_fn=_model_fn) + est.train(_input_fn, steps=1) + scores = est.evaluate(_input_fn, steps=1) + self.assertEqual(7., scores[model_fn_lib.LOSS_METRIC_KEY]) + + def test_with_predict(self): + + def _input_fn(): + return dataset_ops.Dataset.from_tensors([10.]) + + def _model_fn(features, labels, mode): + _ = labels + return model_fn_lib.EstimatorSpec( + mode, + predictions=features, # 10 + loss=features, # 10 + train_op=state_ops.assign_add(training.get_global_step(), 1)) + + est = estimator.Estimator(model_fn=_model_fn) + est.train(_input_fn, steps=1) + self.assertEqual([10.], next(est.predict(input_fn=_input_fn))) + + def test_batching(self): + + def _input_fn(): + return dataset_ops.Dataset.from_tensor_slices(([[1.], [2.]], + [[10.], [20.]])).batch(1) + + def _model_fn(features, labels, mode): + return model_fn_lib.EstimatorSpec( + mode, + predictions=features, + loss=features + (0 if labels is None else labels), # 11, 22 + train_op=state_ops.assign_add(training.get_global_step(), 1)) + + est = estimator.Estimator(model_fn=_model_fn) + est.train(_input_fn) + scores = est.evaluate(_input_fn) + # (11 + 22)/2 = 16.5 + self.assertEqual(16.5, scores[model_fn_lib.LOSS_METRIC_KEY]) + self.assertEqual([1., 2.], list(est.predict(_input_fn))) + + class EstimatorEvaluateTest(test.TestCase): def test_input_fn_args(self): -- GitLab From 49bb801e65caf6afeb7cc7f67a168c9a19582ad1 Mon Sep 17 00:00:00 2001 From: HyoukJoong Lee Date: Tue, 28 Nov 2017 14:11:13 -0800 Subject: [PATCH 0064/1924] Changed to allow removing side-effect instructions from an HLO computation and moved the condition to the hlo_dce pass. PiperOrigin-RevId: 177215395 --- .../compiler/xla/service/hlo_computation.cc | 7 ++----- .../compiler/xla/service/hlo_computation.h | 16 ++++++++++----- tensorflow/compiler/xla/service/hlo_dce.cc | 3 ++- .../compiler/xla/service/hlo_dce_test.cc | 20 +++++++++++++++++++ .../xla/service/while_loop_simplifier.cc | 4 ++-- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index c215cc48d6..014a851c96 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -176,10 +176,6 @@ bool HloComputation::IsRemovable(const HloInstruction* instruction) { return false; } - if (instruction->HasSideEffect()) { - return false; - } - return true; } @@ -207,7 +203,8 @@ Status HloComputation::RemoveInstructionAndUnusedOperands( worklist.pop(); if (removed.count(item) != 0 || item->user_count() != 0 || - item == root_instruction() || !IsRemovable(item)) { + item == root_instruction() || !IsRemovable(item) || + item->HasSideEffect()) { continue; } for (int i = 0; i < item->operand_count(); ++i) { diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index 353b30bc69..ccedda2a03 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -313,11 +313,17 @@ class HloComputation { replacements, HloModule* module = nullptr, const string& suffix = "clone"); - // Returns true if the given instruction can be removed from the - // computation. Instructions such as parameters and send/receive instructions - // cannot be removed without violating invariants of the HLO computation or - // module with the exception of fusion computation. A parameter instruction - // is removable for a fusion computation. + // Returns true if the given instruction can be removed from the computation. + // Parameter instructions cannot be removed without violating invariants of + // the HLO computation with the exception of fusion computation. A parameter + // instruction is removable for a fusion computation. + // + // Note that IsRemovable() is a necessariy condition to remove an instruction + // rather than a sufficient condition. For example, instructions with + // side-effect (e.g., Send, Infeed) may be removed from a computation, but the + // transformation must guarantee the invariants relevant to the instructions + // still hold (e.g., Send and Recv must be removed together to make each + // channel complete). bool IsRemovable(const HloInstruction* instruction); // Returns true if this computation has a side effect. A computation has a diff --git a/tensorflow/compiler/xla/service/hlo_dce.cc b/tensorflow/compiler/xla/service/hlo_dce.cc index 40e67c8780..1e5f0f797a 100644 --- a/tensorflow/compiler/xla/service/hlo_dce.cc +++ b/tensorflow/compiler/xla/service/hlo_dce.cc @@ -55,7 +55,8 @@ StatusOr HloDCE::Run(HloModule* module) { for (auto* instruction : computation->instructions()) { if (instruction->user_count() == 0 && live_instructions.count(instruction) == 0 && - computation->IsRemovable(instruction)) { + computation->IsRemovable(instruction) && + !instruction->HasSideEffect()) { dead_roots.push_back(instruction); } } diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc index d54b9a2708..5a56607a66 100644 --- a/tensorflow/compiler/xla/service/hlo_dce_test.cc +++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc @@ -70,6 +70,26 @@ TEST_F(HloDceTest, NoDeadCode) { EXPECT_EQ(3, computation->instruction_count()); } +TEST_F(HloDceTest, InstructionsWithSideEffect) { + // Verify that side-effect instructions (Send in this test) are not removed. + auto builder = HloComputation::Builder(TestName()); + auto constant = builder.AddInstruction( + HloInstruction::CreateConstant(Literal::CreateR0(42.0f))); + builder.AddInstruction( + HloInstruction::CreateSend(constant, /*channel_id=*/0)); + builder.AddInstruction(HloInstruction::CreateTuple({})); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + + EXPECT_EQ(3, computation->instruction_count()); + + HloDCE dce; + EXPECT_FALSE(dce.Run(module.get()).ValueOrDie()); + + EXPECT_EQ(3, computation->instruction_count()); +} + TEST_F(HloDceTest, DeadParameters) { // Verify that dead parameters are not removed, but use of the dead parameters // are. diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.cc b/tensorflow/compiler/xla/service/while_loop_simplifier.cc index b38ee907d7..b2fd64a4d9 100644 --- a/tensorflow/compiler/xla/service/while_loop_simplifier.cc +++ b/tensorflow/compiler/xla/service/while_loop_simplifier.cc @@ -289,7 +289,7 @@ static StatusOr TryRemoveDeadWhileParams(HloInstruction* while_op) { // Don't try this transformation if the while loop isn't removable, since if // it succeeds ultimately we're going to have to replace the old while loop // with a new one. - if (!while_op->parent()->IsRemovable(while_op)) { + if (!while_op->parent()->IsRemovable(while_op) || while_op->HasSideEffect()) { VLOG(2) << "Can't remove dead parameters from non-removable while op."; return false; } @@ -558,7 +558,7 @@ static StatusOr TryRemoveWhileLoop(HloInstruction* while_op) { // the loop aren't removed, just cloned and added back to the loop. // Nevertheless our infrastructure sees loop simplification as removal of // these nodes and currently doesn't allow it. - if (!while_op->parent()->IsRemovable(while_op)) { + if (!while_op->parent()->IsRemovable(while_op) || while_op->HasSideEffect()) { VLOG(2) << "Not attempting to remove while loop it is not removable: " << while_op->ToShortString(); return false; -- GitLab From b5683d210834fd314410ea4b9c1a756b473fdece Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 14:12:19 -0800 Subject: [PATCH 0065/1924] Change description of sparse_column_with_integerized_feature to make consistent with the _SparseColumn that it creates. Documentation here says that the bucket_size must be an int that is greater than 1. The check performed when creating a _SparseColumn only requires that the bucket_size be at least 1. Hence, bucket_size==1 should be ok. PiperOrigin-RevId: 177215556 --- tensorflow/contrib/layers/python/layers/feature_column.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/layers/python/layers/feature_column.py b/tensorflow/contrib/layers/python/layers/feature_column.py index 226d933d85..092d418c3f 100644 --- a/tensorflow/contrib/layers/python/layers/feature_column.py +++ b/tensorflow/contrib/layers/python/layers/feature_column.py @@ -521,7 +521,7 @@ def sparse_column_with_integerized_feature(column_name, Args: column_name: A string defining sparse column name. - bucket_size: An int that is > 1. The number of buckets. It should be bigger + bucket_size: An int that is >= 1. The number of buckets. It should be bigger than maximum feature. In other words features in this column should be an int64 in range [0, bucket_size) combiner: A string specifying how to reduce if the sparse column is @@ -539,7 +539,7 @@ def sparse_column_with_integerized_feature(column_name, An integerized _SparseColumn definition. Raises: - ValueError: bucket_size is not greater than 1. + ValueError: bucket_size is less than 1. ValueError: dtype is not integer. """ return _SparseColumnIntegerized( -- GitLab From bb33903b4b34e4ac096908c1a08cf5ffa33b6ccf Mon Sep 17 00:00:00 2001 From: Olivia Nordquist Date: Tue, 28 Nov 2017 14:17:25 -0800 Subject: [PATCH 0066/1924] add parentheses because this test is failing in my current CL PiperOrigin-RevId: 177216384 --- tensorflow/python/framework/ops_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index e929cc8abf..371eadcd13 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -1537,7 +1537,7 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase): self.assertEqual(future.calls, 1) else: a = constant_op.constant(1.0) - b = future + b = future() with ops.control_dependencies([a, b]): c = constant_op.constant(3.0) self.assertEqual(future.calls, 1) -- GitLab From e917bf7131b3216f7d09c0251d27a9aafd5b8373 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 14:36:12 -0800 Subject: [PATCH 0067/1924] [TF] Improve LogSoftMax performance In Eigen, eval should be called immediately before a broadcast. Otherwise, broadcast's lazy evaluation causes the broadcasted expression to be evaluated many times. Moving the eval changes the number of calls to log from batch_size * num_classes to batch_size. PiperOrigin-RevId: 177219486 --- tensorflow/core/kernels/softmax_op_functor.h | 33 +++++++++----------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/tensorflow/core/kernels/softmax_op_functor.h b/tensorflow/core/kernels/softmax_op_functor.h index 1f38bdce8c..d3a267ed87 100644 --- a/tensorflow/core/kernels/softmax_op_functor.h +++ b/tensorflow/core/kernels/softmax_op_functor.h @@ -64,23 +64,21 @@ struct SoftmaxEigenImpl { one_by_class.set(1, num_classes); #endif // shifted_logits = logits - max(logits along classes); - auto shifted_logits = (logits - - logits.maximum(along_class) - .eval() - .reshape(batch_by_one) - .broadcast(one_by_class)); + auto shifted_logits = (logits - logits.maximum(along_class) + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)); if (log) { // Calculate the log of the softmax // softmax = logits - max(logits along classes); softmax.device(d) = shifted_logits; // softmax = softmax - log(sum(exp(softmax along classes))); - softmax.device(d) = (softmax - - softmax.exp() - .sum(along_class) - .eval() - .reshape(batch_by_one) - .log() - .broadcast(one_by_class)); + softmax.device(d) = (softmax - softmax.exp() + .sum(along_class) + .log() + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)); } else { // NOTE(touts): If you modify this implementation please run // the BM_ImageNetSoftmaxFwd benchmark in nn_ops_test.cc. @@ -88,12 +86,11 @@ struct SoftmaxEigenImpl { // softmax = exp(logits - max(logits along classes)); softmax.device(d) = shifted_logits.exp(); // softmax = softmax * (1 / sum(softmax along classes)); - softmax.device(d) = (softmax * - softmax.sum(along_class) - .inverse() - .eval() - .reshape(batch_by_one) - .broadcast(one_by_class)); + softmax.device(d) = (softmax * softmax.sum(along_class) + .inverse() + .eval() + .reshape(batch_by_one) + .broadcast(one_by_class)); } } }; -- GitLab From f252ea2d8ac13dd5c558e3862b3885585d3bccfe Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Tue, 28 Nov 2017 14:48:22 -0800 Subject: [PATCH 0068/1924] Deprecating `tf.data.Dataset.from_sparse_tensor_slices`. PiperOrigin-RevId: 177221417 --- tensorflow/python/data/ops/BUILD | 1 + tensorflow/python/data/ops/dataset_ops.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD index 05acfe4de7..695d3ef790 100644 --- a/tensorflow/python/data/ops/BUILD +++ b/tensorflow/python/data/ops/BUILD @@ -21,6 +21,7 @@ py_library( "//tensorflow/python:sparse_tensor", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", + "//tensorflow/python:util", "//tensorflow/python/data/util:nest", "//tensorflow/python/data/util:sparse", "//third_party/py/numpy", diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index dbe29c087a..b5a8622306 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -41,6 +41,7 @@ from tensorflow.python.ops import gen_io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import script_ops from tensorflow.python.ops import sparse_ops +from tensorflow.python.util import deprecation class Dataset(object): @@ -219,6 +220,7 @@ class Dataset(object): return TensorSliceDataset(tensors) @staticmethod + @deprecation.deprecated(None, "Use `tf.data.Dataset.from_tensor_slices()`.") def from_sparse_tensor_slices(sparse_tensor): """Splits each rank-N `tf.SparseTensor` in this dataset row-wise. -- GitLab From efe5658aaa6f1666d4967880311430a70bdb23b9 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Tue, 28 Nov 2017 14:50:43 -0800 Subject: [PATCH 0069/1924] Make 'input_map' argument to import_graph_def work with C API. PiperOrigin-RevId: 177221757 --- tensorflow/python/framework/importer.py | 88 +++++++++++++++----- tensorflow/python/framework/importer_test.py | 29 ++----- 2 files changed, 74 insertions(+), 43 deletions(-) diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index 434cbda7ad..73c35de578 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -179,12 +179,11 @@ def _ProcessInputMapParam(input_map): def _ProcessReturnElementsParam(return_elements): """Type-checks and possibly canonicalizes `return_elements`.""" - if return_elements is not None: - return_elements = tuple(return_elements) - if not all(isinstance(x, compat.bytes_or_text_types) - for x in return_elements): - raise TypeError('return_elements must be a list of strings.') - return return_elements + if return_elements is None: return None + if not all(isinstance(x, compat.bytes_or_text_types) + for x in return_elements): + raise TypeError('return_elements must be a list of strings.') + return tuple(compat.as_str(x) for x in return_elements) def _FindAttrInOpDef(attr_name, op_def): @@ -194,16 +193,60 @@ def _FindAttrInOpDef(attr_name, op_def): return None -def _PopulateTFImportGraphDefOptions(options, prefix, return_elements): +def _ConvertInputMapValues(name, input_map): + """Ensures all input map values are tensors. + + This should be called from inside the import name scope. + + Args: + name: the `name` argument passed to import_graph_def + input_map: the `input_map` argument passed to import_graph_def. + + Returns: + An possibly-updated version of `input_map`. + + Raises: + ValueError: if input map values cannot be converted due to empty name scope. + """ + if not all(isinstance(v, ops.Tensor) for v in input_map.values()): + if name == '': # pylint: disable=g-explicit-bool-comparison + raise ValueError( + 'tf.import_graph_def() requires a non-empty `name` if `input_map` ' + 'contains non-Tensor values. Try calling tf.convert_to_tensor() on ' + '`input_map` values before calling tf.import_graph_def().') + with ops.name_scope('_inputs'): + input_map = {k: ops.convert_to_tensor(v) for k, v in input_map.items()} + return input_map + + +def _PopulateTFImportGraphDefOptions(options, prefix, input_map, + return_elements): """Populates the TF_ImportGraphDefOptions `options`.""" c_api.TF_ImportGraphDefOptionsSetPrefix(options, prefix) + for input_src, input_dst in input_map.items(): + input_src = compat.as_str(input_src) + if input_src.startswith('^'): + src_name = compat.as_bytes(input_src[1:]) + dst_op = input_dst._as_tf_output().oper # pylint: disable=protected-access + c_api.TF_ImportGraphDefOptionsRemapControlDependency(options, src_name, + dst_op) + else: + src_name, src_idx = _ParseTensorName(input_src) + src_name = compat.as_str(src_name) + dst_output = input_dst._as_tf_output() # pylint: disable=protected-access + c_api.TF_ImportGraphDefOptionsAddInputMapping(options, src_name, + src_idx, dst_output) for name in return_elements or []: if ':' in name: op_name, index = _ParseTensorName(name) + op_name = compat.as_str(op_name) c_api.TF_ImportGraphDefOptionsAddReturnOutput(options, op_name, index) else: - c_api.TF_ImportGraphDefOptionsAddReturnOperation(options, name) + c_api.TF_ImportGraphDefOptionsAddReturnOperation(options, + compat.as_str(name)) + + # TODO(skyewm): control dependencies def _ProcessNewOps(graph): @@ -312,17 +355,27 @@ def import_graph_def(graph_def, input_map=None, return_elements=None, else: prefix = '' + # Generate any input map tensors inside name scope + input_map = _ConvertInputMapValues(name, input_map) + scoped_options = c_api_util.ScopedTFImportGraphDefOptions() options = scoped_options.options - _PopulateTFImportGraphDefOptions(options, prefix, return_elements) + _PopulateTFImportGraphDefOptions(options, prefix, input_map, + return_elements) with c_api_util.tf_buffer(graph_def.SerializeToString()) as serialized: - with errors.raise_exception_on_not_ok_status() as status: - results = c_api.TF_GraphImportGraphDefWithResults( - graph._c_graph, serialized, options, status) # pylint: disable=protected-access + try: + with errors.raise_exception_on_not_ok_status() as status: + results = c_api.TF_GraphImportGraphDefWithResults( + graph._c_graph, serialized, options, status) # pylint: disable=protected-access + except errors.InvalidArgumentError as e: + # Convert to ValueError for backwards compatibility. + raise ValueError(str(e)) _ProcessNewOps(graph) + # TODO(skyewm): error if unused input map key + if return_elements is None: return None else: @@ -359,16 +412,7 @@ def import_graph_def(graph_def, input_map=None, return_elements=None, # more nuanced. g.graph_def_versions.CopyFrom(graph_def.versions) - if not all(isinstance(v, ops.Tensor) for v in input_map.values()): - if not scope: - # The caller must have passed `name=''`. - raise ValueError( - 'tf.import_graph_def() requires a non-empty `name` if `input_map`' - ' contains non-Tensor values. Try calling tf.convert_to_tensor() ' - 'on `input_map` values before calling tf.import_graph_def().') - with ops.name_scope('_inputs'): - input_map = {k: ops.convert_to_tensor(v) - for k, v in input_map.items()} + input_map = _ConvertInputMapValues(name, input_map) # NOTE(mrry): We do this in two passes, because there may be a cycle in # `graph_def`. diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py index 5a6187c8a6..000a88bc09 100644 --- a/tensorflow/python/framework/importer_test.py +++ b/tensorflow/python/framework/importer_test.py @@ -201,8 +201,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(outer_inner_c.name, "outer/inner/c_1") def testInputMap(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): feed_a_0 = constant_op.constant(0, dtype=dtypes.int32) feed_b_1 = constant_op.constant(1, dtype=dtypes.int32) @@ -230,8 +228,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(d.inputs[1], feed_b_1) def testInputMapBytes(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): feed_a_0 = constant_op.constant(0, dtype=dtypes.int32) feed_b_1 = constant_op.constant(1, dtype=dtypes.int32) @@ -259,8 +255,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(d.inputs[1], feed_b_1) def testInputMapUnicode(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): feed_a_0 = constant_op.constant(0, dtype=dtypes.int32) feed_b_1 = constant_op.constant(1, dtype=dtypes.int32) @@ -299,8 +293,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(b.inputs[0], a.outputs[0]) def testInputMapImplicitZerothOutput(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): feed_a_0 = constant_op.constant(0, dtype=dtypes.int32) b, = importer.import_graph_def( @@ -453,8 +445,6 @@ class ImportGraphDefTest(test.TestCase): self.assertTrue("Input tensor 'A:0' not found" in str(e.exception)) def testMissingInputOpInGraphDefButAppearsInInputMap(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): feed_a_0 = constant_op.constant(5.0) b, = importer.import_graph_def( @@ -589,19 +579,20 @@ class ImportGraphDefTest(test.TestCase): self.assertTrue("not found in graph_def: [A:2]" in str(e.exception)) def testInputMapTypeMismatch(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - + if ops._USE_C_API: + error_msg = ("Input 0 of node import/B was passed float from Const:0 " + "incompatible with expected int32.") + else: + error_msg = ("Cannot convert a tensor of type float32 to an input of " + "type int32.") with ops.Graph().as_default(): - with self.assertRaises(ValueError) as e: + with self.assertRaisesRegexp(ValueError, error_msg): importer.import_graph_def( self._MakeGraphDef(""" node { name: 'A' op: 'IntOutput' } node { name: 'B' op: 'IntInput' input: 'A:0' } """), input_map={"A:0": constant_op.constant(5.0)}) - self.assertTrue( - "Cannot convert a tensor of type float32 to an input of type int32." - in str(e.exception)) def testNoReturns(self): with ops.Graph().as_default() as g: @@ -825,8 +816,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual("graph_def must be a GraphDef proto.", str(e.exception)) def testInvalidInputForInputMap(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - with ops.Graph().as_default(): with self.assertRaises(TypeError) as e: importer.import_graph_def( @@ -967,7 +956,7 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(2, len(ops_with_two_inputs)) def testGradient(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API + if ops._USE_C_API: return # TODO(skyewm): get_shape() doesn't work with ops.Graph().as_default() as g: inputs = array_ops.placeholder( @@ -1226,8 +1215,6 @@ class ImportGraphDefTest(test.TestCase): self.assertEqual(z_val, -2.0) def testImportGraphWithFunctionTwice(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - g = ops.Graph() with g.as_default(): @function.Defun() -- GitLab From 9306dd922fde7b739c5a4230fdc6d9bd646fb71c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 15:02:11 -0800 Subject: [PATCH 0070/1924] Add bool value type support for gauge metrics. PiperOrigin-RevId: 177223509 --- .../core/lib/monitoring/collected_metrics.h | 1 + .../core/lib/monitoring/collection_registry.h | 6 ++++ tensorflow/core/lib/monitoring/gauge.h | 33 +++++++++++++++++-- tensorflow/core/lib/monitoring/gauge_test.cc | 22 +++++++++++++ tensorflow/core/lib/monitoring/metric_def.h | 13 +++++--- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/lib/monitoring/collected_metrics.h b/tensorflow/core/lib/monitoring/collected_metrics.h index fbef25619f..acdb0d86ed 100644 --- a/tensorflow/core/lib/monitoring/collected_metrics.h +++ b/tensorflow/core/lib/monitoring/collected_metrics.h @@ -88,6 +88,7 @@ struct Point { ValueType value_type; int64 int64_value; string string_value; + bool bool_value; HistogramProto histogram_value; // start_timestamp and end_timestamp indicate the time period over which this diff --git a/tensorflow/core/lib/monitoring/collection_registry.h b/tensorflow/core/lib/monitoring/collection_registry.h index 113d37e07d..2c8e250c56 100644 --- a/tensorflow/core/lib/monitoring/collection_registry.h +++ b/tensorflow/core/lib/monitoring/collection_registry.h @@ -224,6 +224,12 @@ inline void CollectValue(const string& value, Point* const point) { point->string_value = value; } +template <> +inline void CollectValue(const bool& value, Point* const point) { + point->value_type = ValueType::kBool; + point->bool_value = value; +} + template <> inline void CollectValue(const HistogramProto& value, Point* const point) { point->value_type = ValueType::kHistogram; diff --git a/tensorflow/core/lib/monitoring/gauge.h b/tensorflow/core/lib/monitoring/gauge.h index 75471cfb22..ec978a9193 100644 --- a/tensorflow/core/lib/monitoring/gauge.h +++ b/tensorflow/core/lib/monitoring/gauge.h @@ -86,8 +86,29 @@ class GaugeCell { TF_DISALLOW_COPY_AND_ASSIGN(GaugeCell); }; +// Explicit specialization of GaugeCell. Compared to the primary +// template, it uses atomic values as opposed to mutex. This class is +// thread-safe. +template <> +class GaugeCell { + public: + explicit GaugeCell(bool value) : value_(value) {} + ~GaugeCell() {} + + // Atomically sets the value. + void Set(bool value); + + // Retrieves the current value. + bool value() const; + + private: + std::atomic value_; + + TF_DISALLOW_COPY_AND_ASSIGN(GaugeCell); +}; + // A stateful class for updating a gauge-like metric. Allowed ValueType are -// int64 and string. +// int64, string and bool. // // This class encapsulates a set of values (or a single value for a label-less // metric). Each value is identified by a tuple of labels. The class allows the @@ -117,6 +138,9 @@ class Gauge { // // auto* integer_gauge = Gauge::New("/tensorflow/integer_gauge", // "Integer gauge") + // + // auto* bool_gauge = Gauge::New("/tensorflow/bool_gauge", + // "Bool gauge") template static Gauge* New(MetricDefArgs&&... metric_def_args); @@ -172,12 +196,17 @@ inline void GaugeCell::Set(int64 value) { value_ = value; } inline int64 GaugeCell::value() const { return value_; } +inline void GaugeCell::Set(bool value) { value_ = value; } + +inline bool GaugeCell::value() const { return value_; } + template template Gauge* Gauge::New( MetricDefArgs&&... metric_def_args) { static_assert(std::is_same::value || - std::is_same::value, + std::is_same::value || + std::is_same::value, "Gauge only allows int64 and string types."); return new Gauge( MetricDef( diff --git a/tensorflow/core/lib/monitoring/gauge_test.cc b/tensorflow/core/lib/monitoring/gauge_test.cc index f98cfe2a3b..c8f673db38 100644 --- a/tensorflow/core/lib/monitoring/gauge_test.cc +++ b/tensorflow/core/lib/monitoring/gauge_test.cc @@ -87,6 +87,28 @@ TEST(GaugeOfStringValue, GetCell) { EXPECT_EQ("bar", same_cell->value()); } +auto* bool_gauge = + Gauge::New("/tensorflow/test/bool_gauge", "Gauge of bool value."); + +TEST(GaugeOfBoolValue, InitializedWithFalseValue) { + EXPECT_EQ(false, bool_gauge->GetCell()->value()); +} + +TEST(GaugeOfBoolValue, GetCell) { + auto* cell = bool_gauge->GetCell(); + EXPECT_EQ(false, cell->value()); + + cell->Set(true); + EXPECT_EQ(true, cell->value()); + + auto* same_cell = bool_gauge->GetCell(); + EXPECT_EQ(true, cell->value()); + + same_cell->Set(false); + EXPECT_EQ(false, cell->value()); + EXPECT_EQ(false, same_cell->value()); +} + } // namespace } // namespace monitoring } // namespace tensorflow diff --git a/tensorflow/core/lib/monitoring/metric_def.h b/tensorflow/core/lib/monitoring/metric_def.h index a7f14f9c94..f046842618 100644 --- a/tensorflow/core/lib/monitoring/metric_def.h +++ b/tensorflow/core/lib/monitoring/metric_def.h @@ -28,16 +28,16 @@ namespace monitoring { // The different metric kinds available. // // Gauge indicates that the metric's values are instantaneous measurements of a -// (typically) continuously varying quantity or a string value. Examples: a -// process's current heap size, a queue's current length, the name of the binary -// used by a process. +// (typically) continuously varying value. Examples: a process's current heap +// size, a queue's current length, the name of the binary used by a process, +// whether a task is complete. // // Cumulative indicates that the metric's values represent non-negative changes // over specified time periods. Example: the number of rpc calls to a service. enum class MetricKind : int { kGauge = 0, kCumulative }; // The type of the metric values. -enum class ValueType : int { kInt64 = 0, kHistogram, kString }; +enum class ValueType : int { kInt64 = 0, kHistogram, kString, kBool }; // Everything in the internal namespace is implementation details. Do not depend // on this. @@ -61,6 +61,11 @@ inline ValueType GetValueType() { return ValueType::kString; } +template <> +inline ValueType GetValueType() { + return ValueType::kBool; +} + } // namespace internal // Abstract base class for a metric definition. -- GitLab From 8966a794411bd5d17e5ef024a96140f85a9ab500 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Tue, 28 Nov 2017 15:14:30 -0800 Subject: [PATCH 0071/1924] Add a log test for bfloat16. PiperOrigin-RevId: 177225564 --- tensorflow/compiler/xla/tests/bfloat16_test.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorflow/compiler/xla/tests/bfloat16_test.cc b/tensorflow/compiler/xla/tests/bfloat16_test.cc index a1c53ef2aa..ac3f3f4c9d 100644 --- a/tensorflow/compiler/xla/tests/bfloat16_test.cc +++ b/tensorflow/compiler/xla/tests/bfloat16_test.cc @@ -61,6 +61,15 @@ XLA_TEST_F(Bfloat16Test, ScalarOperation) { error_spec_); } +XLA_TEST_F(Bfloat16Test, LogOperation) { + ComputationBuilder builder(client_, TestName()); + auto x = builder.ConstantR0(static_cast(4.0f)); + builder.Log(x); + + ComputeAndCompareR0(&builder, static_cast(1.387f), {}, + error_spec_); +} + XLA_TEST_F(Bfloat16Test, NegateScalarF16) { ComputationBuilder builder(client_, TestName()); builder.Neg(builder.ConstantR0(static_cast(2.1f))); -- GitLab From d72e2a318c6b15d800aa1468dc2af658ea40dffd Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Tue, 28 Nov 2017 15:16:16 -0800 Subject: [PATCH 0072/1924] scatter_nd_update for resource variables PiperOrigin-RevId: 177225812 --- .../api_def_ResourceScatterNdUpdate.pbtxt | 69 ++++++++++++++++++ tensorflow/core/framework/common_shape_fns.cc | 7 +- tensorflow/core/kernels/BUILD | 6 +- tensorflow/core/kernels/scatter_nd_op.cc | 63 ++++++++++++++--- tensorflow/core/ops/state_ops.cc | 56 +++++++++++++++ tensorflow/python/kernel_tests/BUILD | 1 + .../kernel_tests/scatter_nd_ops_test.py | 15 ++++ tensorflow/python/ops/state_ops.py | 70 ++++++++++++++++++- 8 files changed, 274 insertions(+), 13 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt new file mode 100644 index 0000000000..b07ee9fda9 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterNdUpdate.pbtxt @@ -0,0 +1,69 @@ +op { + graph_op_name: "ResourceScatterNdUpdate" + in_arg { + name: "ref" + description: <input(0); + if (c->input_handle_shapes_and_types(0) != nullptr) { + input_shape = (*c->input_handle_shapes_and_types(0))[0].shape; + } ShapeHandle indices_shape; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape)); ShapeHandle updates_shape; @@ -1361,7 +1364,9 @@ Status ScatterNdUpdateShape(InferenceContext* c) { } } - c->set_output(0, input_shape); + if (c->input_handle_shapes_and_types(0) == nullptr) { + c->set_output(0, input_shape); + } return Status::OK(); } diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index b86739eea7..eff15e809a 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -3918,7 +3918,11 @@ tf_kernel_library( "scatter_nd_op.h", "scatter_nd_op_gpu.cu.cc", ], - deps = STATE_DEPS + [":dense_update_functor"], + deps = STATE_DEPS + [ + ":dense_update_functor", + ":training_op_helpers", + ":variable_ops", + ], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc index 484932ab01..98c0181afb 100644 --- a/tensorflow/core/kernels/scatter_nd_op.cc +++ b/tensorflow/core/kernels/scatter_nd_op.cc @@ -21,6 +21,7 @@ limitations under the License. #endif // GOOGLE_CUDA #include "tensorflow/core/kernels/scatter_nd_op.h" + #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" @@ -28,6 +29,8 @@ limitations under the License. #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/dense_update_functor.h" #include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/kernels/training_op_helpers.h" +#include "tensorflow/core/kernels/variable_ops.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" @@ -83,7 +86,10 @@ class ScatterNdUpdateOp : public OpKernel { const DataType dt = DataTypeToEnum::v(); const DataType dt_ref = DataTypeToEnum::ref(); const DataType index_t = DataTypeToEnum::v(); - if (IsRefType(c->input_type(0))) { + dtype_ = c->input_type(0); + if (c->input_type(0) == DT_RESOURCE) { + // TODO(apassos): what to validate here? + } else if (IsRefType(c->input_type(0))) { OP_REQUIRES_OK(c, c->MatchSignature({dt_ref, index_t, dt}, {dt_ref})); OP_REQUIRES_OK(c, c->GetAttr("use_locking", &use_exclusive_lock_)); } else { @@ -93,7 +99,16 @@ class ScatterNdUpdateOp : public OpKernel { } void Compute(OpKernelContext* c) override { - if (use_exclusive_lock_) { + if (dtype_ == DT_RESOURCE) { + if (use_exclusive_lock_) { + Var* v; + OP_REQUIRES_OK(c, LookupResource(c, HandleFromInput(c, 0), &v)); + mutex_lock m(*v->mu()); + DoCompute(c); + } else { + DoCompute(c); + } + } else if (use_exclusive_lock_) { // If we're here, it means the input type is a ref. DCHECK(IsRefType(c->input_dtype(0))); // Hold mutex while we apply updates @@ -105,6 +120,7 @@ class ScatterNdUpdateOp : public OpKernel { } private: + DataType dtype_; bool use_exclusive_lock_; void DoCompute(OpKernelContext* c) { @@ -113,7 +129,20 @@ class ScatterNdUpdateOp : public OpKernel { Tensor params; TensorShape params_shape; - if (IsRefType(c->input_dtype(0))) { + if (dtype_ == DT_RESOURCE) { + Var* v; + OP_REQUIRES_OK(c, LookupResource(c, HandleFromInput(c, 0), &v)); + Tensor* t = v->tensor(); + if (!use_exclusive_lock_) { + // We're not holding the lock in the outer scope so need it here. + mutex_lock m(*v->mu()); + OP_REQUIRES_OK(c, PrepareToUpdateVariable(c, t)); + } else { + OP_REQUIRES_OK(c, PrepareToUpdateVariable(c, t)); + } + params = *t; + params_shape = params.shape(); + } else if (IsRefType(c->input_dtype(0))) { params = c->mutable_input(0, use_exclusive_lock_); params_shape = params.shape(); c->forward_ref_input_to_ref_output(0, 0); @@ -159,6 +188,16 @@ class ScatterNdUpdateOp : public OpKernel { .TypeConstraint("Tindices"), \ ScatterNdUpdateOp) +#define REGISTER_RESOURCE_SCATTER_ND_UPDATE_KERNEL_INDEX(type, index_type, \ + dev, name, op) \ + REGISTER_KERNEL_BUILDER( \ + Name(name) \ + .Device(DEVICE_##dev) \ + .TypeConstraint("T") \ + .TypeConstraint("Tindices") \ + .HostMemory("ref"), \ + ScatterNdUpdateOp) + #define REGISTER_SCATTER_ND_KERNEL(type, dev, name) \ REGISTER_SCATTER_ND_KERNEL_INDEX(type, int32, dev, name); \ REGISTER_SCATTER_ND_KERNEL_INDEX(type, int64, dev, name) @@ -167,6 +206,11 @@ class ScatterNdUpdateOp : public OpKernel { REGISTER_SCATTER_ND_UPDATE_KERNEL_INDEX(type, int32, dev, name, op); \ REGISTER_SCATTER_ND_UPDATE_KERNEL_INDEX(type, int64, dev, name, op) +#define REGISTER_RESOURCE_SCATTER_ND_UPDATE_KERNEL(type, dev, name, op) \ + REGISTER_RESOURCE_SCATTER_ND_UPDATE_KERNEL_INDEX(type, int32, dev, name, \ + op); \ + REGISTER_RESOURCE_SCATTER_ND_UPDATE_KERNEL_INDEX(type, int64, dev, name, op) + #define REGISTER_SCATTER_ND_ADD_SUB(type, dev) \ REGISTER_SCATTER_ND_UPDATE_KERNEL(type, dev, "ScatterNdAdd", \ scatter_nd_op::UpdateOp::ADD); \ @@ -178,9 +222,11 @@ class ScatterNdUpdateOp : public OpKernel { #define REGISTER_SCATTER_ND(type, dev) \ REGISTER_SCATTER_ND_KERNEL(type, dev, "ScatterNd"); -#define REGISTER_SCATTER_ND_UPDATE(type, dev) \ - REGISTER_SCATTER_ND_UPDATE_KERNEL(type, dev, "ScatterNdUpdate", \ - scatter_nd_op::UpdateOp::ASSIGN); +#define REGISTER_SCATTER_ND_UPDATE(type, dev) \ + REGISTER_SCATTER_ND_UPDATE_KERNEL(type, dev, "ScatterNdUpdate", \ + scatter_nd_op::UpdateOp::ASSIGN); \ + REGISTER_RESOURCE_SCATTER_ND_UPDATE_KERNEL( \ + type, dev, "ResourceScatterNdUpdate", scatter_nd_op::UpdateOp::ASSIGN); // Registers CPU kernels. #define REGISTER_SCATTER_ND_ADD_SUB_CPU(type) \ @@ -281,8 +327,7 @@ Status ValidateUpdateShape(const TensorShape& params_shape, } template -Status PrepareAndValidateInputs(OpKernelContext* c, - const TensorShape& params_shape, +Status PrepareAndValidateInputs(const TensorShape& params_shape, const Tensor& indices, const Tensor& updates, int64* slice_dim, Index* num_updates, Index* slice_size) { @@ -396,7 +441,7 @@ Status DoScatterNd(OpKernelContext* c, const Tensor& indices, Index num_updates; Index slice_size; TF_RETURN_IF_ERROR(PrepareAndValidateInputs( - c, shape, indices, updates, &slice_dim, &num_updates, &slice_size)); + shape, indices, updates, &slice_dim, &num_updates, &slice_size)); IndexFlattener index_flattener; auto indices_flat = index_flattener(c, indices); diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc index da5f091e9f..5b1f5d2477 100644 --- a/tensorflow/core/ops/state_ops.cc +++ b/tensorflow/core/ops/state_ops.cc @@ -513,6 +513,62 @@ output_ref: Same as ref. Returned as a convenience for operations that want to use the updated values after the update is done. )doc"); +REGISTER_OP("ResourceScatterNdUpdate") + .Input("ref: resource") + .Input("indices: Tindices") + .Input("updates: T") + .Attr("T: type") + .Attr("Tindices: {int32, int64}") + .Attr("use_locking: bool = true") + .SetShapeFn(shape_inference::ScatterNdUpdateShape) + .Doc(R"doc( +Applies sparse `updates` to individual values or slices within a given +variable according to `indices`. + +`ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. + +`indices` must be integer tensor, containing indices into `ref`. +It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. + +The innermost dimension of `indices` (with length `K`) corresponds to +indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th +dimension of `ref`. + +`updates` is `Tensor` of rank `Q-1+P-K` with shape: + +``` +[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]. +``` + +For example, say we want to update 4 scattered elements to a rank-1 tensor to +8 elements. In Python, that update would look like this: + +```python + ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8]) + indices = tf.constant([[4], [3], [1] ,[7]]) + updates = tf.constant([9, 10, 11, 12]) + update = tf.scatter_nd_update(ref, indices, updates) + with tf.Session() as sess: + print sess.run(update) +``` + +The resulting update to ref would look like this: + + [1, 11, 3, 10, 9, 6, 7, 12] + +See @{tf.scatter_nd} for more details about how to make updates to +slices. + +ref: A resource handle. Must be from a VarHandleOp. +indices: A Tensor. Must be one of the following types: int32, int64. + A tensor of indices into ref. +updates: A Tensor. Must have the same type as ref. A tensor of updated + values to add to ref. +use_locking: An optional bool. Defaults to True. If True, the assignment will + be protected by a lock; otherwise the behavior is undefined, + but may exhibit less contention. +)doc"); + REGISTER_OP("ScatterNdAdd") .Input("ref: Ref(T)") .Input("indices: Tindices") diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 4522520ee4..f15b3baabe 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -676,6 +676,7 @@ cuda_py_test( "//tensorflow/python:gradients", "//tensorflow/python:state_ops", "//tensorflow/python:variables", + "//tensorflow/python:resource_variable_ops", ], tags = ["noasan"], # http://b/32635055 ) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index a79d66e988..d7bde04230 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -27,6 +27,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -157,6 +158,20 @@ class StatefulScatterNdTest(test.TestCase): result = sess.run(scatter) self.assertAllClose(result, expected) + def testSimpleResource(self): + indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) + updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32) + ref = resource_variable_ops.ResourceVariable( + [0, 0, 0, 0, 0, 0, 0, 0], dtype=dtypes.float32) + expected = np.array([0, 11, 0, 10, 9, 0, 0, 12]) + scatter = state_ops.scatter_nd_update(ref, indices, updates) + init = variables.global_variables_initializer() + + with self.test_session(use_gpu=True) as sess: + sess.run(init) + sess.run(scatter) + self.assertAllClose(ref.eval(), expected) + def testSimple2(self): indices = constant_op.constant([[1, 0], [1, 1]], dtype=dtypes.int32) updates = constant_op.constant([11., 12.], dtype=dtypes.float32) diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index dfc657893c..dee495f78f 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -347,5 +347,71 @@ def scatter_update(ref, indices, updates, use_locking=True, name=None): if ref.dtype._is_ref_dtype: return gen_state_ops.scatter_update(ref, indices, updates, use_locking=use_locking, name=name) - return gen_resource_variable_ops.resource_scatter_update( - ref.handle, indices, ops.convert_to_tensor(updates, ref.dtype), name=name) + with ops.control_dependencies( + [gen_resource_variable_ops.resource_scatter_update( + ref.handle, indices, ops.convert_to_tensor(updates, ref.dtype), + name=name)]): + return ref.read_value() + + +def scatter_nd_update(ref, indices, updates, use_locking=True, name=None): + r"""Applies sparse `updates` to individual values or slices in a Variable. + + `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`. + + `indices` must be integer tensor, containing indices into `ref`. + It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`. + + The innermost dimension of `indices` (with length `K`) corresponds to + indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th + dimension of `ref`. + + `updates` is `Tensor` of rank `Q-1+P-K` with shape: + + ``` + [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]]. + ``` + + For example, say we want to update 4 scattered elements to a rank-1 tensor to + 8 elements. In Python, that update would look like this: + + ```python + ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8]) + indices = tf.constant([[4], [3], [1] ,[7]]) + updates = tf.constant([9, 10, 11, 12]) + update = tf.scatter_nd_update(ref, indices, updates) + with tf.Session() as sess: + print sess.run(update) + ``` + + The resulting update to ref would look like this: + + [1, 11, 3, 10, 9, 6, 7, 12] + + See @{tf.scatter_nd} for more details about how to make updates to + slices. + + Args: + ref: A Variable. + indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. + A Tensor. Must be one of the following types: int32, int64. + A tensor of indices into ref. + updates: A `Tensor`. Must have the same type as `ref`. + A Tensor. Must have the same type as ref. A tensor of updated + values to add to ref. + use_locking: An optional `bool`. Defaults to `True`. + An optional bool. Defaults to True. If True, the assignment will + be protected by a lock; otherwise the behavior is undefined, + but may exhibit less contention. + name: A name for the operation (optional). + + Returns: + The value of the variable after the update. + """ + if ref.dtype._is_ref_dtype: + return gen_state_ops.scatter_nd_update( + ref, indices, updates, use_locking, name) + with ops.control_dependencies([gen_state_ops.resource_scatter_nd_update( + ref.handle, indices, ops.convert_to_tensor(updates, dtype=ref.dtype), + use_locking, name)]): + return ref.read_value() -- GitLab From a99e9a2c56a4922e76c367b8d3a9c43ea0a4ef61 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 28 Nov 2017 15:27:57 -0800 Subject: [PATCH 0073/1924] Support tfe.Network.losses Supports only variable regularization losses when executing eagerly. They are stored as zero-argument lambdas and executed when the property is requested. PiperOrigin-RevId: 177227550 --- tensorflow/contrib/eager/python/BUILD | 1 + tensorflow/contrib/eager/python/network.py | 24 +++++- .../contrib/eager/python/network_test.py | 29 +++++++ tensorflow/python/layers/base.py | 85 ++++++++++++------- tensorflow/python/layers/base_test.py | 5 ++ 5 files changed, 114 insertions(+), 30 deletions(-) diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index bf2e883bc5..55d768044b 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -232,6 +232,7 @@ py_test( srcs_version = "PY2AND3", deps = [ ":network", + "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:constant_op", "//tensorflow/python:errors", "//tensorflow/python:framework_test_lib", diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index 0388aaa849..e3c13cbd2e 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -451,8 +451,30 @@ class Network(base.Layer): "at https://github.com/tensorflow/tensorflow/issues/new if this is " "important to you") + def add_loss(self, losses, inputs=None): + raise RuntimeError( + "add_loss is not supported in Network class yet. Please file an issue " + "at https://github.com/tensorflow/tensorflow/issues/new if this is " + "important to you") + + @property + def losses(self): + """Gather losses from `Layer`s in the `Network`. + + Note that when executing eagerly, `Layer.losses` evaluates + regularizers. When using graph execution, variable regularization ops have + already been created and are simply returned here. + + Returns: + A list of tensors. + """ + layer_losses = [] + for layer in self.layers: + layer_losses.extend(layer.losses) + return layer_losses + # TODO(allenl): Support other Layer methods needed for graph mode, such as for - # losses and updates + # updates class Sequential(Network): diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index e7835a63e6..3eb4f5f8b3 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -19,6 +19,7 @@ from __future__ import print_function import gc from tensorflow.contrib.eager.python import network +from tensorflow.contrib.layers.python.layers import regularizers from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.eager import test @@ -45,6 +46,22 @@ class MyNetwork(network.Network): return self.l1(x) +class RegularizedNetwork(network.Network): + + def __init__(self): + super(RegularizedNetwork, self).__init__() + self.l1 = self.track_layer(core.Dense( + 1, + bias_regularizer=regularizers.l1_regularizer(2.0), + kernel_regularizer=regularizers.l1_regularizer(2.0))) + self.l2 = self.track_layer(core.Dense( + 1, + bias_regularizer=regularizers.l1_regularizer(2.0))) + + def call(self, values): + return self.l2(self.l1(values)) + + class NetworkTest(test.TestCase): def _save_modify_load_network_built(self, net, global_step=None): @@ -484,6 +501,18 @@ class NetworkTest(test.TestCase): _check_op_prefixes(expected_prefix="my_network_1/dense/", checked_ops=checked_ops) + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) + def testVariableRegularizers(self): + net = RegularizedNetwork() + net(constant_op.constant([[1.]])) + self.evaluate(net.variables[0].assign([[2.]])) + self.evaluate(net.variables[1].assign([3.])) + self.evaluate(net.variables[2].assign([[-2.]])) + self.evaluate(net.variables[3].assign([4.])) + self.assertAllEqual([4., 6., 8.], self.evaluate(net.losses)) + self.evaluate(net.variables[3].assign([5.])) + self.assertAllEqual([4., 6., 10.], self.evaluate(net.losses)) + @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True) def testDuplicateNameError(self): one = constant_op.constant([[1.]]) diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index 6be2bc3e76..c083f8a5d2 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -103,10 +103,16 @@ class Layer(object): self.built = False self.input_spec = None + if activity_regularizer and context.in_eager_mode(): + raise ValueError( + ('Activity regularization is not supported when executing eagerly. ' + 'Got activity_regularizer=%s') % (activity_regularizer,)) self._activity_regularizer = activity_regularizer self._trainable_weights = [] self._non_trainable_weights = [] self._updates = [] + # When executing eagerly, _losses is a list of zero-argument lambdas which + # return tensors. When using graph execution, _losses is a list of ops. self._losses = [] self._reuse = kwargs.get('_reuse') self._graph = ops.get_default_graph() @@ -287,9 +293,22 @@ class Layer(object): @property def losses(self): + """Losses which are associated with this `Layer`. + + Note that when executing eagerly, getting this property evaluates + regularizers. When using graph execution, variable regularization ops have + already been created and are simply returned here. + + Returns: + A list of tensors. + """ if context.in_eager_mode(): - raise RuntimeError('Layer.losses not supported in Eager mode.') - return self._losses + # _losses may only contain variable regularization losses when executing + # eagerly, and they have been saved as lambdas to be executed when + # requested. + return [regularizer() for regularizer in self._losses] + else: + return self._losses def add_loss(self, losses, inputs=None): """Add loss tensor(s), potentially dependent on layer inputs. @@ -303,6 +322,11 @@ class Layer(object): The `get_losses_for` method allows to retrieve the losses relevant to a specific set of inputs. + Note that `add_loss` is not supported when executing eagerly. Instead, + variable regularizers may be added through `add_variable`. Activity + regularization is not supported directly (but such losses may be returned + from `Layer.call()`). + Arguments: losses: Loss tensor, or list/tuple of tensors. inputs: Optional input tensor(s) that the loss(es) depend on. Must @@ -462,16 +486,8 @@ class Layer(object): Raises: RuntimeError: If called in Eager mode with regularizers. """ - # Note that we currently don't support variable regularization in Eager - # mode. An alternative is for users to directly compute these losses before - # performing a backward pass. if context.in_graph_mode(): existing_variables = set(tf_variables.global_variables()) - else: - existing_variables = [] - if regularizer is not None: - raise RuntimeError('Variable regularization not supported in Eager ' - 'mode.') if dtype is None: dtype = self.dtype or dtypes.float32 @@ -486,28 +502,39 @@ class Layer(object): constraint=constraint, trainable=trainable and self.trainable, partitioner=partitioner) - if (context.in_graph_mode() and trainable and self.trainable - and variable not in tf_variables.trainable_variables()): - # A custom getter / variable scope overrode the trainable flag. - trainable = False - if variable in existing_variables: - return variable - if regularizer: - # To match the behavior of tf.get_variable(), we only - # apply regularization if the variable is newly created. - if isinstance(variable, tf_variables.PartitionedVariable): - for v in variable: - with ops.colocate_with(v.op): + if context.in_graph_mode(): + if (trainable and self.trainable + and variable not in tf_variables.trainable_variables()): + # A custom getter / variable scope overrode the trainable flag. + trainable = False + if variable in existing_variables: + return variable + if regularizer: + # To match the behavior of tf.get_variable(), we only + # apply regularization if the variable is newly created. + if isinstance(variable, tf_variables.PartitionedVariable): + for v in variable: + with ops.colocate_with(v.op): + with ops.name_scope(name + '/Regularizer'): + regularization = regularizer(v) + if regularization is not None: + self.add_loss(regularization) + else: + with ops.colocate_with(variable.op): with ops.name_scope(name + '/Regularizer'): - regularization = regularizer(v) + regularization = regularizer(variable) if regularization is not None: self.add_loss(regularization) - else: - with ops.colocate_with(variable.op): - with ops.name_scope(name + '/Regularizer'): - regularization = regularizer(variable) - if regularization is not None: - self.add_loss(regularization) + elif regularizer: + if isinstance(variable, tf_variables.PartitionedVariable): + raise RuntimeError( + 'Partitioned variable regularization is not yet supported when ' + 'executing eagerly. File a feature request is this is ' + 'important to you.') + # Save a zero-argument lambda which runs the regularizer on the + # variable, to be executed when `Layer.losses` is requested. This + # makes losses responsive to variable updates when executing eagerly. + self._losses.append(lambda: regularizer(variable)) if trainable: self._trainable_weights.append(variable) else: diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index 1eea20deef..3e5a51eb62 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -88,6 +88,11 @@ class BaseLayerTest(test.TestCase): regularizer=regularizer) self.assertEqual(len(layer.losses), 1) + def testNoEagerActivityRegularizer(self): + with context.eager_mode(): + with self.assertRaisesRegexp(ValueError, 'activity_regularizer'): + core_layers.Dense(1, activity_regularizer=lambda *args, **kwargs: 0.) + def testGetVariable(self): with self.test_session(): -- GitLab From a6ee905de83834c35e7cf01182270309ec2425f3 Mon Sep 17 00:00:00 2001 From: Sergio Guadarrama Date: Tue, 28 Nov 2017 15:31:17 -0800 Subject: [PATCH 0074/1924] Add non_trainable_variables to templates. Add aliases for weights, trainable_weights and non_trainable_weights. PiperOrigin-RevId: 177228107 --- .../python/kernel_tests/template_test.py | 37 +++++++++++++++++-- tensorflow/python/ops/template.py | 37 ++++++++++++++++++- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/template_test.py b/tensorflow/python/kernel_tests/template_test.py index 40c0ade62a..f0354374ac 100644 --- a/tensorflow/python/kernel_tests/template_test.py +++ b/tensorflow/python/kernel_tests/template_test.py @@ -34,9 +34,10 @@ from tensorflow.python.platform import test from tensorflow.python.training import gradient_descent -def variable_scoped_function(): +def variable_scoped_function(trainable=True): return variable_scope.get_variable( - "dummy", shape=[1], initializer=init_ops.zeros_initializer()) + "dummy", shape=[1], trainable=trainable, + initializer=init_ops.zeros_initializer()) def internally_variable_scoped_function(scope_name): @@ -413,7 +414,7 @@ class TemplateTest(test.TestCase): self.assertEqual(custom_getter_count[0], 2) # Test that custom getter is called when the variable scope is created - # during construction + # during construction custom_getter_count[0] = 0 tmpl2 = template.make_template( "s2", @@ -539,6 +540,36 @@ class TemplateTest(test.TestCase): # Ensure we can get the scopes before either template is actually called. self.assertEqual(1, len(ta.trainable_variables)) self.assertEqual(1, len(tb.trainable_variables)) + # None non-trainable variable was created. + self.assertEqual([], list(ta.non_trainable_variables)) + self.assertEqual([], list(tb.non_trainable_variables)) + # Ensure variables returns all the variables. + self.assertEqual(1, len(ta.variables)) + self.assertEqual(1, len(tb.variables)) + + @test_util.run_in_graph_and_eager_modes() + def test_non_trainable_variables(self): + # Make sure non_trainable_variables are created. + with variable_scope.variable_scope("foo2"): + ta = template.make_template("a", variable_scoped_function, + trainable=True) + tb = template.make_template("b", variable_scoped_function, + trainable=False) + # Initially there are not variables created. + self.assertEqual([], list(ta.variables)) + self.assertEqual([], list(tb.variables)) + # After calling there are variables created. + ta() + tb() + # Check the trainable and non_trainable variables. + self.assertEqual(1, len(ta.trainable_variables)) + self.assertEqual([], list(ta.non_trainable_variables)) + + self.assertEqual([], list(tb.trainable_variables)) + self.assertEqual(1, len(tb.non_trainable_variables)) + # Ensure variables returns all the variables. + self.assertEqual(1, len(ta.variables)) + self.assertEqual(1, len(tb.variables)) # TODO(apassos) handle local variables in Eager def test_local_variables(self): diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py index 98578b799a..07796b28d9 100644 --- a/tensorflow/python/ops/template.py +++ b/tensorflow/python/ops/template.py @@ -307,6 +307,12 @@ class Template(object): # To prevent partial matches on the scope_name, we add '/' at the end. return name if name[-1] == "/" else name + "/" + @property + def variables(self): + """Returns the list of global and local variables created by the Template. + """ + return self.global_variables + self.local_variables + @property def trainable_variables(self): """Returns the list of trainable variables created by the Template.""" @@ -316,6 +322,14 @@ class Template(object): else: return [] + @property + def non_trainable_variables(self): + """Returns the list of non-trainable variables created by the Template.""" + # TODO(apassos) Make sure it matches Eager when using local variables. + global_variables = self.global_variables + trainable_variables = set(self.trainable_variables) + return [x for x in global_variables if x not in trainable_variables] + @property def global_variables(self): """Returns the list of global variables created by the Template.""" @@ -334,6 +348,21 @@ class Template(object): else: return [] + @property + def weights(self): + """List of weights/variables created by the Template.""" + return self.variables + + @property + def trainable_weights(self): + """List of trainable weights/variables created by the Template.""" + return self.trainable_variables + + @property + def non_trainable_weights(self): + """List of non-trainable weights/variables created by the Template.""" + return self.non_trainable_variables + @property @deprecated( "2017-02-21", "The .var_scope property is deprecated. Please change your " @@ -501,7 +530,7 @@ class EagerTemplate(Template): @property def variables(self): - """Returns the list of trainable variables created by the Template.""" + """Returns the list of variables created by the Template.""" # Currently there is no local variable in Eager mode. return self._eager_variable_store.variables() @@ -511,6 +540,12 @@ class EagerTemplate(Template): # Currently there is no local variable in Eager mode. return self._eager_variable_store.trainable_variables() + @property + def non_trainable_variables(self): + """Returns the list of non-trainable variables created by the Template.""" + # Currently there is no local variable in Eager mode. + return self._eager_variable_store.non_trainable_variables() + @property def global_variables(self): """Returns the list of global variables created by the Template.""" -- GitLab From d8de0d979e9b9dacb20ebf425d54bbc98ed65fad Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 28 Nov 2017 15:31:21 -0800 Subject: [PATCH 0075/1924] Fixing the windows nightly build. PiperOrigin-RevId: 177228112 --- tensorflow/contrib/cmake/tf_core_cpu.cmake | 2 +- tensorflow/contrib/cmake/tf_core_framework.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/cmake/tf_core_cpu.cmake b/tensorflow/contrib/cmake/tf_core_cpu.cmake index 5c01ca382f..e4213ea2a4 100644 --- a/tensorflow/contrib/cmake/tf_core_cpu.cmake +++ b/tensorflow/contrib/cmake/tf_core_cpu.cmake @@ -63,7 +63,7 @@ if (tensorflow_ENABLE_GPU) file(GLOB_RECURSE tf_core_gpu_srcs "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu/*.cc" "${tensorflow_source_dir}/tensorflow/core/platform/default/gpu/cupti_wrapper.cc" - "${tensorflow_source_dir}/tensorflow/core/platform/default/gpu_tracer.cc" + "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc" "${tensorflow_source_dir}/tensorflow/core/common_runtime/gpu_device_factory.cc" "${tensorflow_source_dir}/tensorflow/core/grappler/devices.h" "${tensorflow_source_dir}/tensorflow/core/grappler/devices.cc" diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index c607546f4a..5ec1a8d04f 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -211,7 +211,7 @@ if (NOT tensorflow_ENABLE_GPU) list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_gpu_srcs}) else() file(GLOB tf_core_platform_srcs_exclude - "${tensorflow_source_dir}/tensorflow/core/platform/default/gpu_tracer.cc") + "${tensorflow_source_dir}/tensorflow/core/platform/default/device_tracer.cc") list(REMOVE_ITEM tf_core_platform_srcs ${tf_core_platform_srcs_exclude}) endif() -- GitLab From 5f1b61b5c851409c76015c908d127fbc2f886013 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Tue, 28 Nov 2017 15:37:52 -0800 Subject: [PATCH 0076/1924] Check per HLO instruction only at vlog=1 in non-opt build. PiperOrigin-RevId: 177229069 --- tensorflow/compiler/xla/service/hlo_rematerialization.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc index 017f996bc4..d09de7b528 100644 --- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc +++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc @@ -566,7 +566,9 @@ Status MemoryUsageTracker::BeginInstruction(Item* item) { VLOG(3) << " memory usage = " << memory_usage_; VLOG(10) << ToString(); - DCHECK(Check()); + if (VLOG_IS_ON(1)) { + DCHECK(Check()); + } return Status::OK(); } @@ -603,8 +605,9 @@ Status MemoryUsageTracker::EndInstruction() { VLOG(3) << " memory usage = " << memory_usage_; VLOG(10) << ToString(); - DCHECK(Check()); - + if (VLOG_IS_ON(1)) { + DCHECK(Check()); + } return Status::OK(); } -- GitLab From b8969d12f9260a7b1981b8d22788aa1f8c8cbbb6 Mon Sep 17 00:00:00 2001 From: Martin Wicke Date: Tue, 28 Nov 2017 15:44:48 -0800 Subject: [PATCH 0077/1924] Mark Supervisor deprecated. Please use MonitoredTrainingSession instead. Fixes #6263. PiperOrigin-RevId: 177230053 --- tensorflow/python/training/monitored_session.py | 1 - tensorflow/python/training/supervisor.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py index e931555470..f1cb81981a 100644 --- a/tensorflow/python/training/monitored_session.py +++ b/tensorflow/python/training/monitored_session.py @@ -52,7 +52,6 @@ _PREEMPTION_ERRORS = (errors.AbortedError, errors.UnavailableError) USE_DEFAULT = object() -# TODO(touts): Share that with the Supervisor. class Scaffold(object): """Structure to create or gather pieces commonly needed to train a model. diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index a634a842b6..e4514aaea2 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -36,11 +36,15 @@ from tensorflow.python.training import coordinator from tensorflow.python.training import saver as saver_mod from tensorflow.python.training import session_manager as session_manager_mod from tensorflow.python.training import training_util +from tensorflow.python.util import deprecation class Supervisor(object): """A training helper that checkpoints models and computes summaries. + This class is deprecated. Please use + ${tf.train.MonitoredTrainingSession} instead. + The Supervisor is a small wrapper around a `Coordinator`, a `Saver`, and a `SessionManager` that takes care of common needs of TensorFlow training programs. @@ -198,6 +202,8 @@ class Supervisor(object): # the default behavior should be used. USE_DEFAULT = 0 + @deprecation.deprecated(None, + "Please switch to tf.train.MonitoredTrainingSession") def __init__(self, graph=None, ready_op=USE_DEFAULT, -- GitLab From 5a1e22b753225a7fa14f4ae60c06cf50bce6b9a6 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 28 Nov 2017 15:45:09 -0800 Subject: [PATCH 0078/1924] Remove temp_workaround_http_archive. PiperOrigin-RevId: 177230105 --- tensorflow/workspace.bzl | 51 +++++----------------------------- third_party/aws.BUILD | 16 +++++------ third_party/curl.BUILD | 46 +++++++++++++++--------------- third_party/gif.BUILD | 2 +- third_party/jemalloc.BUILD | 10 +++---- third_party/jpeg/jpeg.BUILD | 2 +- third_party/mkl/build_defs.bzl | 1 - third_party/nccl.BUILD | 8 +++--- third_party/snappy.BUILD | 4 +-- 9 files changed, 51 insertions(+), 89 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index cb77f96be5..68d663acfc 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -57,33 +57,6 @@ def check_version(bazel_version): fail("\nCurrent Bazel version is {}, expected at least {}\n".format( native.bazel_version, bazel_version)) -def _repos_are_siblings(): - return Label("@foo//bar").workspace_root.startswith("../") - -# Temporary workaround to support including TensorFlow as a submodule until this -# use-case is supported in the next Bazel release. -def _temp_workaround_http_archive_impl(repo_ctx): - repo_ctx.template("BUILD", repo_ctx.attr.build_file, { - "%prefix%": ".." if _repos_are_siblings() else "external", - "%ws%": repo_ctx.attr.repository - }, False) - repo_ctx.download_and_extract(repo_ctx.attr.urls, "", repo_ctx.attr.sha256, - "", repo_ctx.attr.strip_prefix) - if repo_ctx.attr.patch_file != None: - _apply_patch(repo_ctx, repo_ctx.attr.patch_file) - -temp_workaround_http_archive = repository_rule( - attrs = { - "build_file": attr.label(), - "repository": attr.string(), - "patch_file": attr.label(default = None), - "urls": attr.string_list(default = []), - "sha256": attr.string(default = ""), - "strip_prefix": attr.string(default = ""), - }, - implementation = _temp_workaround_http_archive_impl, -) - # Executes specified command with arguments and calls 'fail' if it exited with # non-zero code def _execute_and_check_ret_code(repo_ctx, cmd_and_args): @@ -121,8 +94,6 @@ def _patched_http_archive_impl(repo_ctx): patched_http_archive = repository_rule( attrs = { "patch_file": attr.label(), - "build_file": attr.label(), - "repository": attr.string(), "urls": attr.string_list(default = []), "sha256": attr.string(default = ""), "strip_prefix": attr.string(default = ""), @@ -157,7 +128,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "57ba56c4c243f403ff78f417ff854ef50b9eddf4a610a917b7c95e7fa8553a4b", strip_prefix = "mklml_lnx_2018.0.20170720", build_file = str(Label("//third_party/mkl:mkl.BUILD")), - repository = tf_repo_name, ) if path_prefix: @@ -292,7 +262,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): build_file = str(Label("//third_party:nasm.BUILD")), ) - temp_workaround_http_archive( + native.new_http_archive( name = "jpeg", urls = [ "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", @@ -301,7 +271,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77", strip_prefix = "libjpeg-turbo-1.5.1", build_file = str(Label("//third_party/jpeg:jpeg.BUILD")), - repository = tf_repo_name, ) native.new_http_archive( @@ -502,7 +471,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): build_file = str(Label("//third_party:swig.BUILD")), ) - temp_workaround_http_archive( + native.new_http_archive( name = "curl", sha256 = "ff3e80c1ca6a068428726cd7dd19037a47cc538ce58ef61c59587191039b2ca6", urls = [ @@ -511,7 +480,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): ], strip_prefix = "curl-7.49.1", build_file = str(Label("//third_party:curl.BUILD")), - repository = tf_repo_name ) # grpc expects //external:protobuf_clib and //external:protobuf_compiler @@ -575,7 +543,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): # TODO(phawkins): currently, this rule uses an unofficial LLVM mirror. # Switch to an official source of snapshots if/when possible. - temp_workaround_http_archive( + native.new_http_archive( name = "llvm", urls = [ "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/9ab4c272cb604a7f947865428c4ef2169fee2100.tar.gz", @@ -584,7 +552,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "1b1b7d3800a94ca2302e3dd670dbe84238749583027883784b55297059d83da8", strip_prefix = "llvm-9ab4c272cb604a7f947865428c4ef2169fee2100", build_file = str(Label("//third_party/llvm:llvm.BUILD")), - repository = tf_repo_name, ) native.new_http_archive( @@ -650,7 +617,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): build_file = str(Label("//third_party/fft2d:fft2d.BUILD")), ) - temp_workaround_http_archive( + native.new_http_archive( name = "snappy", urls = [ "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz", @@ -659,10 +626,9 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94", strip_prefix = "snappy-1.1.4", build_file = str(Label("//third_party:snappy.BUILD")), - repository = tf_repo_name, ) - temp_workaround_http_archive( + native.new_http_archive( name = "nccl_archive", urls = [ "https://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", @@ -671,10 +637,9 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176", strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7", build_file = str(Label("//third_party:nccl.BUILD")), - repository = tf_repo_name, ) - temp_workaround_http_archive( + native.new_http_archive( name = "aws", urls = [ "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.0.90.tar.gz", @@ -683,7 +648,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "f599b57aec4f03ad696044dd430b2d201864113937353adc346f53ad47991319", strip_prefix = "aws-sdk-cpp-1.0.90", build_file = str(Label("//third_party:aws.BUILD")), - repository = tf_repo_name ) java_import_external( @@ -711,7 +675,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): testonly_ = True, ) - temp_workaround_http_archive( + native.new_http_archive( name = "jemalloc", urls = [ "https://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", @@ -720,7 +684,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8", strip_prefix = "jemalloc-4.4.0", build_file = str(Label("//third_party:jemalloc.BUILD")), - repository = tf_repo_name, ) java_import_external( diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD index bc9e37ffb3..bf5310aa16 100644 --- a/third_party/aws.BUILD +++ b/third_party/aws.BUILD @@ -7,21 +7,21 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") cc_library( name = "aws", srcs = select({ - "@%ws%//tensorflow:linux_x86_64": glob([ + "@org_tensorflow//tensorflow:linux_x86_64": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:darwin": glob([ + "@org_tensorflow//tensorflow:darwin": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:linux_ppc64le": glob([ + "@org_tensorflow//tensorflow:linux_ppc64le": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), - "@%ws%//tensorflow:raspberry_pi_armeabi": glob([ + "@org_tensorflow//tensorflow:raspberry_pi_armeabi": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), "//conditions:default": [], @@ -53,17 +53,17 @@ cc_library( "aws-cpp-sdk-core/include/aws/core/SDKConfig.h", ], defines = select({ - "@%ws%//tensorflow:linux_x86_64": [ + "@org_tensorflow//tensorflow:linux_x86_64": [ "PLATFORM_LINUX", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "PLATFORM_APPLE", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], - "@%ws%//tensorflow:linux_ppc64le": [ + "@org_tensorflow//tensorflow:linux_ppc64le": [ "PLATFORM_LINUX", "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index 805a30d262..e311c7e758 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # MIT/X derivative license exports_files(["COPYING"]) CURL_WIN_COPTS = [ - "/I%prefix%/curl/lib", + "/Iexternal/curl/lib", "/DHAVE_CONFIG_H", "/DCURL_DISABLE_FTP", "/DCURL_DISABLE_NTLM", @@ -224,14 +224,14 @@ cc_library( "lib/wildcard.h", "lib/x509asn1.h", ] + select({ - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "lib/vtls/darwinssl.c", ], - "@%ws%//tensorflow:ios": [ + "@org_tensorflow//tensorflow:ios": [ "lib/vtls/darwinssl.c", ], - "@%ws%//tensorflow:windows": CURL_WIN_SRCS, - "@%ws%//tensorflow:windows_msvc": CURL_WIN_SRCS, + "@org_tensorflow//tensorflow:windows": CURL_WIN_SRCS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_WIN_SRCS, "//conditions:default": [ "lib/vtls/openssl.c", ], @@ -248,10 +248,10 @@ cc_library( "include/curl/typecheck-gcc.h", ], copts = select({ - "@%ws%//tensorflow:windows": CURL_WIN_COPTS, - "@%ws%//tensorflow:windows_msvc": CURL_WIN_COPTS, + "@org_tensorflow//tensorflow:windows": CURL_WIN_COPTS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_WIN_COPTS, "//conditions:default": [ - "-I%prefix%/curl/lib", + "-Iexternal/curl/lib", "-D_GNU_SOURCE", "-DHAVE_CONFIG_H", "-DCURL_DISABLE_FTP", @@ -261,14 +261,14 @@ cc_library( "-Wno-string-plus-int", ], }) + select({ - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-fno-constant-cfstrings", ], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:windows": [ # See curl.h for discussion of write size and Windows "/DCURL_MAX_WRITE_SIZE=16384", ], - "@%ws%//tensorflow:windows_msvc": [ + "@org_tensorflow//tensorflow:windows_msvc": [ # See curl.h for discussion of write size and Windows "/DCURL_MAX_WRITE_SIZE=16384", ], @@ -278,20 +278,20 @@ cc_library( }), includes = ["include"], linkopts = select({ - "@%ws%//tensorflow:android": [ + "@org_tensorflow//tensorflow:android": [ "-pie", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-Wl,-framework", "-Wl,CoreFoundation", "-Wl,-framework", "-Wl,Security", ], - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [ "-Wl,ws2_32.lib", ], - "@%ws%//tensorflow:windows_msvc": [ + "@org_tensorflow//tensorflow:windows_msvc": [ "-Wl,ws2_32.lib", ], "//conditions:default": [ @@ -302,9 +302,9 @@ cc_library( deps = [ "@zlib_archive//:zlib", ] + select({ - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [], - "@%ws%//tensorflow:windows_msvc": [], + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [], + "@org_tensorflow//tensorflow:windows_msvc": [], "//conditions:default": [ "@boringssl//:ssl", ], @@ -312,7 +312,7 @@ cc_library( ) CURL_BIN_WIN_COPTS = [ - "/I%prefix%/curl/lib", + "/Iexternal/curl/lib", "/DHAVE_CONFIG_H", "/DCURL_DISABLE_LIBCURL_OPTION", ] @@ -406,10 +406,10 @@ cc_binary( "src/tool_xattr.h", ], copts = select({ - "@%ws%//tensorflow:windows": CURL_BIN_WIN_COPTS, - "@%ws%//tensorflow:windows_msvc": CURL_BIN_WIN_COPTS, + "@org_tensorflow//tensorflow:windows": CURL_BIN_WIN_COPTS, + "@org_tensorflow//tensorflow:windows_msvc": CURL_BIN_WIN_COPTS, "//conditions:default": [ - "-I%prefix%/curl/lib", + "-Iexternal/curl/lib", "-D_GNU_SOURCE", "-DHAVE_CONFIG_H", "-DCURL_DISABLE_LIBCURL_OPTION", diff --git a/third_party/gif.BUILD b/third_party/gif.BUILD index 27808a9d64..78fbd6c0e0 100644 --- a/third_party/gif.BUILD +++ b/third_party/gif.BUILD @@ -21,7 +21,7 @@ cc_library( ], hdrs = ["lib/gif_lib.h"], defines = select({ - #"@%ws%//tensorflow:android": [ + #"@org_tensorflow//tensorflow:android": [ ":android": [ "S_IREAD=S_IRUSR", "S_IWRITE=S_IWUSR", diff --git a/third_party/jemalloc.BUILD b/third_party/jemalloc.BUILD index a2addf2c66..1b0829b8fe 100644 --- a/third_party/jemalloc.BUILD +++ b/third_party/jemalloc.BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # BSD exports_files(["COPYING"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") cc_library( name = "jemalloc_headers", @@ -97,10 +97,10 @@ cc_library( includes = ["include"], # pthread_atfork() is called for PPC. linkopts = select({ - "@%ws%//tensorflow:linux_ppc64le": [ + "@org_tensorflow//tensorflow:linux_ppc64le": [ "-lpthread", ], - "@%ws%//tensorflow:linux_x86_64": [ + "@org_tensorflow//tensorflow:linux_x86_64": [ "-lpthread", ], "//conditions:default": [ @@ -208,8 +208,8 @@ genrule( name = "size_classes_h", outs = ["include/jemalloc/internal/size_classes.h"], cmd = select({ - "@%ws%//tensorflow:linux_ppc64le": "$(location :size_classes_sh) \"3 4\" 3 16 2 >$@", - "@%ws%//tensorflow:linux_x86_64": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", + "@org_tensorflow//tensorflow:linux_ppc64le": "$(location :size_classes_sh) \"3 4\" 3 16 2 >$@", + "@org_tensorflow//tensorflow:linux_x86_64": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", "//conditions:default": "$(location :size_classes_sh) \"3 4\" 3 12 2 >$@", }), tools = [":size_classes_sh"], diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD index f6078052ec..e431f19382 100644 --- a/third_party/jpeg/jpeg.BUILD +++ b/third_party/jpeg/jpeg.BUILD @@ -5,7 +5,7 @@ licenses(["notice"]) # custom notice-style license, see LICENSE.md exports_files(["LICENSE.md"]) -load("@%ws%//third_party:common.bzl", "template_rule") +load("@org_tensorflow//third_party:common.bzl", "template_rule") libjpegturbo_nocopts = "-[W]error" diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index 533c0766c7..f637873f14 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -60,7 +60,6 @@ mkl_repository = repository_rule( ], attrs = { "build_file": attr.label(), - "repository": attr.string(), "urls": attr.string_list(default = []), "sha256": attr.string(default = ""), "strip_prefix": attr.string(default = ""), diff --git a/third_party/nccl.BUILD b/third_party/nccl.BUILD index 06b9b8ff68..3a2a3afe46 100644 --- a/third_party/nccl.BUILD +++ b/third_party/nccl.BUILD @@ -44,17 +44,17 @@ cc_library( "-O3", ] + cuda_default_copts(), linkopts = select({ - "@%ws%//tensorflow:android": [ + "@org_tensorflow//tensorflow:android": [ "-pie", ], - "@%ws%//tensorflow:darwin": [ + "@org_tensorflow//tensorflow:darwin": [ "-Wl,-framework", "-Wl,CoreFoundation", "-Wl,-framework", "-Wl,Security", ], - "@%ws%//tensorflow:ios": [], - "@%ws%//tensorflow:windows": [ + "@org_tensorflow//tensorflow:ios": [], + "@org_tensorflow//tensorflow:windows": [ "ws2_32.lib", ], "//conditions:default": [ diff --git a/third_party/snappy.BUILD b/third_party/snappy.BUILD index 9c00b7068a..fd48ed8941 100644 --- a/third_party/snappy.BUILD +++ b/third_party/snappy.BUILD @@ -50,8 +50,8 @@ genrule( "-e 's/@ac_cv_have_stddef_h@/1/g' " + "-e 's/@ac_cv_have_stdint_h@/1/g' " + select({ - "@%ws%//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", - "@%ws%//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", + "@org_tensorflow//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", + "@org_tensorflow//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ", "//conditions:default": "-e 's/@ac_cv_have_sys_uio_h@/1/g' ", }) + "-e 's/@SNAPPY_MAJOR@/1/g' " + -- GitLab From 9049b440df17de47baf16d9e24590c3d0761e2c9 Mon Sep 17 00:00:00 2001 From: Andrew Harp Date: Tue, 28 Nov 2017 16:06:37 -0800 Subject: [PATCH 0079/1924] Fix tensorflow-android jcenter link PiperOrigin-RevId: 177233056 --- tensorflow/contrib/android/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/android/README.md b/tensorflow/contrib/android/README.md index f49e5857fe..c7c128bf14 100644 --- a/tensorflow/contrib/android/README.md +++ b/tensorflow/contrib/android/README.md @@ -15,9 +15,9 @@ For prebuilt libraries, see the page for a recent build. The TensorFlow Inference Interface is also available as a -[JCenter package](https://bintray.com/google/tensorflow/tensorflow-android) and -can be included quite simply in your android project with a couple of lines in -the project's `build.gradle` file: +[JCenter package](https://bintray.com/google/tensorflow/tensorflow) +(see the tensorflow-android directory) and can be included quite simply in your +android project with a couple of lines in the project's `build.gradle` file: ``` allprojects { -- GitLab From a80fd2acf08ceba0c8fc7684c3013e8e7d6bd8d3 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Tue, 28 Nov 2017 16:11:48 -0800 Subject: [PATCH 0080/1924] C API: fix bug in ValidateNoCycles(). This change makes ValidateNoCycles() work when the graph has unused node ids (i.e. when Graph::num_nodes() < Graph::num_node_ids()). PiperOrigin-RevId: 177234002 --- tensorflow/c/c_api.cc | 9 ++++----- tensorflow/python/framework/ops_test.py | 8 ++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index bb41f92306..4fb8ec8e4b 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -383,12 +383,11 @@ void TF_Reset_Helper(const TF_SessionOptions* opt, const char** containers, // be less than the total node count. Status ValidateNoCycles(const Graph& g) { // TODO(nolivia): check this on a subset of the graph instead of all of it. - int total_num_nodes = g.num_node_ids(); // A node is ready when all of its inputs have been visited. std::vector ready; - std::vector pending_count(total_num_nodes, 0); + std::vector pending_count(g.num_node_ids(), 0); - for (int i = 0; i < total_num_nodes; ++i) { + for (int i = 0; i < g.num_node_ids(); ++i) { const Node* n = g.FindNodeId(i); if (n == nullptr) continue; pending_count[i] = n->in_edges().size(); @@ -421,7 +420,7 @@ Status ValidateNoCycles(const Graph& g) { } } - if (processed < total_num_nodes) { + if (processed < g.num_nodes()) { std::vector nodes_in_cycle; for (int i = 0; i < pending_count.size() && nodes_in_cycle.size() < 3; ++i) { @@ -430,7 +429,7 @@ Status ValidateNoCycles(const Graph& g) { } } return errors::InvalidArgument( - "Graph is invalid, contains a cycle with ", total_num_nodes - processed, + "Graph is invalid, contains a cycle with ", g.num_nodes() - processed, " nodes, including: ", str_util::Join(nodes_in_cycle, ", ")); } return Status::OK(); diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index 371eadcd13..3eae3b5a25 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -1876,6 +1876,14 @@ class GraphTest(test_util.TensorFlowTestCase): gc.collect() self.assertIsNone(g_ref()) + def testRunnableAfterInvalidShape(self): + with ops.Graph().as_default(): + with self.assertRaises(ValueError): + math_ops.add([1, 2], [1, 2, 3]) + a = constant_op.constant(1) + with session.Session() as sess: + sess.run(a) + @test_util.with_c_api class AttrScopeTest(test_util.TensorFlowTestCase): -- GitLab From f22261e61c2359483ad17465161918856bb86e65 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 16:22:17 -0800 Subject: [PATCH 0081/1924] Add depthwise ops for NAS cell in nn_ops_test to improve the inference time on the particular depthwise ops. PiperOrigin-RevId: 177235744 --- tensorflow/core/kernels/nn_ops_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/kernels/nn_ops_test.cc b/tensorflow/core/kernels/nn_ops_test.cc index 0db7c63b8b..a841291ddd 100644 --- a/tensorflow/core/kernels/nn_ops_test.cc +++ b/tensorflow/core/kernels/nn_ops_test.cc @@ -653,6 +653,8 @@ BM_ConvFloatDepthwiseFwd(32, 7, 7, 1024, 1, 1024, 3, 3, 1, SAME, conv6); // Benchmarks with different stride and padding options. BM_ConvFloatDepthwiseFwd(32, 112, 112, 3, 8, 24, 3, 3, 2, SAME, conv7); BM_ConvFloatDepthwiseFwd(32, 112, 112, 3, 8, 24, 3, 3, 2, VALID, conv8); +BM_ConvFloatDepthwiseFwd(1, 100, 100, 72, 1, 72, 3, 3, 1, SAME, conv9); +BM_ConvFloatDepthwiseFwd(1, 100, 100, 72, 1, 72, 5, 5, 1, SAME, conv10); #define BM_ConvFloatDepthwiseBk(BS, R, C, ID, DM, OD, KR, KC, STR, PAD, LABEL) \ static void BM_ConvFloatDepthwiseBkInCPU1_##LABEL(int iters) { \ -- GitLab From b89251c6300b9941d06071543e5c4974d0db1984 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Tue, 28 Nov 2017 16:31:39 -0800 Subject: [PATCH 0082/1924] [TF:XLA] Implement Cumsum and Cumprod using the XLA ReduceWindow operator. PiperOrigin-RevId: 177236996 --- tensorflow/compiler/tests/BUILD | 14 ++ tensorflow/compiler/tests/scan_ops_test.py | 229 ++++++++++++++++++ tensorflow/compiler/tf2xla/const_analysis.cc | 2 + tensorflow/compiler/tf2xla/kernels/BUILD | 1 + .../compiler/tf2xla/kernels/scan_ops.cc | 140 +++++++++++ tensorflow/compiler/tf2xla/xla_context.cc | 14 ++ tensorflow/compiler/tf2xla/xla_context.h | 8 + tensorflow/compiler/tf2xla/xla_op_kernel.cc | 5 + tensorflow/compiler/tf2xla/xla_op_kernel.h | 5 + 9 files changed, 418 insertions(+) create mode 100644 tensorflow/compiler/tests/scan_ops_test.py create mode 100644 tensorflow/compiler/tf2xla/kernels/scan_ops.cc diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 6cad2b0824..fff1a7f57b 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -416,6 +416,20 @@ tf_xla_py_test( ], ) +tf_xla_py_test( + name = "scan_ops_test", + size = "small", + srcs = ["scan_ops_test.py"], + tags = ["optonly"], + deps = [ + ":xla_test", + "//tensorflow/python:array_ops", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + tf_xla_py_test( name = "segment_reduction_ops_test", size = "medium", diff --git a/tensorflow/compiler/tests/scan_ops_test.py b/tensorflow/compiler/tests/scan_ops_test.py new file mode 100644 index 0000000000..3260e63b23 --- /dev/null +++ b/tensorflow/compiler/tests/scan_ops_test.py @@ -0,0 +1,229 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for scan ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.compiler.tests.xla_test import XLATestCase +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +def numpy_reverse(x, axis): + length = len(x.shape) + if axis < 0: + axis = length + axis + + ix = [ + slice(None, None, -1) if i == axis else slice(None) for i in range(length) + ] + return x[ix] + + +def handle_options(func, x, axis, exclusive, reverse): + """Adds tf options to numpy scan ops.""" + length = len(x.shape) + if axis < 0: + axis = length + axis + + if reverse: + x = numpy_reverse(x, axis) + + if exclusive: + ix_head = [slice(0, 1) if i == axis else slice(None) for i in range(length)] + ix_init = [ + slice(0, -1) if i == axis else slice(None) for i in range(length) + ] + if func == np.cumsum: + init = np.zeros_like(x[ix_head]) + elif func == np.cumprod: + init = np.ones_like(x[ix_head]) + else: + raise ValueError("Unknown scan function.") + x = np.concatenate([init, func(x[ix_init], axis)], axis=axis) + else: + x = func(x, axis=axis) + + if reverse: + x = numpy_reverse(x, axis) + return x + + +class CumsumTest(XLATestCase): + + valid_dtypes = [np.float32] + + def axis_dtypes(self): + return set(self.int_types).intersection([np.int32, np.int64]) + + def _compare(self, x, axis, exclusive, reverse): + np_out = handle_options(np.cumsum, x, axis, exclusive, reverse) + with self.test_session(), self.test_scope(): + p = array_ops.placeholder(x.dtype) + tf_out = math_ops.cumsum(p, axis, exclusive, reverse).eval( + feed_dict={p: x}) + + self.assertAllClose(np_out, tf_out) + + def _compareAll(self, x, axis): + for exclusive in [True, False]: + for reverse in [True, False]: + self._compare(x, axis, exclusive, reverse) + + def testEmpty(self): + for dtype in self.valid_dtypes: + x = np.zeros([0]).astype(dtype) + for axis in (-1, 0): + self._compareAll(x, axis) + + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in self.axis_dtypes(): + with self.test_session(), self.test_scope(): + p = array_ops.placeholder(x.dtype) + axis = constant_op.constant(0, axis_dtype) + math_ops.cumsum(p, axis).eval(feed_dict={p: x}) + + def test1D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis in (-1, 0): + self._compareAll(x, axis) + + def test2D(self): + for dtype in self.valid_dtypes: + x = np.arange(0, 10).reshape([2, 5]).astype(dtype) + for axis in (-2, -1, 0, 1): + self._compareAll(x, axis) + + def test3D(self): + for dtype in self.valid_dtypes: + x = np.arange(0, 20).reshape([2, 2, 5]).astype(dtype) + for axis in (-3, -2, -1, 0, 1, 2): + self._compareAll(x, axis) + + def test6D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype) + for axis in range(-6, 6, 3): + self._compareAll(x, axis) + + def testInvalidAxis(self): + x = np.arange(0, 10).reshape([2, 5]).astype(np.float32) + with self.test_session(), self.test_scope(): + input_tensor = ops.convert_to_tensor(x) + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "Expected scan axis in the range [-2, 2)" in str(e)): + math_ops.cumsum(input_tensor, -3).eval() + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "Expected scan axis in the range [-2, 2)" in str(e)): + math_ops.cumsum(input_tensor, 2).eval() + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "axis must be a scalar" in str(e)): + math_ops.cumsum(input_tensor, [0]).eval() + + +class CumprodTest(XLATestCase): + + valid_dtypes = [np.float32] + + def axis_dtypes(self): + return set(self.int_types).intersection([np.int32, np.int64]) + + def _compare(self, x, axis, exclusive, reverse): + np_out = handle_options(np.cumprod, x, axis, exclusive, reverse) + with self.test_session(), self.test_scope(): + p = array_ops.placeholder(x.dtype) + prod = math_ops.cumprod(p, axis, exclusive, reverse) + tf_out = prod.eval(feed_dict={p: x}) + + self.assertAllClose(np_out, tf_out) + + def _compareAll(self, x, axis): + for exclusive in [True, False]: + for reverse in [True, False]: + self._compare(x, axis, exclusive, reverse) + + def testEmpty(self): + for dtype in self.valid_dtypes: + x = np.zeros([0]).astype(dtype) + for axis in (-1, 0): + self._compareAll(x, axis) + + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in self.axis_dtypes(): + with self.test_session(), self.test_scope(): + p = array_ops.placeholder(x.dtype) + axis = constant_op.constant(0, axis_dtype) + math_ops.cumprod(x, axis).eval(feed_dict={p: x}) + + def test1D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis in (-1, 0): + self._compareAll(x, axis) + + def test2D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 11).reshape([2, 5]).astype(dtype) + for axis in (-2, -1, 0, 1): + self._compareAll(x, axis) + + def test3D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 21).reshape([2, 2, 5]).astype(dtype) + for axis in (-3, -2, -1, 0, 1, 2): + self._compareAll(x, axis) + + def test6D(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 145).reshape([2, 2, 3, 3, 2, 2]).astype(dtype) + for axis in range(-6, 6, 3): + self._compareAll(x, axis) + + def testInvalidAxis(self): + x = np.arange(0, 10).reshape([2, 5]).astype(np.float32) + with self.test_session(), self.test_scope(): + input_tensor = ops.convert_to_tensor(x) + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "Expected scan axis in the range [-2, 2)" in str(e)): + math_ops.cumprod(input_tensor, -3).eval() + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "Expected scan axis in the range [-2, 2)" in str(e)): + math_ops.cumprod(input_tensor, 2).eval() + with self.assertRaisesWithPredicateMatch( + errors_impl.InvalidArgumentError, + lambda e: "axis must be a scalar" in str(e)): + math_ops.cumprod(input_tensor, [0]).eval() + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/compiler/tf2xla/const_analysis.cc b/tensorflow/compiler/tf2xla/const_analysis.cc index d57273d844..6a1a5467e0 100644 --- a/tensorflow/compiler/tf2xla/const_analysis.cc +++ b/tensorflow/compiler/tf2xla/const_analysis.cc @@ -52,6 +52,8 @@ Status BackwardsConstAnalysis(const Graph& g, {"Conv2DBackpropInput", "input_sizes"}, {"Conv3DBackpropFilterV2", "filter_sizes"}, {"Conv3DBackpropInputV2", "input_sizes"}, + {"Cumprod", "axis"}, + {"Cumsum", "axis"}, {"DepthwiseConv2dNativeBackpropFilter", "filter_sizes"}, {"DepthwiseConv2dNativeBackpropInput", "input_sizes"}, {"DynamicStitch", "indices"}, diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index 6302fece1f..a1720ff919 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -54,6 +54,7 @@ tf_kernel_library( "reshape_op.cc", "retval_op.cc", "reverse_op.cc", + "scan_ops.cc", "segment_reduction_ops.cc", "select_op.cc", "sendrecv_ops.cc", diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc new file mode 100644 index 0000000000..3cc9d14411 --- /dev/null +++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc @@ -0,0 +1,140 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/partial_tensor_shape.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/bounds_check.h" +#include "tensorflow/core/kernels/concat_lib.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace { + +class ScanOp : public XlaOpKernel { + public: + ScanOp(OpKernelConstruction* ctx, bool sum) : XlaOpKernel(ctx), sum_(sum) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("reverse", &reverse_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("exclusive", &exclusive_)); + } + + void Compile(XlaOpKernelContext* ctx) override { + const TensorShape input_shape = ctx->InputShape(0); + const TensorShape tensor_axis_shape = ctx->InputShape(1); + + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(tensor_axis_shape), + errors::InvalidArgument("ScanOp: axis must be a scalar, not ", + tensor_axis_shape.DebugString())); + + int64 axis; + OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &axis)); + if (axis < 0) { + axis += input_shape.dims(); + } + OP_REQUIRES( + ctx, FastBoundsCheck(axis, input_shape.dims()), + errors::InvalidArgument("ScanOp: Expected scan axis in the range [", + -input_shape.dims(), ", ", input_shape.dims(), + "), but got ", axis)); + + DataType dtype = ctx->input_type(0); + + if (input_shape.num_elements() == 0) { + // Exit early if there is nothing to compute. + ctx->SetOutput(0, ctx->Input(0)); + return; + } + + xla::ComputationBuilder* builder = ctx->builder(); + + std::vector window_strides(input_shape.dims(), 1); + std::vector window_dims(input_shape.dims(), 1); + window_dims[axis] = input_shape.dim_size(axis); + + std::vector> padding(input_shape.dims(), {0, 0}); + padding[axis].first = input_shape.dim_size(axis) - 1; + // In exclusive mode, add an extra padding element so there is a complete + // window of padding before the data starts. + if (exclusive_) { + ++padding[axis].first; + } + if (reverse_) { + std::swap(padding[axis].first, padding[axis].second); + } + + xla::ComputationDataHandle input = ctx->Input(0); + xla::ComputationDataHandle init; + const xla::Computation* reducer; + if (sum_) { + init = XlaHelpers::Zero(builder, dtype); + reducer = ctx->GetOrCreateAdd(dtype); + } else { + init = XlaHelpers::One(builder, dtype); + reducer = ctx->GetOrCreateMul(dtype); + } + auto output = builder->ReduceWindowWithGeneralPadding( + ctx->Input(0), init, *reducer, window_dims, window_strides, padding); + + // In exclusive mode, we have computed an extra element containing the sum + // of all the input elements. Slice off this extra "last" element. + if (exclusive_) { + if (reverse_) { + output = builder->SliceInDim(output, 1, input_shape.dim_size(axis) + 1, + 1, axis); + + } else { + output = + builder->SliceInDim(output, 0, input_shape.dim_size(axis), 1, axis); + } + } + ctx->SetOutput(0, output); + } + + private: + const bool sum_; // True=cumulative sum. False=cumulative product. + bool reverse_; + bool exclusive_; +}; + +class CumsumOp : public ScanOp { + public: + explicit CumsumOp(OpKernelConstruction* ctx) : ScanOp(ctx, /*sum=*/true) {} +}; +// TODO(phawkins): implement non-float windowed reductions in XLA and remove the +// type constraint. +REGISTER_XLA_OP(Name("Cumsum").TypeConstraint("T", DT_FLOAT), CumsumOp); + +class CumprodOp : public ScanOp { + public: + explicit CumprodOp(OpKernelConstruction* ctx) : ScanOp(ctx, /*sum=*/false) {} +}; +// TODO(phawkins): implement non-float windowed reductions in XLA and remove the +// type constraint. +REGISTER_XLA_OP(Name("Cumprod").TypeConstraint("T", DT_FLOAT), CumprodOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/xla_context.cc b/tensorflow/compiler/tf2xla/xla_context.cc index 651bafd6c5..78e770c62b 100644 --- a/tensorflow/compiler/tf2xla/xla_context.cc +++ b/tensorflow/compiler/tf2xla/xla_context.cc @@ -178,6 +178,20 @@ const xla::Computation* XlaContext::GetOrCreateAdd(const DataType type) { }); } +const xla::Computation* XlaContext::GetOrCreateMul(const DataType type) { + return LookupOrCreate(type, &mul_func_, [this, type] { + const string type_string = DataTypeString(type); + VLOG(1) << "Building Mul() for " << type_string; + xla::ComputationBuilder b(builder()->client(), "mul<" + type_string + ">"); + xla::PrimitiveType xla_type; + TF_CHECK_OK(DataTypeToPrimitiveType(type, &xla_type)); + auto x = b.Parameter(0, xla::ShapeUtil::MakeShape(xla_type, {}), "x"); + auto y = b.Parameter(1, xla::ShapeUtil::MakeShape(xla_type, {}), "y"); + b.Mul(x, y); + return b.Build().ConsumeValueOrDie(); + }); +} + const xla::Computation* XlaContext::LookupOrCreate( DataType type, ComputationMap* out, const std::function& create) { diff --git a/tensorflow/compiler/tf2xla/xla_context.h b/tensorflow/compiler/tf2xla/xla_context.h index de8aafa362..55d2995987 100644 --- a/tensorflow/compiler/tf2xla/xla_context.h +++ b/tensorflow/compiler/tf2xla/xla_context.h @@ -102,6 +102,11 @@ class XlaContext : public ResourceBase { // separate specialization of the computation for each DataType. const xla::Computation* GetOrCreateAdd(const DataType type); + // Get an XLA lambda to compute Mul. This is cached in the + // XlaContext since it may be used by multiple Ops. There is a + // separate specialization of the computation for each DataType. + const xla::Computation* GetOrCreateMul(const DataType type); + // The name of the XlaContext resource during symbolic graph execution. static const char kXlaContextResourceName[]; @@ -155,6 +160,9 @@ class XlaContext : public ResourceBase { // Cached computation to compute Sum of two elements, specialized by type. ComputationMap add_func_; + // Cached computation to compute Mul of two elements, specialized by type. + ComputationMap mul_func_; + // Cached computation to compute Sigmoid of an element, specialized by type. ComputationMap sigmoid_func_; diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index a052bb105e..f16472cac8 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -417,6 +417,11 @@ const xla::Computation* XlaOpKernelContext::GetOrCreateAdd( return XlaContext::Get(context_).GetOrCreateAdd(type); } +const xla::Computation* XlaOpKernelContext::GetOrCreateMul( + const DataType type) { + return XlaContext::Get(context_).GetOrCreateMul(type); +} + XlaOpKernel::XlaOpKernel(OpKernelConstruction* context) : OpKernel(context) {} void XlaOpKernel::Compute(OpKernelContext* context) { diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h index 76bcf594e6..06845a674e 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.h +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h @@ -210,6 +210,11 @@ class XlaOpKernelContext { // separate specialization of the computation for each DataType. const xla::Computation* GetOrCreateAdd(const DataType type); + // Gets an XLA lambda to compute Mul. This is cached in the + // XlaContext since it may be used by multiple Ops. There is a + // separate specialization of the computation for each DataType. + const xla::Computation* GetOrCreateMul(const DataType type); + private: OpKernelContext* const context_; }; -- GitLab From f2f6356f15f4c8b5c560ee8aec7bf1dd097bfbfb Mon Sep 17 00:00:00 2001 From: Sourabh Bajaj Date: Tue, 28 Nov 2017 16:31:57 -0800 Subject: [PATCH 0083/1924] Automated g4 rollback of changelist 177191521 PiperOrigin-RevId: 177237037 --- tensorflow/core/kernels/strided_slice_op.cc | 1 + tensorflow/core/kernels/strided_slice_op_gpu.cu.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow/core/kernels/strided_slice_op.cc b/tensorflow/core/kernels/strided_slice_op.cc index 8fc40db3cc..73b6d4cf6a 100644 --- a/tensorflow/core/kernels/strided_slice_op.cc +++ b/tensorflow/core/kernels/strided_slice_op.cc @@ -427,6 +427,7 @@ REGISTER_STRIDED_SLICE(bfloat16); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU); TF_CALL_complex64(REGISTER_GPU); TF_CALL_complex128(REGISTER_GPU); +TF_CALL_int64(REGISTER_GPU); // A special GPU kernel for int32. // TODO(b/25387198): Also enable int32 in device memory. This kernel diff --git a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc b/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc index a8487f49f4..8ca27e3b92 100644 --- a/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc +++ b/tensorflow/core/kernels/strided_slice_op_gpu.cu.cc @@ -53,6 +53,7 @@ typedef Eigen::GpuDevice GPUDevice; TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_KERNELS); TF_CALL_complex64(DEFINE_GPU_KERNELS); TF_CALL_complex128(DEFINE_GPU_KERNELS); +TF_CALL_int64(DEFINE_GPU_KERNELS); DEFINE_GPU_KERNELS(int32); #undef DEFINE_GPU_KERNELS -- GitLab From 625ae88377b16705378065576cfd6983bb876435 Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Tue, 28 Nov 2017 16:47:47 -0800 Subject: [PATCH 0084/1924] Round-robin variables across local devices with `replicate_model_fn`. When the user specifies `replicate_model_fn(... devices=[d1, d2, ... dN])` all variables are going to be stored on each device an round-robin fashion. They are still going to be shared by all devices. PiperOrigin-RevId: 177239111 --- tensorflow/contrib/estimator/BUILD | 2 +- .../python/estimator/replicate_model_fn.py | 94 +++++++++---- .../estimator/replicate_model_fn_test.py | 130 ++++++++++++++++-- 3 files changed, 188 insertions(+), 38 deletions(-) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 8395e2db5e..e4d51aa148 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -346,7 +346,7 @@ py_library( cuda_py_test( name = "replicate_model_fn_test", - size = "small", + size = "medium", srcs = ["python/estimator/replicate_model_fn_test.py"], additional_deps = [ "//tensorflow/python/estimator", diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py index d9c83aa865..6f7f37473f 100644 --- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py +++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py @@ -42,10 +42,45 @@ from tensorflow.python.ops import sparse_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.platform import tf_logging +from tensorflow.python.training import device_setter as device_setter_lib from tensorflow.python.training import training_util -def replicate_model_fn(model_fn, optimizer_fn, devices=None): +class Mode(object): + """Modes for variables replication used for forcing a particular mode.""" + + AUTO = 0 + """Use internal heuristics for choosing the best Mode value. + + This mode is supposed to be the most appropriate in most cases given what + is known about the system. + """ + # TODO(isaprykin): Query system configuration to choose modes other than + # `SHARED_LOCAL_PARAMETER_SERVER`, even though it is often appropriate. + + SHARED_LOCAL_PARAMETER_SERVER = 2 + """Variables are placed on a single device and shared across all devices. + + Two ways to achieve this replication over available GPUs are supported: + 1) If exactly 1 GPU is detected, then variables and operations are placed + onto GPU. + 2) If more than 1 GPU is detected, then variables are going to be placed on + the CPU. Replicas of operations are placed on each individual GPU. + """ + + SHARED_ROUND_ROBIN = 3 + """Variables are placed on all devices in a round-robin fashion. + + Every subsequent variable is placed on the next device. There is only one + copy of each variable that is shared across all devices. + """ + + # TODO(isaprykin): Implement `REPLICATED_ALL_REDUCE`. + REPLICATED_ALL_REDUCE = 3 + """Variables are mirrored on all devices.""" + + +def replicate_model_fn(model_fn, optimizer_fn, devices=None, mode=Mode.AUTO): """Replicate `Estimator.model_fn` over GPUs within a single host. The given `model_fn` specifies a single forward pass of a model. To replicate @@ -58,14 +93,11 @@ def replicate_model_fn(model_fn, optimizer_fn, devices=None): optimizer. If `devices` are `None`, then all available GPUs are going to be used for - replication. If no GPUs are available, then the model is going to be - placed on the CPU. + replication: `devices=[]`. If no GPUs are available, + then the model is going to be placed on the CPU: `devices=['/device:CPU:0']`. - Two modes of local replication over available GPUs are supported: - 1) If exactly 1 GPU is detected, then variables and operations are placed - onto GPU. - 2) If more than 1 GPU is detected, then variables are going to be placed on - the CPU. Replicas of operations are placed on each individual GPU. + Varibles are placed on to `devices` according to the given `mode`. Operations + are going for each tower are going to be copied on each device. Here is an example of how one might use their `model_fn` to run over GPUs: ```python @@ -127,6 +159,8 @@ def replicate_model_fn(model_fn, optimizer_fn, devices=None): argument can be used to replice only on the subset of available GPUs. If `None`, then all available GPUs are going to be used for replication. If no GPUs are available, then the model is going to be placed on the CPU. + mode: An optional argument that specifies the replication method used for + distributing variables across devices. Returns: A replicated version of the supplied `model_fn`. Returned function that @@ -137,16 +171,21 @@ def replicate_model_fn(model_fn, optimizer_fn, devices=None): devices = _get_local_devices('GPU') or _get_local_devices('CPU') is_a_single_gpu_case = len(devices) == 1 and 'GPU' in devices[0] - local_ps_device = '/{}:0'.format('GPU' if is_a_single_gpu_case else 'CPU') + consolidation_device = '/{}:0'.format('GPU' + if is_a_single_gpu_case else 'CPU') - tf_logging.info('Replicating the `model_fn` across {}. Local parameter ' - 'server device is going to be {}.'.format( - devices, local_ps_device)) + ps_devices = [consolidation_device] + if mode == Mode.SHARED_ROUND_ROBIN: + ps_devices = devices + + tf_logging.info('Replicating the `model_fn` across {}. Variables are going ' + 'to be placed on {}. Consolidation device is going to be {}.' + .format(devices, ps_devices, consolidation_device)) def replicated_model_fn(features, labels, mode, params=None, config=None): """Replicated version of `model_fn` to be used instead.""" feature_shards, label_shards = _split_batch( - features, labels, len(devices), device=local_ps_device) + features, labels, len(devices), device=consolidation_device) tower_specs = _get_loss_towers( model_fn=model_fn, mode=mode, @@ -155,17 +194,17 @@ def replicate_model_fn(model_fn, optimizer_fn, devices=None): params=params, config=config, devices=devices, - local_ps_device=local_ps_device) + local_ps_devices=ps_devices) if mode == model_fn_lib.ModeKeys.TRAIN: train_op = _minimize_towers(tower_specs, _call_optimizer_fn(optimizer_fn, params)) return _train_spec( - tower_specs, train_op, aggregation_device=local_ps_device) + tower_specs, train_op, aggregation_device=consolidation_device) elif mode == model_fn_lib.ModeKeys.EVAL: - return _eval_spec(tower_specs, aggregation_device=local_ps_device) + return _eval_spec(tower_specs, aggregation_device=consolidation_device) elif mode == model_fn_lib.ModeKeys.PREDICT: - return _predict_spec(tower_specs, aggregation_device=local_ps_device) + return _predict_spec(tower_specs, aggregation_device=consolidation_device) return replicated_model_fn @@ -222,7 +261,7 @@ def _get_loss_towers(model_fn, params, config, devices, - local_ps_device, + local_ps_devices, name_scope_pattern=_DEFAULT_NAME_SCOPE_PATTERN): """Replicate the loss computation across devices.""" tower_specs = [] @@ -234,15 +273,22 @@ def _get_loss_towers(model_fn, if 'config' in model_fn_args: optional_params['config'] = copy.deepcopy(config) + # pylint: disable=protected-access + round_robin_strategy = device_setter_lib._RoundRobinStrategy( + num_tasks=len(local_ps_devices)) + # pylint: enable=protected-access + for i, device in enumerate(devices): is_the_first_tower = (i == 0) device_setter = _local_device_setter( - worker_device=device, ps_device=local_ps_device) + worker_device=device, + ps_devices=local_ps_devices, + ps_strategy=round_robin_strategy) - # We would like to preserve the names of the variables and ops that a user - # might be relying on. Names with prefix are going to resolve to variables - # and ops of the first tower. + # We would like to preserve the names of the variables and ops that the user + # might be relying on. Names without a prefix are going to resolve to + # variables and ops of the first tower. name_scope = name_scope_pattern if is_the_first_tower: name_scope = '' @@ -263,7 +309,7 @@ def _get_loss_towers(model_fn, return tower_specs -def _local_device_setter(ps_device, worker_device): +def _local_device_setter(worker_device, ps_devices, ps_strategy): """A device setter that puts distributes Var/Ops to PS/workers.""" ps_ops = ['Variable', 'VariableV2', 'VarHandleOp'] @@ -273,7 +319,7 @@ def _local_device_setter(ps_device, worker_device): node_def = op if isinstance(op, node_def_pb2.NodeDef) else op.node_def if node_def.op in ps_ops: ps_device_spec = framework_device.DeviceSpec.from_string( - '{}'.format(ps_device)) + '{}'.format(ps_devices[ps_strategy(op)])) ps_device_spec.merge_from(current_device) return ps_device_spec.to_string() diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py index ffe69f89b4..662021853d 100644 --- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py +++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py @@ -49,15 +49,29 @@ from tensorflow.python.platform import gfile from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants from tensorflow.python.summary.writer import writer_cache +from tensorflow.python.training import device_setter from tensorflow.python.training import gradient_descent +# TODO(isaprykin): Parametrize all the tests on replicate_model_fn.Mode when +# it's supported. class DNNClassifierIntegrationTest(test_util.TensorFlowTestCase): def setUp(self): self._model_dir = tempfile.mkdtemp() - def test_complete_flow(self): + def test_complete_flow_with_mode_auto(self): + return self._complete_flow_with_mode(replicate_model_fn.Mode.AUTO) + + def test_complete_flow_with_mode_local_ps_server(self): + return self._complete_flow_with_mode( + replicate_model_fn.Mode.SHARED_LOCAL_PARAMETER_SERVER) + + def test_complete_flow_with_mode_round_robin(self): + return self._complete_flow_with_mode( + replicate_model_fn.Mode.SHARED_ROUND_ROBIN) + + def _complete_flow_with_mode(self, mode): n_classes = 3 input_dimension = 2 batch_size = 12 @@ -109,7 +123,8 @@ class DNNClassifierIntegrationTest(test_util.TensorFlowTestCase): model_fn=replicate_model_fn.replicate_model_fn( estimator.model_fn, optimizer_fn, - devices=['/gpu:0', '/gpu:1', '/gpu:2']), + devices=['/gpu:0', '/gpu:1', '/gpu:2'], + mode=mode), model_dir=estimator.model_dir, config=estimator.config, params=estimator.params) @@ -359,7 +374,7 @@ class GetLossTowersTest(test_util.TensorFlowTestCase): params=None, config=None, devices=['/gpu:0', '/gpu:1'], - local_ps_device='/gpu:0', + local_ps_devices=['/gpu:0'], name_scope_pattern='test_tower_{}') session.run(variables.global_variables_initializer()) @@ -382,6 +397,54 @@ class GetLossTowersTest(test_util.TensorFlowTestCase): c = variable_scope.get_variable('c', dtype=dtypes.float64) self.assertEqual(0.25, session.run(c)) + def test_variables_are_round_robined_correctly(self): + """Test that creates multiple variables and tests round-robin placement.""" + + def model_fn(mode, features, labels, params): + del params + for variable_name in ['a', 'b', 'c', 'd']: + c = variable_scope.get_variable( + variable_name, + initializer=constant_op.constant(0.25, dtype=dtypes.float64), + dtype=dtypes.float64) + + predictions = math_ops.add(np.array([0.1, 0.2, 0.3, features[0]]), c) + labels = np.array([0.1, 0.2, 0.3, labels[0]]) + loss = losses.absolute_difference( + labels=labels, + predictions=predictions, + reduction=losses.Reduction.SUM) + return model_fn_lib.EstimatorSpec( + mode=mode, loss=math_ops.reduce_sum(loss)) + + with self.test_session() as session: + tower_specs = replicate_model_fn._get_loss_towers( + model_fn, + mode=None, + features=[[0.6], [1.6], [2.6]], + labels=[[0.6], [0.6], [2.6]], + params=None, + config=None, + devices=['/gpu:0', '/gpu:1', '/gpu:3'], + local_ps_devices=['/gpu:0', '/gpu:1', '/gpu:3'], + name_scope_pattern='test_tower_{}') + session.run(variables.global_variables_initializer()) + + self.assertEqual(len(tower_specs), 3) + self.assertEqual('/device:GPU:0', tower_specs[0].loss.device) + self.assertEqual('/device:GPU:1', tower_specs[1].loss.device) + self.assertEqual('/device:GPU:3', tower_specs[2].loss.device) + + with variable_scope.variable_scope('', reuse=True): + a = variable_scope.get_variable('a', dtype=dtypes.float64) + self.assertEqual('/device:GPU:0', a.device) + b = variable_scope.get_variable('b', dtype=dtypes.float64) + self.assertEqual('/device:GPU:1', b.device) + c = variable_scope.get_variable('c', dtype=dtypes.float64) + self.assertEqual('/device:GPU:3', c.device) + d = variable_scope.get_variable('d', dtype=dtypes.float64) + self.assertEqual('/device:GPU:0', d.device) + class SplitBatchTest(test_util.TensorFlowTestCase): @@ -604,7 +667,7 @@ class PredictSpecTest(test_util.TensorFlowTestCase): params=None, config=None, devices=['/gpu:0', '/gpu:1'], - local_ps_device='/gpu:0', + local_ps_devices=['/gpu:0'], ) session.run(variables.global_variables_initializer()) @@ -850,25 +913,66 @@ class GetLocalDevicesTest(test_util.TensorFlowTestCase): class LocalDeviceSetterTest(test_util.TensorFlowTestCase): def test_vars_are_on_ps_but_ops_are_on_workers(self): + ps_devices = ['/device:GPU:3'] + round_robin = device_setter._RoundRobinStrategy(num_tasks=len(ps_devices)) + + local_device_setter = replicate_model_fn._local_device_setter( + ps_devices=ps_devices, + ps_strategy=round_robin, + worker_device='/device:GPU:2') + + with ops_lib.device(local_device_setter): + a = variables.Variable(0.01) + self.assertEqual('/device:GPU:3', a.device) + + b = variables.Variable(0.02) + self.assertEqual('/device:GPU:3', b.device) + + c = variables.Variable(0.03) + self.assertEqual('/device:GPU:3', c.device) + + a_op = array_ops.concat(a, axis=0) + self.assertEqual('/device:GPU:2', a_op.device) + + b_op = array_ops.concat(b, axis=0) + self.assertEqual('/device:GPU:2', b_op.device) + + def test_round_robin_placement(self): + ps_devices = [ + '/device:GPU:0', '/device:GPU:1', '/device:GPU:3', '/device:GPU:4' + ] + round_robin = device_setter._RoundRobinStrategy(num_tasks=len(ps_devices)) + local_device_setter = replicate_model_fn._local_device_setter( - ps_device='/device:GPU:3', worker_device='/device:GPU:2') + ps_devices=ps_devices, + ps_strategy=round_robin, + worker_device='/device:GPU:2') with ops_lib.device(local_device_setter): - c = variables.Variable(0.01) + a = variables.Variable(0.01) + self.assertEqual('/device:GPU:0', a.device) + + b = variables.Variable(0.02) + self.assertEqual('/device:GPU:1', b.device) + + c = variables.Variable(0.03) self.assertEqual('/device:GPU:3', c.device) - cc = variables.Variable(0.02) - self.assertEqual('/device:GPU:3', cc.device) + a_op = array_ops.concat(a, axis=0) + self.assertEqual('/device:GPU:2', a_op.device) + + b_op = array_ops.concat(b, axis=0) + self.assertEqual('/device:GPU:2', b_op.device) - ccc = variables.Variable(0.03) - self.assertEqual('/device:GPU:3', ccc.device) + c = variables.Variable(0.03) + self.assertEqual('/device:GPU:4', c.device) + + d = variables.Variable(0.03) + self.assertEqual('/device:GPU:0', d.device) c_op = array_ops.concat(c, axis=0) self.assertEqual('/device:GPU:2', c_op.device) - cc_op = array_ops.concat(cc, axis=0) - self.assertEqual('/device:GPU:2', cc_op.device) - class ComputeSumWithDevicePlacementTest(test_util.TensorFlowTestCase): -- GitLab From 57839fbf307fb01a280505f1f964d7331104d8f3 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Tue, 28 Nov 2017 17:02:01 -0800 Subject: [PATCH 0085/1924] Re-using (the more general) DeserializeSparse kernel to implement DeserializeSparseMany and improving documentation. PiperOrigin-RevId: 177241063 --- .../base_api/api_def_DeserializeSparse.pbtxt | 43 +++++ .../core/kernels/serialize_sparse_op.cc | 177 +----------------- tensorflow/core/ops/sparse_ops.cc | 42 +++++ .../sparse_serialization_ops_test.py | 4 +- tensorflow/python/ops/sparse_ops.py | 45 ++++- 5 files changed, 129 insertions(+), 182 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_DeserializeSparse.pbtxt b/tensorflow/core/api_def/base_api/api_def_DeserializeSparse.pbtxt index 00e96c8a15..dfaa531cbc 100644 --- a/tensorflow/core/api_def/base_api/api_def_DeserializeSparse.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_DeserializeSparse.pbtxt @@ -14,4 +14,47 @@ The `dtype` of the serialized `SparseTensor` objects. END } summary: "Deserialize `SparseTensor` objects." + description: <= 0) correspond to a batch. The ranks of the original +`SparseTensor` objects must all match. When the final `SparseTensor` is +created, its rank is the rank of the incoming `SparseTensor` objects plus N; +the sparse tensors have been concatenated along new dimensions, one for each +batch. + +The output `SparseTensor` object's shape values for the original dimensions +are the max across the input `SparseTensor` objects' shape values for the +corresponding dimensions. The new dimensions match the size of the batch. + +The input `SparseTensor` objects' indices are assumed ordered in +standard lexicographic order. If this is not the case, after this +step run `SparseReorder` to restore index ordering. + +For example, if the serialized input is a `[2 x 3]` matrix representing two +original `SparseTensor` objects: + + index = [ 0] + [10] + [20] + values = [1, 2, 3] + shape = [50] + +and + + index = [ 2] + [10] + values = [4, 5] + shape = [30] + +then the final deserialized `SparseTensor` will be: + + index = [0 0] + [0 10] + [0 20] + [1 2] + [1 10] + values = [1, 2, 3, 4, 5] + shape = [2 50] +END } diff --git a/tensorflow/core/kernels/serialize_sparse_op.cc b/tensorflow/core/kernels/serialize_sparse_op.cc index cfb86904d5..f4159da229 100644 --- a/tensorflow/core/kernels/serialize_sparse_op.cc +++ b/tensorflow/core/kernels/serialize_sparse_op.cc @@ -409,186 +409,11 @@ class DeserializeSparseOp : public OpKernel { TF_CALL_ALL_TYPES(REGISTER_KERNELS); #undef REGISTER_KERNELS -template -class DeserializeManySparseOp : public OpKernel { - public: - explicit DeserializeManySparseOp(OpKernelConstruction* context) - : OpKernel(context) {} - - void Compute(OpKernelContext* context) override { - const Tensor& serialized_sparse = context->input(0); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(serialized_sparse.shape()), - errors::InvalidArgument( - "Serialized sparse should be a matrix but received shape ", - serialized_sparse.shape().DebugString())); - OP_REQUIRES( - context, serialized_sparse.shape().dim_size(1) == 3, - errors::InvalidArgument( - "Serialized sparse should have 3 columns but received shape ", - serialized_sparse.shape().DebugString())); - - int num_sparse_tensors = serialized_sparse.shape().dim_size(0); - - OP_REQUIRES( - context, num_sparse_tensors > 0, - errors::InvalidArgument("Must have at least 1 serialized SparseTensor, " - "but input matrix has 0 rows")); - - std::vector indices_to_concat; - std::vector values_to_concat; - std::vector shapes_to_concat; - - const auto& serialized_sparse_t = serialized_sparse.matrix(); - - for (int i = 0; i < num_sparse_tensors; ++i) { - Tensor output_indices(DT_INT64); - Tensor output_values(DataTypeToEnum::value); - Tensor output_shape(DT_INT64); - TensorProto proto_indices; - TensorProto proto_values; - TensorProto proto_shape; - - OP_REQUIRES( - context, - ParseProtoUnlimited(&proto_indices, serialized_sparse_t(i, 0)), - errors::InvalidArgument("Could not parse serialized_sparse[", i, - ", 0]")); - OP_REQUIRES(context, - ParseProtoUnlimited(&proto_values, serialized_sparse_t(i, 1)), - errors::InvalidArgument("Could not parse serialized_sparse[", - i, ", 1]")); - OP_REQUIRES(context, - ParseProtoUnlimited(&proto_shape, serialized_sparse_t(i, 2)), - errors::InvalidArgument("Could not parse serialized_sparse[", - i, ", 2]")); - - OP_REQUIRES(context, output_indices.FromProto(proto_indices), - errors::InvalidArgument( - "Could not construct Tensor serialized_sparse[", i, - ", 0] (indices)")); - OP_REQUIRES(context, TensorShapeUtils::IsMatrix(output_indices.shape()), - errors::InvalidArgument( - "Expected serialized_sparse[", i, - ", 0] to represent an index matrix but received shape ", - output_indices.shape().DebugString())); - OP_REQUIRES(context, output_values.FromProto(proto_values), - errors::InvalidArgument( - "Could not construct Tensor serialized_sparse[", i, - ", 1] (values)")); - OP_REQUIRES(context, TensorShapeUtils::IsVector(output_values.shape()), - errors::InvalidArgument( - "Expected serialized_sparse[", i, - ", 1] to represent a values vector but received shape ", - output_values.shape().DebugString())); - OP_REQUIRES(context, output_shape.FromProto(proto_shape), - errors::InvalidArgument( - "Could not construct Tensor serialized_sparse[", i, - ", 2] (shape)")); - OP_REQUIRES( - context, TensorShapeUtils::IsVector(output_shape.shape()), - errors::InvalidArgument("Expected serialized_sparse[", i, - ", 1] to be a shape vector but its shape is ", - output_shape.shape().DebugString())); - - OP_REQUIRES( - context, DataTypeToEnum::value == output_values.dtype(), - errors::InvalidArgument( - "Requested SparseTensor of type ", - DataTypeString(DataTypeToEnum::value), " but SparseTensor[", i, - "].values.dtype() == ", DataTypeString(output_values.dtype()))); - - int64 num_entries = output_indices.dim_size(0); - OP_REQUIRES(context, num_entries == output_values.dim_size(0), - errors::InvalidArgument( - "Expected row counts of SparseTensor[", i, - "].indices and SparseTensor[", i, - "].values to match but they do not: ", num_entries, - " vs. ", output_values.dim_size(0))); - int rank = output_indices.dim_size(1); - OP_REQUIRES( - context, rank == output_shape.dim_size(0), - errors::InvalidArgument("Expected column counts of SparseTensor[", i, - "].indices to match size of SparseTensor[", i, - "].shape " - "but they do not: ", - rank, " vs. ", output_shape.dim_size(0))); - - // Now we expand each SparseTensors' indices and shape by - // prefixing a dimension - Tensor expanded_indices( - DT_INT64, TensorShape({num_entries, 1 + output_indices.dim_size(1)})); - Tensor expanded_shape(DT_INT64, - TensorShape({1 + output_shape.dim_size(0)})); - const auto& output_indices_t = output_indices.matrix(); - const auto& output_shape_t = output_shape.vec(); - auto expanded_indices_t = expanded_indices.matrix(); - auto expanded_shape_t = expanded_shape.vec(); - expanded_indices_t.chip<1>(0).setZero(); - Eigen::DSizes indices_start(0, 1); - Eigen::DSizes indices_sizes(num_entries, rank); - expanded_indices_t.slice(indices_start, indices_sizes) = output_indices_t; - expanded_shape_t(0) = 1; - std::copy_n(&output_shape_t(0), rank, &expanded_shape_t(1)); - - TensorShape expanded_tensor_shape(expanded_shape.vec()); - - indices_to_concat.push_back(expanded_indices); - values_to_concat.push_back(output_values); - shapes_to_concat.push_back(expanded_tensor_shape); - } - - int rank = -1; - for (int i = 0; i < num_sparse_tensors; ++i) { - if (rank < 0) rank = shapes_to_concat[i].dims(); - OP_REQUIRES(context, rank == shapes_to_concat[i].dims(), - errors::InvalidArgument( - "Inconsistent rank across SparseTensors: rank prior to " - "SparseTensor[", - i, "] was: ", rank, " but rank of SparseTensor[", i, - "] is: ", shapes_to_concat[i].dims())); - } - - // SparseTensor::Concat requires consistent shape for all but the - // primary order dimension (dimension 0 in this case). So we get - // the maximum value across all the input SparseTensors for each - // dimension and use that. - TensorShape preconcat_shape(shapes_to_concat[0]); - for (int i = 0; i < num_sparse_tensors; ++i) { - for (int d = 0; d < rank; ++d) { - preconcat_shape.set_dim(d, std::max(preconcat_shape.dim_size(d), - shapes_to_concat[i].dim_size(d))); - } - } - - // Dimension 0 is the primary dimension. - gtl::InlinedVector std_order(rank); - std::iota(std_order.begin(), std_order.end(), 0); - - std::vector tensors_to_concat; - tensors_to_concat.reserve(num_sparse_tensors); - for (int i = 0; i < num_sparse_tensors; ++i) { - tensors_to_concat.emplace_back(indices_to_concat[i], values_to_concat[i], - preconcat_shape, std_order); - } - - SparseTensor output = SparseTensor::Concat(tensors_to_concat); - - Tensor final_output_shape(DT_INT64, TensorShape({output.dims()})); - - std::copy_n(output.shape().data(), output.dims(), - final_output_shape.vec().data()); - - context->set_output(0, output.indices()); - context->set_output(1, output.values()); - context->set_output(2, final_output_shape); - } -}; - #define REGISTER_KERNELS(type) \ REGISTER_KERNEL_BUILDER(Name("DeserializeManySparse") \ .Device(DEVICE_CPU) \ .TypeConstraint("dtype"), \ - DeserializeManySparseOp) + DeserializeSparseOp) TF_CALL_ALL_TYPES(REGISTER_KERNELS); #undef REGISTER_KERNELS diff --git a/tensorflow/core/ops/sparse_ops.cc b/tensorflow/core/ops/sparse_ops.cc index 8414519f0b..772e2531dc 100644 --- a/tensorflow/core/ops/sparse_ops.cc +++ b/tensorflow/core/ops/sparse_ops.cc @@ -256,6 +256,48 @@ REGISTER_OP("DeserializeSparse") .Doc(R"doc( Deserialize `SparseTensor` objects. +The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where +the last dimension stores serialized `SparseTensor` objects and the other N +dimensions (N >= 0) correspond to a batch. The ranks of the original +`SparseTensor` objects must all match. When the final `SparseTensor` is +created, its rank is the rank of the incoming `SparseTensor` objects plus N; +the sparse tensors have been concatenated along new dimensions, one for each +batch. + +The output `SparseTensor` object's shape values for the original dimensions +are the max across the input `SparseTensor` objects' shape values for the +corresponding dimensions. The new dimensions match the size of the batch. + +The input `SparseTensor` objects' indices are assumed ordered in +standard lexicographic order. If this is not the case, after this +step run `SparseReorder` to restore index ordering. + +For example, if the serialized input is a `[2 x 3]` matrix representing two +original `SparseTensor` objects: + + index = [ 0] + [10] + [20] + values = [1, 2, 3] + shape = [50] + +and + + index = [ 2] + [10] + values = [4, 5] + shape = [30] + +then the final deserialized `SparseTensor` will be: + + index = [0 0] + [0 10] + [0 20] + [1 2] + [1 10] + values = [1, 2, 3, 4, 5] + shape = [2 50] + serialized_sparse: The serialized `SparseTensor` objects. The last dimension must have 3 columns. dtype: The `dtype` of the serialized `SparseTensor` objects. diff --git a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py index 78c113f514..d1a90952c7 100644 --- a/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_serialization_ops_test.py @@ -254,8 +254,8 @@ class SerializeSparseTest(test.TestCase): serialized_concat, dtype=dtypes.int32) with self.assertRaisesOpError( - r"Inconsistent rank across SparseTensors: rank prior to " - r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"): + r"Inconsistent shape across SparseTensors: rank prior to " + r"SparseTensor\[1\] was: 2 but rank of SparseTensor\[1\] is: 3"): sess.run(sp_deserialized, {sp_input0: input0_val, sp_input1: input1_val}) diff --git a/tensorflow/python/ops/sparse_ops.py b/tensorflow/python/ops/sparse_ops.py index cdfe9e1c1e..9bdc124c83 100644 --- a/tensorflow/python/ops/sparse_ops.py +++ b/tensorflow/python/ops/sparse_ops.py @@ -1437,10 +1437,47 @@ def serialize_many_sparse(sp_input, name=None): def deserialize_sparse(serialized_sparse, dtype, rank=None, name=None): """Deserialize `SparseTensor` objects. - The input is expected to have shape [d_1, ..., d_m, 3], where the last - dimension stores a serialized `SparseTensor`. The method deserializes - all input `SparseTensor`s, concatenates them into a single tensor, and - reshapes the sparse tensor to preserve the structure of the input. + The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where + the last dimension stores serialized `SparseTensor` objects and the other N + dimensions (N >= 0) correspond to a batch. The ranks of the original + `SparseTensor` objects must all match. When the final `SparseTensor` is + created, its rank is the rank of the incoming `SparseTensor` objects plus N; + the sparse tensors have been concatenated along new dimensions, one for each + batch. + + The output `SparseTensor` object's shape values for the original dimensions + are the max across the input `SparseTensor` objects' shape values for the + corresponding dimensions. The new dimensions match the size of the batch. + + The input `SparseTensor` objects' indices are assumed ordered in + standard lexicographic order. If this is not the case, after this + step run `SparseReorder` to restore index ordering. + + For example, if the serialized input is a `[2 x 3]` matrix representing two + original `SparseTensor` objects: + + index = [ 0] + [10] + [20] + values = [1, 2, 3] + shape = [50] + + and + + index = [ 2] + [10] + values = [4, 5] + shape = [30] + + then the final deserialized `SparseTensor` will be: + + index = [0 0] + [0 10] + [0 20] + [1 2] + [1 10] + values = [1, 2, 3, 4, 5] + shape = [2 50] Args: serialized_sparse: The serialized `SparseTensor` objects. -- GitLab From 73a803fb854fc842700a865d4742ae893ed236d3 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Tue, 28 Nov 2017 17:03:33 -0800 Subject: [PATCH 0086/1924] Fix flakiness in map_dataset_op_test. PiperOrigin-RevId: 177241314 --- tensorflow/contrib/data/python/kernel_tests/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 0790a4a737..4cb69d7c8e 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -275,7 +275,7 @@ py_test( py_test( name = "map_dataset_op_test", - size = "small", + size = "medium", srcs = ["map_dataset_op_test.py"], srcs_version = "PY2AND3", deps = [ -- GitLab From e2f9107effb0c5c4cee49a71562865d9e919b3d0 Mon Sep 17 00:00:00 2001 From: Tayo Oguntebi Date: Tue, 28 Nov 2017 17:20:39 -0800 Subject: [PATCH 0087/1924] Adds minor-dim pooling tests for cases in which windows exist entirely in padding. Modifies reference util reduce-window 1D implementation to accept general padding. PiperOrigin-RevId: 177243527 --- tensorflow/compiler/xla/reference_util.cc | 18 ++- tensorflow/compiler/xla/reference_util.h | 6 + .../compiler/xla/tests/reduce_window_test.cc | 112 +++++++++++++----- 3 files changed, 102 insertions(+), 34 deletions(-) diff --git a/tensorflow/compiler/xla/reference_util.cc b/tensorflow/compiler/xla/reference_util.cc index 5bb81b80dd..bdf92eaed1 100644 --- a/tensorflow/compiler/xla/reference_util.cc +++ b/tensorflow/compiler/xla/reference_util.cc @@ -195,14 +195,26 @@ ReferenceUtil::ReduceWindow1DGeneric( const tensorflow::gtl::ArraySlice& window, const tensorflow::gtl::ArraySlice& stride, Padding padding) { std::vector dim_lengths{static_cast(operand.size())}; - auto padding_both = xla::MakePadding(dim_lengths, window, stride, padding); + return ReduceWindow1DGeneric( + operand, init, reduce_func, window, stride, + xla::MakePadding(dim_lengths, window, stride, padding)); +} +/* static */ std::unique_ptr> +ReferenceUtil::ReduceWindow1DGeneric( + const tensorflow::gtl::ArraySlice& operand, float init, + const std::function& reduce_func, + const tensorflow::gtl::ArraySlice& window, + const tensorflow::gtl::ArraySlice& stride, + const tensorflow::gtl::ArraySlice>& padding) { + std::vector dim_lengths{static_cast(operand.size())}; std::vector window_counts(window.size(), 0); std::vector pad_low(window.size(), 0); for (int64 i = 0; i < window.size(); ++i) { + int64 padded_width = padding[i].first + dim_lengths[i] + padding[i].second; window_counts[i] = - WindowCount(dim_lengths[i], window[i], stride[i], padding); - pad_low[i] = padding_both[i].first; + window_util::StridedBound(padded_width, window[i], stride[i]); + pad_low[i] = padding[i].first; } auto result = MakeUnique>(window_counts[0]); diff --git a/tensorflow/compiler/xla/reference_util.h b/tensorflow/compiler/xla/reference_util.h index 62d455d71a..ee244e9a66 100644 --- a/tensorflow/compiler/xla/reference_util.h +++ b/tensorflow/compiler/xla/reference_util.h @@ -184,6 +184,12 @@ class ReferenceUtil { const std::function& reduce_func, const tensorflow::gtl::ArraySlice& window, const tensorflow::gtl::ArraySlice& stride, Padding padding); + static std::unique_ptr> ReduceWindow1DGeneric( + const tensorflow::gtl::ArraySlice& operand, float init, + const std::function& reduce_func, + const tensorflow::gtl::ArraySlice& window, + const tensorflow::gtl::ArraySlice& stride, + const tensorflow::gtl::ArraySlice>& padding); static std::unique_ptr> ReduceWindow4DGeneric( const Array4D& operand, float init, const std::function& reduce_func, diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc index 0601a1466b..aa035f0ba5 100644 --- a/tensorflow/compiler/xla/tests/reduce_window_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc @@ -962,68 +962,114 @@ struct R1ReduceWindowTestData { int64 base_bounds[1]; int64 window_bounds[1]; int64 strides[1]; - Padding padding; + int64 pad_low[1]; + int64 pad_high[1]; Reducer reducer; } kR1TestCases[] = { {/*base_bounds=*/{1}, /*window_bounds=*/{1}, /*strides=*/{1}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({1}, {1}, {1}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({1}, {1}, {1}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{3}, /*window_bounds=*/{3}, /*strides=*/{1}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({3}, {3}, {1}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({3}, {3}, {1}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{3}, /*window_bounds=*/{2}, /*strides=*/{1}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({3}, {2}, {1}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({3}, {2}, {1}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{5}, /*window_bounds=*/{1}, /*strides=*/{1}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax}, + /*pad_low=*/{xla::MakePadding({5}, {1}, {1}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({5}, {1}, {1}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kMax}, {/*base_bounds=*/{16}, /*window_bounds=*/{4}, /*strides=*/{4}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kMax}, + /*pad_low=*/{xla::MakePadding({16}, {4}, {4}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({16}, {4}, {4}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kMax}, {/*base_bounds=*/{16}, /*window_bounds=*/{4}, /*strides=*/{3}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({16}, {4}, {3}, Padding::kValid)[0].first}, + /*pad_high=*/{xla::MakePadding({16}, {4}, {3}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, - {/*base_bounds=*/{128 * 2}, /*window_bounds=*/{30}, + {/*base_bounds=*/{128 * 2}, + /*window_bounds=*/{30}, /*strides=*/{27}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, - - {/*base_bounds=*/{128 * 17}, /*window_bounds=*/{7}, + /*pad_low=*/ + {xla::MakePadding({128 * 2}, {30}, {27}, Padding::kValid)[0].first}, + /*pad_high=*/ + {xla::MakePadding({128 * 2}, {30}, {27}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, + + {/*base_bounds=*/{128 * 17}, + /*window_bounds=*/{7}, /*strides=*/{64}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, - - {/*base_bounds=*/{128 * 2}, /*window_bounds=*/{32}, + /*pad_low=*/ + {xla::MakePadding({128 * 17}, {7}, {64}, Padding::kValid)[0].first}, + /*pad_high=*/ + {xla::MakePadding({128 * 17}, {7}, {64}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, + + {/*base_bounds=*/{128 * 2}, + /*window_bounds=*/{32}, /*strides=*/{56}, - /*padding=*/Padding::kValid, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/ + {xla::MakePadding({128 * 2}, {32}, {56}, Padding::kValid)[0].first}, + /*pad_high=*/ + {xla::MakePadding({128 * 2}, {32}, {56}, Padding::kValid)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{3}, /*window_bounds=*/{2}, /*strides=*/{1}, - /*padding=*/Padding::kSame, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({3}, {2}, {1}, Padding::kSame)[0].first}, + /*pad_high=*/{xla::MakePadding({3}, {2}, {1}, Padding::kSame)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{5}, /*window_bounds=*/{3}, /*strides=*/{2}, - /*padding=*/Padding::kSame, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({5}, {3}, {2}, Padding::kSame)[0].first}, + /*pad_high=*/{xla::MakePadding({5}, {3}, {2}, Padding::kSame)[0].second}, + /*reducer=*/Reducer::kAdd}, {/*base_bounds=*/{16}, /*window_bounds=*/{4}, /*strides=*/{3}, - /*padding=*/Padding::kSame, /*reducer=*/Reducer::kAdd}, + /*pad_low=*/{xla::MakePadding({16}, {4}, {3}, Padding::kSame)[0].first}, + /*pad_high=*/{xla::MakePadding({16}, {4}, {3}, Padding::kSame)[0].second}, + /*reducer=*/Reducer::kAdd}, + + {/*base_bounds=*/{5}, /*window_bounds=*/{5}, + /*strides=*/{1}, + /*pad_low=*/{0}, + /*pad_high=*/{5}, + /*reducer=*/Reducer::kAdd}, + + {/*base_bounds=*/{5}, /*window_bounds=*/{5}, + /*strides=*/{1}, + /*pad_low=*/{5}, + /*pad_high=*/{0}, + /*reducer=*/Reducer::kAdd}, }; string R1ReduceWindowTestDataToString( const ::testing::TestParamInfo& data) { string str = tensorflow::strings::StrCat( - "base_bounds_", - tensorflow::str_util::Join(data.param.base_bounds, "x"), // + "base_bounds_", tensorflow::str_util::Join(data.param.base_bounds, "x"), "__window_bounds_", - tensorflow::str_util::Join(data.param.window_bounds, "x"), // - "__strides_", tensorflow::str_util::Join(data.param.strides, "x"), // - "__padding_", data.param.padding == Padding::kSame ? "same" : "valid", // - "__reducer_", data.param.reducer == kAdd ? "add" : "max"); + tensorflow::str_util::Join(data.param.window_bounds, "x"), "__strides_", + tensorflow::str_util::Join(data.param.strides, "x"), "__pad_low_", + tensorflow::str_util::Join(data.param.pad_low, "x"), "__pad_high_", + tensorflow::str_util::Join(data.param.pad_high, "x"), "__reducer_", + data.param.reducer == kAdd ? "add" : "max"); return str; } @@ -1044,15 +1090,18 @@ TEST_P(R1ReduceWindowTest, DoIt) { TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr input_arg, client_->TransferToServer(*input_literal)); + std::vector> padding(1); + padding[0] = {param.pad_low[0], param.pad_high[0]}; + auto computation = param.reducer == kAdd ? CreateScalarAddComputation(F32, &b) : CreateScalarMaxComputation(F32, &b); - b.ReduceWindow(/*operand=*/ - b.Parameter(0, input_literal->shape(), "p0"), - /*init_value=*/b.ConstantR0(kInitValue), - /*computation=*/computation, - /*window_dimensions=*/param.window_bounds, - /*window_strides=*/param.strides, /*padding=*/param.padding); + b.ReduceWindowWithGeneralPadding( + /*operand=*/b.Parameter(0, input_literal->shape(), "p0"), + /*init_value=*/b.ConstantR0(kInitValue), + /*computation=*/computation, + /*window_dimensions=*/param.window_bounds, + /*window_strides=*/param.strides, /*padding=*/padding); auto reduce_func = param.reducer == kAdd ? +[](float a, float b) { return a + b; } @@ -1062,7 +1111,8 @@ TEST_P(R1ReduceWindowTest, DoIt) { /*init=*/kInitValue, /*reduce_func=*/reduce_func, /*window=*/param.window_bounds, - /*stride=*/param.strides, /*padding=*/param.padding); + /*stride=*/param.strides, + /*padding=*/padding); ComputeAndCompareR1(&b, tensorflow::gtl::ArraySlice(*expected), {input_arg.get()}, ErrorSpec(1e-3, 1e-3)); -- GitLab From 782ec4e186943d69e4ad047a835cbbf2eb477359 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Tue, 28 Nov 2017 18:17:01 -0800 Subject: [PATCH 0088/1924] Silenced noisy log PiperOrigin-RevId: 177249675 --- tensorflow/core/grappler/grappler_item_builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc index 36c7f92c49..a186e9a181 100644 --- a/tensorflow/core/grappler/grappler_item_builder.cc +++ b/tensorflow/core/grappler/grappler_item_builder.cc @@ -173,7 +173,7 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( << ", skipping this input."; return nullptr; } - LOG(INFO) << "Will use feed node " << feed_name; + VLOG(1) << "Will use feed node " << feed_name; new_item->feed.emplace_back(feed_name, Tensor()); } -- GitLab From bf05a2d1dce3af9b88dcd5c9253a163353951c99 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 28 Nov 2017 18:39:34 -0800 Subject: [PATCH 0089/1924] Support shape inference (i.e., shapes containing -1) in the Reshape bijector. PiperOrigin-RevId: 177251901 --- .../kernel_tests/bijectors/reshape_test.py | 342 +++++++++++------- .../python/ops/bijectors/reshape_impl.py | 277 +++++++------- 2 files changed, 362 insertions(+), 257 deletions(-) diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py index 38b3a23c2d..49451446b5 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py @@ -28,8 +28,19 @@ from tensorflow.python.ops.distributions.bijector_test_util import assert_biject from tensorflow.python.platform import test -class ReshapeBijectorTest(test.TestCase): - """Tests correctness of the reshape transformation.""" +class _ReshapeBijectorTest(object): + """Base class for testing the reshape transformation. + + Methods defined in this class call a method self.build_shapes() that + is implemented by subclasses defined below, returning respectively + ReshapeBijectorTestStatic: static shapes, + ReshapeBijectorTestDynamic: shape placeholders of known ndims, and + ReshapeBijectorTestDynamicNdims: shape placeholders of unspecified ndims, + so that each test in this base class is automatically run over all + three cases. The subclasses also implement assertRaisesError to test + for either Python exceptions (in the case of static shapes) or + TensorFlow op errors (dynamic shapes). + """ def setUp(self): self._rng = np.random.RandomState(42) @@ -40,9 +51,10 @@ class ReshapeBijectorTest(test.TestCase): expected_y = np.reshape(expected_x, [4, 6]) with self.test_session() as sess: + shape_in, shape_out, feed_dict = self.build_shapes([3, 2], [6,]) bijector = Reshape( - event_shape_out=[6,], - event_shape_in=[3, 2], + event_shape_out=shape_out, + event_shape_in=shape_in, validate_args=True) (x_, y_, @@ -52,66 +64,23 @@ class ReshapeBijectorTest(test.TestCase): bijector.forward(expected_x), bijector.forward_log_det_jacobian(expected_x), bijector.inverse_log_det_jacobian(expected_y), - )) + ), feed_dict=feed_dict) self.assertEqual("reshape", bijector.name) self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) self.assertAllClose(0., fldj_, rtol=1e-6, atol=0) self.assertAllClose(0., ildj_, rtol=1e-6, atol=0) - def testEventShapeDynamicNdims(self): - """Check forward/inverse shape methods with dynamic ndims.""" - - shape_in = tensor_shape.TensorShape([6,]) - shape_in_ph = array_ops.placeholder(dtype=dtypes.int32) - - shape_out = tensor_shape.TensorShape([2, 3]) - shape_out_ph = array_ops.placeholder(dtype=dtypes.int32) - - bijector = Reshape( - event_shape_out=shape_out_ph, - event_shape_in=shape_in_ph, validate_args=True) - - # using the _tensor methods, we should always get a fully-specified - # result since these are evaluated at graph runtime. - with self.test_session() as sess: - (shape_out_, - shape_in_) = sess.run(( - bijector.forward_event_shape_tensor(shape_in), - bijector.inverse_event_shape_tensor(shape_out), - ), feed_dict={ - shape_in_ph: shape_in, - shape_out_ph: shape_out, - }) - self.assertAllEqual(shape_out, shape_out_) - self.assertAllEqual(shape_in, shape_in_) - - def testEventShapeDynamic(self): - """Check shape methods with static ndims but dynamic shape.""" - - shape_in = tensor_shape.TensorShape([6,]) - shape_in_partial = tensor_shape.TensorShape([None,]) - shape_in_ph = array_ops.placeholder( - shape=[1,], dtype=dtypes.int32) - - shape_out = tensor_shape.TensorShape([2, 3]) - shape_out_partial = tensor_shape.TensorShape([None, None]) - shape_out_ph = array_ops.placeholder( - shape=[2,], dtype=dtypes.int32) + def testEventShapeTensor(self): + """Test event_shape_tensor methods when even ndims may be dynamic.""" + shape_in_static = [2, 3] + shape_out_static = [6,] + shape_in, shape_out, feed_dict = self.build_shapes(shape_in_static, + shape_out_static) bijector = Reshape( - event_shape_out=shape_out_ph, - event_shape_in=shape_in_ph, - validate_args=True) - - # if event shapes are not statically available, should - # return partially-specified TensorShapes. - self.assertAllEqual( - bijector.forward_event_shape(shape_in).as_list(), - shape_out_partial.as_list()) - self.assertAllEqual( - bijector.inverse_event_shape(shape_out).as_list(), - shape_in_partial.as_list()) + event_shape_out=shape_out, + event_shape_in=shape_in, validate_args=True) # using the _tensor methods, we should always get a fully-specified # result since these are evaluated at graph runtime. @@ -120,42 +89,9 @@ class ReshapeBijectorTest(test.TestCase): shape_in_) = sess.run(( bijector.forward_event_shape_tensor(shape_in), bijector.inverse_event_shape_tensor(shape_out), - ), feed_dict={ - shape_in_ph: shape_in, - shape_out_ph: shape_out, - }) - self.assertAllEqual(shape_out, shape_out_) - self.assertAllEqual(shape_in, shape_in_) - - def testEventShapeStatic(self): - """Check shape methods when shape is statically known.""" - - shape_in = tensor_shape.TensorShape([6,]) - shape_out = tensor_shape.TensorShape([2, 3]) - - bijector_static = Reshape( - event_shape_out=shape_out, - event_shape_in=shape_in, - validate_args=True) - - # test that forward_ and inverse_event_shape do sensible things - # when shapes are statically known. - self.assertEqual( - bijector_static.forward_event_shape(shape_in), - shape_out) - self.assertEqual( - bijector_static.inverse_event_shape(shape_out), - shape_in) - - with self.test_session() as sess: - (shape_out_static_, - shape_in_static_, - ) = sess.run(( - bijector_static.forward_event_shape_tensor(shape_in), - bijector_static.inverse_event_shape_tensor(shape_out), - )) - self.assertAllEqual(shape_out, shape_out_static_) - self.assertAllEqual(shape_in, shape_in_static_) + ), feed_dict=feed_dict) + self.assertAllEqual(shape_out_static, shape_out_) + self.assertAllEqual(shape_in_static, shape_in_) def testScalarReshape(self): """Test reshaping to and from a scalar shape ().""" @@ -166,11 +102,11 @@ class ReshapeBijectorTest(test.TestCase): expected_x_scalar = np.random.randn(1,) expected_y_scalar = expected_x_scalar[0] + shape_in, shape_out, feed_dict = self.build_shapes([], [1,]) with self.test_session() as sess: bijector = Reshape( - event_shape_out=[], - event_shape_in=[1,], validate_args=True) - + event_shape_out=shape_in, + event_shape_in=shape_out, validate_args=True) (x_, y_, x_scalar_, @@ -180,53 +116,178 @@ class ReshapeBijectorTest(test.TestCase): bijector.forward(expected_x), bijector.inverse(expected_y_scalar), bijector.forward(expected_x_scalar), - )) + ), feed_dict=feed_dict) self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) self.assertAllClose(expected_y_scalar, y_scalar_, rtol=1e-6, atol=0) self.assertAllClose(expected_x_scalar, x_scalar_, rtol=1e-6, atol=0) - def testRaisesOpError(self): - x1 = np.random.randn(4, 2, 3) - x2 = np.random.randn(4, 3, 2) - x3 = np.random.randn(4, 5, 1, 1) + def testMultipleUnspecifiedDimensionsOpError(self): with self.test_session() as sess: - shape_in_ph = array_ops.placeholder(shape=[2,], dtype=dtypes.int32) - shape_out_ph = array_ops.placeholder(shape=[3,], dtype=dtypes.int32) + shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [4, -1, -1,]) bijector = Reshape( - event_shape_out=shape_out_ph, - event_shape_in=shape_in_ph, + event_shape_out=shape_out, + event_shape_in=shape_in, validate_args=True) - with self.assertRaisesOpError( + with self.assertRaisesError( + "elements must have at most one `-1`."): + sess.run(bijector.forward_event_shape_tensor(shape_in), + feed_dict=feed_dict) + + def testInvalidDimensionsOpError(self): + + with self.test_session() as sess: + + shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [1, 2, -2,]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + + with self.assertRaisesError( + "elements must be either positive integers or `-1`."): + sess.run(bijector.forward_event_shape_tensor(shape_in), + feed_dict=feed_dict) + + def testValidButNonMatchingInputOpError(self): + x = np.random.randn(4, 3, 2) + + with self.test_session() as sess: + shape_in, shape_out, feed_dict = self.build_shapes([2, 3], [1, 6, 1,]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + + # Here we pass in a tensor (x) whose shape is compatible with + # the output shape, so tf.reshape will throw no error, but + # doesn't match the expected input shape. + with self.assertRaisesError( "Input `event_shape` does not match `event_shape_in`."): - sess.run(bijector.forward(x2), - feed_dict={shape_out_ph: [1, 6, 1], - shape_in_ph: [2, 3]}) + sess.run(bijector.forward(x), + feed_dict=feed_dict) - with self.assertRaisesOpError( - "event_shape_out entries must be positive."): - sess.run(bijector.forward(x1), - feed_dict={shape_out_ph: [-1, -1, 6], - shape_in_ph: [2, 3]}) + def testValidButNonMatchingInputPartiallySpecifiedOpError(self): + x = np.random.randn(4, 3, 2) + + with self.test_session() as sess: + shape_in, shape_out, feed_dict = self.build_shapes([2, -1], [1, 6, 1,]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + + with self.assertRaisesError( + "Input `event_shape` does not match `event_shape_in`."): + sess.run(bijector.forward(x), + feed_dict=feed_dict) + + def testInputOutputMismatchOpError(self): + x1 = np.random.randn(4, 2, 3) + x2 = np.random.randn(4, 1, 1, 5) + + with self.test_session() as sess: + shape_in, shape_out, fd_mismatched = self.build_shapes([2, 3], + [1, 1, 5]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) # test that *all* methods check basic assertions - fd_mismatched = {shape_out_ph: [1, 1, 5], shape_in_ph: [2, 3]} - with self.assertRaisesOpError( - "Input/output `event_size`s do not match."): + with self.assertRaisesError( + "Input to reshape is a tensor with"): sess.run(bijector.forward(x1), feed_dict=fd_mismatched) - with self.assertRaisesOpError( - "Input/output `event_size`s do not match."): - sess.run(bijector.inverse(x3), feed_dict=fd_mismatched) - with self.assertRaisesOpError( - "Input/output `event_size`s do not match."): - sess.run(bijector.inverse_log_det_jacobian(x3), - feed_dict=fd_mismatched) - with self.assertRaisesOpError( - "Input/output `event_size`s do not match."): - sess.run(bijector.forward_log_det_jacobian(x1), - feed_dict=fd_mismatched) + with self.assertRaisesError( + "Input to reshape is a tensor with"): + sess.run(bijector.inverse(x2), feed_dict=fd_mismatched) + + def testOneShapePartiallySpecified(self): + expected_x = np.random.randn(4, 6) + expected_y = np.reshape(expected_x, [4, 2, 3]) + + with self.test_session() as sess: + # one of input/output shapes is partially specified + shape_in, shape_out, feed_dict = self.build_shapes([-1,], [2, 3]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + (x_, + y_, + ) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + ), feed_dict=feed_dict) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + + def testBothShapesPartiallySpecified(self): + expected_x = np.random.randn(4, 2, 3) + expected_y = np.reshape(expected_x, [4, 3, 2]) + with self.test_session() as sess: + shape_in, shape_out, feed_dict = self.build_shapes([-1, 3], [-1, 2]) + bijector = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + (x_, + y_, + ) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + ), feed_dict=feed_dict) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + + def testDefaultVectorShape(self): + expected_x = np.random.randn(4, 4) + expected_y = np.reshape(expected_x, [4, 2, 2]) + with self.test_session() as sess: + _, shape_out, feed_dict = self.build_shapes([-1,], [-1, 2]) + bijector = Reshape(shape_out, + validate_args=True) + (x_, + y_, + ) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + ), feed_dict=feed_dict) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + + def build_shapes(self, *args, **kwargs): + raise NotImplementedError("Subclass failed to implement `build_shapes`.") + + +class ReshapeBijectorTestStatic(test.TestCase, _ReshapeBijectorTest): + + def build_shapes(self, shape_in, shape_out): + shape_in_static = shape_in + shape_out_static = shape_out + feed_dict = {} + return shape_in_static, shape_out_static, feed_dict + + def assertRaisesError(self, msg): + return self.assertRaisesRegexp(Exception, msg) + + def testEventShape(self): + shape_in_static = tensor_shape.TensorShape([2, 3]) + shape_out_static = tensor_shape.TensorShape([6,]) + bijector = Reshape( + event_shape_out=shape_out_static, + event_shape_in=shape_in_static, validate_args=True) + + # test that forward_ and inverse_event_shape do sensible things + # when shapes are statically known. + self.assertEqual( + bijector.forward_event_shape(shape_in_static), + shape_out_static) + self.assertEqual( + bijector.inverse_event_shape(shape_out_static), + shape_in_static) def testBijectiveAndFinite(self): x = np.random.randn(4, 2, 3) @@ -238,5 +299,32 @@ class ReshapeBijectorTest(test.TestCase): validate_args=True) assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0) + +class ReshapeBijectorTestDynamic(test.TestCase, _ReshapeBijectorTest): + + def build_shapes(self, shape_in, shape_out): + shape_in_ph = array_ops.placeholder(shape=(len(shape_in),), + dtype=dtypes.int32) + shape_out_ph = array_ops.placeholder(shape=(len(shape_out),), + dtype=dtypes.int32) + feed_dict = {shape_in_ph: shape_in, shape_out_ph: shape_out} + return shape_in_ph, shape_out_ph, feed_dict + + def assertRaisesError(self, msg): + return self.assertRaisesOpError(msg) + + +class ReshapeBijectorTestDynamicNdims(test.TestCase, _ReshapeBijectorTest): + + def build_shapes(self, shape_in, shape_out): + shape_in_ph = array_ops.placeholder(shape=None, dtype=dtypes.int32) + shape_out_ph = array_ops.placeholder(shape=None, dtype=dtypes.int32) + feed_dict = {shape_in_ph: shape_in, shape_out_ph: shape_out} + return shape_in_ph, shape_out_ph, feed_dict + + def assertRaisesError(self, msg): + return self.assertRaisesOpError(msg) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py index 93682639aa..1eb8e74fda 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py @@ -36,70 +36,77 @@ __all__ = [ ] +def _static_ndims_from_shape(shape): + return shape.shape.with_rank_at_least(1)[0].value + + +def _ndims_from_shape(shape): + return array_ops.shape(shape)[0] + + class Reshape(bijector_lib.Bijector): """Reshapes the `event_shape` of a `Tensor`. The semantics generally follow that of `tf.reshape()`, with a few differences: - * The user must provide both the input and output shape, so that - the transformation can be inverted. - * The `Reshape` bijector automatically broadcasts over the leftmost - dimensions of its input (`sample_shape` and `batch_shape`); only - the rightmost `event_ndims_in` dimensions are reshaped. The - number of dimensions to reshape is inferred from the provided - `event_shape_in` (`event_ndims_in = len(event_shape_in)`). - * The `Reshape` bijector does not currently support - partially-specified shapes, i.e., those with a dimension - implicitly specified by `-1`. + + * The user must provide both the input and output shape, so that + the transformation can be inverted. If an input shape is not + specified, the default assumes a vector-shaped input, i.e., + event_shape_in = (-1,). + * The `Reshape` bijector automatically broadcasts over the leftmost + dimensions of its input (`sample_shape` and `batch_shape`); only + the rightmost `event_ndims_in` dimensions are reshaped. The + number of dimensions to reshape is inferred from the provided + `event_shape_in` (`event_ndims_in = len(event_shape_in)`). Example usage: ```python bs = tf.contrib.distributions.bijectors - reverse = bs.Reshape(event_shape_out=[1,2], - event_shape_in=[2,]) + r = bs.Reshape(event_shape_out=[1, -1]) - reverse.forward([1., 2.]) # shape [2,] - # ==> [[1., 2.]] # shape [1,2] + r.forward([3., 4.]) # shape [2] + # ==> [[3., 4.]] # shape [1, 2] - reverse.forward([[1., 2.], [3., 4.]]) # shape [2, 2] - # ==> [[[1., 2.]], [[3., 4.]]] # shape [2, 1, 2] + r.forward([[1., 2.], [3., 4.]]) # shape [2, 2] + # ==> [[[1., 2.]], + # [[3., 4.]]] # shape [2, 1, 2] - reverse.inverse([[1., 2.]]) # shape [1,2] - # ==> [1., 2.] # shape [2,] + r.inverse([[3., 4.]]) # shape [1,2] + # ==> [3., 4.] # shape [2] - reverse.forward_log_det_jacobian(any_value) + r.forward_log_det_jacobian(any_value) # ==> 0. - reverse.inverse_log_det_jacobian(any_value) + r.inverse_log_det_jacobian(any_value) # ==> 0. ``` """ - def __init__(self, event_shape_out, event_shape_in, + def __init__(self, event_shape_out, event_shape_in=(-1,), validate_args=False, name=None): """Creates a `Reshape` bijector. Args: event_shape_out: An `int`-like vector-shaped `Tensor` - representing the fully specified (no -1's) event shape of the - transformed output. - event_shape_in: An `int`-like vector-shaped `Tensor` - representing the fully specified (no -1's) event shape of the - input. + representing the event shape of the transformed output. + event_shape_in: An optional `int`-like vector-shape `Tensor` + representing the event shape of the input. This is required in + order to define inverse operations; the default of (-1,) + assumes a vector-shaped input. validate_args: Python `bool` indicating whether arguments should be checked for correctness. name: Python `str`, name given to ops managed by this object. Raises: TypeError: if either `event_shape_in` or `event_shape_out` has - non-vector shape (`rank > 1`), or non-integer `dtype`. - ValueError: if either `event_shape_in` or `event_shape_out` - contains non-positive entries, or if their sizes do not match - (`prod(event_shape_in)` != `prod(event_shape_out)`), or if - their dimensionality(s) cannot be statically inferred. + non-integer `dtype`. + ValueError: if either of `event_shape_in` or `event_shape_out` + has non-vector shape (`rank > 1`), or if their sizes do not + match. """ with ops.name_scope(name, "reshape", values=[event_shape_out, event_shape_in]): @@ -111,105 +118,74 @@ class Reshape(bijector_lib.Bijector): name="event_shape_in", preferred_dtype=dtypes.int32) - # check that input shapes are positive integers assertions = [] - assertions += self._maybe_check_valid_shape( - event_shape_out, "event_shape_out", - validate_args=validate_args) - assertions += self._maybe_check_valid_shape( - event_shape_in, "event_shape_in", validate_args=validate_args) - - # check that prod(event_shape_in) = prod(event_shape_out) - assertions += self._maybe_check_matching_sizes( - event_shape_in, event_shape_out, validate_args=validate_args) + assertions.extend(self._maybe_check_valid_shape( + event_shape_out, validate_args)) + assertions.extend(self._maybe_check_valid_shape( + event_shape_in, validate_args)) self._assertions = assertions self._event_shape_in = event_shape_in self._event_shape_out = event_shape_out - self._event_shape_in_static = tensor_util.constant_value_as_shape( - event_shape_in) - self._event_shape_out_static = tensor_util.constant_value_as_shape( - event_shape_out) super(Reshape, self).__init__(is_constant_jacobian=True, validate_args=validate_args, name=name or "reshape") - def _maybe_check_valid_shape(self, shape_tensor, label, - validate_args=False): - """Check that a shape Tensor is int-type and positive.""" - - assertions = [] - - if not shape_tensor.dtype.is_integer: + def _maybe_check_valid_shape(self, shape, validate_args): + """Check that a shape Tensor is int-type and otherwise sane.""" + if not shape.dtype.is_integer: raise TypeError("{} dtype ({}) should be `int`-like.".format( - label, shape_tensor.dtype.name)) + shape.op.name, shape.dtype.name)) - shape_rank = tensor_util.constant_value(array_ops.rank(shape_tensor)) - if shape_rank is not None and shape_rank > 1: - raise ValueError("{} rank should be <= 1.".format(label)) + assertions = [] - s = tensor_util.constant_value(shape_tensor) - if s is not None: - if (s <= 0).any(): - raise ValueError("{} entries must be positive, but found {}".format( - label, s)) + ndims = array_ops.rank(shape) + ndims_ = tensor_util.constant_value(ndims) + if ndims_ is not None and ndims_ > 1: + raise ValueError("`{}` rank ({}) should be <= 1.".format( + shape.op.name, ndims_)) elif validate_args: - assertions.append(check_ops.assert_positive( - shape_tensor, message="{} entries must be positive".format(label))) - - return assertions - - def _maybe_check_matching_sizes(self, event_shape_in, event_shape_out, - validate_args=False): - """Check that prod(event_shape_in)==prod(event_shape_out).""" + assertions.append(check_ops.assert_less_equal( + ndims, 1, message="`{}` rank should be <= 1.".format(shape.op.name))) - def _get_size_from_shape(shape): - """Computes size from a shape `Tensor`, statically if possible.""" - s = tensor_util.constant_value(shape) - if s is not None: - return [np.int32(np.prod(s))]*2 - return None, math_ops.reduce_prod(shape, name="size") - - # Ensure `event_shape_in` is compatible with `event_shape_out`. - event_size_in_, event_size_in = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking - event_shape_in) - event_size_out_, event_size_out = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking - event_shape_out) - - assertions = [] - if event_size_in_ is not None and event_size_out_ is not None: - if event_size_in_ != event_size_out_: + shape_ = tensor_util.constant_value_as_shape(shape) + if shape_.is_fully_defined(): + es = np.int32(shape_.as_list()) + if sum(es == -1) > 1: + raise ValueError( + "`{}` must have at most one `-1` (given {})" + .format(shape.op.name, es)) + if np.any(es < -1): raise ValueError( - "Input `event_size` ({}) does not match output `event_size` ({}).". - format(event_size_in, event_size_out_)) + "`{}` elements must be either positive integers or `-1`" + "(given {})." + .format(shape.op.name, es)) elif validate_args: - assertions.append(check_ops.assert_equal( - event_size_in, event_size_out, - message="Input/output `event_size`s do not match.")) - + assertions.extend([ + check_ops.assert_less_equal( + math_ops.reduce_sum( + math_ops.cast(math_ops.equal(shape, -1), dtypes.int32)), + 1, + message="`{}` elements must have at most one `-1`." + .format(shape.op.name)), + check_ops.assert_greater_equal( + shape, -1, + message="`{}` elements must be either positive integers or `-1`." + .format(shape.op.name)), + ]) return assertions def _reshape_helper(self, x, event_shape_in, event_shape_out): """Reshape only the event_shape of an input `Tensor`.""" - def _get_rank_from_shape(shape): - """Computes rank from a shape `Tensor`, statically if possible.""" - # Uses fact that rank is "shape of shape". - ndims = shape.shape.with_rank_at_least(1)[0].value - if ndims is not None: - return ndims, ndims - return None, array_ops.shape(shape)[0] - - event_ndims_in_, event_ndims_in = _get_rank_from_shape(event_shape_in) + event_ndims_in_ = _static_ndims_from_shape(event_shape_in) + event_ndims_in = _ndims_from_shape(event_shape_in) + x_ndims_, x_ndims = x.shape.ndims, array_ops.rank(x) assertions = [] - # Ensure x.event_shape is compatible with event_shape_in. - if x.shape.ndims is not None: - x_ndims_, x_ndims = [x.shape.ndims]*2 - else: - x_ndims_, x_ndims = None, array_ops.rank(x) + # Ensure x.event_shape is compatible with event_shape_in. if (event_ndims_in_ is not None and x_ndims_ is not None and x.shape.with_rank_at_least(event_ndims_in_)[ @@ -223,13 +199,35 @@ class Reshape(bijector_lib.Bijector): event_shape_in_ = tensor_util.constant_value(event_shape_in) if x_event_shape_ is not None and event_shape_in_ is not None: - if not np.equal(x_event_shape_, event_shape_in_).all(): + # Compare the shape dimensions that are fully specified in the + # input (i.e., for which event_shape_in is not -1). If x_event_shape + # matches along all of these dimensions, it is compatible with + # the desired input shape and any further mismatches (i.e., + # imcompatibility with the desired *output* shape) will be + # caught inside of array_ops.reshape() below. + x_event_shape_specified_ = x_event_shape_[event_shape_in_ >= 0] + event_shape_in_specified_ = event_shape_in_[event_shape_in_ >= 0] + if not np.equal(x_event_shape_specified_, + event_shape_in_specified_).all(): raise ValueError( - "Input `event_shape` ({}) does not match `event_shape_in` ({}).". + "Input `event_shape` does not match `event_shape_in` ({} vs {}).". format(x_event_shape_, event_shape_in_)) elif self.validate_args: + # Similarly to the static case, we compare the shape dimensions + # that are fully specified in the input. We extract these + # dimensions using boolean_mask(), which requires that the mask + # have known ndims. We can assume that shape Tensors always have + # ndims==1 (this assumption is verified inside of + # _maybe_check_valid_shape), so the reshape operation is just a + # no-op that formally encodes this fact to make boolean_mask() + # happy. + event_shape_mask = array_ops.reshape(event_shape_in >= 0, [-1]) + x_event_shape_specified = array_ops.boolean_mask(x_event_shape, + event_shape_mask) + event_shape_in_specified = array_ops.boolean_mask(event_shape_in, + event_shape_mask) assertions.append(check_ops.assert_equal( - x_event_shape, event_shape_in, + x_event_shape_specified, event_shape_in_specified, message="Input `event_shape` does not match `event_shape_in`.")) if assertions: @@ -243,8 +241,19 @@ class Reshape(bijector_lib.Bijector): sample_and_batch_shape = sample_and_batch_shape[ :(ndims - math_ops.abs(event_ndims_in))] - new_shape = array_ops.concat( - [sample_and_batch_shape, event_shape_out], axis=0) + if (event_ndims_in_ is not None + and x_ndims_ is not None + and event_ndims_in_ == x_ndims_): + # Hack to allow forward/inverse_event_shape to do shape + # inference by calling this helper method with a dummy Tensor of + # shape event_shape_in. In this special case, + # sample_and_batch_shape will be empty so we can preserve static + # shape information by avoiding the concat operation below + # (which would be a no-op). + new_shape = event_shape_out + else: + new_shape = array_ops.concat( + [sample_and_batch_shape, event_shape_out], axis=0) return array_ops.reshape(x, new_shape) @@ -269,29 +278,37 @@ class Reshape(bijector_lib.Bijector): return constant_op.constant(0., dtype=x.dtype) def _forward_event_shape(self, input_shape): - self._event_shape_in_static.assert_is_compatible_with(input_shape) - return self._event_shape_out_static + # NOTE: this method and the other *_event_shape* methods + # compute shape by explicit transformation of a dummy + # variable. This approach is not generally recommended because it + # bloats the graph and could in general trigger side effects. + # + # In this particular case of the Reshape bijector, the + # forward and inverse transforms have no side effects, and we + # believe the reduction in code complexity from delegating the + # heavy lifting to tf.reshape() is worth the added graph ops. + # However, you should think hard before implementing this approach + # in other Bijectors; it is strongly preferred to compute + # shapes explicitly whenever it's feasible to do so. + with ops.control_dependencies(self._assertions): + dummy = array_ops.zeros(dtype=dtypes.float32, shape=input_shape) + dummy_reshaped = self.forward(dummy) + return dummy_reshaped.shape def _inverse_event_shape(self, output_shape): - self._event_shape_out_static.assert_is_compatible_with(output_shape) - return self._event_shape_in_static + with ops.control_dependencies(self._assertions): + dummy = array_ops.zeros(dtype=dtypes.float32, shape=output_shape) + dummy_reshaped = self.inverse(dummy) + return dummy_reshaped.shape def _forward_event_shape_tensor(self, input_shape): - input_assertions = self._maybe_check_valid_shape( - input_shape, "input event shape", validate_args=self.validate_args) - input_assertions += self._maybe_check_matching_sizes( - input_shape, self._event_shape_out, - validate_args=self.validate_args) - - return control_flow_ops.with_dependencies( - input_assertions + self._assertions, self._event_shape_out) + with ops.control_dependencies(self._assertions): + dummy = array_ops.zeros(dtype=dtypes.float32, shape=input_shape) + dummy_reshaped = self.forward(dummy) + return array_ops.shape(dummy_reshaped) def _inverse_event_shape_tensor(self, output_shape): - - output_assertions = self._maybe_check_valid_shape( - output_shape, "output event shape", validate_args=self.validate_args) - output_assertions += self._maybe_check_matching_sizes( - output_shape, self._event_shape_in, validate_args=self.validate_args) - - return control_flow_ops.with_dependencies( - output_assertions + self._assertions, self._event_shape_in) + with ops.control_dependencies(self._assertions): + dummy = array_ops.zeros(dtype=dtypes.float32, shape=output_shape) + dummy_reshaped = self.inverse(dummy) + return array_ops.shape(dummy_reshaped) -- GitLab From 60d2e51254028df73f650abe07fad024c49688bb Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Tue, 28 Nov 2017 19:07:56 -0800 Subject: [PATCH 0090/1924] Introduce tf.contrib.summary.flush This op has been useful while writing benchmarks. PiperOrigin-RevId: 177254316 --- tensorflow/contrib/summary/summary.py | 1 + tensorflow/contrib/summary/summary_ops.py | 21 +++++++++++++++ .../contrib/summary/summary_ops_test.py | 27 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py index f783179f61..9e6af5232f 100644 --- a/tensorflow/contrib/summary/summary.py +++ b/tensorflow/contrib/summary/summary.py @@ -31,6 +31,7 @@ from tensorflow.contrib.summary.summary_ops import audio from tensorflow.contrib.summary.summary_ops import create_summary_db_writer from tensorflow.contrib.summary.summary_ops import create_summary_file_writer from tensorflow.contrib.summary.summary_ops import eval_dir +from tensorflow.contrib.summary.summary_ops import flush from tensorflow.contrib.summary.summary_ops import generic from tensorflow.contrib.summary.summary_ops import graph from tensorflow.contrib.summary.summary_ops import histogram diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index 8e37987cb7..de6f2cd79f 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -516,6 +516,27 @@ def import_event(tensor, name=None): context.context().summary_writer_resource, tensor, name=name) +def flush(writer=None, name=None): + """Forces summary writer to send any buffered data to storage. + + This operation blocks until that finishes. + + Args: + writer: The @{tf.contrib.summary.SummaryWriter} resource to flush. + The thread default will be used if this parameter is None. + Otherwise a @{tf.no_op} is returned. + name: A name for the operation (optional). + + Returns: + The created @{tf.Operation}. + """ + if writer is None: + writer = context.context().summary_writer_resource + if writer is None: + return control_flow_ops.no_op() + return gen_summary_ops.flush_summary_writer(writer, name=name) + + def eval_dir(model_dir, name=None): """Construct a logdir for an eval summary writer.""" return os.path.join(model_dir, "eval" if not name else "eval_" + name) diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index ad89c0c36a..3fe421a7e9 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -109,6 +109,33 @@ class TargetTest(test_util.TensorFlowTestCase): self.assertEqual(len(events), 2) self.assertEqual(events[1].summary.value[0].tag, 'scalar') + def testMaxQueue(self): + logs = tempfile.mkdtemp() + with summary_ops.create_summary_file_writer( + logs, max_queue=2, flush_millis=999999, + name='lol').as_default(), summary_ops.always_record_summaries(): + get_total = lambda: len(summary_test_util.events_from_logdir(logs)) + # Note: First tf.Event is always file_version. + self.assertEqual(1, get_total()) + summary_ops.scalar('scalar', 2.0, step=1) + self.assertEqual(1, get_total()) + summary_ops.scalar('scalar', 2.0, step=2) + self.assertEqual(3, get_total()) + + def testFlush(self): + logs = tempfile.mkdtemp() + with summary_ops.create_summary_file_writer( + logs, max_queue=999999, flush_millis=999999, + name='lol').as_default(), summary_ops.always_record_summaries(): + get_total = lambda: len(summary_test_util.events_from_logdir(logs)) + # Note: First tf.Event is always file_version. + self.assertEqual(1, get_total()) + summary_ops.scalar('scalar', 2.0, step=1) + summary_ops.scalar('scalar', 2.0, step=2) + self.assertEqual(1, get_total()) + summary_ops.flush() + self.assertEqual(3, get_total()) + class DbTest(summary_test_internal.SummaryDbTest): -- GitLab From d2e7a2e4bf295a23d6a2e86aa7e0636f00cc2d75 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Tue, 28 Nov 2017 19:42:30 -0800 Subject: [PATCH 0091/1924] Add VLOG-ging to gcs_file_system PiperOrigin-RevId: 177256727 --- tensorflow/core/platform/cloud/gcs_file_system.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 54d38fe962..45e9b05092 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -697,6 +697,9 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& filename, size_t offset, TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when reading gs://", bucket, "/", object); + VLOG(1) << "Successful read of gs://" << bucket << "/" << object << " @ " + << offset << " of size: " << out->size(); + if (out->size() < block_size()) { // Check stat cache to see if we encountered an interrupted read. FileStatistics stat; @@ -706,6 +709,8 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& filename, size_t offset, "File contents are inconsistent for file: %s @ %lu.", filename.c_str(), offset)); } + VLOG(2) << "Successful integrity check for: gs://" << bucket << "/" + << object << " @ " << offset; } } @@ -868,6 +873,11 @@ Status GcsFileSystem::StatForObject(const string& fname, const string& bucket, TF_RETURN_IF_ERROR(GetStringValue(root, "updated", &updated)); TF_RETURN_IF_ERROR(ParseRfc3339Time(updated, &(stat->mtime_nsec))); + VLOG(1) << "Stat of: gs://" << bucket << "/" << object << " -- " + << " length: " << stat->length + << "; mtime_nsec: " << stat->mtime_nsec + << "; updated: " << updated; + stat->is_directory = false; return Status::OK(); }; -- GitLab From bdde4d040cf01ef241ad349cf222c227b9a88814 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Tue, 28 Nov 2017 20:41:47 -0800 Subject: [PATCH 0092/1924] [XLA] Support transposing the spatial dimensions of a convolution's activations PiperOrigin-RevId: 177260886 --- .../compiler/xla/service/transpose_folding.cc | 9 ++- .../xla/service/transpose_folding_test.cc | 64 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc index fb55d4e543..42b616f4c3 100644 --- a/tensorflow/compiler/xla/service/transpose_folding.cc +++ b/tensorflow/compiler/xla/service/transpose_folding.cc @@ -102,6 +102,10 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) { auto& convolution = *pair.first; auto& operand_indices = pair.second; + if (operand_indices.empty()) { + return false; + } + const ConvolutionDimensionNumbers& dnums = convolution.convolution_dimension_numbers(); ConvolutionDimensionNumbers new_dnums = dnums; @@ -121,8 +125,9 @@ bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) { transpose_dimensions[dnums.input_batch_dimension()]); new_dnums.set_input_feature_dimension( transpose_dimensions[dnums.input_feature_dimension()]); - for (const auto& spatial_dimension : dnums.input_spatial_dimensions()) { - CHECK_EQ(spatial_dimension, transpose_dimensions[spatial_dimension]); + for (auto& input_spatial_dimension : + *new_dnums.mutable_input_spatial_dimensions()) { + input_spatial_dimension = transpose_dimensions[input_spatial_dimension]; } new_lhs = &transpose_operand; } else { diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc index 6ac32e88f1..ba99852905 100644 --- a/tensorflow/compiler/xla/service/transpose_folding_test.cc +++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc @@ -376,5 +376,69 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { new_conv->convolution_dimension_numbers().output_spatial_dimensions(1)); } +// Test that a transpose of every dimension in the activations gets folded into +// convolution. +TEST_F(TransposeFoldingTest, FoldConvComplexTransposeLhs) { + auto builder = HloComputation::Builder("entry_computation"); + HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter( + /*parameter_number=*/0, ShapeUtil::MakeShape(F32, {3, 2, 1, 1}), + /*name=*/"x")); + HloInstruction* y = builder.AddInstruction(HloInstruction::CreateParameter( + /*parameter_number=*/1, ShapeUtil::MakeShape(F32, {2, 3, 1, 1}), + /*name=*/"y")); + HloInstruction* transpose_x = + builder.AddInstruction(HloInstruction::CreateTranspose( + ShapeUtil::MakeShape(F32, {2, 3, 1, 1}), x, {1, 0, 3, 2})); + auto dnums = ComputationBuilder::CreateDefaultConvDimensionNumbers(); + Window window; + for (int i = 0; i < 2; ++i) { + WindowDimension* dim = window.add_dimensions(); + dim->set_padding_low(0); + dim->set_padding_high(0); + dim->set_base_dilation(1); + dim->set_window_dilation(1); + dim->set_stride(1); + dim->set_size(y->shape().dimensions(dnums.kernel_spatial_dimensions(i))); + } + StatusOr conv_shape = ShapeInference::InferConvolveShape( + transpose_x->shape(), y->shape(), window, dnums); + EXPECT_IS_OK(conv_shape); + HloInstruction* conv = builder.AddInstruction(HloInstruction::CreateConvolve( + conv_shape.ValueOrDie(), transpose_x, y, window, dnums)); + + HloModule module("test_module"); + HloComputation* entry_computation = + module.AddEntryComputation(builder.Build(conv)); + FoldTranspose(&module); + + // Instructions after folding: x, y, and the convolution. + std::unordered_set instruction_set( + entry_computation->instructions().begin(), + entry_computation->instructions().end()); + EXPECT_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation."; + EXPECT_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation."; + EXPECT_EQ(1, instruction_set.size()) + << "entry_computation should contain exactly 3 instructions."; + HloInstruction* new_conv = *instruction_set.begin(); + EXPECT_EQ(HloOpcode::kConvolution, new_conv->opcode()); + EXPECT_EQ(dnums.input_feature_dimension(), + new_conv->convolution_dimension_numbers().input_batch_dimension()); + EXPECT_EQ( + dnums.input_batch_dimension(), + new_conv->convolution_dimension_numbers().input_feature_dimension()); + EXPECT_EQ( + dnums.input_spatial_dimensions(0), + new_conv->convolution_dimension_numbers().input_spatial_dimensions(1)); + EXPECT_EQ( + dnums.input_spatial_dimensions(1), + new_conv->convolution_dimension_numbers().input_spatial_dimensions(0)); + EXPECT_EQ( + dnums.output_spatial_dimensions(0), + new_conv->convolution_dimension_numbers().output_spatial_dimensions(0)); + EXPECT_EQ( + dnums.output_spatial_dimensions(1), + new_conv->convolution_dimension_numbers().output_spatial_dimensions(1)); +} + } // namespace } // namespace xla -- GitLab From a55ee58c89d5bf6a8cd70b706dc3af90d7d6efc4 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Tue, 28 Nov 2017 20:54:54 -0800 Subject: [PATCH 0093/1924] Include the filename when we encounter EOF PiperOrigin-RevId: 177261696 --- tensorflow/core/platform/cloud/file_block_cache.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/cloud/file_block_cache.cc b/tensorflow/core/platform/cloud/file_block_cache.cc index a472ae52fc..e1afc7b308 100644 --- a/tensorflow/core/platform/cloud/file_block_cache.cc +++ b/tensorflow/core/platform/cloud/file_block_cache.cc @@ -181,7 +181,9 @@ Status FileBlockCache::Read(const string& filename, size_t offset, size_t n, // The requested offset is at or beyond the end of the file. This can // happen if `offset` is not block-aligned, and the read returns the last // block in the file, which does not extend all the way out to `offset`. - return errors::OutOfRange("EOF at offset ", offset); + return errors::OutOfRange("EOF at offset ", offset, " in file ", filename, + " at position ", pos, "with data size ", + data.size()); } auto begin = data.begin(); if (offset > pos) { -- GitLab From 05f57851d4657ec6c09a454b157cf17d89d0cfe2 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 28 Nov 2017 21:10:16 -0800 Subject: [PATCH 0094/1924] Bugfixes: Gather's gradient for 2+ dimensional indices with eager execution. And shape inference function for the VariableShape operation. PiperOrigin-RevId: 177262783 --- tensorflow/core/ops/resource_variable_ops.cc | 5 ++- .../python/ops/resource_variable_ops.py | 21 ++++------ tensorflow/python/training/momentum_test.py | 41 +++++++++++++------ 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tensorflow/core/ops/resource_variable_ops.cc b/tensorflow/core/ops/resource_variable_ops.cc index cdfbec85cf..bf9e673e8e 100644 --- a/tensorflow/core/ops/resource_variable_ops.cc +++ b/tensorflow/core/ops/resource_variable_ops.cc @@ -204,7 +204,10 @@ Status VariableShapeShapeFn(InferenceContext* c) { if (handle_data == nullptr || handle_data->empty()) { return errors::InvalidArgument("Handle doesn't have shape information."); } - c->set_output(0, (*handle_data)[0].shape); + ShapeHandle var_shape = (*handle_data)[0].shape; + int64 rank = c->RankKnown(var_shape) ? c->Rank(var_shape) + : InferenceContext::kUnknownDim; + c->set_output(0, c->Vector(rank)); return Status::OK(); } diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 343e38f960..652bfa1ebc 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -887,26 +887,19 @@ def _ReadGrad(_, grad): def _GatherGrad(op, grad): """Gradient for gather op.""" # Build appropriately shaped IndexedSlices - # Walk graph back until the original handle is found. - # TODO(apassos): more robust way of getting the shape. - # TODO(apassos): implement this for EAGER mode. - if context.in_eager_mode(): - dense_shape = gen_resource_variable_ops.variable_shape(op.inputs[0]) - return (ops.IndexedSlices(grad, - op.inputs[1], - dense_shape=dense_shape), - None) handle = op.inputs[0] - while handle.op.type != "VarHandleOp": - handle = handle.op.inputs[0] - params_shape = ops.convert_to_tensor( - tensor_shape.TensorShape(handle.op.get_attr("shape"))) indices = op.inputs[1] + if context.in_graph_mode(): + # Walk graph back until the original handle is found. + # TODO(apassos): implement this for EAGER mode. + while handle.op.type != "VarHandleOp": + handle = handle.op.inputs[0] + params_shape = gen_resource_variable_ops.variable_shape(handle) size = array_ops.expand_dims(array_ops.size(indices), 0) values_shape = array_ops.concat([size, params_shape[1:]], 0) values = array_ops.reshape(grad, values_shape) indices = array_ops.reshape(indices, size) - return [ops.IndexedSlices(values, indices, params_shape), None] + return (ops.IndexedSlices(values, indices, params_shape), None) def _to_proto_fn(v, export_scope=None): diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py index 7268b3abc9..6865513b0e 100644 --- a/tensorflow/python/training/momentum_test.py +++ b/tensorflow/python/training/momentum_test.py @@ -234,23 +234,38 @@ class MomentumOptimizerTest(test.TestCase): self.assertAllClose(var0_np, var0.eval()) self.assertAllClose(var1_np, var1.eval()) + @test_util.run_in_graph_and_eager_modes(reset_test=True) def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype) + + # pylint: disable=cell-var-from-loop + def loss(): x = constant_op.constant([[4.0], [5.0]], dtype=dtype) pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - loss = pred * pred - sgd_op = momentum_lib.MomentumOptimizer( - learning_rate=1.0, momentum=0.0).minimize(loss) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval()) - # Run 1 step of sgd - sgd_op.run() - # Validate updated params - self.assertAllCloseAccordingToType( - [[-111, -138]], var0.eval()) + return pred * pred + # pylint: enable=cell-var-from-loop + + opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0) + sgd_op = opt.minimize(loss if context.in_eager_mode() else loss()) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) + + @test_util.run_in_graph_and_eager_modes(reset_test=True) + def testMinimizeWith2DIndiciesForEmbeddingLookup(self): + var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2])) + + def loss(): + return math_ops.reduce_sum(embedding_ops.embedding_lookup(var0, [[1]])) + + opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0) + sgd_op = opt.minimize(loss if context.in_eager_mode() else loss()) + self.evaluate(variables.global_variables_initializer()) + self.evaluate(sgd_op) + self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) def testTensorLearningRateAndMomentum(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: -- GitLab From 2ab7e9dad284bd15d69779ee0bcf8a2c894c2a2a Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Tue, 28 Nov 2017 21:22:20 -0800 Subject: [PATCH 0095/1924] Go: Add some more detail to an error message. Closes #14806 PiperOrigin-RevId: 177263469 --- tensorflow/go/tensor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/go/tensor.go b/tensorflow/go/tensor.go index 1326a95278..cd05e2aa0a 100644 --- a/tensorflow/go/tensor.go +++ b/tensorflow/go/tensor.go @@ -270,7 +270,7 @@ func typeOf(dt DataType, shape []int64) reflect.Type { } } if ret == nil { - panic(bug("DataType %v is not supported", dt)) + panic(bug("DataType %v is not supported (see https://www.tensorflow.org/code/tensorflow/core/framework/types.proto)", dt)) } for range shape { ret = reflect.SliceOf(ret) -- GitLab From a7c13b33d6df91e25fc793043bf748b30e311c73 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Tue, 28 Nov 2017 21:45:51 -0800 Subject: [PATCH 0096/1924] Update TF Android build instructions to warn about NDK 16 See https://github.com/bazelbuild/bazel/issues/4068 PiperOrigin-RevId: 177264756 --- tensorflow/contrib/lite/java/demo/README.md | 8 +++++++- tensorflow/examples/android/README.md | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/java/demo/README.md b/tensorflow/contrib/lite/java/demo/README.md index 71b633c577..5d13a798e2 100644 --- a/tensorflow/contrib/lite/java/demo/README.md +++ b/tensorflow/contrib/lite/java/demo/README.md @@ -8,7 +8,12 @@ It's easiest with Android Studio. - You'll need at least SDK version 23. + - Make sure to install the latest version of Bazel. Some distributions + ship with Bazel 0.5.4, which is too old. - Bazel requires Android Build Tools `26.0.1` or higher. + - **Bazel is incompatible with NDK revisions 15 and above,** with revision + 16 being a compile-breaking change. [Download an older version manually + instead of using the SDK Manager.](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android#install-bazel-and-android-prerequisites) - You also need to install the Android Support Repository, available through Android Studio under `Android SDK Manager -> SDK Tools -> Android Support Repository`. @@ -19,7 +24,8 @@ - Make sure the `api_level` in `WORKSPACE` is set to an SDK version that you have installed. - By default, Android Studio will install the SDK to `~/Android/Sdk` and - the NDK to `~/Android/Sdk/ndk-bundle`. + the NDK to `~/Android/Sdk/ndk-bundle` (but the NDK should be a manual + download until Bazel supports NDK 16. See bullet points under (1)). 2. Build the app with Bazel. The demo needs C++11: diff --git a/tensorflow/examples/android/README.md b/tensorflow/examples/android/README.md index 79202a38d7..881a975e60 100644 --- a/tensorflow/examples/android/README.md +++ b/tensorflow/examples/android/README.md @@ -126,6 +126,10 @@ the Android NDK and SDK must be installed on your system. 2. The Android NDK is required to build the native (C/C++) TensorFlow code. The current recommended version is 14b, which may be found [here](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads). + + * NDK 16, the revision released in November 2017, is **incompatible** with + Bazel. See [here](https://github.com/tensorflow/tensorflow/issues/14918). + 3. The Android SDK and build tools may be obtained [here](https://developer.android.com/tools/revisions/build-tools.html), or alternatively as part of [Android @@ -133,6 +137,10 @@ the Android NDK and SDK must be installed on your system. 23 is required to build the TF Android demo (though it will run on API >= 21 devices). + - The Android Studio SDK Manager's NDK installer will install the latest + revision of the NDK, which is **incompatible** with Bazel. You'll need + to download an older version manually, as (2) suggests. + ##### Edit WORKSPACE The Android entries in -- GitLab From 9aeb0eef9188a48a02078128d3d1ca6f78f0f438 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Tue, 28 Nov 2017 22:12:34 -0800 Subject: [PATCH 0097/1924] Add logging to help differentiate multiple stacktraces PiperOrigin-RevId: 177266569 --- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index fe17664d7f..84a4208be3 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -514,6 +514,7 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController): exc_info=1 ) time.sleep(120) + logging.error('Closing the failed session.') session.close() def join(self): -- GitLab From 4c2ca8b0cbfdbdc9f7525b1d9ad0c057cb513749 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Tue, 28 Nov 2017 22:48:10 -0800 Subject: [PATCH 0098/1924] Fix typo in GradientTape.persistent_ comment. PiperOrigin-RevId: 177268420 --- tensorflow/c/eager/tape.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h index f52248e7d5..191e9c3413 100644 --- a/tensorflow/c/eager/tape.h +++ b/tensorflow/c/eager/tape.h @@ -161,7 +161,7 @@ class GradientTape { // the tape refer to it); to aid in tape garbage collection. std::unordered_map tensor_usage_; - // If true, all activations are deleted in the first call to ComputeGradient. + // If false, all activations are deleted in the first call to ComputeGradient. // Else, only when this is destructed. bool persistent_; }; -- GitLab From bc87c28c60dddc6137b11f8a1fd31fa79bcf0c1f Mon Sep 17 00:00:00 2001 From: James Qin Date: Wed, 29 Nov 2017 00:34:54 -0800 Subject: [PATCH 0099/1924] Register fp16 Reduce min on GPU. PiperOrigin-RevId: 177274800 --- tensorflow/core/kernels/reduction_ops_min.cc | 1 + tensorflow/core/kernels/reduction_ops_test.cc | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc index 807ac0a456..5c537c5b9c 100644 --- a/tensorflow/core/kernels/reduction_ops_min.cc +++ b/tensorflow/core/kernels/reduction_ops_min.cc @@ -50,6 +50,7 @@ TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); .TypeConstraint("Tidx") \ .HostMemory("reduction_indices"), \ ReductionOp>); +REGISTER_GPU_KERNELS(Eigen::half); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); diff --git a/tensorflow/core/kernels/reduction_ops_test.cc b/tensorflow/core/kernels/reduction_ops_test.cc index 9bbe993a2f..fe8ea59f1b 100644 --- a/tensorflow/core/kernels/reduction_ops_test.cc +++ b/tensorflow/core/kernels/reduction_ops_test.cc @@ -174,6 +174,11 @@ static void BM_Min2DToScalarGPU(int iters, int num_x, int num_y) { } BENCHMARK(BM_Min2DToScalarGPU)->RangePair(2048, 8192, 2048, 8192); +static void BM_Min2DToScalarGPUHalf(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Min", num_x, num_y); +} +BENCHMARK(BM_Min2DToScalarGPUHalf)->RangePair(2048, 8192, 2048, 8192); + static void BM_Bool2DToScalarGPU(int iters, int num_x, int num_y) { ReduceToScalar(iters, "gpu", "All", num_x, num_y); } -- GitLab From 6196d30cf8498c428bdb7fbd4b4ab9cb83853457 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 02:51:49 -0800 Subject: [PATCH 0100/1924] Add alpha support for dealing with shardings to batchnorm rewriter PiperOrigin-RevId: 177285265 --- .../xla/service/batchnorm_rewriter.cc | 328 ++++++++++-------- .../compiler/xla/service/hlo_sharding.cc | 3 +- 2 files changed, 186 insertions(+), 145 deletions(-) diff --git a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc index c6193b3fbb..2bbae25aee 100644 --- a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc +++ b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc @@ -149,6 +149,15 @@ Status BatchNormRewriterVisitor::HandleBatchNormTraining( if (!rewrite_training_op_) { return Status::OK(); } + + std::vector added_instructions; + auto add = [&](std::unique_ptr inst) { + HloInstruction* added_inst = computation_->AddInstruction(std::move(inst)); + added_instructions.push_back(added_inst); + return added_inst; + }; + int64 instruction_count_before = computation_->instruction_count(); + // Expand batch norm training into smaller HLO ops. HloInstruction* operand = batch_norm->mutable_operand(0); const Shape operand_shape = operand->shape(); @@ -160,7 +169,7 @@ Status BatchNormRewriterVisitor::HandleBatchNormTraining( Literal::CreateR0(size_in_elements / feature_count); TF_ASSIGN_OR_RETURN(elements_per_feature_literal, elements_per_feature_literal->Convert(ptype)); - auto elements_per_feature = computation_->AddInstruction( + auto elements_per_feature = add( HloInstruction::CreateConstant(std::move(elements_per_feature_literal))); HloInstruction* scale = batch_norm->mutable_operand(1); @@ -169,14 +178,12 @@ Status BatchNormRewriterVisitor::HandleBatchNormTraining( auto zero_literal = Literal::CreateR0(0.0f); TF_ASSIGN_OR_RETURN(zero_literal, zero_literal->Convert(ptype)); - auto zero = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(zero_literal))); + auto zero = add(HloInstruction::CreateConstant(std::move(zero_literal))); auto epsilon_literal = Literal::CreateR0(batch_norm->epsilon()); TF_ASSIGN_OR_RETURN(epsilon_literal, epsilon_literal->Convert(ptype)); - auto epsilon = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(epsilon_literal))); - + auto epsilon = + add(HloInstruction::CreateConstant(std::move(epsilon_literal))); std::vector dimensions_without_feature; for (int64 i = 0; i < ShapeUtil::Rank(operand_shape); ++i) { @@ -185,105 +192,110 @@ Status BatchNormRewriterVisitor::HandleBatchNormTraining( } } - auto scale_broadcasted = computation_->AddInstruction( + auto scale_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, scale, {feature_index})); - auto offset_broadcasted = computation_->AddInstruction( + auto offset_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, offset, {feature_index})); HloComputation* add_reduce_computation = GetScalarBinaryComputation(ptype, HloOpcode::kAdd); // X^2. - auto operand_squared = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kMultiply, operand, operand)); + auto operand_squared = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kMultiply, operand, operand)); // Sum[X]. - auto sum = computation_->AddInstruction(HloInstruction::CreateReduce( - feature_shape, operand, zero, dimensions_without_feature, - add_reduce_computation)); + auto sum = add(HloInstruction::CreateReduce(feature_shape, operand, zero, + dimensions_without_feature, + add_reduce_computation)); // Sum[X^2]. - auto squared_sum = computation_->AddInstruction(HloInstruction::CreateReduce( + auto squared_sum = add(HloInstruction::CreateReduce( feature_shape, operand_squared, zero, dimensions_without_feature, add_reduce_computation)); // Fuse two parallel reduces together to improve performance. - if (use_fusion_) { - auto tuple = computation_->AddInstruction( - HloInstruction::CreateTuple({sum, squared_sum})); + if (use_fusion_ && !batch_norm->has_sharding()) { + auto tuple = add(HloInstruction::CreateTuple({sum, squared_sum})); auto fused = computation_->CreateFusionInstruction( {tuple, sum, squared_sum, operand_squared}, HloInstruction::FusionKind::kInput); - sum = computation_->AddInstruction( - HloInstruction::CreateGetTupleElement(feature_shape, fused, 0)); + sum = add(HloInstruction::CreateGetTupleElement(feature_shape, fused, 0)); - squared_sum = computation_->AddInstruction( - HloInstruction::CreateGetTupleElement(feature_shape, fused, 1)); + squared_sum = + add(HloInstruction::CreateGetTupleElement(feature_shape, fused, 1)); } // E[X]. - auto mean = computation_->AddInstruction(HloInstruction::CreateBinary( + auto mean = add(HloInstruction::CreateBinary( feature_shape, HloOpcode::kDivide, sum, elements_per_feature)); - auto mean_broadcasted = computation_->AddInstruction( + auto mean_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, mean, {feature_index})); // E[X^2]. - auto square_mean = computation_->AddInstruction(HloInstruction::CreateBinary( + auto square_mean = add(HloInstruction::CreateBinary( feature_shape, HloOpcode::kDivide, squared_sum, elements_per_feature)); // E^2[X]. - auto mean_square = computation_->AddInstruction(HloInstruction::CreateBinary( + auto mean_square = add(HloInstruction::CreateBinary( feature_shape, HloOpcode::kMultiply, mean, mean)); // Var[X]. - auto var = computation_->AddInstruction(HloInstruction::CreateBinary( + auto var = add(HloInstruction::CreateBinary( feature_shape, HloOpcode::kSubtract, square_mean, mean_square)); - auto var_broadcasted = computation_->AddInstruction( - HloInstruction::CreateBroadcast(operand_shape, var, {feature_index})); + auto var_broadcasted = + add(HloInstruction::CreateBroadcast(operand_shape, var, {feature_index})); // Var[X] + epsilon. - auto var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kAdd, var_broadcasted, epsilon)); + auto var_add_epsilon = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kAdd, var_broadcasted, epsilon)); auto neg_half_literal = Literal::CreateR0(-0.5f); TF_ASSIGN_OR_RETURN(neg_half_literal, neg_half_literal->Convert(ptype)); - auto neg_half = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(neg_half_literal))); + auto neg_half = + add(HloInstruction::CreateConstant(std::move(neg_half_literal))); // 1 / Sqrt[Var[X] + epsilon]. - auto rsqrt_var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kPower, var_add_epsilon, neg_half)); + auto rsqrt_var_add_epsilon = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kPower, var_add_epsilon, neg_half)); // X - E[X]. - auto operand_minus_mean = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kSubtract, operand, mean_broadcasted)); + auto operand_minus_mean = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kSubtract, operand, mean_broadcasted)); // (X - E[X]) / Sqrt[Var[X] + epsilon]. - auto normalized = computation_->AddInstruction( + auto normalized = add( HloInstruction::CreateBinary(operand_shape, HloOpcode::kMultiply, operand_minus_mean, rsqrt_var_add_epsilon)); // (X - E[X]) / Sqrt[Var[X] + epsilon] * scale. - auto scaled_normalized = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kMultiply, normalized, scale_broadcasted)); + auto scaled_normalized = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kMultiply, normalized, scale_broadcasted)); // (X - E[X]) / Sqrt[Var[X] + epsilon] * scale + offset. - auto shifted_normalized = computation_->AddInstruction( - HloInstruction::CreateBinary(operand_shape, HloOpcode::kAdd, - scaled_normalized, offset_broadcasted)); - - TF_CHECK_OK(ReplaceWithNewInstruction( - batch_norm, - HloInstruction::CreateTuple({shifted_normalized, mean, var}))); + auto shifted_normalized = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kAdd, scaled_normalized, offset_broadcasted)); + + auto tuple = HloInstruction::CreateTuple({shifted_normalized, mean, var}); + + if (batch_norm->has_sharding()) { + int64 instruction_count_after = computation_->instruction_count(); + CHECK_EQ(instruction_count_after, + instruction_count_before + added_instructions.size()); + for (HloInstruction* inst : added_instructions) { + if (ShapeUtil::Equal(inst->shape(), operand_shape)) { + inst->set_sharding(batch_norm->sharding()); + } else { + inst->set_sharding(HloSharding::Replicate()); + } + } + tuple->set_sharding(batch_norm->sharding()); + } + TF_CHECK_OK(ReplaceWithNewInstruction(batch_norm, std::move(tuple))); return Status::OK(); } @@ -317,52 +329,69 @@ Status BatchNormRewriterVisitor::HandleBatchNormInference( } } - auto scale_broadcasted = computation_->AddInstruction( + std::vector added_instructions; + auto add = [&](std::unique_ptr inst) { + HloInstruction* added_inst = computation_->AddInstruction(std::move(inst)); + added_instructions.push_back(added_inst); + return added_inst; + }; + int64 instruction_count_before = computation_->instruction_count(); + + auto scale_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, scale, {feature_index})); - auto offset_broadcasted = computation_->AddInstruction( + auto offset_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, offset, {feature_index})); - auto mean_broadcasted = computation_->AddInstruction( + auto mean_broadcasted = add( HloInstruction::CreateBroadcast(operand_shape, mean, {feature_index})); - auto var_broadcasted = computation_->AddInstruction( - HloInstruction::CreateBroadcast(operand_shape, var, {feature_index})); + auto var_broadcasted = + add(HloInstruction::CreateBroadcast(operand_shape, var, {feature_index})); // Var[X] + epsilon. - auto var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kAdd, var_broadcasted, epsilon)); + auto var_add_epsilon = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kAdd, var_broadcasted, epsilon)); auto neg_half_literal = Literal::CreateR0(-0.5f); TF_ASSIGN_OR_RETURN(neg_half_literal, neg_half_literal->Convert(ptype)); - auto neg_half = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(neg_half_literal))); + auto neg_half = + add(HloInstruction::CreateConstant(std::move(neg_half_literal))); // 1 / Sqrt[Var[X] + epsilon]. - auto rsqrt_var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kPower, var_add_epsilon, neg_half)); + auto rsqrt_var_add_epsilon = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kPower, var_add_epsilon, neg_half)); // X - E[X]. - auto operand_minus_mean = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kSubtract, operand, mean_broadcasted)); + auto operand_minus_mean = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kSubtract, operand, mean_broadcasted)); // (X - E[X]) / Sqrt[Var[X] + epsilon]. - auto normalized = computation_->AddInstruction( + auto normalized = add( HloInstruction::CreateBinary(operand_shape, HloOpcode::kMultiply, operand_minus_mean, rsqrt_var_add_epsilon)); // (X - E[X]) / Sqrt[Var[X] + epsilon] * scale. - auto scaled_normalized = - computation_->AddInstruction(HloInstruction::CreateBinary( - operand_shape, HloOpcode::kMultiply, normalized, scale_broadcasted)); + auto scaled_normalized = add(HloInstruction::CreateBinary( + operand_shape, HloOpcode::kMultiply, normalized, scale_broadcasted)); // (X - E[X]) / Sqrt[Var[X] + epsilon] * scale + offset. auto shifted_normalized = HloInstruction::CreateBinary( operand_shape, HloOpcode::kAdd, scaled_normalized, offset_broadcasted); + int64 instruction_count_after = computation_->instruction_count(); + CHECK_EQ(instruction_count_after, + instruction_count_before + added_instructions.size()); + if (batch_norm->has_sharding()) { + for (HloInstruction* inst : added_instructions) { + if (ShapeUtil::Equal(inst->shape(), operand_shape)) { + inst->set_sharding(batch_norm->sharding()); + } else { + inst->set_sharding(HloSharding::Replicate()); + } + } + shifted_normalized->set_sharding(batch_norm->sharding()); + } TF_CHECK_OK( ReplaceWithNewInstruction(batch_norm, std::move(shifted_normalized))); return Status::OK(); @@ -385,6 +414,13 @@ Status BatchNormRewriterVisitor::HandleBatchNormGrad( if (!rewrite_grad_op_) { return Status::OK(); } + std::vector added_instructions; + auto add = [&](std::unique_ptr inst) { + HloInstruction* added_inst = computation_->AddInstruction(std::move(inst)); + added_instructions.push_back(added_inst); + return added_inst; + }; + int64 instruction_count_before = computation_->instruction_count(); HloInstruction* activation = batch_norm->mutable_operand(0); const Shape activation_shape = activation->shape(); @@ -403,23 +439,22 @@ Status BatchNormRewriterVisitor::HandleBatchNormGrad( Literal::CreateR0(size_in_elements / feature_count); TF_ASSIGN_OR_RETURN(elements_per_feature_literal, elements_per_feature_literal->Convert(ptype)); - auto elements_per_feature = computation_->AddInstruction( + auto elements_per_feature = add( HloInstruction::CreateConstant(std::move(elements_per_feature_literal))); auto zero_literal = Literal::CreateR0(0.0f); TF_ASSIGN_OR_RETURN(zero_literal, zero_literal->Convert(ptype)); - auto zero = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(zero_literal))); + auto zero = add(HloInstruction::CreateConstant(std::move(zero_literal))); auto neg_half_literal = Literal::CreateR0(-0.5f); TF_ASSIGN_OR_RETURN(neg_half_literal, neg_half_literal->Convert(ptype)); - auto neg_half = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(neg_half_literal))); + auto neg_half = + add(HloInstruction::CreateConstant(std::move(neg_half_literal))); auto epsilon_literal = Literal::CreateR0(batch_norm->epsilon()); TF_ASSIGN_OR_RETURN(epsilon_literal, epsilon_literal->Convert(ptype)); - auto epsilon = computation_->AddInstruction( - HloInstruction::CreateConstant(std::move(epsilon_literal))); + auto epsilon = + add(HloInstruction::CreateConstant(std::move(epsilon_literal))); std::vector dimensions_without_feature; @@ -429,126 +464,131 @@ Status BatchNormRewriterVisitor::HandleBatchNormGrad( } } - auto scale_broadcasted = - computation_->AddInstruction(HloInstruction::CreateBroadcast( - activation_shape, scale, {feature_index})); - auto variance_broadcasted = - computation_->AddInstruction(HloInstruction::CreateBroadcast( - activation_shape, variance, {feature_index})); + auto scale_broadcasted = add(HloInstruction::CreateBroadcast( + activation_shape, scale, {feature_index})); + auto variance_broadcasted = add(HloInstruction::CreateBroadcast( + activation_shape, variance, {feature_index})); // E[X]. - auto mean_broadcasted = computation_->AddInstruction( + auto mean_broadcasted = add( HloInstruction::CreateBroadcast(activation_shape, mean, {feature_index})); // rsqrt[Var[X] + epsilon]. - auto rsqrt_var_add_epsilon_broadcasted = - computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kPower, - computation_->AddInstruction( - HloInstruction::CreateBinary(activation_shape, HloOpcode::kAdd, - variance_broadcasted, epsilon)), - neg_half)); - - auto rsqrt_var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - feature_shape, HloOpcode::kPower, - computation_->AddInstruction(HloInstruction::CreateBinary( - feature_shape, HloOpcode::kAdd, variance, epsilon)), - neg_half)); + auto rsqrt_var_add_epsilon_broadcasted = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kPower, + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kAdd, + variance_broadcasted, epsilon)), + neg_half)); + + auto rsqrt_var_add_epsilon = add(HloInstruction::CreateBinary( + feature_shape, HloOpcode::kPower, + add(HloInstruction::CreateBinary(feature_shape, HloOpcode::kAdd, variance, + epsilon)), + neg_half)); // X - E[X]. - auto activation_minus_mean = computation_->AddInstruction( - HloInstruction::CreateBinary(activation_shape, HloOpcode::kSubtract, - activation, mean_broadcasted)); + auto activation_minus_mean = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kSubtract, activation, mean_broadcasted)); // Grad[Y] * (X - E[X]). - auto grad_output_times_activiation_minus_mean = computation_->AddInstruction( - HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, - grad_output, activation_minus_mean)); + auto grad_output_times_activiation_minus_mean = + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, + grad_output, activation_minus_mean)); HloComputation* add_reduce_computation = GetScalarBinaryComputation(ptype, HloOpcode::kAdd); // sum(Grad[Y] * (X - E[X])). auto sum_grad_output_times_activiation_minus_mean = - computation_->AddInstruction(HloInstruction::CreateReduce( + add(HloInstruction::CreateReduce( feature_shape, grad_output_times_activiation_minus_mean, zero, dimensions_without_feature, add_reduce_computation)); // Grad[beta] = Sum(Grad[Y]). - auto grad_beta = computation_->AddInstruction(HloInstruction::CreateReduce( + auto grad_beta = add(HloInstruction::CreateReduce( feature_shape, grad_output, zero, dimensions_without_feature, add_reduce_computation)); - if (use_fusion_) { - auto tuple = computation_->AddInstruction(HloInstruction::CreateTuple( + if (use_fusion_ && !batch_norm->has_sharding()) { + auto tuple = add(HloInstruction::CreateTuple( {sum_grad_output_times_activiation_minus_mean, grad_beta})); auto fused = computation_->CreateFusionInstruction( {tuple, sum_grad_output_times_activiation_minus_mean, grad_beta}, HloInstruction::FusionKind::kInput); - sum_grad_output_times_activiation_minus_mean = computation_->AddInstruction( - HloInstruction::CreateGetTupleElement(feature_shape, fused, 0)); + sum_grad_output_times_activiation_minus_mean = + add(HloInstruction::CreateGetTupleElement(feature_shape, fused, 0)); - grad_beta = computation_->AddInstruction( - HloInstruction::CreateGetTupleElement(feature_shape, fused, 1)); + grad_beta = + add(HloInstruction::CreateGetTupleElement(feature_shape, fused, 1)); } // Grad[scale] = Sum(Grad[Y] * (X - E[X]) * rsqrt[Var[X] + epsilon]). - auto grad_scale = computation_->AddInstruction(HloInstruction::CreateBinary( + auto grad_scale = add(HloInstruction::CreateBinary( feature_shape, HloOpcode::kMultiply, sum_grad_output_times_activiation_minus_mean, rsqrt_var_add_epsilon)); // I2 = Sum(Grad[Y]) - auto I2 = computation_->AddInstruction(HloInstruction::CreateBroadcast( - activation_shape, grad_beta, {feature_index})); + auto i2 = add(HloInstruction::CreateBroadcast(activation_shape, grad_beta, + {feature_index})); // I3 = Sum(Grad[Y] * (X - E[X])) - auto I3 = computation_->AddInstruction(HloInstruction::CreateBroadcast( + auto i3 = add(HloInstruction::CreateBroadcast( activation_shape, sum_grad_output_times_activiation_minus_mean, {feature_index})); // I4 = (X - E[X]) * I3 - auto I4 = computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kMultiply, I3, activation_minus_mean)); + auto i4 = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kMultiply, i3, activation_minus_mean)); // I5 = I4 / (Var[X] + epsilon) - auto I5 = computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kDivide, I4, - computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kAdd, variance_broadcasted, epsilon)))); + auto i5 = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kDivide, i4, + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kAdd, + variance_broadcasted, epsilon)))); // scale * rsqrt[Var[X] + epsilon] * 1/N - auto scale_times_rsqrt_var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kMultiply, scale_broadcasted, - rsqrt_var_add_epsilon_broadcasted)); + auto scale_times_rsqrt_var_add_epsilon = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kMultiply, scale_broadcasted, + rsqrt_var_add_epsilon_broadcasted)); - scale_times_rsqrt_var_add_epsilon = - computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kDivide, - scale_times_rsqrt_var_add_epsilon, elements_per_feature)); + scale_times_rsqrt_var_add_epsilon = add(HloInstruction::CreateBinary( + activation_shape, HloOpcode::kDivide, scale_times_rsqrt_var_add_epsilon, + elements_per_feature)); - auto I1 = computation_->AddInstruction( - HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, - grad_output, elements_per_feature)); + auto i1 = + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, + grad_output, elements_per_feature)); // I6 = I1 - I2 - I5 - auto I6 = computation_->AddInstruction(HloInstruction::CreateBinary( + auto i6 = add(HloInstruction::CreateBinary( activation_shape, HloOpcode::kSubtract, - computation_->AddInstruction(HloInstruction::CreateBinary( - activation_shape, HloOpcode::kSubtract, I1, I2)), - I5)); + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kSubtract, + i1, i2)), + i5)); // Grad[X] = scale * rsqrt[Var[X] + epsilon] * 1/N * I6. - auto grad_activation = computation_->AddInstruction( - HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, - scale_times_rsqrt_var_add_epsilon, I6)); + auto grad_activation = + add(HloInstruction::CreateBinary(activation_shape, HloOpcode::kMultiply, + scale_times_rsqrt_var_add_epsilon, i6)); + auto tuple = + HloInstruction::CreateTuple({grad_activation, grad_scale, grad_beta}); + if (batch_norm->has_sharding()) { + int64 instruction_count_after = computation_->instruction_count(); + CHECK_EQ(instruction_count_after, + instruction_count_before + added_instructions.size()); + for (HloInstruction* inst : added_instructions) { + if (ShapeUtil::Equal(inst->shape(), activation_shape)) { + inst->set_sharding(batch_norm->sharding()); + } else { + inst->set_sharding(HloSharding::Replicate()); + } + } + tuple->set_sharding(batch_norm->sharding()); + } - TF_CHECK_OK(ReplaceWithNewInstruction( - batch_norm, - HloInstruction::CreateTuple({grad_activation, grad_scale, grad_beta}))); + TF_CHECK_OK(ReplaceWithNewInstruction(batch_norm, std::move(tuple))); return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index d1adec31c2..447c244666 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -246,7 +246,8 @@ Status HloSharding::ValidateNonTuple(const Shape& shape, // The tile rank must be the same as the input rank. if (ShapeUtil::Rank(shape) != ShapeUtil::Rank(tile_shape_)) { return tensorflow::errors::InvalidArgument( - "Tile rank is different to the input rank"); + "Tile rank is different to the input rank. sharding=", ToString(), + ", input_shape=", ShapeUtil::HumanString(shape)); } // The tile shape must not be the same as the input shape without maximal_ -- GitLab From 4b60f92986ef1a3e4456aa26911df449a68251a5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 04:40:56 -0800 Subject: [PATCH 0101/1924] Change "safe pointers" to make the deleters stateless (i.e. a type, not a value). This decreases the size of each safe pointer object and allows more inlining of the destructor. PiperOrigin-RevId: 177292948 --- tensorflow/python/lib/core/safe_ptr.cc | 16 ++++------ tensorflow/python/lib/core/safe_ptr.h | 42 +++++++++++++++++--------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/tensorflow/python/lib/core/safe_ptr.cc b/tensorflow/python/lib/core/safe_ptr.cc index 456ea3348b..ce34b6d004 100644 --- a/tensorflow/python/lib/core/safe_ptr.cc +++ b/tensorflow/python/lib/core/safe_ptr.cc @@ -16,25 +16,21 @@ limitations under the License. #include "tensorflow/python/lib/core/safe_ptr.h" namespace tensorflow { -namespace { -inline void Py_DECREF_wrapper(PyObject* o) { Py_DECREF(o); } - -} // namespace - -Safe_PyObjectPtr make_safe(PyObject* o) { - return Safe_PyObjectPtr(o, Py_DECREF_wrapper); +Safe_PyObjectPtr make_safe(PyObject* object) { + return Safe_PyObjectPtr(object); } Safe_TF_TensorPtr make_safe(TF_Tensor* tensor) { - return Safe_TF_TensorPtr(tensor, TF_DeleteTensor); + return Safe_TF_TensorPtr(tensor); } Safe_TFE_TensorHandlePtr make_safe(TFE_TensorHandle* handle) { - return Safe_TFE_TensorHandlePtr(handle, TFE_DeleteTensorHandle); + return Safe_TFE_TensorHandlePtr(handle); } Safe_TF_StatusPtr make_safe(TF_Status* status) { - return Safe_TF_StatusPtr(status, TF_DeleteStatus); + return Safe_TF_StatusPtr(status); } + } // namespace tensorflow diff --git a/tensorflow/python/lib/core/safe_ptr.h b/tensorflow/python/lib/core/safe_ptr.h index 70cd2fdf6c..80db840aeb 100644 --- a/tensorflow/python/lib/core/safe_ptr.h +++ b/tensorflow/python/lib/core/safe_ptr.h @@ -17,39 +17,51 @@ limitations under the License. #define THIRD_PARTY_TENSORFLOW_PYTHON_LIB_CORE_SAFE_PTR_H_ #include -#include +#include #include "tensorflow/c/c_api.h" #include "tensorflow/c/eager/c_api.h" namespace tensorflow { +namespace detail { + +struct PyDecrefDeleter { + void operator()(PyObject* p) const { Py_DECREF(p); } +}; + +struct TFTensorDeleter { + void operator()(TF_Tensor* p) const { TF_DeleteTensor(p); } +}; + +struct TFETensorHandleDeleter { + void operator()(TFE_TensorHandle* p) const { TFE_DeleteTensorHandle(p); } +}; + +struct TFStatusDeleter { + void operator()(TF_Status* p) const { TF_DeleteStatus(p); } +}; + +} // namespace detail // Safe container for an owned PyObject. On destruction, the reference count of // the contained object will be decremented. -typedef void (*Py_DECREF_wrapper_type)(PyObject*); -typedef std::unique_ptr Safe_PyObjectPtr; +using Safe_PyObjectPtr = std::unique_ptr; Safe_PyObjectPtr make_safe(PyObject* o); // Safe containers for an owned TF_Tensor. On destruction, the tensor will be // deleted by TF_DeleteTensor. -// Note: can't use decltype(&TF_DeleteTensor) due to SWIG -typedef void (*TF_DeleteTensor_type)(TF_Tensor*); -typedef std::unique_ptr Safe_TF_TensorPtr; +using Safe_TF_TensorPtr = std::unique_ptr; Safe_TF_TensorPtr make_safe(TF_Tensor* tensor); // Safe containers for an owned TFE_TensorHandle. On destruction, the handle -// will be deleted by TFE_DeleteTensorHandle. Note: can't use -// decltype(&TFE_DeleteTensorHandle) due to SWIG -typedef void (*TFE_DeleteTensorHandle_type)(TFE_TensorHandle*); -typedef std::unique_ptr - Safe_TFE_TensorHandlePtr; +// will be deleted by TFE_DeleteTensorHandle. +using Safe_TFE_TensorHandlePtr = + std::unique_ptr; Safe_TFE_TensorHandlePtr make_safe(TFE_TensorHandle* handle); // Safe containers for an owned TF_Status. On destruction, the handle -// will be deleted by TF_DeleteStatus. Note: can't use -// decltype(&TF_DeleteStatus) due to SWIG -typedef void (*TF_DeleteStatus_type)(TF_Status*); -typedef std::unique_ptr Safe_TF_StatusPtr; +// will be deleted by TF_DeleteStatus. +using Safe_TF_StatusPtr = std::unique_ptr; Safe_TF_StatusPtr make_safe(TF_Status* status); } // namespace tensorflow -- GitLab From e6d823dd19a5768d0dcd651c14a6ebf4bb023180 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 07:32:31 -0800 Subject: [PATCH 0102/1924] Make NCCL code ready for NVIDIA's NCCL 2. PiperOrigin-RevId: 177306507 --- .../contrib/nccl/kernels/nccl_manager.cc | 32 +++++++++++++++++-- .../contrib/nccl/kernels/nccl_manager_test.cc | 2 ++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/nccl/kernels/nccl_manager.cc b/tensorflow/contrib/nccl/kernels/nccl_manager.cc index 31a35b0d53..913935b382 100644 --- a/tensorflow/contrib/nccl/kernels/nccl_manager.cc +++ b/tensorflow/contrib/nccl/kernels/nccl_manager.cc @@ -258,9 +258,37 @@ NcclManager::Communicator* NcclManager::GetCommunicator( devices[i] = collective->participants[i]->gpu_device_id; } + int device_count = num_devices; +#if NCCL_MAJOR >= 2 + // NCCL2 prevents InitAll for more communicators than devices (but doesn't + // check that device ids are unique). Work around it by initializing each + // rank individually. + cudaGetDeviceCount(&device_count); +#endif std::vector nccl_comms(num_devices); - auto result = ncclCommInitAll(nccl_comms.data(), num_devices, devices.data()); - CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result); + if (num_devices <= device_count) { + auto result = + ncclCommInitAll(nccl_comms.data(), num_devices, devices.data()); + CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result); + } else { + int savedDevice = 0; + CHECK_EQ(cudaGetDevice(&savedDevice), cudaSuccess); + ncclUniqueId commId; + ncclGetUniqueId(&commId); +#if NCCL_MAJOR >= 2 + CHECK_EQ(ncclGroupStart(), ncclSuccess); +#endif + for (int rank = 0; rank < num_devices; ++rank) { + cudaSetDevice(devices[rank]); + auto result = + ncclCommInitRank(nccl_comms.data() + rank, num_devices, commId, rank); + CHECK_EQ(result, ncclSuccess) << ncclGetErrorString(result); + } +#if NCCL_MAJOR >= 2 + CHECK_EQ(ncclGroupEnd(), ncclSuccess); +#endif + cudaSetDevice(savedDevice); + } for (int rank = 0; rank < num_devices; ++rank) { members[rank].nccl_comm = nccl_comms[rank]; } diff --git a/tensorflow/contrib/nccl/kernels/nccl_manager_test.cc b/tensorflow/contrib/nccl/kernels/nccl_manager_test.cc index 505c4b0d71..abafe4b407 100644 --- a/tensorflow/contrib/nccl/kernels/nccl_manager_test.cc +++ b/tensorflow/contrib/nccl/kernels/nccl_manager_test.cc @@ -30,6 +30,8 @@ namespace tensorflow { static std::vector GetGPUDevices() { std::vector devices; SessionOptions session_options; + session_options.config.mutable_gpu_options() + ->set_per_process_gpu_memory_fraction(0.1); session_options.env = Env::Default(); Status s = DeviceFactory::GetFactory(DEVICE_GPU) ->AddDevices(session_options, "", &devices); -- GitLab From 18a36a823141c675658d218fa78ed5e2bf19ea8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 08:05:58 -0800 Subject: [PATCH 0103/1924] [XLA:CPU] Factor IR function building logic out of IrEmitter into its own file (no functional changes, just code movement). This will enable building parallel IR functions from other emitters, and remove the requirement that parallel IR functions are associated with a sub-computation. PiperOrigin-RevId: 177309875 --- tensorflow/compiler/xla/service/cpu/BUILD | 17 ++ .../compiler/xla/service/cpu/ir_emitter.cc | 180 +++------------- .../compiler/xla/service/cpu/ir_emitter.h | 20 +- .../compiler/xla/service/cpu/ir_function.cc | 195 ++++++++++++++++++ .../compiler/xla/service/cpu/ir_function.h | 109 ++++++++++ 5 files changed, 352 insertions(+), 169 deletions(-) create mode 100644 tensorflow/compiler/xla/service/cpu/ir_function.cc create mode 100644 tensorflow/compiler/xla/service/cpu/ir_function.h diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index e1eed498f6..e64b313ffc 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -250,6 +250,7 @@ cc_library( ":dot_op_emitter", ":external_constant_pool", ":ir_emission_utils", + ":ir_function", ":shape_partition", ":simple_orc_jit", "//tensorflow/compiler/xla:shape_util", @@ -280,6 +281,22 @@ cc_library( ], ) +cc_library( + name = "ir_function", + srcs = ["ir_function.cc"], + hdrs = ["ir_function.h"], + deps = [ + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla/service/llvm_ir:ir_array", + "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop", + "//tensorflow/compiler/xla/service/llvm_ir:llvm_util", + "//tensorflow/compiler/xla/service/llvm_ir:vector_support_library", + "@llvm//:core", + ], +) + cc_library( name = "dot_op_emitter", srcs = ["dot_op_emitter.cc"], diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 502dd2e738..f087329c6d 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -42,6 +42,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h" #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h" +#include "tensorflow/compiler/xla/service/cpu/ir_function.h" #include "tensorflow/compiler/xla/service/cpu/shape_partition.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h" @@ -124,131 +125,27 @@ StatusOr IrEmitter::EmitComputation( } else { TF_RETURN_IF_ERROR(computation->AcceptOrdered(this, *instruction_order)); } - InsertOrDie(&emitted_functions_, computation, compute_function_); - - return compute_function_; -} - -static llvm::Argument* GetArg(llvm::Function* f, int idx) { - llvm::Function::arg_iterator arg_iter = f->arg_begin(); - std::advance(arg_iter, idx); - return &*arg_iter; + llvm::Function* ir_function = compute_function_->function(); + InsertOrDie(&emitted_functions_, computation, ir_function); + // Delete 'compute_function', finalizing 'ir_function' and restoring caller + // IR insert point. + compute_function_.reset(); + return ir_function; } void IrEmitter::InitializeIrFunction(const string& function_name) { - // The function signature is: - // void function(i8* retval, i8* run_options, i8** params, i8** temps, - // i64* dynamic_loop_bounds, i64* prof_counters) - // - // retval: points to the returned value. - // params: address of an array with pointers to parameters. - // temps: address of an array with pointers to temporary buffers. - // - // Therefore, the generated function's signature (FunctionType) is statically - // determined - parameter unpacking is done in code generated into the - // function, rather than by a prologue dictated by the platform ABI. - // - // /--------------\ - // retval ----------> | return value | - // \--------------/ - // - // /-------------------------------\ - // run_options -----> | xla::ExecutableRunOptions | - // \-------------------------------/ - // - // /---------------------------------------------\ - // params --------> | param 0 | param 1 | ..... | param N-1 | - // | addr | addr | | addr | - // \---------------------------------------------/ - // | | | - // | | | - // V V V - // /---------\ /---------\ /-----------\ - // | param 0 | | param 1 | | param N-1 | - // \---------/ \---------/ \-----------/ - // - // /---------------------------------------------\ - // temps ---------> | temp 0 | temp 1 | ..... | temp N-1 | - // | addr | addr | | addr | - // \---------------------------------------------/ - // | | | - // | | | - // V V V - // /---------\ /---------\ /-----------\ - // | temp 0 | | temp 1 | | temp N-1 | - // \---------/ \---------/ \-----------/ - // - // /--------------------------------------------\ - // dynamic loop bounds -> | outer_dim0_start | outer_dim0_limit | .....| - // (elided for aot) \--------------------------------------------/ - // - // /---------------------------------------------\ - // prof counters -> | counter 0 | counter 1 | ..... | counter N-1 | - // (elided for aot) \---------------------------------------------/ - - // Even though the type of params and temps is void** in the host's view, in - // LLVM IR this is represented by i8*, similarly to void*. It's up to the code - // to use GEPs to unravel the indirection layers. - llvm::FunctionType* compute_function_type = llvm::FunctionType::get( - /*Result=*/llvm::Type::getVoidTy(module_->getContext()), - /*Params=*/GetComputeFunctionParams(), - /*isVarArg=*/false); - // Functions with local linkage get an inlining bonus. Because we know // a-priori that embedded functions (non-entry functions) will not have its // name resolved, give it local linkage. llvm::Function::LinkageTypes linkage = is_top_level_computation_ ? llvm::GlobalValue::ExternalLinkage : llvm::GlobalValue::InternalLinkage; - compute_function_ = - llvm::Function::Create(/*Ty=*/compute_function_type, - /*Linkage=*/linkage, - /*Name=*/AsStringRef(function_name), - /*Module=*/module_); - compute_function_->setCallingConv(llvm::CallingConv::C); - - // Set meaningful names for the function's arguments: useful for debugging. - llvm::Function::arg_iterator arg_iter = compute_function_->arg_begin(); - arg_iter->setName("retval"); - (++arg_iter)->setName("run_options"); - (++arg_iter)->setName("params"); - (++arg_iter)->setName("temps"); - if (num_dynamic_loop_bounds_ > 0) { - (++arg_iter)->setName("dynamic_loop_bounds"); - } - (++arg_iter)->setName("prof_counters"); - - // We know a-priori that the function arguments are guaranteed to point to - // disjoint objects. - llvm::Argument* retval = GetResultArgument(); - for (llvm::Argument& argument : compute_function_->args()) { - // However, the return buffer aliases the temporaries and thus cannot be - // marked noalias. - if (&argument == retval) { - continue; - } - compute_function_->addAttribute(argument.getArgNo() + 1, - llvm::Attribute::NoAlias); - } - - // Add the optize attribute to the function if optimizing for size. This - // controls internal behavior of some optimization passes (e.g. loop - // unrolling). - if (options::OptimizeForSizeRequested(hlo_module_config_)) { - compute_function_->addFnAttr(llvm::Attribute::OptimizeForSize); - } - - if (hlo_module_config_.debug_options().xla_enable_fast_math()) { - compute_function_->addFnAttr("unsafe-fp-math", "true"); - compute_function_->addFnAttr("no-infs-fp-math", "true"); - compute_function_->addFnAttr("no-nans-fp-math", "true"); - compute_function_->addFnAttr("no-signed-zeros-fp-math", "true"); - } - - ir_builder_.SetInsertPoint(llvm::BasicBlock::Create( - /*Context=*/module_->getContext(), - /*Name=*/"entry", - /*Parent=*/compute_function_)); + // Create and initialize new IrFunction. + compute_function_.reset( + new IrFunction(function_name, linkage, + options::OptimizeForSizeRequested(hlo_module_config_), + hlo_module_config_.debug_options().xla_enable_fast_math(), + module_, &ir_builder_, num_dynamic_loop_bounds_)); } IrEmitter::~IrEmitter() {} @@ -1452,7 +1349,7 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) { // // Where Param is the actual element type of the underlying buffer (for // example, float for an XLA F32 element type). - llvm::Argument* params = GetArg(compute_function_, 2); + llvm::Argument* params = compute_function_->parameters_arg(); llvm::Value* param_address_offset = llvm_ir::EmitBufferIndexingGEP(params, param_number, &ir_builder_); llvm::LoadInst* param_address_untyped = @@ -1590,7 +1487,7 @@ IrEmitter::ShardedVectorType IrEmitter::CreateShardedVectorType( // Here we assume that the largest register is a vector register. int max_vector_register_size_in_bytes = target_machine_features_.largest_register_size_in_bytes( - compute_function_); + compute_function_->function()); int vector_register_size_in_elements = max_vector_register_size_in_bytes / @@ -2410,7 +2307,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) { // Terminates the current block with a branch to a while header. llvm::BasicBlock* header_bb = llvm::BasicBlock::Create( module_->getContext(), AsStringRef(IrName(xla_while, "header")), - compute_function_); + compute_function_->function()); ir_builder_.CreateBr(header_bb); ir_builder_.SetInsertPoint(header_bb); @@ -2427,7 +2324,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) { // Branches to the body or to the while exit depending on the condition. llvm::BasicBlock* body_bb = llvm::BasicBlock::Create( module_->getContext(), AsStringRef(IrName(xla_while, "body")), - compute_function_); + compute_function_->function()); llvm::BasicBlock* exit_bb = llvm::BasicBlock::Create( module_->getContext(), AsStringRef(IrName(xla_while, "exit"))); ir_builder_.CreateCondBr(while_predicate, body_bb, exit_bb); @@ -2442,7 +2339,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) { ir_builder_.CreateBr(header_bb); // Adds the exit block to the function and sets the insert point there. - compute_function_->getBasicBlockList().push_back(exit_bb); + compute_function_->function()->getBasicBlockList().push_back(exit_bb); ir_builder_.SetInsertPoint(exit_bb); return Status::OK(); @@ -2642,7 +2539,6 @@ Status IrEmitter::FinishVisit(HloInstruction* root) { if (prof_counter) { profiling_state_.RecordCompleteComputation(&ir_builder_, prof_counter); } - ir_builder_.CreateRetVoid(); return Status::OK(); } @@ -2783,43 +2679,16 @@ llvm::Type* IrEmitter::IrShapeType(const Shape& shape) { return llvm_ir::ShapeToIrType(shape, module_); } -std::vector IrEmitter::GetComputeFunctionParams() { - llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext()); - llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); - llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext()); - std::vector compute_function_params( - {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); - if (num_dynamic_loop_bounds_ > 0) { - compute_function_params.push_back(i64_ptr_type); - } - compute_function_params.push_back(i64_ptr_type); - return compute_function_params; -} - -llvm::Argument* IrEmitter::GetResultArgument() { - return GetArg(compute_function_, 0); -} - llvm::Argument* IrEmitter::GetProfileCountersArgument() { - const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4; - return GetArg(compute_function_, arg_index); + return compute_function_->profile_counters_arg(); } llvm::Value* IrEmitter::GetTempBuffersArgument() { - return GetArg(compute_function_, 3); -} - -llvm::Value* IrEmitter::GetDynamicLoopBound(const int64 offset) { - CHECK_GT(num_dynamic_loop_bounds_, 0); - CHECK_LT(offset, num_dynamic_loop_bounds_ * 2); - llvm::Argument* loop_bounds_arg = GetArg(compute_function_, 4); - string name = tensorflow::strings::StrCat("dynamic_loop_bound_", offset); - return ir_builder_.CreateLoad(ir_builder_.CreateGEP( - loop_bounds_arg, ir_builder_.getInt64(offset), AsStringRef(name))); + return compute_function_->temp_buffers_arg(); } llvm::Value* IrEmitter::GetExecutableRunOptionsArgument() { - return GetArg(compute_function_, 1); + return compute_function_->exec_run_options_arg(); } llvm::Value* IrEmitter::EmitTempBufferPointer( @@ -2965,7 +2834,8 @@ Status IrEmitter::EmitParallelForkJoin( HloInstruction* root = computation->root_instruction(); // Build ParallelForkJoin function type. - std::vector compute_function_params = GetComputeFunctionParams(); + std::vector compute_function_params = + compute_function_->GetComputeFunctionParams(); // Number of parallel compute functions. compute_function_params.push_back(ir_builder_.getInt32Ty()); // Array of partitions. There is an array element for each @@ -3066,7 +2936,7 @@ Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) { if (op == op->parent()->root_instruction()) { // For the root node, we write directly to the output buffer of the // function. - llvm::Argument* retval = GetResultArgument(); + llvm::Argument* retval = compute_function_->result_arg(); if (!ShapeUtil::IsNil(target_shape)) { llvm::AttrBuilder attr_builder; attr_builder.addAlignmentAttr(MinimumAlignmentForShape(target_shape)); @@ -3148,7 +3018,7 @@ Status IrEmitter::EmitParallelTargetElementLoop( // Emit code to read dynamic loop bounds from function argument 4. std::vector dynamic_loop_bounds(2 * num_dynamic_loop_bounds_); for (int i = 0; i < 2 * num_dynamic_loop_bounds_; ++i) { - dynamic_loop_bounds[i] = GetDynamicLoopBound(i); + dynamic_loop_bounds[i] = compute_function_->GetDynamicLoopBound(i); } llvm_ir::ForLoopNest loop_nest(loop_name, &ir_builder_); diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 351c95278c..9e5595052f 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include #include #include #include @@ -30,6 +31,7 @@ limitations under the License. #include "llvm/Target/TargetMachine.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/cpu/external_constant_pool.h" +#include "tensorflow/compiler/xla/service/cpu/ir_function.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" @@ -233,13 +235,6 @@ class IrEmitter : public DfsHloVisitorWithDefault { // Convenience function to get the IR type matching the given shape. llvm::Type* IrShapeType(const Shape& shape); - // Returns an array of compute function parameter types. - std::vector GetComputeFunctionParams(); - - // Get the llvm::Value* that represents the "retval" argument of the - // computation function being emitted by this emitter. - llvm::Argument* GetResultArgument(); - // Get the llvm::Value* that represents the "prof_counters" argument of the // computation function being emitted by this emitter. llvm::Argument* GetProfileCountersArgument(); @@ -252,11 +247,6 @@ class IrEmitter : public DfsHloVisitorWithDefault { // computation function being emitted by this emitter. llvm::Value* GetTempBuffersArgument(); - // Emit ir to read and return the ir value for the dynamic loop bound at - // 'offset' from the "dynamic_loop_bounds" argument of the computation - // function being emitted by this emitter. - llvm::Value* GetDynamicLoopBound(const int64 offset); - // Emits code that computes the address of the given temporary buffer to the // function. target_shape is the shape of this temporary buffer. // The returned Value's type is a pointer to element_type. @@ -476,8 +466,10 @@ class IrEmitter : public DfsHloVisitorWithDefault { thread_local_buffers_; // The following fields track the IR emission state. According to LLVM memory - // management rules, their memory is owned by the module. - llvm::Function* compute_function_; + // management rules, their memory is owned by the module (Note that IrFunction + // creates the encapsulated llvm::Function s.t. it is added to the llvm + // module's function list). + std::unique_ptr compute_function_; llvm::IRBuilder<> ir_builder_; // Maps HLOs to their index into the profile counter array. diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.cc b/tensorflow/compiler/xla/service/cpu/ir_function.cc new file mode 100644 index 0000000000..fa88627156 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/ir_function.cc @@ -0,0 +1,195 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/xla/service/cpu/ir_function.h" + +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" +#include "tensorflow/compiler/xla/status_macros.h" + +namespace xla { + +namespace { +using llvm_ir::AsStringRef; +} // namespace + +namespace cpu { + +IrFunction::IrFunction(const string& function_name, + llvm::Function::LinkageTypes linkage, + const bool optimize_for_size_requested, + const bool enable_fast_math, llvm::Module* llvm_module, + llvm::IRBuilder<>* ir_builder, + int64 num_dynamic_loop_bounds) + : ir_builder_(ir_builder), + llvm_module_(llvm_module), + caller_insert_point_guard_(*ir_builder), + num_dynamic_loop_bounds_(num_dynamic_loop_bounds) { + Initialize(function_name, linkage, optimize_for_size_requested, + enable_fast_math); +} + +IrFunction::~IrFunction() { + // Emit function return value. + ir_builder_->CreateRetVoid(); +} + +void IrFunction::Initialize(const string& function_name, + llvm::Function::LinkageTypes linkage, + const bool optimize_for_size_requested, + const bool enable_fast_math) { + // The function signature is: + // void function(i8* retval, i8* run_options, i8** params, i8** temps, + // i64* dynamic_loop_bounds, i64* prof_counters) + // + // retval: points to the returned value. + // params: address of an array with pointers to parameters. + // temps: address of an array with pointers to temporary buffers. + // + // Therefore, the generated function's signature (FunctionType) is statically + // determined - parameter unpacking is done in code generated into the + // function, rather than by a prologue dictated by the platform ABI. + // + // /--------------\ + // retval ----------> | return value | + // \--------------/ + // + // /-------------------------------\ + // run_options -----> | xla::ExecutableRunOptions | + // \-------------------------------/ + // + // /---------------------------------------------\ + // params --------> | param 0 | param 1 | ..... | param N-1 | + // | addr | addr | | addr | + // \---------------------------------------------/ + // | | | + // | | | + // V V V + // /---------\ /---------\ /-----------\ + // | param 0 | | param 1 | | param N-1 | + // \---------/ \---------/ \-----------/ + // + // /---------------------------------------------\ + // temps ---------> | temp 0 | temp 1 | ..... | temp N-1 | + // | addr | addr | | addr | + // \---------------------------------------------/ + // | | | + // | | | + // V V V + // /---------\ /---------\ /-----------\ + // | temp 0 | | temp 1 | | temp N-1 | + // \---------/ \---------/ \-----------/ + // + // /--------------------------------------------\ + // dynamic loop bounds -> | outer_dim0_start | outer_dim0_limit | .....| + // (elided for aot) \--------------------------------------------/ + // + // /---------------------------------------------\ + // prof counters -> | counter 0 | counter 1 | ..... | counter N-1 | + // (elided for aot) \---------------------------------------------/ + + // Even though the type of params and temps is void** in the host's view, in + // LLVM IR this is represented by i8*, similarly to void*. It's up to the code + // to use GEPs to unravel the indirection layers. + llvm::FunctionType* function_type = llvm::FunctionType::get( + /*Result=*/llvm::Type::getVoidTy(llvm_module_->getContext()), + /*Params=*/GetComputeFunctionParams(), + /*isVarArg=*/false); + + // Functions with local linkage get an inlining bonus. Because we know + // a-priori that embedded functions (non-entry functions) will not have its + // name resolved, give it local linkage. + function_ = llvm::Function::Create(/*Ty=*/function_type, + /*Linkage=*/linkage, + /*N=*/AsStringRef(function_name), + /*M=*/llvm_module_); + function_->setCallingConv(llvm::CallingConv::C); + + // Set meaningful names for the function's arguments: useful for debugging. + llvm::Function::arg_iterator arg_iter = function_->arg_begin(); + arg_iter->setName("retval"); + result_arg_ = &*arg_iter; + (++arg_iter)->setName("run_options"); + exec_run_options_arg_ = &*arg_iter; + (++arg_iter)->setName("params"); + parameters_arg_ = &*arg_iter; + (++arg_iter)->setName("temps"); + temp_buffers_arg_ = &*arg_iter; + if (num_dynamic_loop_bounds_ > 0) { + (++arg_iter)->setName("dynamic_loop_bounds"); + dynamic_loop_bounds_arg_ = &*arg_iter; + } + (++arg_iter)->setName("prof_counters"); + profile_counters_arg_ = &*arg_iter; + + // We know a-priori that the function arguments are guaranteed to point to + // disjoint objects. + llvm::Argument* retval = result_arg(); + for (llvm::Argument& argument : function_->args()) { + // However, the return buffer aliases the temporaries and thus cannot be + // marked noalias. + if (&argument == retval) { + continue; + } + function_->addAttribute(argument.getArgNo() + 1, llvm::Attribute::NoAlias); + } + + // Add the optize attribute to the function if optimizing for size. This + // controls internal behavior of some optimization passes (e.g. loop + // unrolling). + if (optimize_for_size_requested) { + function_->addFnAttr(llvm::Attribute::OptimizeForSize); + } + + if (enable_fast_math) { + function_->addFnAttr("unsafe-fp-math", "true"); + function_->addFnAttr("no-infs-fp-math", "true"); + function_->addFnAttr("no-nans-fp-math", "true"); + function_->addFnAttr("no-signed-zeros-fp-math", "true"); + } + + ir_builder_->SetInsertPoint(llvm::BasicBlock::Create( + /*Context=*/llvm_module_->getContext(), + /*Name=*/"entry", + /*Parent=*/function_)); +} + +std::vector IrFunction::GetComputeFunctionParams() { + llvm::Type* i8_ptr_type = + llvm::Type::getInt8PtrTy(llvm_module_->getContext()); + llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); + llvm::Type* i64_ptr_type = + llvm::Type::getInt64PtrTy(llvm_module_->getContext()); + std::vector compute_function_params( + {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); + if (num_dynamic_loop_bounds_ > 0) { + compute_function_params.push_back(i64_ptr_type); + } + compute_function_params.push_back(i64_ptr_type); + return compute_function_params; +} + +llvm::Value* IrFunction::GetDynamicLoopBound(const int64 offset) { + CHECK_GT(num_dynamic_loop_bounds_, 0); + CHECK_LT(offset, num_dynamic_loop_bounds_ * 2); + string name = tensorflow::strings::StrCat("dynamic_loop_bound_", offset); + return ir_builder_->CreateLoad( + ir_builder_->CreateGEP(CHECK_NOTNULL(dynamic_loop_bounds_arg_), + ir_builder_->getInt64(offset), AsStringRef(name))); +} + +} // namespace cpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.h b/tensorflow/compiler/xla/service/cpu/ir_function.h new file mode 100644 index 0000000000..b7516b403e --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/ir_function.h @@ -0,0 +1,109 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_IR_FUNCTION_H_ +#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_IR_FUNCTION_H_ + +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Value.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" + +namespace xla { +namespace cpu { + +// IrFunction creates and encapsulates an llvm::Function, exposing methods to +// emitters for function and function argument access. +// The llvm::Function is created with the standard function signature +// used in the XLA CPU backend (see ir_function.cc for argument details). +// In addtion IrFunction saves the callers IR insert point during contruction, +// and restores it after desctruction. +// +// Example usage: +// +// // Create and initialize new IrFunction. +// std::unique_ptr compute_function(new IrFunction(...)); +// // Emit IR for function body using IrFunction helper methods. +// ... +// // Store reference to llvm::Function for future invocation. +// ir_functions.push_back(compute_function.function()); +// // Delete IrFunction (finalizes IR function and restores caller insertion +// // point). +// compute_function.reset(); +// + +class IrFunction { + public: + IrFunction(const string& function_name, llvm::Function::LinkageTypes linkage, + const bool optimize_for_size_requested, + const bool enable_fast_math, llvm::Module* llvm_module, + llvm::IRBuilder<>* ir_builder, int64 num_dynamic_loop_bounds); + ~IrFunction(); + + // Returns an array of compute function parameter types. + std::vector GetComputeFunctionParams(); + + // Emit ir to read and return the ir value for the dynamic loop bound at + // 'offset' from the "dynamic_loop_bounds" argument of this function. + llvm::Value* GetDynamicLoopBound(int64 offset); + + // Returns the encapculated llvm::Function. + llvm::Function* function() { return function_; } + + // Get the llvm::Value* that represents this functions "retval" argument. + llvm::Argument* result_arg() { return result_arg_; } + + // Get the xla::ExecutableRunOptions that represents this functions + // "run_options" argument. + llvm::Value* exec_run_options_arg() { return exec_run_options_arg_; } + + // Get the llvm::Argument that represents this functions parameters argument. + llvm::Argument* parameters_arg() { return parameters_arg_; } + + // Get the llvm::Value* that represents this functions "temps" argument. + llvm::Value* temp_buffers_arg() { return temp_buffers_arg_; } + + // Get the llvm::Value* that represents this functions "prof_counters" + // argument. + llvm::Argument* profile_counters_arg() { return profile_counters_arg_; } + + private: + // Initialize an llvm::Function with standard signature based on arguments. + void Initialize(const string& function_name, + llvm::Function::LinkageTypes linkage, + bool optimize_for_size_requested, bool enable_fast_math); + + llvm::IRBuilder<>* ir_builder_; + llvm::Module* llvm_module_; + llvm::IRBuilder<>::InsertPointGuard caller_insert_point_guard_; + + int64 num_dynamic_loop_bounds_ = 0; + // Encapsulated llvm::Function. + llvm::Function* function_; + // Function argument IR values. + llvm::Argument* result_arg_; + llvm::Value* exec_run_options_arg_; + llvm::Argument* parameters_arg_; + llvm::Value* temp_buffers_arg_; + llvm::Argument* dynamic_loop_bounds_arg_ = nullptr; + llvm::Argument* profile_counters_arg_; +}; + +} // namespace cpu +} // namespace xla + +#endif // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_IR_FUNCTION_H_ -- GitLab From e7e1cab9fe66f00716ffaae8a180c5c08a2a050e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 09:13:54 -0800 Subject: [PATCH 0104/1924] [XLA:CPU] Factor out parallel loop emission into its own file so it can be called by other emitters (no functional change, just code movement). PiperOrigin-RevId: 177317764 --- tensorflow/compiler/xla/service/cpu/BUILD | 17 +++++ .../compiler/xla/service/cpu/ir_emitter.cc | 72 ++++-------------- .../compiler/xla/service/cpu/ir_emitter.h | 11 +-- .../xla/service/cpu/parallel_loop_emitter.cc | 76 +++++++++++++++++++ .../xla/service/cpu/parallel_loop_emitter.h | 75 ++++++++++++++++++ 5 files changed, 184 insertions(+), 67 deletions(-) create mode 100644 tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc create mode 100644 tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index e64b313ffc..bf41d5ce07 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -251,6 +251,7 @@ cc_library( ":external_constant_pool", ":ir_emission_utils", ":ir_function", + ":parallel_loop_emitter", ":shape_partition", ":simple_orc_jit", "//tensorflow/compiler/xla:shape_util", @@ -297,6 +298,22 @@ cc_library( ], ) +cc_library( + name = "parallel_loop_emitter", + srcs = ["parallel_loop_emitter.cc"], + hdrs = ["parallel_loop_emitter.h"], + deps = [ + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service/llvm_ir:ir_array", + "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop", + "//tensorflow/compiler/xla/service/llvm_ir:llvm_util", + "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter", + "//tensorflow/core:lib", + "@llvm//:core", + ], +) + cc_library( name = "dot_op_emitter", srcs = ["dot_op_emitter.cc"], diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index f087329c6d..3f991c03e9 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -43,6 +43,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h" #include "tensorflow/compiler/xla/service/cpu/ir_function.h" +#include "tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h" #include "tensorflow/compiler/xla/service/cpu/shape_partition.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h" @@ -1892,7 +1893,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice) { VLOG(2) << "HandleSlice: " << slice->ToString(); auto operand = slice->operand(0); // The code below emits a sequential loop nest. For the parallel backend, use - // EmitParallelTargetElementLoop() which respects dynamic loop bounds. + // ParallelLoopEmitter which respects dynamic loop bounds. if (ShouldEmitParallelLoopFor(*slice)) { return DefaultAction(slice); } @@ -2997,8 +2998,19 @@ Status IrEmitter::EmitTargetElementLoop( } else { if (ShouldEmitParallelLoopFor(*target_op)) { - TF_RETURN_IF_ERROR(EmitParallelTargetElementLoop( - target_shape, element_generator, IrName(target_op), &target_array)); + // Emit code to read dynamic loop bounds from compute function argument. + ParallelLoopEmitter::LoopBounds dynamic_loop_bounds( + num_dynamic_loop_bounds_); + for (int i = 0; i < num_dynamic_loop_bounds_; ++i) { + dynamic_loop_bounds[i].first = + compute_function_->GetDynamicLoopBound(i * 2 + 0); + dynamic_loop_bounds[i].second = + compute_function_->GetDynamicLoopBound(i * 2 + 1); + } + // Emit parallel loop with dynamic loop bounds for most-major dimensions. + TF_RETURN_IF_ERROR(ParallelLoopEmitter(element_generator, target_array, + &dynamic_loop_bounds, &ir_builder_) + .EmitLoop(IrName(target_op))); } else { TF_RETURN_IF_ERROR( llvm_ir::LoopEmitter(element_generator, target_array, &ir_builder_) @@ -3008,60 +3020,6 @@ Status IrEmitter::EmitTargetElementLoop( return Status::OK(); } -Status IrEmitter::EmitParallelTargetElementLoop( - const Shape& target_shape, - const llvm_ir::ElementGenerator& element_generator, - tensorflow::StringPiece loop_name, llvm_ir::IrArray* target_array) { - CHECK(!ShapeUtil::IsTuple(target_shape)); - CHECK(!ShapeUtil::IsScalar(target_shape)); - - // Emit code to read dynamic loop bounds from function argument 4. - std::vector dynamic_loop_bounds(2 * num_dynamic_loop_bounds_); - for (int i = 0; i < 2 * num_dynamic_loop_bounds_; ++i) { - dynamic_loop_bounds[i] = compute_function_->GetDynamicLoopBound(i); - } - - llvm_ir::ForLoopNest loop_nest(loop_name, &ir_builder_); - const int64 num_dims = target_shape.dimensions_size(); - llvm_ir::IrArray::Index array_index(num_dims); - - // Add loops from outer-most to inner-most dimensions. - for (int i = target_shape.layout().minor_to_major_size() - 1; i >= 0; --i) { - const int64 dimension = target_shape.layout().minor_to_major(i); - const int bounds_index = num_dims - 1 - i; - if (bounds_index < num_dynamic_loop_bounds_) { - // Emit dynamic loop bounds for this dimension. Dynamic loop bounds - // are read from ir function dynamic loop bounds argument. - llvm::Value* start_index = dynamic_loop_bounds[bounds_index * 2 + 0]; - llvm::Value* end_index = dynamic_loop_bounds[bounds_index * 2 + 1]; - - std::unique_ptr loop = loop_nest.AddLoop( - /*suffix=*/tensorflow::strings::Printf("dim.%lld", dimension), - start_index, end_index); - array_index[dimension] = loop->GetIndVarValue(); - } else { - // Emit static loop bounds for this dimension. - std::unique_ptr loop = loop_nest.AddLoop( - /*start_index=*/0, - /*end_index=*/target_shape.dimensions(dimension), - /*suffix=*/tensorflow::strings::Printf("dim.%lld", dimension)); - array_index[dimension] = loop->GetIndVarValue(); - } - } - // Point IR builder at inner loop BB. - SetToFirstInsertPoint(loop_nest.GetInnerLoopBodyBasicBlock(), &ir_builder_); - - // Emit loop body. - TF_ASSIGN_OR_RETURN(llvm::Value * target_element, - element_generator(array_index)); - target_array->EmitWriteArrayElement(array_index, target_element, - &ir_builder_); - // Point IR builder at outer loop exit BB. - SetToFirstInsertPoint(loop_nest.GetOuterLoopExitBasicBlock(), &ir_builder_); - - return Status::OK(); -} - Status IrEmitter::EmitMemcpy(const HloInstruction& source, const HloInstruction& destination) { llvm::Value* source_value = GetEmittedValueFor(&source); diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 9e5595052f..6b576d16bb 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -336,15 +336,6 @@ class IrEmitter : public DfsHloVisitorWithDefault { HloInstruction* target_op, tensorflow::StringPiece desc, const llvm_ir::ElementGenerator& element_generator); - // Emit IR to perform a computation for every element in a partition/slice of - // 'target_shape'. The loop bounds for the outer-dimension partitions are - // passed into the compute function as a runtime argument (accessible from - // GetDynamicLoopBound). - Status EmitParallelTargetElementLoop( - const Shape& target_shape, - const llvm_ir::ElementGenerator& element_generator, - tensorflow::StringPiece loop_name, llvm_ir::IrArray* target_array); - // Emits a memcpy from the source instruction's result value to the // destination's. Both source and destination must have an entry in the // emitted_value_ table. @@ -482,7 +473,7 @@ class IrEmitter : public DfsHloVisitorWithDefault { llvm_ir::AliasAnalysis alias_analysis_; // The number of root instruction outer dimensions used in parallel loop - // emission (EmitParallelTargetElementLoop). + // emission (ParallelLoopEmitter). int64 num_dynamic_loop_bounds_ = 0; // Returns whether the given instruction should be emitted as a parallel loop. diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc new file mode 100644 index 0000000000..91e704e3d0 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.cc @@ -0,0 +1,76 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h" + +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h" +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" +#include "tensorflow/core/lib/strings/stringprintf.h" + +namespace xla { +namespace cpu { + +ParallelLoopEmitter::ParallelLoopEmitter( + const llvm_ir::ElementGenerator& target_element_generator, + const llvm_ir::IrArray& target_array, const LoopBounds* dynamic_loop_bounds, + llvm::IRBuilder<>* ir_builder) + : LoopEmitter(target_element_generator, target_array, ir_builder), + dynamic_loop_bounds_(dynamic_loop_bounds) {} + +llvm_ir::IrArray::Index ParallelLoopEmitter::EmitIndexAndSetExitBasicBlock( + tensorflow::StringPiece loop_name) { + CHECK(!ShapeUtil::IsTuple(shape_)); + CHECK(!ShapeUtil::IsScalar(shape_)); + + llvm_ir::ForLoopNest loop_nest(loop_name, ir_builder_); + const int64 num_dims = shape_.dimensions_size(); + llvm_ir::IrArray::Index array_index(num_dims); + + // Add loops from outer-most to inner-most dimensions. + for (int i = shape_.layout().minor_to_major_size() - 1; i >= 0; --i) { + const int64 dimension = shape_.layout().minor_to_major(i); + const int bounds_index = num_dims - 1 - i; + if (bounds_index < dynamic_loop_bounds_->size()) { + // Emit dynamic loop bounds for this dimension. Dynamic loop bounds + // are read from ir function dynamic loop bounds argument. + llvm::Value* start_index = (*dynamic_loop_bounds_)[bounds_index].first; + llvm::Value* end_index = (*dynamic_loop_bounds_)[bounds_index].second; + + std::unique_ptr loop = loop_nest.AddLoop( + /*suffix=*/tensorflow::strings::Printf("dim.%lld", dimension), + start_index, end_index); + array_index[dimension] = loop->GetIndVarValue(); + } else { + // Emit static loop bounds for this dimension. + std::unique_ptr loop = loop_nest.AddLoop( + /*start_index=*/0, + /*end_index=*/shape_.dimensions(dimension), + /*suffix=*/tensorflow::strings::Printf("dim.%lld", dimension)); + array_index[dimension] = loop->GetIndVarValue(); + } + } + // Point IR builder at inner loop BB. + llvm_ir::SetToFirstInsertPoint(loop_nest.GetInnerLoopBodyBasicBlock(), + ir_builder_); + + // Set exit_bb_ to the exit block of the loop nest. + exit_bb_ = loop_nest.GetOuterLoopExitBasicBlock(); + CHECK(exit_bb_ != nullptr); + + return array_index; +} + +} // namespace cpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h new file mode 100644 index 0000000000..492d5953c4 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/parallel_loop_emitter.h @@ -0,0 +1,75 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_PARALLEL_LOOP_EMITTER_H_ +#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_PARALLEL_LOOP_EMITTER_H_ + +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Value.h" +#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h" +#include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h" + +namespace xla { +namespace cpu { + +// ParallelLoopEmitter emits a loop nest for the target array shape. +// The outer loop bounds of the loop nest are passed as ir values at runtime +// (specified in 'dynamic_loop_bounds'), and the inner loop bounds are static. +// Dynamic loop bounds are specified as an array of dimension index +// [start, limit) pairs of ir values (one for each partitioned outer dimension). +// +// EX: Let 'shape' = [8, 16, 32], with the loop bounds of the two-most major +// dimensions dynamic. +// Then 'dynamic_loop_bounds' will contain the following ir values for +// the two most-major dimenions: +// [dim0_index_start_ir_value, dim0_index_limit_ir_value] +// [dim1_index_start_ir_value, dim1_index_limit_ir_value] +// +// Code emitted by ParallelLoopEmitter will be called in a multi-threaded +// context where each thread will be assigned a different set of outer dimension +// partitions, and where all threads will collectively iterate over the +// entire target array shape. +// +// Outer dimension partitions can be generated using the ShapePartitionAssigner +// and ShapePartitionIterator utility classes from shape_partition.cc. +// +class ParallelLoopEmitter : public llvm_ir::LoopEmitter { + public: + using LoopBounds = std::vector>; + + // Constructs a ParallelLoopEmitter which uses 'target_element_generator' to + // generate elements, 'dynamic_loop_bounds' to set the loop bounds of the + // most-major dimensions, and 'target_array.' shape to set the static loop + // bounds for the most-minor dimensions. + ParallelLoopEmitter(const llvm_ir::ElementGenerator& target_element_generator, + const llvm_ir::IrArray& target_array, + const LoopBounds* dynamic_loop_bounds, + llvm::IRBuilder<>* ir_builder); + + ParallelLoopEmitter(const ParallelLoopEmitter&) = delete; + ParallelLoopEmitter& operator=(const ParallelLoopEmitter&) = delete; + ~ParallelLoopEmitter() override = default; + + llvm_ir::IrArray::Index EmitIndexAndSetExitBasicBlock( + tensorflow::StringPiece loop_name) override; + + private: + const LoopBounds* dynamic_loop_bounds_; +}; + +} // namespace cpu +} // namespace xla + +#endif // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_PARALLEL_LOOP_EMITTER_H_ -- GitLab From 667282eb0e62bef03bbe527bef88c656532444bb Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 29 Nov 2017 09:30:56 -0800 Subject: [PATCH 0105/1924] [TFXLA] Return nullopt if no merge node found. PiperOrigin-RevId: 177319722 --- tensorflow/compiler/tf2xla/functionalize_control_flow.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc index 5726d8294a..267268298c 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc @@ -1067,6 +1067,10 @@ FunctionalizeCond::CreateCorrespondingMergeCluster(Cluster* switch_cluster) { enqueue_or_update_merge(out); } } + // Return if there are no merge nodes. + if (merges.empty()) { + return gtl::nullopt; + } auto it = merges.begin(); Cluster* merge_cluster = *it; for (++it; it != merges.end(); ++it) { -- GitLab From 537ecc56cf09d5dcb2b328b322d9f8b195abcc6c Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 29 Nov 2017 09:48:08 -0800 Subject: [PATCH 0106/1924] [tf.data] Remove GraphDefBuilder and NodeBuilder dependencies from "dataset.h". This is a step towards making a header-only library on which external op implementations can depend. To do this "dataset.h" cannot depend on any headers in "tensorflow/core/graph/...". PiperOrigin-RevId: 177322011 --- tensorflow/core/kernels/BUILD | 2 +- tensorflow/core/kernels/dataset.cc | 140 ++++++++++++++++ tensorflow/core/kernels/dataset.h | 155 +++--------------- tensorflow/core/kernels/filter_dataset_op.cc | 2 +- .../core/kernels/flat_map_dataset_op.cc | 2 +- .../core/kernels/interleave_dataset_op.cc | 2 +- tensorflow/core/kernels/map_dataset_op.cc | 2 +- .../core/kernels/padded_batch_dataset_op.cc | 4 +- tensorflow/core/kernels/tensor_dataset_op.cc | 2 +- .../core/kernels/tensor_slice_dataset_op.cc | 2 +- tensorflow/core/kernels/zip_dataset_op.cc | 2 +- 11 files changed, 171 insertions(+), 144 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index eff15e809a..fd36e6ca1f 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -5832,11 +5832,11 @@ cc_library( srcs = ["dataset.cc"], hdrs = ["dataset.h"], deps = [ + "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", - "//tensorflow/core/util/tensor_bundle", ], ) diff --git a/tensorflow/core/kernels/dataset.cc b/tensorflow/core/kernels/dataset.cc index fcfa2956f7..0972129787 100644 --- a/tensorflow/core/kernels/dataset.cc +++ b/tensorflow/core/kernels/dataset.cc @@ -15,6 +15,9 @@ limitations under the License. #include "tensorflow/core/kernels/dataset.h" +#include "tensorflow/core/graph/graph_def_builder.h" +#include "tensorflow/core/graph/node_builder.h" + namespace tensorflow { namespace { @@ -70,6 +73,143 @@ class DatasetVariantWrapper { } // namespace +Status GraphDefBuilderWrapper::AddDataset( + const GraphDatasetBase* dataset, + const std::vector>& inputs, + const std::vector>>& list_inputs, + const std::vector>& attrs, + Node** output) { + const string& op_type_name = dataset->op_name(); + std::unique_ptr opts( + new GraphDefBuilder::Options(b_->opts())); + // TODO(srbs|mrry): Not all datasets have output_types and output_shapes + // attributes defined. It will be nice to have a consistent pattern. + bool has_output_types_attr = HasAttr(op_type_name, "output_types"); + bool has_output_shapes_attr = HasAttr(op_type_name, "output_shapes"); + if (has_output_shapes_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_shapes", dataset->output_shapes()))); + } + if (has_output_types_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_types", dataset->output_dtypes()))); + } + for (auto attr : attrs) { + opts.reset( + new GraphDefBuilder::Options(opts->WithAttr(attr.first, attr.second))); + } + if (opts->HaveError()) { + return errors::Internal("AddDataset: Failed to build Options with error ", + opts->StatusToString()); + } + NodeBuilder node_builder(opts->GetNameForOp(op_type_name), op_type_name, + opts->op_registry()); + { + size_t total_size = inputs.size() + list_inputs.size(); + auto inputs_iter = inputs.begin(); + auto list_inputs_iter = list_inputs.begin(); + for (int i = 0; i < total_size; i++) { + if (inputs_iter != inputs.end() && inputs_iter->first == i) { + node_builder.Input(NodeBuilder::NodeOut(inputs_iter->second)); + inputs_iter++; + } else if (list_inputs_iter != list_inputs.end() && + list_inputs_iter->first == i) { + std::vector nodeout_inputs; + nodeout_inputs.reserve(list_inputs_iter->second.size()); + for (Node* n : list_inputs_iter->second) { + nodeout_inputs.emplace_back(n); + } + node_builder.Input(nodeout_inputs); + list_inputs_iter++; + } else { + return errors::InvalidArgument("No input found for index ", i); + } + } + } + *output = opts->FinalizeBuilder(&node_builder); + if (*output == nullptr) { + return errors::Internal("AddDataset: Failed to build ", op_type_name, + " op with error ", opts->StatusToString()); + } + return Status::OK(); +} + +Status GraphDefBuilderWrapper::AddFunction(OpKernelContext* ctx, + const string& function_name) { + if (b_->HasFunction(function_name)) { + LOG(INFO) << "Function with name " << function_name << "already exists in" + << " the graph. It will not be added again."; + return Status::OK(); + } + TF_RETURN_IF_ERROR(EnsureFunctionIsStateless(ctx, function_name)); + const FunctionLibraryDefinition* flib_def = + ctx->function_library()->GetFunctionLibraryDefinition(); + const FunctionDef* f_def = flib_def->Find(function_name); + if (f_def == nullptr) { + return errors::InvalidArgument("Unable to find FunctionDef for ", + function_name, " in the registry."); + } + FunctionDefLibrary def; + *def.add_function() = *f_def; + const string gradient_func = flib_def->FindGradient(function_name); + if (!gradient_func.empty()) { + GradientDef* g_def = def.add_gradient(); + g_def->set_function_name(function_name); + g_def->set_gradient_func(gradient_func); + } + TF_RETURN_IF_ERROR(b_->AddFunctionLibrary(def)); + + // Recursively add functions in inputs of function_name. + for (const NodeDef& node_def : f_def->node_def()) { + const OpRegistrationData* op_reg_data = nullptr; + TF_RETURN_IF_ERROR(flib_def->LookUp(node_def.op(), &op_reg_data)); + if (op_reg_data->is_function_op) { + TF_RETURN_IF_ERROR(AddFunction(ctx, op_reg_data->op_def.name())); + } + // Recursively add functions in attrs of this NodeDef. + for (const auto& pair : node_def.attr()) { + TF_RETURN_IF_ERROR(AddAttrFunctions(pair.second, ctx)); + } + } + + // Recursively add functions in attrs of function_name. + for (auto iter = f_def->attr().begin(); iter != f_def->attr().end(); iter++) { + TF_RETURN_IF_ERROR(AddAttrFunctions(iter->second, ctx)); + } + return Status::OK(); +} + +void GraphDefBuilderWrapper::AddTensorInternal(const Tensor& val, + Node** output) { + *output = ops::SourceOp( + "Const", + b_->opts().WithAttr("dtype", val.dtype()).WithAttr("value", val)); +} + +bool GraphDefBuilderWrapper::HasAttr(const string& op_type_name, + const string& attr_name) const { + const OpDef* op_def = nullptr; + Status s = b_->opts().op_registry()->LookUpOpDef(op_type_name, &op_def); + if (!s.ok() || op_def == nullptr) { + return false; + } + return HasAttr(op_def, attr_name); +} + +Status GraphDatasetBase::Serialize(OpKernelContext* ctx, + string* serialized_graph_def, + string* output_node) const { + GraphDefBuilder b; + DatasetGraphDefBuilder db(&b); + Node* node = nullptr; + TF_RETURN_IF_ERROR(AsGraphDefInternal(ctx, &db, &node)); + *output_node = node->name(); + GraphDef graph_def; + TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + graph_def.SerializeToString(serialized_graph_def); + return Status::OK(); +} + Status GetDatasetFromVariantTensor(const Tensor& tensor, DatasetBase** out_dataset) { if (!(tensor.dtype() == DT_VARIANT || diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h index afbebb0692..504a88a309 100644 --- a/tensorflow/core/kernels/dataset.h +++ b/tensorflow/core/kernels/dataset.h @@ -19,12 +19,13 @@ limitations under the License. #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/attr_value_util.h" +#include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/variant_encode_decode.h" #include "tensorflow/core/framework/variant_tensor_data.h" -#include "tensorflow/core/graph/graph_def_builder.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/tracing.h" @@ -59,6 +60,12 @@ class IteratorStateWriter { virtual ~IteratorStateWriter() {} }; +// Forward declarations to avoid introducing a dependency on headers in +// "tensorflow/core/graph/...". +class GraphDefBuilder; +class GraphDatasetBase; +class Node; + // Wrapper around GraphDefBuilder. Used to serialize Dataset graph. class GraphDefBuilderWrapper { public: @@ -110,10 +117,8 @@ class GraphDefBuilderWrapper { return Status::OK(); } - template - Status AddDataset(const DatasetType* dataset, - const std::vector& inputs, - Node** output) { + Status AddDataset(const GraphDatasetBase* dataset, + const std::vector& inputs, Node** output) { return AddDataset(dataset, inputs, {}, output); } @@ -125,77 +130,23 @@ class GraphDefBuilderWrapper { // `*output` contains a pointer to the output `Node`. It is guaranteed to be // non-null if the method returns with an OK status. // The returned Node pointer is owned by the backing Graph of GraphDefBuilder. - template - Status AddDataset(const DatasetType* dataset, - const std::vector& inputs, + Status AddDataset(const GraphDatasetBase* dataset, + const std::vector& inputs, const std::vector>& attrs, Node** output) { - std::vector> enumerated_inputs( - inputs.size()); + std::vector> enumerated_inputs(inputs.size()); for (int i = 0; i < inputs.size(); i++) { enumerated_inputs[i] = std::make_pair(i, inputs[i]); } return AddDataset(dataset, enumerated_inputs, {}, attrs, output); } - template Status AddDataset( - const DatasetType* dataset, - const std::vector>& inputs, - const std::vector< - std::pair>>& - list_inputs, + const GraphDatasetBase* dataset, + const std::vector>& inputs, + const std::vector>>& list_inputs, const std::vector>& attrs, - Node** output) { - const string& op_type_name = dataset->op_name(); - std::unique_ptr opts( - new GraphDefBuilder::Options(b_->opts())); - // TODO(srbs|mrry): Not all datasets have output_types and output_shapes - // attributes defined. It will be nice to have a consistent pattern. - bool has_output_types_attr = HasAttr(op_type_name, "output_types"); - bool has_output_shapes_attr = HasAttr(op_type_name, "output_shapes"); - if (has_output_shapes_attr) { - opts.reset(new GraphDefBuilder::Options( - opts->WithAttr("output_shapes", dataset->output_shapes()))); - } - if (has_output_types_attr) { - opts.reset(new GraphDefBuilder::Options( - opts->WithAttr("output_types", dataset->output_dtypes()))); - } - for (auto attr : attrs) { - opts.reset(new GraphDefBuilder::Options( - opts->WithAttr(attr.first, attr.second))); - } - if (opts->HaveError()) { - return errors::Internal("AddDataset: Failed to build Options with error ", - opts->StatusToString()); - } - NodeBuilder node_builder(opts->GetNameForOp(op_type_name), op_type_name, - opts->op_registry()); - { - size_t total_size = inputs.size() + list_inputs.size(); - auto inputs_iter = inputs.begin(); - auto list_inputs_iter = list_inputs.begin(); - for (int i = 0; i < total_size; i++) { - if (inputs_iter != inputs.end() && inputs_iter->first == i) { - node_builder.Input(inputs_iter->second); - inputs_iter++; - } else if (list_inputs_iter != list_inputs.end() && - list_inputs_iter->first == i) { - node_builder.Input(list_inputs_iter->second); - list_inputs_iter++; - } else { - return errors::InvalidArgument("No input found for index ", i); - } - } - } - *output = opts->FinalizeBuilder(&node_builder); - if (*output == nullptr) { - return errors::Internal("AddDataset: Failed to build ", op_type_name, - " op with error ", opts->StatusToString()); - } - return Status::OK(); - } + Node** output); // Adds a user-defined function with name `function_name` to the graph and // recursively adds all functions it references. If a function with a matching @@ -203,50 +154,7 @@ class GraphDefBuilderWrapper { // name `function_name` is not found in the FunctionLibraryDefinition, returns // an InvalidArgumentError. If the function with name `function_name` or any // of its dependent functions are stateful, returns an InvalidArgument error. - Status AddFunction(OpKernelContext* ctx, const string& function_name) { - if (b_->HasFunction(function_name)) { - LOG(INFO) << "Function with name " << function_name << "already exists in" - << " the graph. It will not be added again."; - return Status::OK(); - } - TF_RETURN_IF_ERROR(EnsureFunctionIsStateless(ctx, function_name)); - const FunctionLibraryDefinition* flib_def = - ctx->function_library()->GetFunctionLibraryDefinition(); - const FunctionDef* f_def = flib_def->Find(function_name); - if (f_def == nullptr) { - return errors::InvalidArgument("Unable to find FunctionDef for ", - function_name, " in the registry."); - } - FunctionDefLibrary def; - *def.add_function() = *f_def; - const string gradient_func = flib_def->FindGradient(function_name); - if (!gradient_func.empty()) { - GradientDef* g_def = def.add_gradient(); - g_def->set_function_name(function_name); - g_def->set_gradient_func(gradient_func); - } - TF_RETURN_IF_ERROR(b_->AddFunctionLibrary(def)); - - // Recursively add functions in inputs of function_name. - for (const NodeDef& node_def : f_def->node_def()) { - const OpRegistrationData* op_reg_data = nullptr; - TF_RETURN_IF_ERROR(flib_def->LookUp(node_def.op(), &op_reg_data)); - if (op_reg_data->is_function_op) { - TF_RETURN_IF_ERROR(AddFunction(ctx, op_reg_data->op_def.name())); - } - // Recursively add functions in attrs of this NodeDef. - for (const auto& pair : node_def.attr()) { - TF_RETURN_IF_ERROR(AddAttrFunctions(pair.second, ctx)); - } - } - - // Recursively add functions in attrs of function_name. - for (auto iter = f_def->attr().begin(); iter != f_def->attr().end(); - iter++) { - TF_RETURN_IF_ERROR(AddAttrFunctions(iter->second, ctx)); - } - return Status::OK(); - } + Status AddFunction(OpKernelContext* ctx, const string& function_name); template void BuildAttrValue(const T& value, AttrValue* attr) { @@ -254,11 +162,7 @@ class GraphDefBuilderWrapper { } private: - void AddTensorInternal(const Tensor& val, Node** output) { - *output = ops::SourceOp( - "Const", - b_->opts().WithAttr("dtype", val.dtype()).WithAttr("value", val)); - } + void AddTensorInternal(const Tensor& val, Node** output); Status EnsureFunctionIsStateless(OpKernelContext* ctx, const string& function_name) const { @@ -294,14 +198,7 @@ class GraphDefBuilderWrapper { HasAttr(op_def, "output_shapes"); } - bool HasAttr(const string& op_type_name, const string& attr_name) const { - const OpDef* op_def = nullptr; - Status s = b_->opts().op_registry()->LookUpOpDef(op_type_name, &op_def); - if (!s.ok() || op_def == nullptr) { - return false; - } - return HasAttr(op_def, attr_name); - } + bool HasAttr(const string& op_type_name, const string& attr_name) const; bool HasAttr(const OpDef* op_def, const string& attr_name) const { for (auto attr : op_def->attr()) { @@ -548,17 +445,7 @@ class GraphDatasetBase : public DatasetBase { private: Status Serialize(OpKernelContext* ctx, string* serialized_graph_def, - string* output_node) const { - GraphDefBuilder b; - DatasetGraphDefBuilder db(&b); - Node* node = nullptr; - TF_RETURN_IF_ERROR(AsGraphDefInternal(ctx, &db, &node)); - *output_node = node->name(); - GraphDef graph_def; - TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); - graph_def.SerializeToString(serialized_graph_def); - return Status::OK(); - } + string* output_node) const; const string op_name_; }; diff --git a/tensorflow/core/kernels/filter_dataset_op.cc b/tensorflow/core/kernels/filter_dataset_op.cc index e4d80e4ce3..0ac6cd9a98 100644 --- a/tensorflow/core/kernels/filter_dataset_op.cc +++ b/tensorflow/core/kernels/filter_dataset_op.cc @@ -95,7 +95,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel { DataTypeVector other_arguments_types; other_arguments_types.reserve(captured_func_->captured_inputs().size()); - std::vector other_arguments; + std::vector other_arguments; other_arguments.reserve(captured_func_->captured_inputs().size()); for (const Tensor& t : captured_func_->captured_inputs()) { Node* node; diff --git a/tensorflow/core/kernels/flat_map_dataset_op.cc b/tensorflow/core/kernels/flat_map_dataset_op.cc index ac1689e5bf..8fe8489371 100644 --- a/tensorflow/core/kernels/flat_map_dataset_op.cc +++ b/tensorflow/core/kernels/flat_map_dataset_op.cc @@ -102,7 +102,7 @@ class FlatMapDatasetOp : public UnaryDatasetOpKernel { DataTypeVector other_arguments_types; other_arguments_types.reserve(captured_func_->captured_inputs().size()); - std::vector other_arguments; + std::vector other_arguments; other_arguments.reserve(captured_func_->captured_inputs().size()); for (const Tensor& t : captured_func_->captured_inputs()) { Node* node; diff --git a/tensorflow/core/kernels/interleave_dataset_op.cc b/tensorflow/core/kernels/interleave_dataset_op.cc index cbee68b2db..833e8cb9c5 100644 --- a/tensorflow/core/kernels/interleave_dataset_op.cc +++ b/tensorflow/core/kernels/interleave_dataset_op.cc @@ -126,7 +126,7 @@ class InterleaveDatasetOp : public UnaryDatasetOpKernel { TF_RETURN_IF_ERROR(b->AddScalar(block_length_, &block_length_node)); DataTypeVector other_arguments_types; other_arguments_types.reserve(captured_func_->captured_inputs().size()); - std::vector other_arguments; + std::vector other_arguments; other_arguments.reserve(captured_func_->captured_inputs().size()); for (const Tensor& t : captured_func_->captured_inputs()) { Node* node; diff --git a/tensorflow/core/kernels/map_dataset_op.cc b/tensorflow/core/kernels/map_dataset_op.cc index 4ba09bc335..23148f122d 100644 --- a/tensorflow/core/kernels/map_dataset_op.cc +++ b/tensorflow/core/kernels/map_dataset_op.cc @@ -100,7 +100,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel { DataTypeVector other_arguments_types( captured_func_->captured_inputs().size()); - std::vector other_arguments( + std::vector other_arguments( captured_func_->captured_inputs().size()); for (const Tensor& t : captured_func_->captured_inputs()) { Node* node; diff --git a/tensorflow/core/kernels/padded_batch_dataset_op.cc b/tensorflow/core/kernels/padded_batch_dataset_op.cc index 7c28d955e1..cef5bde156 100644 --- a/tensorflow/core/kernels/padded_batch_dataset_op.cc +++ b/tensorflow/core/kernels/padded_batch_dataset_op.cc @@ -242,7 +242,7 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel { Node* batch_size = nullptr; TF_RETURN_IF_ERROR(b->AddScalar(batch_size_, &batch_size)); - std::vector padded_shapes; + std::vector padded_shapes; padded_shapes.reserve(padded_shapes_.size()); for (int i = 0; i < padded_shapes_.size(); i++) { Node* node; @@ -254,7 +254,7 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel { padded_shapes.emplace_back(node); } - std::vector padding_values; + std::vector padding_values; padding_values.reserve(padding_values_.size()); for (const Tensor& t : padding_values_) { Node* node; diff --git a/tensorflow/core/kernels/tensor_dataset_op.cc b/tensorflow/core/kernels/tensor_dataset_op.cc index fe53434d17..5cf9931188 100644 --- a/tensorflow/core/kernels/tensor_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_dataset_op.cc @@ -70,7 +70,7 @@ class TensorDatasetOp : public DatasetOpKernel { protected: Status AsGraphDefInternal(DatasetGraphDefBuilder* b, Node** output) const override { - std::vector components; + std::vector components; components.reserve(tensors_.size()); for (const Tensor& t : tensors_) { Node* node; diff --git a/tensorflow/core/kernels/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/tensor_slice_dataset_op.cc index e85f59b584..19d4816ff3 100644 --- a/tensorflow/core/kernels/tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_slice_dataset_op.cc @@ -86,7 +86,7 @@ class TensorSliceDatasetOp : public DatasetOpKernel { protected: Status AsGraphDefInternal(DatasetGraphDefBuilder* b, Node** output) const override { - std::vector components; + std::vector components; components.reserve(tensors_.size()); for (const Tensor& t : tensors_) { Node* node; diff --git a/tensorflow/core/kernels/zip_dataset_op.cc b/tensorflow/core/kernels/zip_dataset_op.cc index 9381915ae9..31e5737f62 100644 --- a/tensorflow/core/kernels/zip_dataset_op.cc +++ b/tensorflow/core/kernels/zip_dataset_op.cc @@ -80,7 +80,7 @@ class ZipDatasetOp : public DatasetOpKernel { protected: Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b, Node** output) const override { - std::vector input_graph_nodes; + std::vector input_graph_nodes; input_graph_nodes.reserve(inputs_.size()); for (const auto& input : inputs_) { Node* input_node; -- GitLab From fa8bfa89cd2d7b57bb119afcabdf67ce1539081d Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 29 Nov 2017 09:52:17 -0800 Subject: [PATCH 0107/1924] Add feature_columns doc. PiperOrigin-RevId: 177322632 --- .../docs_src/get_started/feature_columns.md | 570 ++++++++++++++++++ 1 file changed, 570 insertions(+) create mode 100644 tensorflow/docs_src/get_started/feature_columns.md diff --git a/tensorflow/docs_src/get_started/feature_columns.md b/tensorflow/docs_src/get_started/feature_columns.md new file mode 100644 index 0000000000..f9537927b7 --- /dev/null +++ b/tensorflow/docs_src/get_started/feature_columns.md @@ -0,0 +1,570 @@ +# Feature Columns + +This document details feature columns. Think of **feature columns** as the +intermediaries between raw data and Estimators. Feature columns are very rich, +enabling you to transform a diverse range of raw data into formats that +Estimators can use, allowing easy experimentation. + +In @{$get_started/estimator$Premade Estimators}, we used the premade Estimator, +@{tf.estimator.DNNClassifier$`DNNClassifier`} to train a model to predict +different types of Iris flowers from four input features. That example created +only numerical feature columns (of type @{tf.feature_column.numeric_column}). +Although numerical feature columns model the lengths of petals and sepals +effectively, real world data sets contain all kinds of features, many of which +are non-numerical. + +
+ +
+
+Some real-world features (such as, longitude) are numerical, but many are not. +
+ +## Input to a Deep Neural Network + +What kind of data can a deep neural network operate on? The answer +is, of course, numbers (for example, `tf.float32`). After all, every neuron in +a neural network performs multiplication and addition operations on weights and +input data. Real-life input data, however, often contains non-numerical +(categorical) data. For example, consider a `product_class` feature that can +contain the following three non-numerical values: + +* `kitchenware` +* `electronics` +* `sports` + +ML models generally represent categorical values as simple vectors in which a +1 represents the presence of a value and a 0 represents the absence of a value. +For example, when `product_class` is set to `sports`, an ML model would usually +represent `product_class` as `[0, 0, 1]`, meaning: + +* `0`: `kitchenware` is absent +* `0`: `electronics` is absent +* `1`: `sports` is present + +So, although raw data can be numerical or categorical, an ML model represents +all features as numbers. + +## Feature Columns + +As the following figure suggests, you specify the input to a model through the +`feature_columns` argument of an Estimator (`DNNClassifier` for Iris). +Feature Columns bridge input data (as returned by `input_fn`) with your model. + +
+ +
+
+Feature columns bridge raw data with the data your model needs. +
+ +To create feature columns, call functions from the +@{tf.feature_column} module. This document explains nine of the functions in +that module. As the following figure shows, all nine functions return either a +Categorical-Column or a Dense-Column object, except `bucketized_column`, which +inherits from both classes: + +
+ +
+
+Feature column methods fall into two main categories and one hybrid category. +
+ +Let's look at these functions in more detail. + +### Numeric column + +The Iris classifier calls the @{tf.feature_column.numeric_column} function for +all input features: + + * `SepalLength` + * `SepalWidth` + * `PetalLength` + * `PetalWidth` + +Although `tf.numeric_column` provides optional arguments, calling +`tf.numeric_column` without any arguments, as follows, is a fine way to specify +a numerical value with the default data type (`tf.float32`) as input to your +model: + +```python +# Defaults to a tf.float32 scalar. +numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength") +``` + +To specify a non-default numerical data type, use the `dtype` argument. For +example: + +``` python +# Represent a tf.float64 scalar. +numeric_feature_column = tf.feature_column.numeric_column(key="SepalLength", + dtype=tf.float64) +``` + +By default, a numeric column creates a single value (scalar). Use the shape +argument to specify another shape. For example: + + +```python +# Represent a 10-element vector in which each cell contains a tf.float32. +vector_feature_column = tf.feature_column.numeric_column(key="Bowling", + shape=10) + +# Represent a 10x5 matrix in which each cell contains a tf.float32. +matrix_feature_column = tf.feature_column.numeric_column(key="MyMatrix", + shape=[10,5]) +``` +### Bucketized column + +Often, you don't want to feed a number directly into the model, but instead +split its value into different categories based on numerical ranges. To do so, +create a @{tf.feature_column.bucketized_column$bucketized column}. For +example, consider raw data that represents the year a house was built. Instead +of representing that year as a scalar numeric column, we could split the year +into the following four buckets: + +
+ +
+
+Dividing year data into four buckets. +
+ +The model will represent the buckets as follows: + +|Date Range |Represented as... | +|:----------|:-----------------| +|< 1960 | [1, 0, 0, 0] | +|>= 1960 but < 1980 | [0, 1, 0, 0] | +|>= 1980 but < 2000 | [0, 0, 1, 0] | +|> 2000 | [0, 0, 0, 1] | + +Why would you want to split a number—a perfectly valid input to your +model—into a categorical value? Well, notice that the categorization splits a +single input number into a four-element vector. Therefore, the model now can +learn _four individual weights_ rather than just one; four weights creates a +richer model than one weight. More importantly, bucketizing enables the model +to clearly distinguish between different year categories since only one of the +elements is set (1) and the other three elements are cleared (0). When we just +use a single number (a year) as input, the model can only learn a linear +relationship. So, bucketing provides the model with additional flexibility that +the model can use to learn. + +The following code demonstrates how to create a bucketized feature: + + +```python +# First, convert the raw input to a numeric column. +numeric_feature_column = tf.feature_column.numeric_column("Year") + +# Then, bucketize the numeric column on the years 1960, 1980, and 2000. +bucketized_feature_column = tf.feature_column.bucketized_column( + source_column = numeric_feature_column, + boundaries = [1960, 1980, 2000]) +``` +Note that specifying a _three_-element boundaries vector creates a +_four_-element bucketized vector. + + +### Categorical identity column + +**Categorical identity columns** can be seen as a special case of bucketized +columns. In traditional bucketized columns, each bucket represents a range of +values (for example, from 1960 to 1979). In a categorical identity column, each +bucket represents a single, unique integer. For example, let's say you want to +represent the integer range `[0, 4)`. That is, you want to represent the +integers 0, 1, 2, or 3. In this case, the categorical identity mapping looks +like this: + +
+ +
+
+A categorical identity column mapping. Note that this is a one-hot +encoding, not a binary numerical encoding. +
+ +As with bucketized columns, a model can learn a separate weight for each class +in a categorical identity column. For example, instead of using a string to +represent the `product_class`, let's represent each class with a unique integer +value. That is: + +* `0="kitchenware"` +* `1="electronics"` +* `2="sport"` + +Call @{tf.feature_column.categorical_column_with_identity} to implement a +categorical identity column. For example: + +``` python +# Create categorical output for an integer feature named "my_feature_b", +# The values of my_feature_b must be >= 0 and < num_buckets +identity_feature_column = tf.feature_column.categorical_column_with_identity( + key='my_feature_b', + num_buckets=4) # Values [0, 4) + +# In order for the preceding call to work, the input_fn() must return +# a dictionary containing 'my_feature_b' as a key. Furthermore, the values +# assigned to 'my_feature_b' must belong to the set [0, 4). +def input_fn(): + ... + return ({ 'my_feature_a':[7, 9, 5, 2], 'my_feature_b':[3, 1, 2, 2] }, + [Label_values]) +``` + +### Categorical vocabulary column + +We cannot input strings directly to a model. Instead, we must first map strings +to numeric or categorical values. Categorical vocabulary columns provide a good +way to represent strings as a one-hot vector. For example: + +
+ +
+
+Mapping string values to vocabulary columns. +
+ +As you can see, categorical vocabulary columns are kind of an enum version of +categorical identity columns. TensorFlow provides two different functions to +create categorical vocabulary columns: + +* @{tf.feature_column.categorical_column_with_vocabulary_list} +* @{tf.feature_column.categorical_column_with_vocabulary_file} + +`categorical_column_with_vocabulary_list` maps each string to an integer based +on an explicit vocabulary list. For example: + +```python +# Given input "feature_name_from_input_fn" which is a string, +# create a categorical feature by mapping the input to one of +# the elements in the vocabulary list. +vocabulary_feature_column = + tf.feature_column.categorical_column_with_vocabulary_list( + key="a feature returned by input_fn()", + vocabulary_list=["kitchenware", "electronics", "sports"]) +``` + +The preceding function is pretty straightforward, but it has a significant +drawback. Namely, there's way too much typing when the vocabulary list is long. +For these cases, call +`tf.feature_column.categorical_column_with_vocabulary_file` instead, which lets +you place the vocabulary words in a separate file. For example: + +```python + +# Given input "feature_name_from_input_fn" which is a string, +# create a categorical feature to our model by mapping the input to one of +# the elements in the vocabulary file +vocabulary_feature_column = + tf.feature_column.categorical_column_with_vocabulary_file( + key="a feature returned by input_fn()", + vocabulary_file="product_class.txt", + vocabulary_size=3) +``` + +`product_class.txt` should contain one line for each vocabulary element. In our +case: + +```None +kitchenware +electronics +sports +``` + +### Hashed Column + +So far, we've worked with a naively small number of categories. For example, +our product_class example has only 3 categories. Often though, the number of +categories can be so big that it's not possible to have individual categories +for each vocabulary word or integer because that would consume too much memory. +For these cases, we can instead turn the question around and ask, "How many +categories am I willing to have for my input?" In fact, the +@{tf.feature_column.categorical_column_with_hash_bucket} function enables you +to specify the number of categories. For this type of feature column the model +calculates a hash value of the input, then puts it into one of +the `hash_bucket_size` categories using the modulo operator, as in the following +pseudocode: + +```python +# pseudocode +feature_id = hash(raw_feature) % hash_buckets_size +``` + +The code to create the `feature_column` might look something like this: + +``` python +hashed_feature_column = + tf.feature_column.categorical_column_with_hash_bucket( + key = "some_feature", + hash_buckets_size = 100) # The number of categories +``` +At this point, you might rightfully think: "This is crazy!" After all, we are +forcing the different input values to a smaller set of categories. This means +that two probably unrelated inputs will be mapped to the same +category, and consequently mean the same thing to the neural network. The +following figure illustrates this dilemma, showing that kitchenware and sports +both get assigned to category (hash bucket) 12: + +
+ +
+
+Representing data with hash buckets. +
+ +As with many counterintuitive phenomena in machine learning, it turns out that +hashing often works well in practice. That's because hash categories provide +the model with some separation. The model can use additional features to further +separate kitchenware from sports. + +### Crossed column + +Combining features into a single feature, better known as +[feature crosses](https://developers.google.com/machine-learning/glossary/#feature_cross), +enables the model to learn separate weights for each combination of +features. + +More concretely, suppose we want our model to calculate real estate prices in +Atlanta, GA. Real-estate prices within this city vary greatly depending on +location. Representing latitude and longitude as separate features isn't very +useful in identifying real-estate location dependencies; however, crossing +latitude and longitude into a single feature can pinpoint locations. Suppose we +represent Atlanta as a grid of 100x100 rectangular sections, identifying each +of the 10,000 sections by a feature cross of latitude and longitude. This +feature cross enables the model to train on pricing conditions related to each +individual section, which is a much stronger signal than latitude and longitude +alone. + +The following figure shows our plan, with the latitude & longitude values for +the corners of the city in red text: + +
+ +
+
+Map of Atlanta. Imagine this map divided into 10,000 sections of +equal size. +
+ +For the solution, we used a combination of the `bucketized_column` we looked at +earlier, with the @{tf.feature_column.crossed_column} function. + + + +``` python +def make_dataset(latitude, longitude, labels): + assert latitude.shape == longitude.shape == labels.shape + + features = {'latitude': latitude.flatten(), + 'longitude': longitude.flatten()} + labels=labels.flatten() + + return tf.data.Dataset.from_tensor_slices((features, labels)) + + +# Bucketize the latitude and longitude usig the `edges` +latitude_bucket_fc = tf.feature_column.bucketized_column( + tf.feature_column.numeric_column('latitude'), + list(atlanta.latitude.edges)) + +longitude_bucket_fc = tf.feature_column.bucketized_column( + tf.feature_column.numeric_column('longitude'), + list(atlanta.longitude.edges)) + +# Cross the bucketized columns, using 5000 hash bins. +crossed_lat_lon_fc = tf.feature_column.crossed_column( + [latitude_bucket_fc, longitude_bucket_fc], 5000) + +fc = [ + latitude_bucket_fc, + longitude_bucket_fc, + crossed_lat_lon_fc] + +# Build and train the Estimator. +est = tf.estimator.LinearRegressor(fc, ...) +``` + +You may create a feature cross from either of the following: + +* Feature names; that is, names from the `dict` returned from `input_fn`. +* Any categorical column, except `categorical_column_with_hash_bucket` + (since `crossed_column` hashes the input). + +When the feature columns `latitude_bucket_fc` and `longitude_bucket_fc` are +crossed, TensorFlow will create `(latitude_fc, longitude_fc)` pairs for each +example. This would produce a full grid of possibilities as follows: + +``` None + (0,0), (0,1)... (0,99) + (1,0), (1,1)... (1,99) + ... ... ... +(99,0), (99,1)...(99, 99) +``` + +Except that a full grid would only be tractable for inputs with limited +vocabularies. Instead of building this, potentially huge, table of inputs, +the `crossed_column` only builds the number requested by the `hash_bucket_size` +argument. The feature column assigns an example to a index by running a hash +function on the tuple of inputs, followed by a modulo operation with +`hash_bucket_size`. + +As discussed earlier, performing the +hash and modulo function limits the number of categories, but can cause category +collisions; that is, multiple (latitude, longitude) feature crosses will end +up in the same hash bucket. In practice though, performing feature crosses +still adds significant value to the learning capability of your models. + +Somewhat counterintuitively, when creating feature crosses, you typically still +should include the original (uncrossed) features in your model (as in the +preceding code snippet). The independent latitude and longitude features help the +model distinguish between examples where a hash collision has occured in the +crossed feature. + +## Indicator and embedding columns + +Indicator columns and embedding columns never work on features directly, but +instead take categorical columns as input. + +When using an indicator column, we're telling TensorFlow to do exactly what +we've seen in our categorical product_class example. That is, an +**indicator column** treats each category as an element in a one-hot vector, +where the matching category has value 1 and the rest have 0s: + +
+ +
+
+Representing data in indicator columns. +
+ +Here's how you create an indicator column by calling +@{tf.feature_column.indicator_column}: + +``` python +categorical_column = ... # Create any type of categorical column. + +# Represent the categorical column as an indicator column. +indicator_column = tf.feature_column.indicator_column(categorical_column) +``` + +Now, suppose instead of having just three possible classes, we have a million. +Or maybe a billion. For a number of reasons, as the number of categories grow +large, it becomes infeasible to train a neural network using indicator columns. + +We can use an embedding column to overcome this limitation. Instead of +representing the data as a one-hot vector of many dimensions, an +**embedding column** represents that data as a lower-dimensional, ordinary +vector in which each cell can contain any number, not just 0 or 1. By +permitting a richer palette of numbers for every cell, an embedding column +contains far fewer cells than an indicator column. + +Let's look at an example comparing indicator and embedding columns. Suppose our +input examples consists of different words from a limited palette of only 81 +words. Further suppose that the data set provides provides the following input +words in 4 separate examples: + +* `"dog"` +* `"spoon"` +* `"scissors"` +* `"guitar"` + +In that case, the following figure illustrates the processing path for +embedding columns or indicator columns. + +
+ +
+
+An embedding column stores categorical data in a lower-dimensional +vector than an indicator column. (We just placed random numbers into the +embedding vectors; training determines the actual numbers.) +
+ +When an example is processed, one of the `categorical_column_with...` functions +maps the example string to a numerical categorical value. For example, a +function maps "spoon" to `[32]`. (The 32 comes from our imagination—the actual +values depend on the mapping function.) You may then represent these numerical +categorical values in either of the following two ways: + +* As an indicator column. A function converts each numeric categorical value + into an 81-element vector (because our palette consists of 81 words), placing + a 1 in the index of the categorical value (0, 32, 79, 80) and a 0 in all the + other positions. + +* As an embedding column. A function uses the numerical categorical values + `(0, 32, 79, 80)` as indices to a lookup table. Each slot in that lookup table + contains a 3-element vector. + +How do the values in the embeddings vectors magically get assigned? Actually, +the assignments happen during training. That is, the model learns the best way +to map your input numeric categorical values to the embeddings vector value in +order to solve your problem. Embedding columns increase your model's +capabilities, since an embeddings vector learns new relationships between +categories from the training data. + +Why is the embedding vector size 3 in our example? Well, the following "formula" +provides a general rule of thumb about the number of embedding dimensions: + +```python +embedding_dimensions = number_of_categories**0.25 +``` + +That is, the embedding vector dimension should be the 4th root of the number of +categories. Since our vocabulary size in this example is 81, the recommended +number of dimensions is 3: + +``` python +3 = 81**0.25 +``` +Note that this is just a general guideline; you can set the number of embedding +dimensions as you please. + +Call @{tf.feature_column.embedding_column} to create an `embedding_column` as +suggested by the following snippet: + +``` python +categorical_column = ... # Create any categorical column + +# Represent the categorical column as an embedding column. +# This means creating a one-hot vector with one element for each category. +embedding_column = tf.feature_column.embedding_column( + categorical_column=categorical_column, + dimension=dimension_of_embedding_vector) +``` + +@{$programmers_guide/embedding$Embeddings} is a significant topic within machine +learning. This information was just to get you started using them as feature +columns. + +## Passing feature columns to Estimators + +As the following list indicates, not all Estimators permit all types of +`feature_columns` argument(s): + +* @{tf.estimator.LinearClassifier$`LinearClassifier`} and + @{tf.estimator.LinearRegressor$`LinearRegressor`}: Accept all types of + feature column. +* @{tf.estimator.DNNClassifier$`DNNClassifier`} and + @{tf.estimator.DNNRegressor$`DNNRegressor`}: Only accept dense columns. Other + column types must be wrapped in either an `indicator_column` or + `embedding_column`. +* @{tf.estimator.DNNLinearCombinedClassifier$`DNNLinearCombinedClassifier`} and + @{tf.estimator.DNNLinearCombinedRegressor$`DNNLinearCombinedRegressor`}: + * The `linear_feature_columns` argument accepts any feature column type. + * The `dnn_feature_columns` argument only accepts dense columns. + +## Other Sources + +For more examples on feature columns, view the following: + +* The @{$wide_and_deep$Wide & Deep Tutorial} +* [Examples](https://github.com/tensorflow/models/tree/master/samples/cookbook/regression) + of DNNs and linear models that use feature columns. + +To learn more about embeddings, see the following: + +* [Deep Learning, NLP, and representations](http://colah.github.io/posts/2014-07-NLP-RNNs-Representations/) + (Chris Olah's blog) +* The TensorFlow [Embedding Projector](http://projector.tensorflow.org) -- GitLab From c27a90d2195545c9147ec79094d7bca3176deb44 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Wed, 29 Nov 2017 09:59:34 -0800 Subject: [PATCH 0108/1924] [TF:XLA] VariableShape op support. PiperOrigin-RevId: 177323587 --- .../compiler/tf2xla/kernels/variable_ops.cc | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tensorflow/compiler/tf2xla/kernels/variable_ops.cc b/tensorflow/compiler/tf2xla/kernels/variable_ops.cc index b19ea22f50..2346c62ad1 100644 --- a/tensorflow/compiler/tf2xla/kernels/variable_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/variable_ops.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/no_op.h" namespace tensorflow { @@ -121,5 +122,31 @@ class ResourceGatherOp : public XlaOpKernel { REGISTER_XLA_OP(Name("ResourceGather").TypeConstraint("dtype", kNumericTypes), ResourceGatherOp); +class VariableShapeOp : public XlaOpKernel { + public: + explicit VariableShapeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + + void Compile(XlaOpKernelContext* ctx) override { + DataType dtype; + TensorShape shape; + OP_REQUIRES_OK(ctx, ctx->GetVariableTypeAndShape(0, &dtype, &shape)); + const int rank = shape.dims(); + Tensor shape_constant(DT_INT32, TensorShape({rank})); + auto vec = shape_constant.vec(); + // TODO(dga): support int64. b/28119922. + for (int i = 0; i < rank; ++i) { + int64 dim_size = shape.dim_size(i); + OP_REQUIRES( + ctx, FastBoundsCheck(dim_size, std::numeric_limits::max()), + errors::InvalidArgument("Shape does not support tensors > int32max", + " but dim ", i, " is ", dim_size)); + vec(i) = static_cast(dim_size); + } + + ctx->SetConstantOutput(0, shape_constant); + } +}; + +REGISTER_XLA_OP(Name("VariableShape"), VariableShapeOp); } // namespace } // namespace tensorflow -- GitLab From 2229a6cbbe27b3c42fbcd4aff0bb3de1925a8768 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 10:04:47 -0800 Subject: [PATCH 0109/1924] Internal Change PiperOrigin-RevId: 177324488 --- tensorflow/contrib/lite/python/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py index 0fd70f842b..982ea90f2b 100644 --- a/tensorflow/contrib/lite/python/lite.py +++ b/tensorflow/contrib/lite/python/lite.py @@ -50,7 +50,7 @@ GRAPHVIZ_DOT = _toco_flags_pb2.GRAPHVIZ_DOT # to protect against crashes. However, it breaks some dependent targets because # it forces us to depend on an external py_binary. The experimental API doesn't # have that drawback. -EXPERIMENTAL_USE_TOCO_API_DIRECTLY = True +EXPERIMENTAL_USE_TOCO_API_DIRECTLY = False # Find the toco_from_protos binary using the resource loader if using from # bazel, otherwise we are in a pip where console_scripts already has -- GitLab From 7921d01ec8fed3e5c62264b99b09440ea09796fe Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 29 Nov 2017 10:06:59 -0800 Subject: [PATCH 0110/1924] Raise an exception when converting lists with invalid lengths to Tensors instead of CHECK failing PiperOrigin-RevId: 177324815 --- tensorflow/contrib/cmake/tf_python.cmake | 2 + tensorflow/python/BUILD | 13 ++++ .../kernel_tests/constant_op_eager_test.py | 33 +++++++++ tensorflow/python/lib/core/py_func.cc | 54 ++------------ tensorflow/python/lib/core/py_seq_tensor.cc | 18 ++++- tensorflow/python/lib/core/py_util.cc | 70 +++++++++++++++++++ tensorflow/python/lib/core/py_util.h | 27 +++++++ tensorflow/tools/ci_build/ci_sanity.sh | 3 +- 8 files changed, 169 insertions(+), 51 deletions(-) create mode 100644 tensorflow/python/lib/core/py_util.cc create mode 100644 tensorflow/python/lib/core/py_util.h diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 0128946e45..819b6213ea 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -899,6 +899,8 @@ set (pywrap_tensorflow_internal_src "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.cc" "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.h" "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.cc" + "${tensorflow_source_dir}/tensorflow/python/lib/core/py_util.h" + "${tensorflow_source_dir}/tensorflow/python/lib/core/py_util.cc" "${tensorflow_source_dir}/tensorflow/python/lib/core/safe_ptr.h" "${tensorflow_source_dir}/tensorflow/python/lib/core/safe_ptr.cc" "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_reader.h" diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 9d3974b98e..5e7a6c0b59 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -268,6 +268,7 @@ cc_library( deps = [ ":ndarray_tensor_bridge", ":numpy_lib", + ":py_util", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", @@ -309,6 +310,7 @@ cc_library( hdrs = ["lib/core/py_seq_tensor.h"], deps = [ ":numpy_lib", + ":py_util", ":safe_ptr", "//tensorflow/core:framework", "//tensorflow/core:lib", @@ -316,6 +318,17 @@ cc_library( ], ) +cc_library( + name = "py_util", + srcs = ["lib/core/py_util.cc"], + hdrs = ["lib/core/py_util.h"], + deps = [ + "//tensorflow/core:lib", + "//tensorflow/core:script_ops_op_lib", + "//util/python:python_headers", + ], +) + cc_library( name = "py_record_reader_lib", srcs = ["lib/io/py_record_reader.cc"], diff --git a/tensorflow/python/kernel_tests/constant_op_eager_test.py b/tensorflow/python/kernel_tests/constant_op_eager_test.py index 3b71586b55..8e9d75667d 100644 --- a/tensorflow/python/kernel_tests/constant_op_eager_test.py +++ b/tensorflow/python/kernel_tests/constant_op_eager_test.py @@ -237,6 +237,39 @@ class ConstantTest(test.TestCase): self._testAll((1, x)) self._testAll((x, 1)) + def testInvalidLength(self): + + class BadList(list): + + def __init__(self): + super(BadList, self).__init__([1, 2, 3]) # pylint: disable=invalid-length-returned + + def __len__(self): + return -1 + + with self.assertRaisesRegexp(ValueError, "should return >= 0"): + constant_op.constant([BadList()]) + with self.assertRaisesRegexp(ValueError, "mixed types"): + constant_op.constant([1, 2, BadList()]) + with self.assertRaisesRegexp(ValueError, "should return >= 0"): + constant_op.constant(BadList()) + with self.assertRaisesRegexp(ValueError, "should return >= 0"): + constant_op.constant([[BadList(), 2], 3]) + with self.assertRaisesRegexp(ValueError, "should return >= 0"): + constant_op.constant([BadList(), [1, 2, 3]]) + with self.assertRaisesRegexp(ValueError, "should return >= 0"): + constant_op.constant([BadList(), []]) + + # TODO(allenl, josh11b): These cases should return exceptions rather than + # working (currently shape checking only checks the first element of each + # sequence recursively). Maybe the first one is fine, but the second one + # silently truncating is rather bad. + + # with self.assertRaisesRegexp(ValueError, "should return >= 0"): + # constant_op.constant([[3, 2, 1], BadList()]) + # with self.assertRaisesRegexp(ValueError, "should return >= 0"): + # constant_op.constant([[], BadList()]) + def testSparseValuesRaiseErrors(self): with self.assertRaisesRegexp(ValueError, "non-rectangular Python sequence"): constant_op.constant([[1, 2], [3]], dtype=dtypes_lib.int32) diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index 8bf831f8ba..a42282b055 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -22,11 +22,11 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/threadpool.h" -#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/python/lib/core/ndarray_tensor_bridge.h" +#include "tensorflow/python/lib/core/py_util.h" #include namespace tensorflow { @@ -133,48 +133,6 @@ bool IsSingleNone(PyObject* obj) { return item == Py_None; } -// py.__class__.__name__ -const char* ClassName(PyObject* py) { -/* PyPy doesn't have a separate C API for old-style classes. */ -#if PY_MAJOR_VERSION < 3 && !defined(PYPY_VERSION) - if (PyClass_Check(py)) - return PyString_AS_STRING( - CHECK_NOTNULL(reinterpret_cast(py)->cl_name)); - if (PyInstance_Check(py)) - return PyString_AS_STRING(CHECK_NOTNULL( - reinterpret_cast(py)->in_class->cl_name)); -#endif - if (Py_TYPE(py) == &PyType_Type) { - return reinterpret_cast(py)->tp_name; - } - return Py_TYPE(py)->tp_name; -} - -string PyExcFetch() { - CHECK(PyErr_Occurred()) << "Must only call PyExcFetch after an exception."; - PyObject* ptype; - PyObject* pvalue; - PyObject* ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); - string err = ClassName(ptype); - if (pvalue) { - PyObject* str = PyObject_Str(pvalue); - if (str) { -#if PY_MAJOR_VERSION < 3 - strings::StrAppend(&err, ": ", PyString_AS_STRING(str)); -#else - strings::StrAppend(&err, ": ", PyUnicode_AsUTF8(str)); -#endif - Py_DECREF(str); - } - Py_DECREF(pvalue); - } - Py_DECREF(ptype); - Py_XDECREF(ptraceback); - return err; -} - // Calls the registered py function through the trampoline. Status DoCallPyFunc(PyCall* call, bool* out_log_on_error) { *out_log_on_error = true; @@ -195,18 +153,18 @@ Status DoCallPyFunc(PyCall* call, bool* out_log_on_error) { if (PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_ValueError) || PyErr_ExceptionMatches(PyExc_TypeError)) { - return errors::InvalidArgument(PyExcFetch()); + return errors::InvalidArgument(PyExceptionFetch()); } else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { *out_log_on_error = false; - return errors::OutOfRange(PyExcFetch()); + return errors::OutOfRange(PyExceptionFetch()); } else if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - return errors::ResourceExhausted(PyExcFetch()); + return errors::ResourceExhausted(PyExceptionFetch()); } else if (PyErr_ExceptionMatches(PyExc_NotImplementedError)) { - return errors::Unimplemented(PyExcFetch()); + return errors::Unimplemented(PyExceptionFetch()); } else { // TODO(ebrevdo): Check if exception is an OpError and use the // OpError.error_code property to map it back in the Status. - return errors::Unknown(PyExcFetch()); + return errors::Unknown(PyExceptionFetch()); } } else { return errors::Internal("Failed to run py callback ", call->token, diff --git a/tensorflow/python/lib/core/py_seq_tensor.cc b/tensorflow/python/lib/core/py_seq_tensor.cc index 71cb38f8fd..317bdc2e14 100644 --- a/tensorflow/python/lib/core/py_seq_tensor.cc +++ b/tensorflow/python/lib/core/py_seq_tensor.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/python/lib/core/numpy.h" +#include "tensorflow/python/lib/core/py_util.h" #include "tensorflow/python/lib/core/safe_ptr.h" namespace tensorflow { @@ -89,12 +90,25 @@ Status InferShapeAndType(PyObject* obj, TensorShape* shape, DataType* dtype) { *dtype = DT_STRING; } else if (PySequence_Check(obj)) { auto length = PySequence_Length(obj); - shape->AddDim(length); if (length > 0) { + shape->AddDim(length); obj = PySequence_GetItem(obj, 0); continue; - } else { + } else if (length == 0) { + shape->AddDim(length); *dtype = DT_INVALID; // Invalid dtype for empty tensors. + } else { + // The sequence does not have a valid length (PySequence_Length < 0). + if (PyErr_Occurred()) { + // PySequence_Length failed and set an exception. Fetch the message + // and convert it to a failed status. + return errors::InvalidArgument(PyExceptionFetch()); + } else { + // This is almost certainly dead code: PySequence_Length failed but + // did not set an exception. + return errors::InvalidArgument( + "Attempted to convert an invalid sequence to a Tensor."); + } } } else if (IsPyFloat(obj)) { *dtype = DT_DOUBLE; diff --git a/tensorflow/python/lib/core/py_util.cc b/tensorflow/python/lib/core/py_util.cc new file mode 100644 index 0000000000..2635694e23 --- /dev/null +++ b/tensorflow/python/lib/core/py_util.cc @@ -0,0 +1,70 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/python/lib/core/py_util.h" + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include + +namespace tensorflow { +namespace { + +// py.__class__.__name__ +const char* ClassName(PyObject* py) { +/* PyPy doesn't have a separate C API for old-style classes. */ +#if PY_MAJOR_VERSION < 3 && !defined(PYPY_VERSION) + if (PyClass_Check(py)) + return PyString_AS_STRING( + CHECK_NOTNULL(reinterpret_cast(py)->cl_name)); + if (PyInstance_Check(py)) + return PyString_AS_STRING(CHECK_NOTNULL( + reinterpret_cast(py)->in_class->cl_name)); +#endif + if (Py_TYPE(py) == &PyType_Type) { + return reinterpret_cast(py)->tp_name; + } + return Py_TYPE(py)->tp_name; +} + +} // end namespace + +string PyExceptionFetch() { + CHECK(PyErr_Occurred()) + << "Must only call PyExceptionFetch after an exception."; + PyObject* ptype; + PyObject* pvalue; + PyObject* ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); + string err = ClassName(ptype); + if (pvalue) { + PyObject* str = PyObject_Str(pvalue); + if (str) { +#if PY_MAJOR_VERSION < 3 + strings::StrAppend(&err, ": ", PyString_AS_STRING(str)); +#else + strings::StrAppend(&err, ": ", PyUnicode_AsUTF8(str)); +#endif + Py_DECREF(str); + } + Py_DECREF(pvalue); + } + Py_DECREF(ptype); + Py_XDECREF(ptraceback); + return err; +} + +} // end namespace tensorflow diff --git a/tensorflow/python/lib/core/py_util.h b/tensorflow/python/lib/core/py_util.h new file mode 100644 index 0000000000..44dfe7ba21 --- /dev/null +++ b/tensorflow/python/lib/core/py_util.h @@ -0,0 +1,27 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_PYTHON_LIB_CORE_UTIL_H_ +#define TENSORFLOW_PYTHON_LIB_CORE_UTIL_H_ + +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +// Fetch the exception message as a string. An exception must be set +// (PyErr_Occurred() must be true). +string PyExceptionFetch(); +} // end namespace tensorflow + +#endif // TENSORFLOW_PYTHON_LIB_CORE_UTIL_H_ diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 404a9a6b62..4021d794b6 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -99,7 +99,8 @@ do_pylint() { "^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\ "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\ "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable "\ -"^tensorflow/python/keras/_impl/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition" +"^tensorflow/python/keras/_impl/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition "\ +"^tensorflow/python/kernel_tests/constant_op_eager_test.py.*\[E0303.*invalid-length-returned" echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\"" -- GitLab From c572bc4fd7c73f4b8014ae43cdf9da5b99592f59 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 29 Nov 2017 10:37:28 -0800 Subject: [PATCH 0111/1924] Outline generated LLVM IR matrix-vector dot kernels This is a code size optimization for cases that dot matrix-vectors of the same shape repeatedly, but is also a slight performance improvment (most likely due to better icache behavior). PiperOrigin-RevId: 177329302 --- .../xla/service/cpu/dot_op_emitter.cc | 62 +++++++++++++------ .../service/llvm_ir/kernel_support_library.cc | 44 +++++++++++++ .../service/llvm_ir/kernel_support_library.h | 32 ++++++++++ 3 files changed, 118 insertions(+), 20 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc index 8f7b478cee..4ccff756a3 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc @@ -522,8 +522,10 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() { return false; } - if (!primitive_util::IsFloatingPointType(dot_.shape().element_type()) && - !primitive_util::IsIntegralType(dot_.shape().element_type())) { + PrimitiveType primitive_type = dot_.shape().element_type(); + + if (!primitive_util::IsFloatingPointType(primitive_type) && + !primitive_util::IsIntegralType(primitive_type)) { return false; } @@ -573,30 +575,50 @@ bool DotOpEmitter::EmitLlvmIrDotIfProfitable() { int64 tiling_factor = GetGemvTilingFactor(); CHECK_GT(tiling_factor, 0); + llvm::Value* result_op = target_array_.GetBasePointer(); + llvm::Value* lhs_op = + swap_operands ? rhs_array_.GetBasePointer() : lhs_array_.GetBasePointer(); + llvm::Value* rhs_op = + swap_operands ? lhs_array_.GetBasePointer() : rhs_array_.GetBasePointer(); + if (is_column_major_matrix_vector) { VLOG(2) << "Emitting column major matrix-vector multiply with m = " << m << " and k = " << k; - ColumnMajorMatrixVectorProductEmitter emitter( - dot_.shape().element_type(), /*tile_rows=*/8, - /*tile_cols=*/tiling_factor, m, k, - swap_operands ? rhs_array_.GetBasePointer() - : lhs_array_.GetBasePointer(), - swap_operands ? lhs_array_.GetBasePointer() - : rhs_array_.GetBasePointer(), - target_array_.GetBasePointer(), ir_builder_); - emitter.Emit(); + int64 tile_rows = 8; + int64 tile_cols = tiling_factor; + + string kernel_name = tensorflow::strings::StrCat( + "col_major_gemv_", PrimitiveType_Name(primitive_type), "_", tile_rows, + "_", tile_cols, "_", m, "_", k); + + KernelSupportLibrary::EmitAndCallOutlinedKernel( + ir_builder_, kernel_name, lhs_op, rhs_op, result_op, + [this, tile_rows, tile_cols, m, k, primitive_type]( + llvm::Value* lhs_op, llvm::Value* rhs_op, llvm::Value* result_op) { + ColumnMajorMatrixVectorProductEmitter emitter( + primitive_type, tile_rows, tile_cols, m, k, lhs_op, rhs_op, + result_op, ir_builder_); + emitter.Emit(); + }); } else { VLOG(2) << "Emitting row major matrix-vector multiply with m = " << m << " and k = " << k; - RowMajorMatrixVectorProductEmitter emitter( - dot_.shape().element_type(), /*tile_rows=*/tiling_factor, - /*tile_cols=*/8, m, k, - swap_operands ? rhs_array_.GetBasePointer() - : lhs_array_.GetBasePointer(), - swap_operands ? lhs_array_.GetBasePointer() - : rhs_array_.GetBasePointer(), - target_array_.GetBasePointer(), ir_builder_); - emitter.Emit(); + int64 tile_rows = tiling_factor; + int64 tile_cols = 8; + + string kernel_name = tensorflow::strings::StrCat( + "row_major_gemv_", PrimitiveType_Name(primitive_type), "_", tile_rows, + "_", tile_cols, "_", m, "_", k); + + KernelSupportLibrary::EmitAndCallOutlinedKernel( + ir_builder_, kernel_name, lhs_op, rhs_op, result_op, + [this, tile_rows, tile_cols, m, k, primitive_type]( + llvm::Value* lhs_op, llvm::Value* rhs_op, llvm::Value* result_op) { + RowMajorMatrixVectorProductEmitter emitter( + primitive_type, tile_rows, tile_cols, m, k, lhs_op, rhs_op, + result_op, ir_builder_); + emitter.Emit(); + }); } return true; diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc index 29cc0f81bd..d951a37d5d 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h" #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h" +#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h" namespace xla { void KernelSupportLibrary::For( @@ -62,4 +63,47 @@ void KernelSupportLibrary::If( false_block_generator(); llvm_ir::SetToLastInsertPoint(if_data.after_block, ir_builder_); } + +void KernelSupportLibrary::EmitAndCallOutlinedKernel( + llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name, + KernelSupportLibrary::ArgumentVector arguments, + const std::function& + kernel_body_generator) { + llvm::Module* module = ir_builder->GetInsertBlock()->getModule(); + llvm::Function* function = + module->getFunction(llvm_ir::AsStringRef(kernel_name)); + if (!function) { + VLOG(2) << "Generating kernel for " << kernel_name; + std::vector arg_types; + std::transform(arguments.begin(), arguments.end(), + std::back_inserter(arg_types), + [](llvm::Value* arg) { return arg->getType(); }); + + auto* function_type = llvm::FunctionType::get( + ir_builder->getVoidTy(), arg_types, /*isVarArg=*/false); + + function = llvm::Function::Create( + function_type, llvm::GlobalValue::InternalLinkage, + llvm_ir::AsStringRef(kernel_name), module); + + llvm::IRBuilder<>::InsertPointGuard guard(*ir_builder); + + auto* entry_bb = + llvm::BasicBlock::Create(ir_builder->getContext(), "entry", function); + auto* return_inst = llvm::ReturnInst::Create(ir_builder->getContext(), + /*retVal=*/nullptr, entry_bb); + // Set the insert point to before return_inst. + ir_builder->SetInsertPoint(return_inst); + + std::vector arg_values; + std::transform(function->arg_begin(), function->arg_end(), + std::back_inserter(arg_values), std::addressof); + kernel_body_generator(arg_values); + } else { + VLOG(3) << "Re-using kernel for " << kernel_name; + } + + ir_builder->CreateCall(function, llvm_ir::AsArrayRef(arguments)); +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h index 9bafb7b577..997b84bb27 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h +++ b/tensorflow/compiler/xla/service/llvm_ir/kernel_support_library.h @@ -118,6 +118,38 @@ class KernelSupportLibrary { const std::function& true_block_generator, const std::function& false_block_generator = []() {}); + using ArgumentVector = tensorflow::gtl::ArraySlice; + + // Generates the following control flow structure: + // + // define @`kernel_name`(arg0, arg1, ... arg`arguments.size()`) { + // kernel_body_generator({arg0, arg1, ... arg`arguments.size()`}); + // } + // + // ... + // call @`kernel_name`(arguments[0], arguments[1] ...) + // ... + // + // If a function called `kernel_name` is already present in the module then + // that function is re-used. In that sense we're using the llvm::Module as a + // cache of outlined kernels, keyed by function name. + static void EmitAndCallOutlinedKernel( + llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name, + ArgumentVector arguments, + const std::function& kernel_body_generator); + + // Thin wrapper around the more general EmitAndCallOutlinedKernel above. + static void EmitAndCallOutlinedKernel( + llvm::IRBuilder<>* ir_builder, tensorflow::StringPiece kernel_name, + llvm::Value* arg0, llvm::Value* arg1, llvm::Value* arg2, + const std::function& + kernel_body_generator) { + EmitAndCallOutlinedKernel( + ir_builder, kernel_name, {arg0, arg1, arg2}, [&](ArgumentVector args) { + kernel_body_generator(args[0], args[1], args[2]); + }); + } + private: llvm::IRBuilder<>* ir_builder_; bool prevent_unrolling_; -- GitLab From 78a4873cfa4562cf071492636f03e13fcb188bd8 Mon Sep 17 00:00:00 2001 From: Asim Shankar Date: Wed, 29 Nov 2017 11:18:38 -0800 Subject: [PATCH 0112/1924] Go: Bugfix: Make list-of-shape attributes in an operation work. By respecting cgo rules on pointers. Without the change to graph.go, the newly added test would fail with: panic: runtime error: cgo argument has Go pointer to Go pointer in the call to the C function TF_SetAttrShapeList. Fixes #14891 PiperOrigin-RevId: 177336663 --- tensorflow/go/graph.go | 64 +++++++++++++++++++------------- tensorflow/go/op/op_test.go | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 25 deletions(-) diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go index 46c600eab1..f200a8e00a 100644 --- a/tensorflow/go/graph.go +++ b/tensorflow/go/graph.go @@ -20,6 +20,24 @@ package tensorflow // // #include // #include +// +// void TF_SetAttrShapeList_Helper(TF_OperationDescription* desc, +// const char* attr_name, +// const int64_t* flat_dims, +// const int* num_dims, +// int num_shapes) { +// const int64_t** dims = +// (const int64_t**)malloc(sizeof(const int64_t*) * num_shapes); +// for (int i = 0; i < num_shapes; i++) { +// dims[i] = flat_dims; +// if (num_dims[i] > 0) { +// // flat_dims will be NULL iff num_shapes is 0 or all elements in num_dims are <= 0. +// flat_dims += num_dims[i]; +// } +// } +// TF_SetAttrShapeList(desc, attr_name, dims, num_dims, num_shapes); +// free(dims); +// } import "C" import ( @@ -289,41 +307,37 @@ func setAttr(cdesc *C.TF_OperationDescription, status *status, name string, valu return fmt.Errorf("bad value for attribute %q: %v", name, err) } case Shape: - ndims, dims := cshape(value) + ndims := C.int(value.NumDimensions()) var dimsp *C.int64_t if ndims > 0 { + dims := make([]C.int64_t, ndims) + for i, d := range value.dims { + dims[i] = C.int64_t(d) + } dimsp = &dims[0] } C.TF_SetAttrShape(cdesc, cAttrName, dimsp, ndims) case []Shape: - ndims := make([]C.int, len(value)) - dims := make([][]C.int64_t, len(value)) - dimsp := make([]*C.int64_t, len(value)) - for i, s := range value { - ndims[i], dims[i] = cshape(s) - if ndims[i] > 0 { - dimsp[i] = &dims[i][0] - } - } - if len(value) > 0 { - C.TF_SetAttrShapeList(cdesc, cAttrName, &dimsp[0], &ndims[0], C.int(len(value))) - } else { + if len(value) == 0 { C.TF_SetAttrShapeList(cdesc, cAttrName, nil, nil, 0) + } else { + var flatDims []C.int64_t + ndims := make([]C.int, len(value)) + for i, s := range value { + nd := s.NumDimensions() + ndims[i] = C.int(nd) + for _, d := range s.dims { + flatDims = append(flatDims, C.int64_t(d)) + } + } + var flatDimsp *C.int64_t + if len(flatDims) > 0 { + flatDimsp = &flatDims[0] + } + C.TF_SetAttrShapeList_Helper(cdesc, cAttrName, flatDimsp, &ndims[0], C.int(len(value))) } default: return fmt.Errorf("attribute %q has a type (%T) which is not valid for operation attributes", name, value) } return nil } - -func cshape(s Shape) (C.int, []C.int64_t) { - ndims := C.int(s.NumDimensions()) - if ndims < 0 { - return -1, nil - } - dims := make([]C.int64_t, ndims) - for i, s := range s.dims { - dims[i] = C.int64_t(s) - } - return ndims, dims -} diff --git a/tensorflow/go/op/op_test.go b/tensorflow/go/op/op_test.go index 2451ba3606..842dee9ffe 100644 --- a/tensorflow/go/op/op_test.go +++ b/tensorflow/go/op/op_test.go @@ -58,3 +58,76 @@ func TestAddOperationFailure(t *testing.T) { _ = resize.Shape() t.Errorf("resize.Shape() should have paniced since the underlying Operation was not created") } + +func TestShapeAttribute(t *testing.T) { + s := NewScope() + x := Placeholder(s.SubScope("x"), tf.Int32, PlaceholderShape(tf.MakeShape(1))) + y := Placeholder(s.SubScope("y"), tf.Int32, PlaceholderShape(tf.Shape{})) + z := Add(s, x, y) + graph, err := s.Finalize() + if err != nil { + t.Fatal(err) + } + sess, err := tf.NewSession(graph, nil) + if err != nil { + t.Fatal(err) + } + + value, err := tf.NewTensor([]int32{7}) + if err != nil { + t.Fatal(err) + } + feeds := map[tf.Output]*tf.Tensor{ + x: value, + y: value, + } + fetched, err := sess.Run(feeds, []tf.Output{z}, nil) + if err != nil { + t.Fatal(err) + } + if got, want := len(fetched), 1; got != want { + t.Fatalf("Fetched %d tensors, expected %d", got, want) + } + if got, want := fetched[0].Value().([]int32), []int32{14}; len(got) != len(want) || len(got) != 1 || got[0] != want[0] { + t.Fatalf("Got %v, want %v", got, want) + } +} + +func TestDataset(t *testing.T) { + var ( + s = NewScope() + + // The use of a non-scalar here is inspired by + // https://github.com/tensorflow/tensorflow/issues/14891 + c = Const(s, []int32{21718, 31415}) + types = []tf.DataType{c.DataType()} + shapes = []tf.Shape{c.Shape()} + dataset = TensorDataset(s, []tf.Output{c}, shapes) + + iterator = Iterator(s, "", "", types, shapes) + next = IteratorGetNext(s, iterator, types, shapes) + init = MakeIterator(s, dataset, iterator) + ) + graph, err := s.Finalize() + if err != nil { + t.Fatal(err) + } + sess, err := tf.NewSession(graph, nil) + if err != nil { + t.Fatal(err) + } + if _, err := sess.Run(nil, nil, []*tf.Operation{init}); err != nil { + t.Fatal(err) + } + results, err := sess.Run(nil, next, nil) + if err != nil { + t.Fatal(err) + } + got := results[0].Value().([]int32) + if len(got) != 2 || got[0] != 21718 || got[1] != 31415 { + t.Errorf("Got %v, want {21718, 31415}", got) + } + if _, err := sess.Run(nil, next, nil); err == nil { + t.Errorf("Expected sess.Run() to fail since the iterator should have reached the end of the dataset") + } +} -- GitLab From 71f22bbab05e25c5f026c4343664091cc117b5ab Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Wed, 29 Nov 2017 11:27:08 -0800 Subject: [PATCH 0113/1924] (Temporarily) call Graph._add_op outside of Operation.__init__ again. This change partially undoes my previous commit (https://github.com/tensorflow/tensorflow/commit/f4c18a0eb05e21bae397c9c16527ff8080cae6b8). Without this change, if an op is added that has invalid input shapes and also requires a kernel label, the op will be added to the graph before shape inference is run, but then the shape inference error will prevent the kernel label from being applied. The placer will then complain about the missing label when the graph is run. This is only a problem with the C API disabled. With the C API enabled, shape inference is performed when the TF_Operation is created in Operation.__init__. Thus we can and should move the _add_op call back to Operation.__init__ once the _USE_C_API flag is removed. PiperOrigin-RevId: 177338123 --- .../copy_graph/python/util/copy_elements.py | 1 + tensorflow/contrib/graph_editor/transform.py | 3 +++ tensorflow/python/framework/ops.py | 5 ++--- tensorflow/python/framework/ops_test.py | 13 +++++++++++++ tensorflow/python/framework/test_ops.cc | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py index d060eda0a7..bae66ffd42 100644 --- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py +++ b/tensorflow/contrib/copy_graph/python/util/copy_elements.py @@ -225,6 +225,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, new_original_op, op_def) #Use Graph's hidden methods to add the op + to_graph._add_op(new_op) # pylint: disable=protected-access to_graph._record_op_seen_by_control_dependencies(new_op) for device_function in reversed(to_graph._device_function_stack): new_op._set_device(device_function(new_op)) diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py index 2a97a79070..14ac529665 100644 --- a/tensorflow/contrib/graph_editor/transform.py +++ b/tensorflow/contrib/graph_editor/transform.py @@ -173,6 +173,9 @@ def copy_op_handler(info, op, copy_shape=True): if op._original_op: op_._original_op = op._original_op + # Add op to the graph + info.graph_._add_op(op_) + return op_, op_.outputs diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index cfef5e35f4..2217513966 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -1635,8 +1635,6 @@ class Operation(object): self._id_value = self._graph._next_id() # pylint: disable=protected-access self._recompute_node_def() - self._graph._add_op(self) # pylint: disable=protected-access - def _reconstruct_sequence_inputs(self, op_def, inputs, attrs): """Regroups a flat list of input tensors into scalar and sequence inputs. @@ -3100,7 +3098,6 @@ class Graph(object): input_types=input_types, original_op=self._default_original_op, op_def=op_def) - self._create_op_helper(ret, compute_shapes=compute_shapes, compute_device=compute_device) return ret @@ -3139,6 +3136,8 @@ class Graph(object): # compute_shapes argument. if op._c_op or compute_shapes: # pylint: disable=protected-access set_shapes_for_outputs(op) + # TODO(b/XXXX): move to Operation.__init__ once _USE_C_API flag is removed. + self._add_op(op) # Apply any additional attributes requested. Do not overwrite any existing # attributes. diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index 3eae3b5a25..b1ad6ad744 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -274,6 +274,7 @@ class OperationTest(test_util.TensorFlowTestCase): op1 = ops.Operation( ops._NodeDef("RefOutputFloatOutput", "op1"), g, [], [dtypes.float32_ref, dtypes.float32]) + g._add_op(op1) self.assertProtoEquals("op:'RefOutputFloatOutput' name:'op1'", op1.node_def) self.assertEquals([], list(op1.inputs)) ref_t, nonref_t = op1.values() @@ -282,12 +283,14 @@ class OperationTest(test_util.TensorFlowTestCase): ops._NodeDef("RefInputFloatInput", "op2"), g, [ref_t, nonref_t], [], input_types=[dtypes.float32_ref, dtypes.float32]) + g._add_op(op2) self.assertProtoEquals( "op:'RefInputFloatInput' name:'op2' input:'op1' input:'op1:1'", op2.node_def) self.assertEquals([ref_t, nonref_t], list(op2.inputs)) op3 = ops.Operation( ops._NodeDef("TwoFloatInputs", "op3"), g, [ref_t, nonref_t], []) + g._add_op(op3) self.assertProtoEquals( "op:'TwoFloatInputs' name:'op3' input:'op1' input:'op1:1'", op3.node_def) @@ -1884,6 +1887,16 @@ class GraphTest(test_util.TensorFlowTestCase): with session.Session() as sess: sess.run(a) + def testRunnableAfterInvalidShapeWithKernelLabelMap(self): + g = ops.Graph() + with g.as_default(): + with g._kernel_label_map({"KernelLabelRequired": "overload_1"}): + with self.assertRaises(ValueError): + test_ops.kernel_label_required(1) + a = constant_op.constant(1) + with session.Session() as sess: + sess.run(a) + @test_util.with_c_api class AttrScopeTest(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc index 25bb7af20c..dbabce0962 100644 --- a/tensorflow/python/framework/test_ops.cc +++ b/tensorflow/python/framework/test_ops.cc @@ -26,6 +26,16 @@ REGISTER_OP("KernelLabel") .Output("result: string") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("KernelLabelRequired") + .Input("input: int32") + .Output("result: string") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle out; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &out)); + c->set_output(0, c->Scalar()); + return Status::OK(); + }); + REGISTER_OP("GraphDefVersion") .Output("version: int32") .SetIsStateful() @@ -104,6 +114,14 @@ REGISTER_KERNEL_BUILDER(Name("KernelLabel") .Label("overload_2"), KernelLabelOp); +// All "KernelLabelRequired" kernels have labels +REGISTER_KERNEL_BUILDER( + Name("KernelLabelRequired").Device(DEVICE_CPU).Label("overload_1"), + KernelLabelOp); +REGISTER_KERNEL_BUILDER( + Name("KernelLabelRequired").Device(DEVICE_CPU).Label("overload_2"), + KernelLabelOp); + class GraphDefVersionOp : public OpKernel { public: explicit GraphDefVersionOp(OpKernelConstruction* ctx) -- GitLab From ad1310a87caa14c495ad7ab47db7572443b2e7ef Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Wed, 29 Nov 2017 11:36:36 -0800 Subject: [PATCH 0114/1924] Add RandomDataset which generates pseudo random number of type int64. Add tf.contrib.data.shuffle_and_repeat which reshuffles its input on each epoch. Going forward, this will replace reshuffle_each_iteration=true. PiperOrigin-RevId: 177339570 --- tensorflow/contrib/data/BUILD | 1 + tensorflow/contrib/data/__init__.py | 1 + .../contrib/data/python/kernel_tests/BUILD | 4 +- .../kernel_tests/shuffle_dataset_op_test.py | 80 +++++++++ tensorflow/contrib/data/python/ops/BUILD | 32 ++++ .../contrib/data/python/ops/random_ops.py | 67 ++++++++ .../contrib/data/python/ops/shuffle_ops.py | 69 ++++++++ .../base_api/api_def_RandomDataset.pbtxt | 18 ++ tensorflow/core/kernels/BUILD | 13 ++ tensorflow/core/kernels/random_dataset_op.cc | 154 ++++++++++++++++++ tensorflow/core/ops/dataset_ops.cc | 18 ++ tensorflow/python/data/ops/dataset_ops.py | 33 +++- 12 files changed, 486 insertions(+), 4 deletions(-) create mode 100644 tensorflow/contrib/data/python/ops/random_ops.py create mode 100644 tensorflow/contrib/data/python/ops/shuffle_ops.py create mode 100644 tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt create mode 100644 tensorflow/core/kernels/random_dataset_op.cc diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD index f7d8a084d9..3b1c33063f 100644 --- a/tensorflow/contrib/data/BUILD +++ b/tensorflow/contrib/data/BUILD @@ -18,6 +18,7 @@ py_library( "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:readers", + "//tensorflow/contrib/data/python/ops:shuffle_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:util", "//tensorflow/python/data/ops:iterator_ops", diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 7c6244f22b..c9ad091bd4 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -66,6 +66,7 @@ from tensorflow.contrib.data.python.ops.readers import TextLineDataset from tensorflow.contrib.data.python.ops.readers import TFRecordDataset from tensorflow.contrib.data.python.ops.resampling import rejection_resample from tensorflow.contrib.data.python.ops.scan_ops import scan +from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat from tensorflow.python.data.ops.iterator_ops import Iterator # pylint: enable=unused-import diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 4cb69d7c8e..43431ca2c5 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -414,12 +414,14 @@ py_test( py_test( name = "shuffle_dataset_op_test", - size = "small", + size = "medium", srcs = ["shuffle_dataset_op_test.py"], srcs_version = "PY2AND3", deps = [ + ":dataset_serialization_test", "//tensorflow/contrib/data/python/ops:dataset_ops", "//tensorflow/contrib/data/python/ops:iterator_ops", + "//tensorflow/contrib/data/python/ops:shuffle_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", diff --git a/tensorflow/contrib/data/python/kernel_tests/shuffle_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/shuffle_dataset_op_test.py index 6b5b53cc0f..ba1be0690f 100644 --- a/tensorflow/contrib/data/python/kernel_tests/shuffle_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/shuffle_dataset_op_test.py @@ -22,8 +22,10 @@ import os import numpy as np +from tensorflow.contrib.data.python.kernel_tests import dataset_serialization_test_base from tensorflow.contrib.data.python.ops import dataset_ops as contrib_dataset_ops from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops +from tensorflow.contrib.data.python.ops import shuffle_ops from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.framework import constant_op @@ -156,6 +158,13 @@ class ShuffleDatasetTest(test.TestCase): for i in range(5): self.assertEqual(10, counts[i]) + def testSeedNoneSeed2NonNone(self): + with self.assertRaises(ValueError): + dataset_ops.ShuffleDataset(dataset_ops.Dataset.range(5), + buffer_size=1, + seed=None, + seed2=10) + class ShuffleDatasetSerializationTest(test.TestCase): @@ -474,5 +483,76 @@ class ShuffleDatasetSerializationTest(test.TestCase): self.assertEqual(expected_outputs_sorted, sorted(actual)) +class ShuffleAndRepeatTest( + dataset_serialization_test_base.DatasetSerializationTestBase): + + def _build_ds(self, seed, count=5): + return dataset_ops.Dataset.range(20).apply( + shuffle_ops.shuffle_and_repeat(buffer_size=5, count=count, seed=seed)) + + def testCorrectOutput(self): + output = self.gen_outputs(lambda: self._build_ds(10), [], 100) + self.assertSequenceEqual( + sorted(output), sorted( + np.array([range(20) for _ in range(5)]).flatten())) + for i in range(5): + self.assertSequenceEqual(sorted(output[i * 20:(i + 1) * 20]), range(20)) + + def testReshuffling(self): + # Check that the output orders of different epochs are indeed different. + output = self.gen_outputs(lambda: self._build_ds(10), [], 100) + for i in range(4): + epoch1 = output[i * 20:(i + 1) * 20] + epoch2 = output[(i + 1) * 20:(i + 2) * 20] + self.assertNotEqual(epoch1, epoch2) + + def testSameOrderForSameSeeds(self): + output1 = self.gen_outputs(lambda: self._build_ds(10), [], 100) + output2 = self.gen_outputs(lambda: self._build_ds(10), [], 100) + self.assertEqual(output1, output2) + + def testDifferentOrderForDifferentSeeds(self): + output1 = self.gen_outputs(lambda: self._build_ds(10), [], 100) + output2 = self.gen_outputs(lambda: self._build_ds(20), [], 100) + self.assertNotEqual(output1, output2) + self.assertEqual(sorted(output1), sorted(output2)) + + def testCountNone(self): + output1 = self.gen_outputs( + lambda: self._build_ds(10, count=None), [], 100, verify_exhausted=False) + output2 = self.gen_outputs( + lambda: self._build_ds(20, count=None), [], 100, verify_exhausted=False) + self.assertNotEqual(output1, output2) + self.assertEqual(sorted(output1), sorted(output2)) + + def testCountMinusOne(self): + output1 = self.gen_outputs( + lambda: self._build_ds(10, count=-1), [], 100, verify_exhausted=False) + output2 = self.gen_outputs( + lambda: self._build_ds(20, count=-1), [], 100, verify_exhausted=False) + self.assertNotEqual(output1, output2) + self.assertEqual(sorted(output1), sorted(output2)) + + def testInfiniteOutputs(self): + # Asserting that the iterator is exhausted after producing 100 items should + # fail. + with self.assertRaises(AssertionError): + self.gen_outputs(lambda: self._build_ds(10, count=None), [], 100) + with self.assertRaises(AssertionError): + self.gen_outputs(lambda: self._build_ds(10, count=-1), [], 100) + + +class ShuffleAndRepeatSerializationTest( + dataset_serialization_test_base.DatasetSerializationTestBase): + + def _build_ds(self, seed): + return dataset_ops.Dataset.range(20).apply( + shuffle_ops.shuffle_and_repeat(buffer_size=5, count=5, seed=seed)) + + def testCore(self): + self.run_core_tests(lambda: self._build_ds(10), lambda: self._build_ds(20), + 100) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index 25ed58cdf5..1f35ee056b 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -40,6 +40,25 @@ py_library( ], ) +py_library( + name = "random_ops", + srcs = [ + "random_ops.py", + ], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:constant_op", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:random_seed", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + "//tensorflow/python/data/util:sparse", + ], +) + py_library( name = "readers", srcs = [ @@ -62,6 +81,19 @@ py_library( ], ) +py_library( + name = "shuffle_ops", + srcs = [ + "shuffle_ops.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":random_ops", + ":transformation_ops", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + py_library( name = "transformation_ops", srcs = [ diff --git a/tensorflow/contrib/data/python/ops/random_ops.py b/tensorflow/contrib/data/python/ops/random_ops.py new file mode 100644 index 0000000000..7d727165fe --- /dev/null +++ b/tensorflow/contrib/data/python/ops/random_ops.py @@ -0,0 +1,67 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Datasets for random number generators.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.util import nest +from tensorflow.python.data.util import sparse +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import random_seed +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import gen_dataset_ops + + +class RandomDataset(dataset_ops.Dataset): + """A `Dataset` of pseudorandom values.""" + + def __init__(self, seed=None): + """A `Dataset` of pseudorandom values.""" + super(RandomDataset, self).__init__() + seed, seed2 = random_seed.get_seed(seed) + if seed is None: + self._seed = constant_op.constant(0, dtype=dtypes.int64, name="seed") + else: + self._seed = ops.convert_to_tensor(seed, dtype=dtypes.int64, name="seed") + if seed2 is None: + self._seed2 = constant_op.constant(0, dtype=dtypes.int64, name="seed2") + else: + self._seed2 = ops.convert_to_tensor( + seed2, dtype=dtypes.int64, name="seed2") + + def _as_variant_tensor(self): + return gen_dataset_ops.random_dataset( + seed=self._seed, + seed2=self._seed2, + output_shapes=nest.flatten( + sparse.as_dense_shapes(self.output_shapes, self.output_classes)), + output_types=nest.flatten( + sparse.as_dense_types(self.output_types, self.output_classes))) + + @property + def output_classes(self): + return ops.Tensor + + @property + def output_shapes(self): + return tensor_shape.scalar() + + @property + def output_types(self): + return dtypes.int64 diff --git a/tensorflow/contrib/data/python/ops/shuffle_ops.py b/tensorflow/contrib/data/python/ops/shuffle_ops.py new file mode 100644 index 0000000000..460732d65e --- /dev/null +++ b/tensorflow/contrib/data/python/ops/shuffle_ops.py @@ -0,0 +1,69 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental shuffle ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.data.python.ops import batching +from tensorflow.contrib.data.python.ops import random_ops +from tensorflow.python.data.ops import dataset_ops + + +def shuffle_and_repeat(buffer_size, count=None, seed=None): + """Shuffles and repeats a Dataset returning a new permutation for each epoch. + + `dataset.apply(tf.contrib.data.shuffle_and_repeat(buffer_size, count))` + + is equivalent to + + `dataset.shuffle(buffer_size, reshuffle_each_iteration=True).repeat(count)` + + The difference is that the latter dataset is not serializable. So, + if you need to checkpoint an input pipeline with reshuffling you must use + this implementation. + + Args: + buffer_size: A `tf.int64` scalar `tf.Tensor`, representing the + maximum number elements that will be buffered when prefetching. + count: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the + number of times the dataset should be repeated. The default behavior + (if `count` is `None` or `-1`) is for the dataset be repeated + indefinitely. + seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the + random seed that will be used to create the distribution. See + @{tf.set_random_seed} for behavior. + + Returns: + A `Dataset` transformation function, which can be passed to + @{tf.contrib.data.Dataset.apply}. + """ + def _apply_fn(dataset): # pylint: disable=missing-docstring + random_ds = random_ops.RandomDataset(seed).apply( + batching.batch_and_drop_remainder(2)) + if count is not None and count is not -1: + random_ds = random_ds.take(count) + + def map_fn(seeds): + return dataset_ops.ShuffleDataset( + input_dataset=dataset, + buffer_size=buffer_size, + seed=seeds[0], + reshuffle_each_iteration=False, + seed2=seeds[1]) + + return random_ds.flat_map(map_fn) + + return _apply_fn diff --git a/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt new file mode 100644 index 0000000000..0466b40f85 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_RandomDataset.pbtxt @@ -0,0 +1,18 @@ +op { + graph_op_name: "RandomDataset" + in_arg { + name: "seed" + description: <(ctx, "seed", &seed)); + + int64 seed2; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "seed2", &seed2)); + + // By TensorFlow convention, passing 0 for both seeds indicates + // that the shuffling should be seeded non-deterministically. + if (seed == 0 && seed2 == 0) { + seed = random::New64(); + seed2 = random::New64(); + } + + *output = new Dataset(ctx, seed, seed2); + } + + private: + class Dataset : public GraphDatasetBase { + public: + Dataset(OpKernelContext* ctx, int64 seed, int64 seed2) + : GraphDatasetBase(ctx), seed_(seed), seed2_(seed2) {} + + std::unique_ptr MakeIterator( + const string& prefix) const override { + return std::unique_ptr( + new Iterator({this, strings::StrCat(prefix, "::Random")})); + } + + const DataTypeVector& output_dtypes() const override { + static DataTypeVector* dtypes = new DataTypeVector({DT_INT64}); + return *dtypes; + } + + const std::vector& output_shapes() const override { + static std::vector* shapes = + new std::vector({{}}); + return *shapes; + } + + string DebugString() override { + return strings::StrCat("RandomDatasetOp(", seed_, ", ", seed2_, + ")::Dataset"); + } + + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + Node* seed = nullptr; + Node* seed2 = nullptr; + TF_RETURN_IF_ERROR(b->AddScalar(seed_, &seed)); + TF_RETURN_IF_ERROR(b->AddScalar(seed2_, &seed2)); + TF_RETURN_IF_ERROR(b->AddDataset(this, {seed, seed2}, output)); + return Status::OK(); + } + + private: + class Iterator : public DatasetIterator { + public: + explicit Iterator(const Params& params) + : DatasetIterator(params), + parent_generator_(dataset()->seed_, dataset()->seed2_), + generator_(&parent_generator_) {} + + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + Tensor value_tensor(cpu_allocator(), DT_INT64, {}); + value_tensor.scalar()() = Random(); + out_tensors->emplace_back(std::move(value_tensor)); + *end_of_sequence = false; + return Status::OK(); + } + + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("num_random_samples"), + num_random_samples_)); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("num_random_samples"), + &num_random_samples_)); + parent_generator_ = + random::PhiloxRandom(dataset()->seed_, dataset()->seed2_); + generator_ = random::SingleSampleAdapter( + &parent_generator_); + generator_.Skip(num_random_samples_); + return Status::OK(); + } + + private: + random::SingleSampleAdapter::ResultType Random() + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + num_random_samples_++; + auto out = generator_(); + return out; + } + mutex mu_; + random::PhiloxRandom parent_generator_ GUARDED_BY(mu_); + random::SingleSampleAdapter generator_ + GUARDED_BY(mu_); + int64 num_random_samples_ GUARDED_BY(mu_) = 0; + }; + + const int64 seed_; + const int64 seed2_; + }; +}; + +REGISTER_KERNEL_BUILDER(Name("RandomDataset").Device(DEVICE_CPU), + RandomDatasetOp); + +} // namespace + +} // namespace tensorflow diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 6bf226e7a5..be41531347 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -469,6 +469,24 @@ stop: corresponds to stop in python's xrange(). step: corresponds to step in python's xrange(). )doc"); +REGISTER_OP("RandomDataset") + .Input("seed: int64") + .Input("seed2: int64") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetIsStateful() // TODO(b/65524810): Source dataset ops must be marked + // stateful to inhibit constant folding. + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( +Creates a Dataset that returns pseudorandom numbers. + +seed: A scalar seed for the random number generator. If either seed or + seed2 is set to be non-zero, the random number generator is seeded + by the given seed. Otherwise, a random seed is used. +seed2: A second scalar seed to avoid seed collision. +)doc"); + REGISTER_OP("ShuffleDataset") .Input("input_dataset: variant") .Input("buffer_size: int64") diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index b5a8622306..927c6d5c02 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -1234,13 +1234,40 @@ class ShuffleDataset(Dataset): input_dataset, buffer_size, seed=None, - reshuffle_each_iteration=None): - """See `Dataset.shuffle()` for details.""" + reshuffle_each_iteration=None, + seed2=None): + """Randomly shuffles the elements of this dataset. + + Args: + input_dataset: The input dataset. + buffer_size: A `tf.int64` scalar `tf.Tensor`, representing the + number of elements from this dataset from which the new + dataset will sample. + seed: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the + random seed that will be used to create the distribution. See + @{tf.set_random_seed} for behavior. + reshuffle_each_iteration: (Optional.) A boolean, which if true indicates + that the dataset should be pseudorandomly reshuffled each time it is + iterated over. (Defaults to `True`.) + seed2: (Optional.) A `tf.int64` scalar `tf.Tensor` used to avoid seed + collision. Users should generally not need to specify this. This is + supposed to be used when both the seeds for the Dataset op need to be + manually specified. If not None, seed must also be non-None. + + Returns: + A `Dataset`. + + Raises: + ValueError: if invalid arguments are provided. + """ super(ShuffleDataset, self).__init__() self._input_dataset = input_dataset self._buffer_size = ops.convert_to_tensor( buffer_size, dtype=dtypes.int64, name="buffer_size") - seed, seed2 = random_seed.get_seed(seed) + if seed2 is None: + seed, seed2 = random_seed.get_seed(seed) + elif seed is None: + raise ValueError("seed must be non-None if seed2 is non-None.") if seed is None: self._seed = constant_op.constant(0, dtype=dtypes.int64, name="seed") else: -- GitLab From c0bd9dffccb29d4c01a2a18fc23b0ecad41aa4c6 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 29 Nov 2017 11:59:47 -0800 Subject: [PATCH 0115/1924] [tf.data] Fix compiler warnings about unused captures in lambda expressions. PiperOrigin-RevId: 177343020 --- tensorflow/core/kernels/dataset_utils.cc | 2 +- tensorflow/core/kernels/filter_dataset_op.cc | 2 +- tensorflow/core/kernels/group_by_window_dataset_op.cc | 6 +++--- tensorflow/core/kernels/map_and_batch_dataset_op.cc | 4 ++-- tensorflow/core/kernels/map_dataset_op.cc | 2 +- tensorflow/core/kernels/parallel_map_dataset_op.cc | 4 ++-- tensorflow/core/kernels/scan_dataset_op.cc | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/kernels/dataset_utils.cc b/tensorflow/core/kernels/dataset_utils.cc index cd58c80912..bd20e20cad 100644 --- a/tensorflow/core/kernels/dataset_utils.cc +++ b/tensorflow/core/kernels/dataset_utils.cc @@ -32,7 +32,7 @@ Status MakeIteratorFromInputElement( // is always 0, so a negative random step ID should suffice. opts.step_id = CapturedFunction::generate_step_id(); ScopedStepContainer step_container( - opts.step_id, [captured_func, ctx](const string& name) { + opts.step_id, [captured_func](const string& name) { captured_func->resource_manager()->Cleanup(name).IgnoreError(); }); opts.step_container = &step_container; diff --git a/tensorflow/core/kernels/filter_dataset_op.cc b/tensorflow/core/kernels/filter_dataset_op.cc index 0ac6cd9a98..67417d467d 100644 --- a/tensorflow/core/kernels/filter_dataset_op.cc +++ b/tensorflow/core/kernels/filter_dataset_op.cc @@ -149,7 +149,7 @@ class FilterDatasetOp : public UnaryDatasetOpKernel { FunctionLibraryRuntime::Options opts; opts.step_id = CapturedFunction::generate_step_id(); ScopedStepContainer step_container( - opts.step_id, [this, ctx](const string& name) { + opts.step_id, [this](const string& name) { dataset() ->captured_func_->resource_manager() ->Cleanup(name) diff --git a/tensorflow/core/kernels/group_by_window_dataset_op.cc b/tensorflow/core/kernels/group_by_window_dataset_op.cc index 8644bcf9b5..604555a560 100644 --- a/tensorflow/core/kernels/group_by_window_dataset_op.cc +++ b/tensorflow/core/kernels/group_by_window_dataset_op.cc @@ -169,7 +169,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { opts.step_id = CapturedFunction::generate_step_id(); opts.runner = ctx->runner(); ScopedStepContainer step_container( - opts.step_id, [this, ctx](const string& name) { + opts.step_id, [this](const string& name) { dataset() ->captured_key_func_->resource_manager() ->Cleanup(name) @@ -198,7 +198,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { opts2.step_id = CapturedFunction::generate_step_id(); opts2.runner = ctx->runner(); ScopedStepContainer step_container2( - opts2.step_id, [this, ctx](const string& name) { + opts2.step_id, [this](const string& name) { dataset() ->captured_window_size_func_->resource_manager() ->Cleanup(name) @@ -257,7 +257,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { opts.step_id = CapturedFunction::generate_step_id(); opts.runner = ctx->runner(); ScopedStepContainer step_container( - opts.step_id, [this, ctx](const string& name) { + opts.step_id, [this](const string& name) { dataset() ->captured_reduce_func_->resource_manager() ->Cleanup(name) diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc index ad1e356dbd..9bd66e681f 100644 --- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc @@ -239,8 +239,8 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { // to unblock a consumer. FunctionLibraryRuntime::Options opts; opts.step_id = CapturedFunction::generate_step_id(); - ScopedStepContainer* step_container = new ScopedStepContainer( - opts.step_id, [this, ctx](const string& name) { + ScopedStepContainer* step_container = + new ScopedStepContainer(opts.step_id, [this](const string& name) { dataset() ->captured_func_->resource_manager() ->Cleanup(name) diff --git a/tensorflow/core/kernels/map_dataset_op.cc b/tensorflow/core/kernels/map_dataset_op.cc index 23148f122d..29899a987e 100644 --- a/tensorflow/core/kernels/map_dataset_op.cc +++ b/tensorflow/core/kernels/map_dataset_op.cc @@ -146,7 +146,7 @@ class MapDatasetOp : public UnaryDatasetOpKernel { FunctionLibraryRuntime::Options opts; opts.step_id = CapturedFunction::generate_step_id(); ScopedStepContainer step_container( - opts.step_id, [this, ctx](const string& name) { + opts.step_id, [this](const string& name) { dataset() ->captured_func_->resource_manager() ->Cleanup(name) diff --git a/tensorflow/core/kernels/parallel_map_dataset_op.cc b/tensorflow/core/kernels/parallel_map_dataset_op.cc index 2be87f4bde..b9175fe904 100644 --- a/tensorflow/core/kernels/parallel_map_dataset_op.cc +++ b/tensorflow/core/kernels/parallel_map_dataset_op.cc @@ -195,8 +195,8 @@ class ParallelMapDatasetOp : public UnaryDatasetOpKernel { FunctionLibraryRuntime::Options opts; opts.step_id = CapturedFunction::generate_step_id(); - ScopedStepContainer* step_container = new ScopedStepContainer( - opts.step_id, [this, ctx](const string& name) { + ScopedStepContainer* step_container = + new ScopedStepContainer(opts.step_id, [this](const string& name) { dataset() ->captured_func_->resource_manager() ->Cleanup(name) diff --git a/tensorflow/core/kernels/scan_dataset_op.cc b/tensorflow/core/kernels/scan_dataset_op.cc index 76c219f1ae..bc52322022 100644 --- a/tensorflow/core/kernels/scan_dataset_op.cc +++ b/tensorflow/core/kernels/scan_dataset_op.cc @@ -132,7 +132,7 @@ class ScanDatasetOp : public UnaryDatasetOpKernel { FunctionLibraryRuntime::Options opts; opts.step_id = CapturedFunction::generate_step_id(); ScopedStepContainer step_container( - opts.step_id, [this, ctx](const string& name) { + opts.step_id, [this](const string& name) { dataset() ->captured_func_->resource_manager() ->Cleanup(name) -- GitLab From dcf9b035a09904322020d87a9324f04bcaf89eec Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 29 Nov 2017 12:06:55 -0800 Subject: [PATCH 0116/1924] Made sure the unknown shapes of placeholders always propagate to their fanouts PiperOrigin-RevId: 177344207 --- .../core/grappler/costs/graph_properties.cc | 12 ++++--- .../grappler/costs/graph_properties_test.cc | 31 ++++++++++++------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index fb7e20fca0..fbc52e9bd1 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -313,15 +313,17 @@ class SymbolicShapeRefiner { Status UpdateNode(const Node* node, bool relax, bool* refined) { return shape_refiner_->UpdateNode(node, relax, refined); } - Status SetShape(const Node* node, int output_port, - shape_inference::ShapeHandle shape) { - return shape_refiner_->SetShape(node, output_port, shape); - } Status SetUnknownShape(const Node* node, int output_port) { shape_inference::ShapeHandle shape = GetUnknownOutputShape(node, output_port); - return shape_refiner_->SetShape(node, output_port, shape); + InferenceContext* ctx = GetContext(node); + if (ctx == nullptr) { + return errors::InvalidArgument("Missing context"); + } + ctx->set_output(output_port, shape); + return Status::OK(); } + struct ShapeId { const Node* node; int port_id; diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index ad8e768f1f..cc40ff2cfc 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -856,7 +856,6 @@ TEST_F(GraphPropertiesTest, FedNodes) { cluster_->GetDeviceNames()); GrapplerItem item; CHECK(fake_input.NextItem(&item)); - item.feed.emplace_back("AddN", Tensor()); { // Conservative shape analysis: the shape of fed ports should be unknown @@ -864,17 +863,27 @@ TEST_F(GraphPropertiesTest, FedNodes) { Status s = properties.InferStatically(false); TF_CHECK_OK(s); for (const auto& node : item.graph.node()) { - if (node.name() == "AddN") { - const auto in_props = properties.GetInputProperties(node.name()); - EXPECT_EQ(1, in_props.size()); - const OpInfo::TensorProperties& in_prop = in_props[0]; - EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + if (node.op() == "Const") { + continue; + } + const auto in_props = properties.GetInputProperties(node.name()); + EXPECT_EQ(1, in_props.size()); + const OpInfo::TensorProperties& in_prop = in_props[0]; + const auto out_props = properties.GetOutputProperties(node.name()); + EXPECT_EQ(1, out_props.size()); + const OpInfo::TensorProperties& out_prop = out_props[0]; + + if (node.name() == "x") { + // x is fed: its input should have a known shape, while its output + // doesn't EXPECT_FALSE(in_prop.shape().unknown_rank()); - EXPECT_EQ(2, in_prop.shape().dim_size()); - const auto out_props = properties.GetOutputProperties(node.name()); - EXPECT_EQ(1, out_props.size()); - EXPECT_EQ(DT_FLOAT, in_prop.dtype()); + EXPECT_EQ(1, in_prop.shape().dim_size()); + EXPECT_EQ(2, in_prop.shape().dim(0).size()); + EXPECT_TRUE(out_prop.shape().unknown_rank()); + } else if (node.op() == "Square" || node.op() == "AddN") { + // These nodes are in the fanout of x: their shapes should be unknown. EXPECT_TRUE(in_prop.shape().unknown_rank()); + EXPECT_TRUE(out_prop.shape().unknown_rank()); } } } @@ -885,7 +894,7 @@ TEST_F(GraphPropertiesTest, FedNodes) { Status s = properties.InferStatically(true); TF_CHECK_OK(s); for (const auto& node : item.graph.node()) { - if (node.name() == "AddN") { + if (node.op() == "Square" || node.op() == "AddN") { const auto in_props = properties.GetInputProperties(node.name()); EXPECT_EQ(1, in_props.size()); const OpInfo::TensorProperties& in_prop = in_props[0]; -- GitLab From 97da160010a47ba37afa1afca914038d3ab0ba55 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 12:31:41 -0800 Subject: [PATCH 0117/1924] Allow the toolchain defaults to be used instead of hard-coding -Os. For example toolchains with clang may set -Oz which is more analogous to gcc's -Os. -Os for clang is closer to -O2. PiperOrigin-RevId: 177347371 --- .../contrib/android/cmake/CMakeLists.txt | 2 ++ tensorflow/core/BUILD | 10 ++++----- tensorflow/tensorflow.bzl | 22 ++++++++++++------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/tensorflow/contrib/android/cmake/CMakeLists.txt b/tensorflow/contrib/android/cmake/CMakeLists.txt index aba356d616..a115d1610e 100644 --- a/tensorflow/contrib/android/cmake/CMakeLists.txt +++ b/tensorflow/contrib/android/cmake/CMakeLists.txt @@ -34,6 +34,8 @@ add_library(lib_tf STATIC IMPORTED ) set_target_properties(lib_tf PROPERTIES IMPORTED_LOCATION ${PREBUILT_DIR}/lib/libtensorflow-core.a) # Change to compile flags should be replicated into bazel build file +# TODO: Consider options other than -O2 for binary size. +# e.g. -Os for gcc, and -Oz for clang. set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DIS_SLIM_BUILD \ -std=c++11 -fno-rtti -fno-exceptions \ -O2 -Wno-narrowing -fomit-frame-pointer \ diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 4ca6fb1631..a1d61a7932 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1008,7 +1008,7 @@ filegroup( cc_library( name = "android_tensorflow_lib_lite", srcs = if_android(["//tensorflow/core:android_srcs"]), - copts = tf_copts() + if_not_android_mips_and_mips64(["-Os"]), + copts = tf_copts(android_optimization_level_override = None), linkopts = ["-lz"], tags = [ "manual", @@ -1096,8 +1096,7 @@ cc_library( cc_library( name = "android_tensorflow_lib_selective_registration", srcs = if_android(["//tensorflow/core:android_srcs"]), - copts = tf_copts() + [ - "-Os", + copts = tf_copts(android_optimization_level_override = None) + [ "-DSUPPORT_SELECTIVE_REGISTRATION", ], tags = [ @@ -1118,8 +1117,7 @@ cc_library( cc_library( name = "android_tensorflow_lib_selective_registration_nortti", srcs = if_android(["//tensorflow/core:android_srcs"]), - copts = tf_copts() + tf_opts_nortti_if_android() + [ - "-Os", + copts = tf_copts(android_optimization_level_override = None) + tf_opts_nortti_if_android() + [ "-DSUPPORT_SELECTIVE_REGISTRATION", ], tags = [ @@ -1198,7 +1196,7 @@ cc_library( "framework/tensor_testutil.h", "util/reporter.h", ], - copts = tf_copts() + ["-Os"], + copts = tf_copts(android_optimization_level_override = None), tags = [ "manual", "notap", diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 8d392fb36d..76ef59484f 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -167,7 +167,19 @@ WIN_COPTS = [ ] # LINT.IfChange -def tf_copts(): +def tf_copts(android_optimization_level_override="-O2"): + # For compatibility reasons, android_optimization_level_override + # is currently only being set for Android. + # To clear this value, and allow the CROSSTOOL default + # to be used, pass android_optimization_level_override=None + android_copts = [ + "-std=c++11", + "-DTF_LEAN_BINARY", + "-Wno-narrowing", + "-fomit-frame-pointer", + ] + if android_optimization_level_override: + android_copts.append(android_optimization_level_override) return ( if_not_windows([ "-DEIGEN_AVOID_STL_ARRAY", @@ -180,13 +192,7 @@ def tf_copts(): + if_android_arm(["-mfpu=neon"]) + if_linux_x86_64(["-msse3"]) + select({ - clean_dep("//tensorflow:android"): [ - "-std=c++11", - "-DTF_LEAN_BINARY", - "-O2", - "-Wno-narrowing", - "-fomit-frame-pointer", - ], + clean_dep("//tensorflow:android"): android_copts, clean_dep("//tensorflow:darwin"): [], clean_dep("//tensorflow:windows"): WIN_COPTS, clean_dep("//tensorflow:windows_msvc"): WIN_COPTS, -- GitLab From 037acadcfc5f2b96a2e9c1653f28131bb91858aa Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 29 Nov 2017 12:56:52 -0800 Subject: [PATCH 0118/1924] Deleted unused method arguments PiperOrigin-RevId: 177350575 --- .../grappler/optimizers/constant_folding.cc | 25 +++---------------- .../grappler/optimizers/constant_folding.h | 6 ++--- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 33a9dddba7..03eaa4a84a 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -190,8 +190,7 @@ Status ConvertShapeToConstant(const string& op, const DataType& type, return Status::OK(); } -Status ConstantFolding::MaterializeShapes(const GrapplerItem& item, - const GraphProperties& properties) { +Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { // We may add some nodes to the graph to encode control dependencies: there is // no need to process these, so only iterate over the nodes of the input // graph. @@ -285,22 +284,6 @@ Status ConstantFolding::MaterializeShapes(const GrapplerItem& item, return Status::OK(); } -bool ShapesEqual(const TensorShapeProto& shape1, - const TensorShapeProto& shape2) { - if (shape1.unknown_rank() || shape2.unknown_rank()) { - return false; - } - if (shape1.dim_size() != shape2.dim_size()) { - return false; - } - for (int i = 0; i < shape1.dim_size(); ++i) { - if (shape1.dim(i).size() != shape2.dim(i).size()) { - return false; - } - } - return true; -} - namespace { bool ExtractShape(const NodeDef& shape_node, const GraphProperties& properties, BCast::Vec* shape, int64* min_id) { @@ -504,7 +487,7 @@ Status ConstantFolding::MaterializeReductionIndices( } Status ConstantFolding::MaterializeConstants( - const GrapplerItem& item, const GraphProperties& properties) { + const GraphProperties& properties) { const int node_count = graph_.node_size(); for (int i = 0; i < node_count; ++i) { NodeDef& node = *graph_.mutable_node(i); @@ -1171,10 +1154,10 @@ Status ConstantFolding::RunOptimizationPass(Cluster* cluster, // graph. That's because it's possible to feed a placeholder with a tensor // of any shape, which could make the static information inconsistent with // the shapes actually fed. - TF_RETURN_IF_ERROR(MaterializeShapes(item, properties)); + TF_RETURN_IF_ERROR(MaterializeShapes(properties)); } if (opt_level_ == RewriterConfig::AGGRESSIVE && s.ok()) { - TF_RETURN_IF_ERROR(MaterializeConstants(item, properties)); + TF_RETURN_IF_ERROR(MaterializeConstants(properties)); } TF_RETURN_IF_ERROR(FoldGraph(output)); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h index f04f413c10..7c5db2a70f 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.h +++ b/tensorflow/core/grappler/optimizers/constant_folding.h @@ -51,16 +51,14 @@ class ConstantFolding : public GraphOptimizer { const GraphDef& optimize_output, double result) override; private: - Status MaterializeShapes(const GrapplerItem& item, - const GraphProperties& properties); + Status MaterializeShapes(const GraphProperties& properties); Status MaterializeBroadcastGradientArgs(const NodeDef& node, const GraphProperties& properties); Status MaterializeReductionIndices(NodeDef* node, const GraphProperties& properties); - Status MaterializeConstants(const GrapplerItem& item, - const GraphProperties& properties); + Status MaterializeConstants(const GraphProperties& properties); bool IsFoldable(const NodeDef& node) const; Status EvaluateNode(const NodeDef& node, -- GitLab From d1aea3be42f6153a970319a298eb55372ab9aa2e Mon Sep 17 00:00:00 2001 From: Igor Saprykin Date: Wed, 29 Nov 2017 13:02:30 -0800 Subject: [PATCH 0119/1924] Clarify the role of replicate_model_fn.Mode better. PiperOrigin-RevId: 177351409 --- .../estimator/python/estimator/replicate_model_fn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py index 6f7f37473f..f5154231da 100644 --- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py +++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py @@ -47,7 +47,11 @@ from tensorflow.python.training import training_util class Mode(object): - """Modes for variables replication used for forcing a particular mode.""" + """Modes for variables replication used for forcing a particular mode. + + Forcing a mode is meant for performance experimentation purposes rather than + for general use cases. + """ AUTO = 0 """Use internal heuristics for choosing the best Mode value. -- GitLab From 197850fb12015f8e01a3b9c4d3e3546fc54aaa0b Mon Sep 17 00:00:00 2001 From: Olivia Nordquist Date: Wed, 29 Nov 2017 13:21:31 -0800 Subject: [PATCH 0120/1924] enabling Tensor._set_shape() to work with the C API PiperOrigin-RevId: 177353959 --- tensorflow/python/client/tf_session.i | 43 ++++++++++++++ tensorflow/python/client/tf_session_helper.cc | 19 +++++++ tensorflow/python/client/tf_session_helper.h | 14 +++++ tensorflow/python/framework/ops.py | 57 +++++++++++++------ 4 files changed, 117 insertions(+), 16 deletions(-) diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 5fa1a7e8fc..c286d5fe47 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -532,6 +532,49 @@ def TF_Reset(target, containers=None, config=None): %unignore TF_GraphGetTensorShapeHelper; %ignore TF_GraphGetTensorShape; +// We use TF_GraphSetTensorShape_wrapper instead of +// TF_GraphSetTensorShape +%ignore TF_GraphSetTensorShape; +%unignore tensorflow; +%unignore TF_GraphSetTensorShape_wrapper; + +// $input is a Python list of ints to a vector for TF_GraphSetTensorShape_wrapper +%typemap(in) (const std::vector& dims) + (std::vector dims_local){ + if ($input != Py_None) { + if (!PyList_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, tensorflow::strings::Printf( + "$symname: expected list but got %s ", Py_TYPE($input)->tp_name).c_str()); + } + size_t size = PyList_Size($input); + for (int i = 0; i < size; ++i) { + PyObject* item = PyList_GetItem($input, i); + dims_local.push_back(PyInt_AS_LONG(item)); + } + $1 = &dims_local; + } else { + $1 = nullptr; + } +} + +// We use TF_GraphGetTensorShape_wrapper instead of +// TF_GraphGetTensorShape +%ignore TF_GraphGetTensorShape; +%unignore tensorflow; +%unignore TF_GraphGetTensorShape_wrapper; + +// Build a Python list of ints and return it. +%typemap(out) std::vector tensorflow::TF_GraphGetTensorShape_wrapper { + $result = PyList_New($1.size()); + if (!$result) { + SWIG_exception_fail(SWIG_MemoryError, "$symname: couldn't create list"); + } + + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, PyInt_FromLong($1[i])); + } +} + %include "tensorflow/python/client/tf_session_helper.h" %unignoreall diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc index ad982e5dd8..e4bf09a0ca 100644 --- a/tensorflow/python/client/tf_session_helper.cc +++ b/tensorflow/python/client/tf_session_helper.cc @@ -407,4 +407,23 @@ TF_Function* TF_GraphToFunction_wrapper( opts, description, out_status); } +void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, + const std::vector& dims, + bool unknown_shape, TF_Status* status) { + if (unknown_shape) { + TF_GraphSetTensorShape(graph, output, nullptr, -1, status); + return; + } + TF_GraphSetTensorShape(graph, output, dims.data(), dims.size(), status); +} + +std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, + TF_Output output, + int num_dims, + TF_Status* status) { + std::vector dims(num_dims); + TF_GraphGetTensorShape(graph, output, dims.data(), num_dims, status); + return dims; +} + } // namespace tensorflow diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h index 6ed08d3a58..bb7171db31 100644 --- a/tensorflow/python/client/tf_session_helper.h +++ b/tensorflow/python/client/tf_session_helper.h @@ -168,6 +168,20 @@ TF_Function* TF_GraphToFunction_wrapper( const std::vector& inputs, const std::vector& outputs, const NameVector& output_names, const TF_FunctionOptions* opts, const char* description, TF_Status* out_status); + +// Set the shape of output. If unknown is true, `num_dims` must be set to +// -1 and `dims` is set to nullptr. +void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, + const std::vector& dims, + bool unknown_shape, TF_Status* status); + +// Return the shape of output. `num_dims` should be the output of +// TF_GraphGetTensorNumDims. If `num_dims = -1`, this should not be called. +std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, + TF_Output output, + int num_dims, + TF_Status* status); + } // namespace tensorflow #endif // TENSORFLOW_PYTHON_CLIENT_TF_SESSION_HELPER_H_ diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 2217513966..975a1c87ec 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -374,6 +374,19 @@ class Tensor(_TensorLike): A `TensorShape` representing the shape of this tensor. """ + if _USE_C_API: + graph = self._op._graph._c_graph # pylint: disable=protected-access + with errors.raise_exception_on_not_ok_status() as status: + num_dims = c_api.TF_GraphGetTensorNumDims(graph, self._as_tf_output(), + status) + if num_dims == -1: + dim_list = None + else: + with errors.raise_exception_on_not_ok_status() as status: + dim_list = c_api.TF_GraphGetTensorShape_wrapper( + graph, self._as_tf_output(), num_dims, status) + dim_list = [None if i == -1 else i for i in dim_list] + return tensor_shape.TensorShape(dim_list) return self._shape def __iter__(self): @@ -393,8 +406,8 @@ class Tensor(_TensorLike): yield self[i] def _shape_as_list(self): - if self._shape.ndims is not None: - return [dim.value for dim in self._shape.dims] + if self.shape.ndims is not None: + return [dim.value for dim in self.shape.dims] else: return None @@ -410,7 +423,7 @@ class Tensor(_TensorLike): Returns: Integer rank or None """ - return self._shape.ndims + return self.shape.ndims def get_shape(self): """Alias of Tensor.shape.""" @@ -441,14 +454,35 @@ class Tensor(_TensorLike): ``` Args: - shape: A `TensorShape` representing the shape of this tensor. + shape: A `TensorShape` representing the shape of this tensor, a + `TensorShapeProto`, a list, a tuple, or None. Raises: ValueError: If `shape` is not compatible with the current shape of this tensor. """ - # TODO(skyewm): call C API - self._shape = self._shape.merge_with(shape) + if not _USE_C_API: + self._shape = self._shape.merge_with(shape) # pylint: disable=protected-access + return + if not isinstance(shape, tensor_shape.TensorShape): + shape = tensor_shape.TensorShape(shape) + dim_list = [] + if shape.dims is None: + unknown_shape = True + else: + unknown_shape = False + for dim in shape.dims: + if dim.value is None: + dim_list.append(-1) + else: + dim_list.append(dim.value) + with errors.raise_exception_on_not_ok_status() as status: + c_api.TF_GraphSetTensorShape_wrapper( + self._op._graph._c_graph, # pylint: disable=protected-access + self._as_tf_output(), + dim_list, + unknown_shape, + status) @property def value_index(self): @@ -4517,15 +4551,11 @@ def control_dependencies(control_inputs): See @{tf.Graph.control_dependencies} for more details. - When eager execution is enabled, any callable object in the `control_inputs` - list will be called. - Args: control_inputs: A list of `Operation` or `Tensor` objects which must be executed or computed before running the operations defined in the context. Can also be `None` to clear the control - dependencies. If eager execution is enabled, any callable object in the - `control_inputs` list will be called. + dependencies. Returns: A context manager that specifies control dependencies for all @@ -4534,11 +4564,6 @@ def control_dependencies(control_inputs): if context.in_graph_mode(): return get_default_graph().control_dependencies(control_inputs) else: - if control_inputs: - # Excute any pending callables. - for control in control_inputs: - if callable(control): - control() return _NullContextmanager() -- GitLab From e00156b36d91019039c9148dc86b64017154564e Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 29 Nov 2017 13:24:28 -0800 Subject: [PATCH 0121/1924] Proper deallocation in the thread-local tape stack. PiperOrigin-RevId: 177354350 --- tensorflow/python/eager/pywrap_tfe_src.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index ce823cb567..b52d71dc6c 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -531,12 +531,9 @@ static PyTypeObject TFE_Py_Tape_Type = { // xcode 7 doesn't define thread_local, so for compatibility we implement our // own. TODO(apassos) remove once we can deprecate xcode 7. #ifndef __APPLE__ -thread_local std::vector* tape_stack = nullptr; std::vector* GetTapeStack() { - if (tape_stack == nullptr) { - tape_stack = new std::vector; - } - return tape_stack; + thread_local std::vector tape_stack; + return &tape_stack; } #else static tensorflow::mutex stack_mu(tensorflow::LINKER_INITIALIZED); -- GitLab From d3a8bf0783754b8f4bbc24274ecd79d4cc3217f0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 13:28:59 -0800 Subject: [PATCH 0122/1924] Added comment/TODO concerning memory use of extract_images_patches. PiperOrigin-RevId: 177354924 --- tensorflow/contrib/kfac/python/ops/fisher_factors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index fbc192f1dc..6c1dd0ae40 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -580,6 +580,9 @@ class ConvDiagonalFactor(DiagonalFactor): # the target entry of _outputs_grads changes with idx.) with _maybe_colocate_with(inputs, self._colocate_cov_ops_with_inputs): filter_height, filter_width, _, _ = self._filter_shape + + # TODO(b/64144716): there is potential here for a big savings in terms of + # memory use. patches = array_ops.extract_image_patches( inputs, ksizes=[1, filter_height, filter_width, 1], @@ -739,6 +742,9 @@ class ConvInputKroneckerFactor(InverseProvidingFactor): # TODO(jamesmartens): factor this patches stuff out into a utility function with _maybe_colocate_with(self._inputs, self._colocate_cov_ops_with_inputs): filter_height, filter_width, in_channels, _ = self._filter_shape + + # TODO(b/64144716): there is potential here for a big savings in terms of + # memory use. patches = array_ops.extract_image_patches( self._inputs, ksizes=[1, filter_height, filter_width, 1], -- GitLab From 19f62f62e5dab41b62b60ac66e7d07c09d55e17a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 13:41:03 -0800 Subject: [PATCH 0123/1924] Re-enable Mul hoisting for aggregations other than Add when input shapes match. PiperOrigin-RevId: 177356621 --- .../optimizers/arithmetic_optimizer.cc | 107 ++++++++++++------ .../optimizers/arithmetic_optimizer_test.cc | 106 +++++++++-------- 2 files changed, 135 insertions(+), 78 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 930d122234..6861a51795 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -253,6 +253,30 @@ bool IsNumberType(DataType dtype) { const char kOutputShapesAttr[] = "_output_shapes"; +PartialTensorShape GetInputShape(const string& input, const NodeMap& node_map) { + int output_pos; + string node_name = ParseNodeName(input, &output_pos); + const NodeDef* input_node = node_map.GetNode(node_name); + return input_node->attr().at(kOutputShapesAttr).list().shape(output_pos); +} + +bool ShapesEqual(const string& input_x, const string& input_y, + const NodeMap& node_map) { + PartialTensorShape x_shape = GetInputShape(input_x, node_map); + PartialTensorShape y_shape = GetInputShape(input_y, node_map); + if (x_shape.unknown_rank() || y_shape.unknown_rank() || + x_shape.dims() != y_shape.dims()) { + return false; + } + for (int i = 0; i < x_shape.dims(); ++i) { + if (x_shape.dim_size(i) == -1 || y_shape.dim_size(i) == -1 || + x_shape.dim_size(i) != y_shape.dim_size(i)) { + return false; + } + } + return true; +} + // Returns whether `reshape` is an identity op. The tensor that `reshape` // reshapes is the `output_pos`-th output of node `input`. bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input, @@ -868,8 +892,11 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses( // multiplication over addition to hoist common factors out of aggregate nodes // where all the inputs are Mul nodes. This pattern occurs frequently in // regularization terms for the gradients during training. - // TODO(rmlarsen): Check shapes and enable for AddN. - if (IsAdd(*node) && NumNonControlInputs(*node) > 1 && + // For example, we can rewrite an expression of the form: + // AddN(Mul(x, y1), Mul(y2, x), Mul(x, y3), ... Mul(x, yn)) + // to the following: + // Mul(x, AddN(y1, y2, y3, ... yn)) + if (IsAggregate(*node) && NumNonControlInputs(*node) > 1 && !OptimizedNodeExists(StrCat(node->name(), "_hoist_add"))) { // Determine the set of common factors if the input nodes are all Mul nodes. std::set common_factors; @@ -899,24 +926,15 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses( } if (common_factors.size() == 1) { const string& common_factor = *common_factors.begin(); - // In this case we have an expression of the form - // AddN(Mul(x, y1), Mul(y2, x), Mul(x, y3), ... Mul(x, yn)) - // that can be rewritten as - // Mul(x, AddN(y1, y2, y3, ... yn)) - - // 1. Use a copy of the first Mul node for the outer multiplication. - NodeDef* new_mul_node = AddNode(StrCat(node->name(), "_hoist_mul"), - node_map_->GetNode(node->input(0))); - NodeDef* new_add_node = AddNode(StrCat(node->name(), "_hoist_add"), node); - new_mul_node->set_device(node->device()); - new_mul_node->set_input(0, common_factor); - node_map_->AddOutput(common_factor, new_mul_node->name()); - new_mul_node->set_input(1, new_add_node->name()); - node_map_->AddOutput(new_add_node->name(), new_mul_node->name()); - - // 2. Hoist non-shared factors up into the new AddN node. - nodes_to_simplify->PushBack(new_add_node); - for (int i = 0; i < node->input_size(); ++i) { + + // Gather up the non-shared factors (the y's in the example). + // Unless the aggregation is Add, we have to make sure that all the y's + // have the same shape since the other aggregation ops do not support + // broadcasting. + std::vector unique_factors; + unique_factors.reserve(node->input_size()); + bool shapes_match = true; + for (int i = 0; i < node->input_size() && shapes_match; ++i) { const string& input = node->input(i); if (IsControlInput(input)) { break; @@ -924,15 +942,41 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses( const NodeDef* mul_node = node_map_->GetNode(input); const int unique_factor_index = mul_node->input(0) == common_factor ? 1 : 0; - const string unique_factor = mul_node->input(unique_factor_index); - new_add_node->set_input(i, unique_factor); + unique_factors.push_back(mul_node->input(unique_factor_index)); + if (i > 0 && !IsAdd(*node)) { + shapes_match = ShapesEqual(unique_factors.front(), + unique_factors.back(), *node_map_); + } } - // 4. Add frame dependencies that the original node might have had. - AddFrameControlDeps(node, {new_add_node, new_mul_node}, common_factor, - {new_add_node}); + if (shapes_match) { + // 1. Use a copy of the first Mul node for the outer multiplication. + NodeDef* new_mul_node = AddNode(StrCat(node->name(), "_hoist_mul"), + node_map_->GetNode(node->input(0))); + NodeDef* new_add_node = + AddNode(StrCat(node->name(), "_hoist_add"), node); + new_mul_node->set_device(node->device()); + new_mul_node->set_input(0, common_factor); + node_map_->AddOutput(common_factor, new_mul_node->name()); + new_mul_node->set_input(1, new_add_node->name()); + node_map_->AddOutput(new_add_node->name(), new_mul_node->name()); + + // 2. Hoist non-shared factors up into the new AddN node. + nodes_to_simplify->PushBack(new_add_node); + for (int i = 0; i < node->input_size(); ++i) { + const string& input = node->input(i); + if (IsControlInput(input)) { + break; + } + new_add_node->set_input(i, unique_factors[i]); + } - return new_mul_node->name(); + // 3. Add frame dependencies that the original node might have had. + AddFrameControlDeps(node, {new_add_node, new_mul_node}, common_factor, + {new_add_node}); + + return new_mul_node->name(); + } } } @@ -1064,13 +1108,10 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/, int num_frames; TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_, &frame_map_, &num_frames)); - if (opt_level_ == RewriterConfig::AGGRESSIVE) { - graph_properties_.reset(new GraphProperties(item)); - // Shapes are only needed in aggressive mode. - TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false)); - TF_RETURN_IF_ERROR( - graph_properties_->AnnotateOutputShapes(optimized_graph_)); - } + graph_properties_.reset(new GraphProperties(item)); + // Shapes are only needed in aggressive mode. + TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false)); + TF_RETURN_IF_ERROR(graph_properties_->AnnotateOutputShapes(optimized_graph_)); // Perform the optimizations. DedupComputations(); diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index e8a18ff9d9..80f42694d9 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -32,6 +32,21 @@ string OptimizedName(const string& name) { return AddPrefixToNodeName(name, kArithmeticOptimizer); } +void VerifyGraphsMatch(const GraphDef& original_graph, + const GraphDef& optimized_graph, int line) { + EXPECT_EQ(original_graph.node_size(), optimized_graph.node_size()) << line; + for (int i = 0; i < original_graph.node_size(); ++i) { + const NodeDef& original = original_graph.node(i); + const NodeDef& optimized = optimized_graph.node(i); + EXPECT_EQ(original.name(), optimized.name()) << line; + EXPECT_EQ(original.op(), optimized.op()) << line; + EXPECT_EQ(original.input_size(), optimized.input_size()) << line; + for (int j = 0; j < original.input_size(); ++j) { + EXPECT_EQ(original.input(j), optimized.input(j)) << line; + } + } +} + class ArithmeticOptimizerTest : public ::testing::Test {}; TEST_F(ArithmeticOptimizerTest, NoOp) { @@ -44,18 +59,7 @@ TEST_F(ArithmeticOptimizerTest, NoOp) { GraphDef output; Status status = optimizer.Optimize(nullptr, item, &output); TF_EXPECT_OK(status); - - EXPECT_EQ(item.graph.node_size(), output.node_size()); - for (int i = 0; i < item.graph.node_size(); ++i) { - const NodeDef& original = item.graph.node(i); - const NodeDef& optimized = output.node(i); - EXPECT_EQ(original.name(), optimized.name()); - EXPECT_EQ(original.op(), optimized.op()); - EXPECT_EQ(original.input_size(), optimized.input_size()); - for (int j = 0; j < original.input_size(); ++j) { - EXPECT_EQ(original.input(j), optimized.input(j)); - } - } + VerifyGraphsMatch(item.graph, output, __LINE__); } TEST_F(ArithmeticOptimizerTest, OpDedupping) { @@ -398,39 +402,51 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) { } TEST_F(ArithmeticOptimizerTest, HoistFactor) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2}); - Output y1 = ops::Const(s.WithOpName("y1"), {3.0f, 4.0f}, {1, 2}); - Output y2 = ops::Const(s.WithOpName("y2"), {5.0f, 6.0f}, {1, 2}); - Output mul1 = ops::Mul(s.WithOpName("mul1"), x, y1); - Output mul2 = ops::Mul(s.WithOpName("mul2"), y2, x); - Output add = ops::Add(s.WithOpName("add"), mul1, mul2); - Output id = ops::Identity(s.WithOpName("id"), add); - - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - - ArithmeticOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(nullptr, item, &output); - TF_EXPECT_OK(status); - // Run the optimizer twice to make sure the rewrite is idempotent. - item.graph.Swap(&output); - status = optimizer.Optimize(nullptr, item, &output); - TF_EXPECT_OK(status); - - EXPECT_EQ(9, output.node_size()); - const NodeDef& new_add = output.node(8); - EXPECT_EQ(OptimizedName("add_hoist_add"), new_add.name()); - EXPECT_EQ("y1", new_add.input(0)); - EXPECT_EQ("y2", new_add.input(1)); - const NodeDef& new_mul = output.node(7); - EXPECT_EQ(OptimizedName("add_hoist_mul"), new_mul.name()); - EXPECT_EQ("x", new_mul.input(0)); - EXPECT_EQ(OptimizedName("add_hoist_add"), new_mul.input(1)); - const NodeDef& new_id = output.node(6); - EXPECT_EQ("id", new_id.name()); - EXPECT_EQ(OptimizedName("add_hoist_mul"), new_id.input(0)); + for (bool matching_shapes : {true, false}) { + for (bool use_addn : {true, false}) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2}); + Output y1 = ops::Const(s.WithOpName("y1"), {3.0f, 4.0f}, {1, 2}); + Output y2 = matching_shapes + ? ops::Const(s.WithOpName("y2"), {5.0f, 6.0f}, {1, 2}) + : ops::Const(s.WithOpName("y2"), {5.0f}, {1, 1}); + Output mul1 = ops::Mul(s.WithOpName("mul1"), x, y1); + Output mul2 = ops::Mul(s.WithOpName("mul2"), y2, x); + Output id = + use_addn ? ops::Identity(s.WithOpName("id"), + ops::AddN(s.WithOpName("add"), {mul1, mul2})) + : ops::Identity(s.WithOpName("id"), + ops::Add(s.WithOpName("add"), mul1, mul2)); + + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + ArithmeticOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + // Run the optimizer twice to make sure the rewrite is idempotent. + item.graph.Swap(&output); + status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + if (use_addn && !matching_shapes) { + VerifyGraphsMatch(item.graph, output, __LINE__); + } else { + EXPECT_EQ(9, output.node_size()); + const NodeDef& new_add = output.node(8); + EXPECT_EQ(OptimizedName("add_hoist_add"), new_add.name()); + EXPECT_EQ("y1", new_add.input(0)); + EXPECT_EQ("y2", new_add.input(1)); + const NodeDef& new_mul = output.node(7); + EXPECT_EQ(OptimizedName("add_hoist_mul"), new_mul.name()); + EXPECT_EQ("x", new_mul.input(0)); + EXPECT_EQ(OptimizedName("add_hoist_add"), new_mul.input(1)); + const NodeDef& new_id = output.node(6); + EXPECT_EQ("id", new_id.name()); + EXPECT_EQ(OptimizedName("add_hoist_mul"), new_id.input(0)); + } + } + } } TEST_F(ArithmeticOptimizerTest, FuseConjAndTranspose) { -- GitLab From 48347ee4105d78d8f36ba8645953b75cb5280c4c Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Wed, 29 Nov 2017 13:46:24 -0800 Subject: [PATCH 0124/1924] Simplify const node creation. PiperOrigin-RevId: 177357416 --- tensorflow/core/grappler/optimizers/BUILD | 5 + .../grappler/optimizers/layout_optimizer.cc | 218 +++++++----------- .../optimizers/layout_optimizer_test.cc | 75 +++++- .../python/grappler/layout_optimizer_test.py | 2 +- 4 files changed, 169 insertions(+), 131 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 5d9eb8e0b1..24e6f8847a 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -332,6 +332,11 @@ tf_cc_test( deps = [ ":layout_optimizer", "//tensorflow/cc:cc_ops", + "//tensorflow/cc:cc_ops_internal", + "//tensorflow/core:all_kernels", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", "//tensorflow/core:test_main", diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index 1b8046b787..ef4b015295 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -69,6 +69,8 @@ std::set GetOpsFormatSupported() { return ops_format_supported; } +// TODO(yaozhang): enable SumProcessor with auto-tuning. Currently disabled +// because of the worse performance in some cases. std::set GetOpsFormatAgnostic() { std::set ops_format_agnostic = {"Add", "AddN", @@ -88,7 +90,7 @@ std::set GetOpsFormatAgnostic() { "Split", "SquaredDifference", "Squeeze", - "Sub"}; + /*"Sum",*/ "Sub"}; return ops_format_agnostic; } @@ -186,33 +188,6 @@ class GraphProcessor { return node; } - NodeDef* AddNodeReductionConst(const string& name, const string& device) { - NodeDef* node = graph_->add_node(); - node_map_->AddNode(name, node); - node->set_name(name); - node->set_op("Const"); - AttrValue attr_data_type; - attr_data_type.set_type(DT_INT32); - node->mutable_attr()->insert({"dtype", attr_data_type}); - - AttrValue attr_tensor; - Tensor tensor(DT_INT32, TensorShape({3})); - std::vector axis = {0, 2, 3}; - for (int i = 0; static_cast(i) < axis.size(); i++) { - tensor.flat()(i) = axis[i]; - } - tensor.AsProtoTensorContent(attr_tensor.mutable_tensor()); - node->mutable_attr()->insert({"value", attr_tensor}); - string device_name; - if (device.empty()) { - device_name = virtual_placer_.get_canonical_device_name(*node); - } else { - device_name = device; - } - node->set_device(device_name); - return node; - } - const VirtualPlacer& virtual_placer_; const std::unordered_set& nodes_to_preserve_; GraphDef* graph_; @@ -370,10 +345,20 @@ class NodeProcessor : public GraphProcessor { LOG(ERROR) << "Failed to parse TensorProto."; } if (tensor.dims() == 1) { - int c = tensor.flat()(3); - tensor.flat()(3) = tensor.flat()(2); - tensor.flat()(2) = tensor.flat()(1); - tensor.flat()(1) = c; + if (tensor.flat().size() == 4) { + int c = tensor.flat()(3); + tensor.flat()(3) = tensor.flat()(2); + tensor.flat()(2) = tensor.flat()(1); + tensor.flat()(1) = c; + } else if (tensor.flat().size() == 3) { + tensor.flat()(0) = 0; + tensor.flat()(1) = 2; + tensor.flat()(2) = 3; + } else { + return Status(error::INVALID_ARGUMENT, + strings::StrCat("Unsupported tensor size: ", + tensor.flat().size())); + } } else if (tensor.dims() == 2) { for (int i = 0; i < 2; i++) { int c = tensor.matrix()(3, i); @@ -394,7 +379,9 @@ class NodeProcessor : public GraphProcessor { Status UpdateAttrValueOfInput(int input_index) { auto input_node = node_map_->GetNode(node_->input(input_index)); // We created a copy of the node, so that we don't modify the original node, - // which might be used elsewhere. + // which might be used elsewhere. Note that this copy also copies the + // control dependency input in the case this node is inside a loop, + // to ensure added_node is in the same frame with node_. NodeDef* added_node = graph_->add_node(); *added_node = *input_node; string base_name = strings::StrCat(node_->name(), "-", input_node->name()); @@ -411,6 +398,14 @@ class NodeProcessor : public GraphProcessor { return input_pos; } + virtual std::set GetOutputPos() const { + // For most nodes, no need to process control nodes or nodes that use an + // output other than the first output: only the first output is of + // 4D NCHW/NHWC format and thus relevant here. + std::set output_pos = {0}; + return output_pos; + } + NodeDef* AddNodeTranspose(const string& node_name, const string& input_name, const string& const_name, DataType data_type, const TensorShapeProto& input_shape, @@ -476,37 +471,28 @@ class NodeProcessor : public GraphProcessor { auto outputs = node_map_->GetOutputs(node_->name()); string const_name = GetOrAddNodePermNCHWToNHWC(); for (const auto& output : outputs) { - string base_name = strings::StrCat(node_->name(), "-", output->name()); - string node_name = - AddPrefixToNodeName(base_name, kTransposeNCHWToNHWC, "-"); - // TODO(yaozhang): handle the rare case where node A is connected to more - // than one input of node B. - auto it = std::find_if(output->mutable_input()->begin(), - output->mutable_input()->end(), - [this](const string& input) { - string node_name = NodeName(input); - return node_name.compare(node_->name()) == 0; - }); - if (it == output->mutable_input()->end()) { - return Status(error::INVALID_ARGUMENT, - strings::StrCat("Expect ", node_->name(), - " to be an input of ", output->name())); - } - int output_pos = NodePosition(*it); - // No need to process control nodes or nodes that use an output - // other than the first output: only the first output is of 4D NCHW/NHWC - // format and thus relevant here. - if (output_pos != 0) { - continue; + for (int i = 0; i < output->input_size(); i++) { + auto& input = *output->mutable_input(i); + int input_port; + string input_name = ParseNodeName(input, &input_port); + auto output_pos = GetOutputPos(); + if (input_name == node_->name() && + output_pos.find(input_port) != output_pos.end()) { + string base_name = + strings::StrCat(node_->name(), "-", output->name(), "-", i); + string node_name = + AddPrefixToNodeName(base_name, kTransposeNCHWToNHWC, "-"); + TF_RETURN_IF_ERROR(HasAttribute(*node_, "T")); + TF_RETURN_IF_ERROR(HasAttribute(*node_, "_output_shapes")); + AddNodeTranspose( + node_name, input, const_name, node_->attr().at("T").type(), + node_->attr().at("_output_shapes").list().shape(0), false); + input = node_name; + node_map_->AddOutput(node_->name(), node_name); + node_map_->AddOutput(node_name, output->name()); + } } - TF_RETURN_IF_ERROR(HasAttribute(*node_, "T")); - TF_RETURN_IF_ERROR(HasAttribute(*node_, "_output_shapes")); - AddNodeTranspose( - node_name, node_->name(), const_name, node_->attr().at("T").type(), - node_->attr().at("_output_shapes").list().shape(0), false); - *it = node_name; - node_map_->UpdateOutput(node_->name(), output->name(), node_name); - node_map_->AddOutput(node_name, output->name()); + node_map_->RemoveOutput(node_->name(), output->name()); } return Status::OK(); } @@ -948,7 +934,7 @@ class ConcatProcessor : public AgnosticNodeProcessor { } Status CustomizedProcessing() override { - string concat_const_name = GetOrAddNodeConcatConst(); + string concat_const_name = AddNodeConcatConst()->name(); node_map_->AddOutput(concat_const_name, node_->name()); *node_->mutable_input(axis_node_pos_) = concat_const_name; return Status::OK(); @@ -956,8 +942,14 @@ class ConcatProcessor : public AgnosticNodeProcessor { bool IsAlongDimC() const { auto axis_node = node_map_->GetNode(node_->input(axis_node_pos_)); + if (!IsConstant(*axis_node)) { + return false; + } if (axis_node->attr().find("value") != axis_node->attr().end()) { - return axis_node->attr().at("value").tensor().int_val(0) == 3; + auto tensor = axis_node->attr().at({"value"}).tensor(); + if (tensor.tensor_shape().dim_size() == 0 && tensor.int_val_size() == 1) { + return tensor.int_val(0) == 3; + } } return false; } @@ -965,28 +957,18 @@ class ConcatProcessor : public AgnosticNodeProcessor { int axis_node_pos_; private: - NodeDef* AddNodeConcatConst(const string& suffix, const string& depended_node, - const string& device) { - auto const_node = AddNodeConstScalar( - strings::StrCat(kConcatConst, "-", suffix), device, DT_INT32, 1); - // This is to ensure the concat node and the const node are - // in the same frame. - *const_node->add_input() = AsControlDependency(depended_node); - return const_node; - } - - string GetOrAddNodeConcatConst() { - string const_name; - if (is_in_frame_) { - int value_node_pos = (axis_node_pos_ == 0) ? 1 : 0; - auto const_node = AddNodeConcatConst( - node_->name(), NodeName(node_->input(value_node_pos)), - node_->device()); - const_name = const_node->name(); - } else { - const_name = kConcatConst; - } - return const_name; + NodeDef* AddNodeConcatConst() { + auto axis_node = node_map_->GetNode(node_->input(axis_node_pos_)); + // We created a copy of the node, so that we don't modify the original node, + // which might be used elsewhere. Note that this copy also copies the + // control dependency input in the case this node is inside a loop, + // to ensure added_node is in the same frame with node_. + auto added_node = graph_->add_node(); + *added_node = *axis_node; + added_node->set_name(strings::StrCat(kConcatConst, "-", node_->name())); + added_node->mutable_attr()->at({"value"}).mutable_tensor()->set_int_val(0, + 1); + return added_node; } }; @@ -1036,6 +1018,16 @@ class SplitProcessor : public AgnosticNodeProcessor { return input_pos; } + std::set GetOutputPos() const override { + std::set output_pos{0}; + if (HasAttribute(*node_, "num_split").ok()) { + for (int i = 1; i < node_->attr().at("num_split").i(); i++) { + output_pos.insert(i); + } + } + return output_pos; + } + Status CustomizedProcessing() override { string split_const_name = AddNodeSplitConst()->name(); node_map_->AddOutput(split_const_name, node_->name()); @@ -1073,7 +1065,7 @@ class SplitProcessor : public AgnosticNodeProcessor { // We created a copy of the node, so that we don't modify the original node, // which might be used elsewhere. Note that this copy also copies the // control dependency input in the case this node is inside a loop, - // to ensure added_node is in the same frame with the Split node. + // to ensure added_node is in the same frame with node_. NodeDef* added_node = graph_->add_node(); *added_node = *dim_node; added_node->set_name(strings::StrCat(kSplitConst, "-", node_->name())); @@ -1329,20 +1321,21 @@ class SumProcessor : public AgnosticNodeProcessor { Status AddLayoutTransposeToOutputs() override { return Status::OK(); } - Status CustomizedProcessing() override { - node_map_->AddOutput(kReductionConst, node_->name()); - *node_->mutable_input(1) = GetOrAddNodeReductionConst(); - return Status::OK(); - } + Status CustomizedProcessing() override { return UpdateAttrValueOfInput(1); } private: bool IsAlongDimNHW() const { - NodeDef* node = node_map_->GetNode(node_->input(1)); + NodeDef* reduction_indices = node_map_->GetNode(node_->input(1)); + if (!IsConstant(*reduction_indices)) { + return false; + } Tensor tensor; - if (node->attr().find({"value"}) == node->attr().end()) { + if (reduction_indices->attr().find({"value"}) == + reduction_indices->attr().end()) { return false; } - auto success = tensor.FromProto(node->attr().at({"value"}).tensor()); + auto success = + tensor.FromProto(reduction_indices->attr().at({"value"}).tensor()); if (!success) { LOG(ERROR) << "Failed to parse TensorProto."; return false; @@ -1356,29 +1349,6 @@ class SumProcessor : public AgnosticNodeProcessor { } return false; } - - NodeDef* AddNodeReductionConst(const string& suffix, - const string& depended_node, - const string& device) { - auto const_node = GraphProcessor::AddNodeReductionConst( - strings::StrCat(kReductionConst, "-", suffix), device); - // This is to ensure the Sum node and the const node are in the - // same frame. - *const_node->add_input() = AsControlDependency(depended_node); - return const_node; - } - - string GetOrAddNodeReductionConst() { - string const_name; - if (is_in_frame_) { - auto const_node = AddNodeReductionConst( - node_->name(), NodeName(node_->input(0)), node_->device()); - const_name = const_node->name(); - } else { - const_name = kReductionConst; - } - return const_name; - } }; class DataLayoutOptimizer : GraphProcessor { @@ -1409,18 +1379,10 @@ class DataLayoutOptimizer : GraphProcessor { return AddNodePermConst(kPermNCHWToNHWC, "", {0, 2, 3, 1}); } - NodeDef* AddNodeConcatConst() { - return AddNodeConstScalar(kConcatConst, "", DT_INT32, 1); - } - NodeDef* AddNodeGatherAxisConst() { return AddNodeConstScalar(kGatherAxisConst, "", DT_INT32, 0); } - NodeDef* AddNodeReductionConst() { - return GraphProcessor::AddNodeReductionConst(kReductionConst, ""); - } - // Expand all nodes which is in NHWC, but supports NCHW or is layout agnostic. Status Expand() { int node_size_original = graph_->node_size(); @@ -1474,9 +1436,7 @@ class DataLayoutOptimizer : GraphProcessor { if (graph_->node_size() > node_size_original) { NodeDef* n = AddNodePermNHWCToNCHW(); n = AddNodePermNCHWToNHWC(); - n = AddNodeConcatConst(); n = AddNodeGatherAxisConst(); - n = AddNodeReductionConst(); std::set ops_format_agnostic = GetOpsFormatAgnostic(); for (int i = 0; i < graph_->node_size(); i++) { if (ops_format_agnostic.find(graph_->node(i).op()) != diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index 8c89f6744b..e8f7b8ac3c 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -495,7 +495,80 @@ TEST_F(LayoutOptimizerTest, SplitNonConstDim) { auto split_node = node_map.GetNode("split"); EXPECT_EQ(split_node->input(0), "i1"); EXPECT_EQ(split_node->input(1), - "LayoutOptimizerTransposeNCHWToNHWC-Conv2D-split"); + "LayoutOptimizerTransposeNCHWToNHWC-Conv2D-split-1"); +} + +TEST_F(LayoutOptimizerTest, SplitSamePortToMultipleInputsOfSameNode) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto axis = ops::Const(s.WithOpName("axis"), 3); + auto split = ops::Split(s.WithOpName("split"), axis, conv, 2); + auto concat = + ops::Concat(s.WithOpName("concat"), {split[1], split[1], split[1]}, axis); + auto o = ops::Identity(s.WithOpName("o"), concat); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto concat_node = node_map.GetNode("concat"); + EXPECT_EQ(concat_node->input(0), "split:1"); + EXPECT_EQ(concat_node->input(1), "split:1"); + EXPECT_EQ(concat_node->input(2), "split:1"); + EXPECT_EQ(concat_node->input(3), "LayoutOptimizerConcatConst-concat"); + auto concat_dim = node_map.GetNode("LayoutOptimizerConcatConst-concat"); + EXPECT_EQ(concat_dim->attr().at({"value"}).tensor().int_val(0), 1); +} + +TEST_F(LayoutOptimizerTest, Concat) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto axis = ops::Const(s.WithOpName("axis"), 3); + auto split = ops::Split(s.WithOpName("split"), axis, conv, 2); + auto concat = ops::Concat(s.WithOpName("concat"), {split[0], split[1]}, axis); + auto o = ops::Identity(s.WithOpName("o"), concat); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto concat_node = node_map.GetNode("concat"); + EXPECT_EQ(concat_node->input(0), "split"); + EXPECT_EQ(concat_node->input(1), "split:1"); + EXPECT_EQ(concat_node->input(2), "LayoutOptimizerConcatConst-concat"); + auto concat_dim = node_map.GetNode("LayoutOptimizerConcatConst-concat"); + EXPECT_EQ(concat_dim->attr().at({"value"}).tensor().int_val(0), 1); +} + +TEST_F(LayoutOptimizerTest, Sum) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto reduction_indices = + ops::Const(s.WithOpName("reduction_indices"), {0, 1, 2}, {3}); + auto sum = ops::Sum(s.WithOpName("sum"), conv, reduction_indices); + auto o = ops::Identity(s.WithOpName("o"), sum); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + // TODO(yaozhang): enable SumProcessor with auto-tuning. Currently disabled + // because of the worse performance in some cases. + /* + NodeMap node_map(&output); + auto sum_node = node_map.GetNode("sum"); + EXPECT_EQ(sum_node->input(0), "Conv2D"); + EXPECT_EQ(sum_node->input(1), "LayoutOptimizer-sum-reduction_indices"); + auto sum_const = node_map.GetNode("LayoutOptimizer-sum-reduction_indices"); + Tensor tensor; + EXPECT_TRUE( + tensor.FromProto(sum_const->mutable_attr()->at({"value"}).tensor())); + Tensor tensor_expected(DT_INT32, {3}); + test::FillValues(&tensor_expected, {0, 2, 3}); + test::ExpectTensorEqual(tensor_expected, tensor); + */ } } // namespace diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py index 626e0502cb..50735fb567 100644 --- a/tensorflow/python/grappler/layout_optimizer_test.py +++ b/tensorflow/python/grappler/layout_optimizer_test.py @@ -190,7 +190,7 @@ class LayoutOptimizerTest(test.TestCase): self.assertEqual(expected_num_transposes, num_transposes) self.assertIn('LayoutOptimizerTransposeNHWCToNCHW-Conv2D-Reshape-0', nodes) - self.assertIn('LayoutOptimizerTransposeNCHWToNHWC-Relu_1-MaxPool_1', + self.assertIn('LayoutOptimizerTransposeNCHWToNHWC-Relu_1-MaxPool_1-0', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3) -- GitLab From 1d0b07351d901334b33565595d4c23607f11cc27 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 29 Nov 2017 13:58:55 -0800 Subject: [PATCH 0125/1924] Add a way to query a batch scheduler to determine the max task size. A layer on top of the batcher could use this interface to pre-split large tasks that exceed the max batch size. PiperOrigin-RevId: 177359263 --- .../contrib/batching/adaptive_shared_batch_scheduler.h | 2 ++ .../contrib/batching/adaptive_shared_batch_scheduler_test.cc | 1 + tensorflow/contrib/batching/basic_batch_scheduler.h | 4 ++++ tensorflow/contrib/batching/basic_batch_scheduler_test.cc | 1 + tensorflow/contrib/batching/batch_scheduler.h | 4 ++++ tensorflow/contrib/batching/shared_batch_scheduler.h | 5 +++++ tensorflow/contrib/batching/shared_batch_scheduler_test.cc | 1 + 7 files changed, 18 insertions(+) diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h index 6ed177e001..9e32bee505 100644 --- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h @@ -208,6 +208,8 @@ class ASBSQueue : public BatchScheduler { // place any more tasks in this batch. void ReleaseBatch(const ASBSBatch* batch); + size_t max_task_size() const override { return options_.max_batch_size; } + private: std::shared_ptr> scheduler_; const QueueOptions options_; diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc index a07cd6d834..e2aac54eeb 100644 --- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc @@ -186,6 +186,7 @@ TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) { queue_options.max_enqueued_batches = 2; TF_ASSERT_OK( scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); + EXPECT_EQ(10, queue_0->max_task_size()); queue_options.max_batch_size = 0; // Queue must have max_batch_size > 0. EXPECT_FALSE( diff --git a/tensorflow/contrib/batching/basic_batch_scheduler.h b/tensorflow/contrib/batching/basic_batch_scheduler.h index 9d3805fbaf..91065db249 100644 --- a/tensorflow/contrib/batching/basic_batch_scheduler.h +++ b/tensorflow/contrib/batching/basic_batch_scheduler.h @@ -192,6 +192,10 @@ class BasicBatchScheduler : public BatchScheduler { size_t NumEnqueuedTasks() const override; size_t SchedulingCapacity() const override; + size_t max_task_size() const override { + return shared_scheduler_queue_->max_task_size(); + } + private: explicit BasicBatchScheduler( std::unique_ptr> shared_scheduler_queue); diff --git a/tensorflow/contrib/batching/basic_batch_scheduler_test.cc b/tensorflow/contrib/batching/basic_batch_scheduler_test.cc index e020301795..187823151c 100644 --- a/tensorflow/contrib/batching/basic_batch_scheduler_test.cc +++ b/tensorflow/contrib/batching/basic_batch_scheduler_test.cc @@ -73,6 +73,7 @@ TEST(BasicBatchSchedulerTest, Basic) { std::unique_ptr> scheduler; TF_ASSERT_OK( BasicBatchScheduler::Create(options, callback, &scheduler)); + EXPECT_EQ(10, scheduler->max_task_size()); EXPECT_EQ(0, scheduler->NumEnqueuedTasks()); EXPECT_EQ(3 * 10, scheduler->SchedulingCapacity()); TF_ASSERT_OK(ScheduleTask(3, scheduler.get())); diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h index a5072f439a..e18cf6c350 100644 --- a/tensorflow/contrib/batching/batch_scheduler.h +++ b/tensorflow/contrib/batching/batch_scheduler.h @@ -178,6 +178,10 @@ class BatchScheduler { // This method is useful for monitoring, or for guaranteeing a future slot in // the schedule (but being mindful about the caveats listed above). virtual size_t SchedulingCapacity() const = 0; + + // Returns the maximum allowed size of tasks submitted to the scheduler. (This + // is typically equal to a configured maximum batch size.) + virtual size_t max_task_size() const = 0; }; ////////// diff --git a/tensorflow/contrib/batching/shared_batch_scheduler.h b/tensorflow/contrib/batching/shared_batch_scheduler.h index 41a3f99137..1d2158062e 100644 --- a/tensorflow/contrib/batching/shared_batch_scheduler.h +++ b/tensorflow/contrib/batching/shared_batch_scheduler.h @@ -248,6 +248,9 @@ class Queue { // BatchScheduler::SchedulingCapacity(). size_t SchedulingCapacity() const; + // Returns the maximum allowed size of tasks submitted to the queue. + size_t max_task_size() const { return options_.max_batch_size; } + // Called by a thread that is ready to process a batch, to request one from // this queue. Either returns a batch that is ready to be processed, or // nullptr if the queue declines to schedule a batch at this time. If it @@ -338,6 +341,8 @@ class QueueHandle : public BatchScheduler { size_t NumEnqueuedTasks() const override; size_t SchedulingCapacity() const override; + size_t max_task_size() const override { return queue_->max_task_size(); } + private: // The scheduler that owns 'queue_'. std::shared_ptr> scheduler_; diff --git a/tensorflow/contrib/batching/shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/shared_batch_scheduler_test.cc index 3e924ae5f1..3ac79a8fdc 100644 --- a/tensorflow/contrib/batching/shared_batch_scheduler_test.cc +++ b/tensorflow/contrib/batching/shared_batch_scheduler_test.cc @@ -429,6 +429,7 @@ TEST(SharedBatchSchedulerTest, ConstMethods) { queue_options.max_enqueued_batches = max_enqueued_batches; std::unique_ptr> queue; TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); + EXPECT_EQ(2, queue->max_task_size()); EXPECT_EQ(0, queue->NumEnqueuedTasks()); EXPECT_EQ(max_enqueued_batches * 2, queue->SchedulingCapacity()); -- GitLab From cb5a63d8d2b6e049a0a128ba47560f842497db8b Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Wed, 29 Nov 2017 14:01:29 -0800 Subject: [PATCH 0126/1924] Check when session cannot run because its graph was modified With current tensorflow code, if user modifies some operation after session.run() was called, this modification will never make it to the C++ runtime and no errors will be raised leading to silent wrong results. This change adds checks for such cases when C API is enabled. We don't change the code path for C API being disabled because C API should be enabled by default soon. PiperOrigin-RevId: 177359630 --- tensorflow/c/c_api.cc | 38 +++++-- tensorflow/c/c_api_internal.h | 21 +++- tensorflow/c/python_api.cc | 10 ++ tensorflow/python/client/session_test.py | 133 +++++++++++++++++++++++ 4 files changed, 190 insertions(+), 12 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 4fb8ec8e4b..c8b4bfffd4 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -624,6 +624,23 @@ Status MessageToBuffer(const tensorflow::protobuf::Message& in, return Status::OK(); } +void RecordMutation(TF_Graph* graph, const TF_Operation& op, + const char* mutation_type) + EXCLUSIVE_LOCKS_REQUIRED(graph->mu) { + // If any session has already run this node_id, mark this session as + // unrunnable. + for (auto it : graph->sessions) { + if (it.first->last_num_graph_nodes > op.node.id()) { + it.second = FailedPrecondition( + "Operation '", op.node.DebugString(), "' was changed by ", + mutation_type, + " after it was run by a session. Nodes can be mutated " + "only before they are executed by a session. Either don't modify " + "nodes after running them or create a new session."); + } + } +} + // Helpers for loading a TensorFlow plugin (a .so file). Status LoadLibrary(const char* library_filename, void** result, const void** buf, size_t* len); @@ -1744,7 +1761,6 @@ void TF_OperationToNodeDef(TF_Operation* oper, TF_Buffer* output_node_def, TF_Graph::TF_Graph() : graph(tensorflow::OpRegistry::Global()), refiner(graph.versions().producer(), graph.op_registry()), - num_sessions(0), delete_requested(false), parent(nullptr), parent_inputs(nullptr) {} @@ -1754,7 +1770,7 @@ TF_Graph* TF_NewGraph() { return new TF_Graph; } void TF_DeleteGraph(TF_Graph* g) { g->mu.lock(); g->delete_requested = true; - const bool del = g->num_sessions == 0; + const bool del = g->sessions.empty(); g->mu.unlock(); if (del) delete g; } @@ -2324,11 +2340,12 @@ TF_Session* TF_NewSession(TF_Graph* graph, const TF_SessionOptions* opt, Session* session; status->status = NewSession(opt->options, &session); if (status->status.ok()) { + TF_Session* new_session = new TF_Session(session, graph); if (graph != nullptr) { mutex_lock l(graph->mu); - graph->num_sessions += 1; + graph->sessions[new_session] = Status::OK(); } - return new TF_Session(session, graph); + return new_session; } else { DCHECK_EQ(nullptr, session); return nullptr; @@ -2392,7 +2409,7 @@ TF_Session* TF_LoadSessionFromSavedModel( TF_Session* session = new TF_Session(bundle.session.release(), graph); - graph->num_sessions += 1; + graph->sessions[session] = Status::OK(); session->last_num_graph_nodes = graph->graph.num_node_ids(); return session; #endif // __ANDROID__ @@ -2407,8 +2424,8 @@ void TF_DeleteSession(TF_Session* s, TF_Status* status) { TF_Graph* const graph = s->graph; if (graph != nullptr) { graph->mu.lock(); - graph->num_sessions -= 1; - const bool del = graph->delete_requested && graph->num_sessions == 0; + graph->sessions.erase(s); + const bool del = graph->delete_requested && graph->sessions.empty(); graph->mu.unlock(); if (del) delete graph; } @@ -2424,6 +2441,13 @@ static bool ExtendSessionGraphHelper(TF_Session* session, TF_Status* status) { mutex_lock session_lock(session->mu); session->graph->mu.lock(); const Graph& graph = session->graph->graph; + + status->status = session->graph->sessions[session]; + if (!status->status.ok()) { + session->graph->mu.unlock(); + return false; + } + const auto num_nodes = graph.num_node_ids(); if (session->last_num_graph_nodes < num_nodes) { status->status = tensorflow::ValidateNoCycles(session->graph->graph); diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h index bb04e01bee..aac333d9e2 100644 --- a/tensorflow/c/c_api_internal.h +++ b/tensorflow/c/c_api_internal.h @@ -81,12 +81,20 @@ struct TF_Graph { std::unordered_map name_map GUARDED_BY(mu); - // TF_Graph may only / must be deleted when - // num_sessions == 0 && delete_requested == true - - // num_sessions incremented by TF_NewSession, and decremented by + // The keys of this map are all the active sessions using this graph. + // Each value is the current "runnability" status of the corresponding + // session. Under normal conditions all statuses are Status::OK(), but + // if some operation is mutated after it was run by a session (this + // is detected in RecordMutation function), that session is no longer + // safe to run. Its status will contain the error that will be returned + // to the user, should she try running this session. + // + // Sessions are added to this map in TF_NewSession, and removed in // TF_DeleteSession. - int num_sessions GUARDED_BY(mu); + // TF_Graph may only / must be deleted when + // sessions.size() == 0 && delete_requested == true + tensorflow::gtl::FlatMap sessions + GUARDED_BY(mu); bool delete_requested GUARDED_BY(mu); // set true by TF_DeleteGraph // Used to link graphs contained in TF_WhileParams to the parent graph that @@ -167,6 +175,9 @@ TF_Tensor* TF_TensorFromTensor(const Tensor& src, TF_Status* status); Status MessageToBuffer(const tensorflow::protobuf::Message& in, TF_Buffer* out); +void RecordMutation(TF_Graph* graph, const TF_Operation& op, + const char* mutation_type); + } // end namespace tensorflow #endif // TENSORFLOW_C_C_API_INTERNAL_H_ diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc index ba5a9268b4..37629a74ba 100644 --- a/tensorflow/c/python_api.cc +++ b/tensorflow/c/python_api.cc @@ -22,6 +22,7 @@ namespace tensorflow { void AddControlInput(TF_Graph* graph, TF_Operation* op, TF_Operation* input) { mutex_lock l(graph->mu); graph->graph.AddControlEdge(&input->node, &op->node); + RecordMutation(graph, *op, "adding control input"); } void SetAttr(TF_Graph* graph, TF_Operation* op, const char* attr_name, @@ -36,11 +37,13 @@ void SetAttr(TF_Graph* graph, TF_Operation* op, const char* attr_name, mutex_lock l(graph->mu); op->node.AddAttr(attr_name, attr_val); + RecordMutation(graph, *op, "setting attribute"); } void SetRequestedDevice(TF_Graph* graph, TF_Operation* op, const char* device) { mutex_lock l(graph->mu); op->node.set_requested_device(device); + RecordMutation(graph, *op, "setting device"); } void UpdateEdge(TF_Graph* graph, TF_Output new_src, TF_Input dst, @@ -75,6 +78,13 @@ void UpdateEdge(TF_Graph* graph, TF_Output new_src, TF_Input dst, } status->status = graph->graph.UpdateEdge(&new_src.oper->node, new_src.index, &dst.oper->node, dst.index); + + if (status->status.ok()) { + // This modification only updates the destination node for + // the purposes of running this graph in a session. Thus, we don't + // record the source node as being modified. + RecordMutation(graph, *dst.oper, "updating input tensor"); + } } } // namespace tensorflow diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py index f4b0271195..e4545d287b 100644 --- a/tensorflow/python/client/session_test.py +++ b/tensorflow/python/client/session_test.py @@ -28,6 +28,8 @@ import numpy as np import six from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.core.framework import attr_value_pb2 +from tensorflow.core.framework import types_pb2 from tensorflow.core.lib.core import error_codes_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 @@ -1742,5 +1744,136 @@ class SessionTest(test_util.TensorFlowTestCase): self.runTestAddFunctionToSession(server.target) +class GraphMutationTest(test_util.TensorFlowTestCase): + + def testUpdateInputAfterRunning(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess: + self.assertAllEqual(3.0, sess.run(c)) + c.op._update_input(1, a) # pylint: disable=protected-access + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by updating input tensor after it was run'): + sess.run(c) + + # Check that running the graph with a new session is fine + with session.Session(graph=g) as sess2: + self.assertAllEqual(2.0, sess2.run(c)) + + def testSetDeviceAfterRunning(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess: + self.assertAllEqual(3.0, sess.run(c)) + c.op._set_device('/cpu:0') # pylint: disable=protected-access + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by setting device after it was run'): + sess.run(c) + + def testSetAttrAfterRunning(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(1.0, dtype=dtypes.float32) + b = math_ops.cast(a, dtypes.float64) + + with session.Session(graph=g) as sess: + self.assertAllEqual(1.0, sess.run(b)) + b.op._set_attr('DstT', + attr_value_pb2.AttrValue(type=types_pb2.DT_FLOAT)) + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'Cast.*was changed by setting attribute after it was run'): + sess.run(b) + + def testRunModifyRun(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess: + self.assertAllEqual(3.0, sess.run(c)) + + d = b + c + d.op._update_input(0, a) # pylint: disable=protected-access + self.assertAllEqual(3.0, sess.run(c)) + self.assertAllEqual(4.0, sess.run(d)) + + def testRunModifyRunTwoSessions(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess1: + with session.Session(graph=g) as sess2: + self.assertAllEqual(3.0, sess1.run(c)) + self.assertAllEqual(3.0, sess2.run(c)) + + d = b + c + d.op._update_input(0, a) # pylint: disable=protected-access + self.assertAllEqual(3.0, sess2.run(c)) + self.assertAllEqual(4.0, sess2.run(d)) + + d.op._update_input(0, b) # pylint: disable=protected-access + self.assertAllEqual(3.0, sess1.run(c)) + self.assertAllEqual(5.0, sess1.run(d)) + + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by updating input tensor after it was run'): + sess2.run(c) + + def testTwoSessionsOneRunBeforeModification(self): + with ops.Graph().as_default() as g, ops.device('/cpu:0'): + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess1: + with session.Session(graph=g) as sess2: + sess1.run(c) + + c.op._set_device('/cpu:0') # pylint: disable=protected-access + + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by setting device after it was run'): + sess1.run(c) + + # sess2 was not run before modification + self.assertAllEqual(3.0, sess2.run(c)) + + def testTwoSessionsBothRunBeforeModification(self): + with ops.Graph().as_default() as g, ops.device('/cpu:0'): + a = constant_op.constant(1.0) + b = constant_op.constant(2.0) + c = a + b + + with session.Session(graph=g) as sess1: + with session.Session(graph=g) as sess2: + sess1.run(c) + sess2.run(c) + + c.op._set_device('/cpu:0') # pylint: disable=protected-access + + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by setting device after it was run'): + sess1.run(c) + + with self.assertRaisesRegexp( + errors.FailedPreconditionError, + 'add.*was changed by setting device after it was run'): + sess2.run(c) + + if __name__ == '__main__': googletest.main() -- GitLab From d0d85965f3dc92a1572bd0853526c657395dff99 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 29 Nov 2017 14:21:53 -0800 Subject: [PATCH 0127/1924] Add R1 slice tests. PiperOrigin-RevId: 177362829 --- tensorflow/compiler/xla/tests/slice_test.cc | 88 +++++++++++++++------ 1 file changed, 65 insertions(+), 23 deletions(-) diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc index c21124750a..981d075089 100644 --- a/tensorflow/compiler/xla/tests/slice_test.cc +++ b/tensorflow/compiler/xla/tests/slice_test.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -211,6 +212,13 @@ class SliceR1Test : public ClientLibraryTestBase, } }; +string SliceR1TestDataToString(const ::testing::TestParamInfo& data) { + const R1Spec& spec = data.param; + return ::tensorflow::strings::Printf("%lld_%lld_%lld_%lld", spec.input_dim0, + spec.slice_start, spec.slice_limit, + spec.slice_stride); +} + XLA_TEST_P(SliceR1Test, DoIt_F32) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_F64) { Run(GetParam()); } @@ -223,30 +231,64 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_S64) { Run(GetParam()); } -INSTANTIATE_TEST_CASE_P( // - SliceR1TestInstantiation, // - SliceR1Test, // - ::testing::Values( // - R1Spec{10, 0, 0, 1}, // - R1Spec{10, 7, 7, 1}, // - R1Spec{10, 2, 4, 1}, // - R1Spec{10, 2, 4, 2}, // - R1Spec{10, 0, 10, 1}, // - R1Spec{1024, 1024 - 4, 1024, 1}, // - R1Spec{4096, 7, 7 + 1024, 1}, // - R1Spec{10, 0, 10, 2}, // - R1Spec{10, 0, 10, 3}, // - R1Spec{10, 0, 10, 4}, // - R1Spec{10, 0, 10, 5}, // - R1Spec{10, 0, 10, 10}, // - R1Spec{500, 200, 400, 7}, // - R1Spec{4096, 1, 4095, 3}, // - R1Spec{2047, 1024 - 24, 1024 + 160, 31}, // - R1Spec{2047, 1, 2046, 3 * 128}, // - R1Spec{4096, 1024 + 3, 4095, 500}, // - R1Spec{8192, 0, 8192, 1024 * 3 + 400} // - ) // +// Tests for R1 slice ops. +// The format for each testcase is {input size, start, limit, stride}. +// clang-format off +INSTANTIATE_TEST_CASE_P( + SliceR1TestInstantiation, + SliceR1Test, + ::testing::Values( + R1Spec{10, 0, 0, 1}, + R1Spec{10, 7, 7, 1}, + R1Spec{10, 0, 5, 1}, + R1Spec{10, 3, 5, 1}, + R1Spec{10, 0, 10, 1}, + R1Spec{1024, 0, 5, 1}, + R1Spec{1024, 3, 5, 1}, + R1Spec{1024 + 17, 0, 5, 1}, + R1Spec{1024 + 17, 3, 5, 1}, + R1Spec{1024 + 17, 1024, 1024 + 6, 1}, + R1Spec{1024 + 17, 1024 + 1, 1024 + 6, 1}, + R1Spec{1024, 1024 - 4, 1024, 1}, + R1Spec{4 * 1024, 7, 7 + 1024, 1}, + R1Spec{4 * 1024, 0, 4 * 1024, 1}, + R1Spec{4 * 1024, 1, 4 * 1024 - 1, 1}, + R1Spec{4 * 1024, 1024, 3 * 1024, 1}, + R1Spec{4 * 1024, 1024 + 1, 3 * 1024 - 1, 1}, + R1Spec{16 * 1024, 0, 5, 1}, + R1Spec{16 * 1024, 3, 5, 1}, + R1Spec{16 * 1024 + 17, 0, 5, 1}, + R1Spec{16 * 1024 + 17, 3, 5, 1}, + R1Spec{16 * 1024 + 17, 16 * 1024, 16 * 1024 + 6, 1}, + R1Spec{16 * 1024 + 17, 16 * 1024 + 1, 16 * 1024 + 6, 1}, + R1Spec{64 * 1024, 0, 64 * 1024, 1}, + R1Spec{64 * 1024, 1, 64 * 1024 - 1, 1}, + R1Spec{64 * 1024, 1024, 63 * 1024, 1}, + R1Spec{64 * 1024, 1024 + 1, 63 * 1024 - 1, 1}, + R1Spec{64 * 1024, 32 * 1024, 33 * 1024, 1}, + R1Spec{64 * 1024, 32 * 1024 + 1, 33 * 1024 - 1, 1}, +// TODO(b/69425338): This uses too much memory on GPU. +#ifndef XLA_TEST_BACKEND_GPU + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1}, + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1}, + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1}, +#endif + R1Spec{10, 2, 4, 2}, + R1Spec{10, 0, 10, 2}, + R1Spec{10, 0, 10, 3}, + R1Spec{10, 0, 10, 4}, + R1Spec{10, 0, 10, 5}, + R1Spec{10, 0, 10, 10}, + R1Spec{500, 200, 400, 7}, + R1Spec{4096, 1, 4095, 3}, + R1Spec{2047, 1024 - 24, 1024 + 160, 31}, + R1Spec{2047, 1, 2046, 3 * 128}, + R1Spec{4096, 1024 + 3, 4095, 500}, + R1Spec{8192, 0, 8192, 1024 * 3 + 400} + ), + SliceR1TestDataToString ); +// clang-format on struct R2Spec { int64 input_dim0; -- GitLab From aeba52380f1b3bdf2ff9bd2256129e209bab08ca Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Wed, 29 Nov 2017 14:48:23 -0800 Subject: [PATCH 0128/1924] Updating references to the `tf.data` API to `tf.data` from `Datasets`. PiperOrigin-RevId: 177367024 --- tensorflow/docs_src/programmers_guide/datasets.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md index f458cbcef2..073bdb7baa 100644 --- a/tensorflow/docs_src/programmers_guide/datasets.md +++ b/tensorflow/docs_src/programmers_guide/datasets.md @@ -1,16 +1,16 @@ # Importing Data -The @{tf.data.Dataset$`Dataset`} API enables you to build complex input pipelines from +The `tf.data` API enables you to build complex input pipelines from simple, reusable pieces. For example, the pipeline for an image model might aggregate data from files in a distributed file system, apply random perturbations to each image, and merge randomly selected images into a batch for training. The pipeline for a text model might involve extracting symbols from raw text data, converting them to embedding identifiers with a lookup -table, and batching together sequences of different lengths. The `Dataset` API +table, and batching together sequences of different lengths. The `tf.data` API makes it easy to deal with large amounts of data, different data formats, and complicated transformations. -The `Dataset` API introduces two new abstractions to TensorFlow: +The `tf.data` API introduces two new abstractions to TensorFlow: * A `tf.data.Dataset` represents a sequence of elements, in which each element contains one or more `Tensor` objects. For example, in an image @@ -121,7 +121,7 @@ dataset3 = dataset3.filter(lambda x, (y, z): ...) ### Creating an iterator Once you have built a `Dataset` to represent your input data, the next step is to -create an `Iterator` to access elements from that dataset. The `Dataset` API +create an `Iterator` to access elements from that dataset. The `tf.data` API currently supports the following iterators, in increasing level of sophistication: @@ -379,7 +379,7 @@ sess.run(iterator.initializer, feed_dict={features_placeholder: features, ### Consuming TFRecord data -The `Dataset` API supports a variety of file formats so that you can process +The `tf.data` API supports a variety of file formats so that you can process large datasets that do not fit in memory. For example, the TFRecord file format is a simple record-oriented binary format that many TensorFlow applications use for training data. The `tf.data.TFRecordDataset` class enables you to @@ -628,7 +628,7 @@ TODO(mrry): Add this section. ### Processing multiple epochs -The `Dataset` API offers two main ways to process multiple epochs of the same +The `tf.data` API offers two main ways to process multiple epochs of the same data. The simplest way to iterate over a dataset in multiple epochs is to use the @@ -693,7 +693,7 @@ dataset = dataset.repeat() The @{tf.train.MonitoredTrainingSession} API simplifies many aspects of running TensorFlow in a distributed setting. `MonitoredTrainingSession` uses the @{tf.errors.OutOfRangeError} to signal that training has completed, so to use it -with the `Dataset` API, we recommend using +with the `tf.data` API, we recommend using `Dataset.make_one_shot_iterator()`. For example: ```python -- GitLab From 32b861d3d4f920b46954a2e02aee1fbf46a81c63 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Wed, 29 Nov 2017 15:17:06 -0800 Subject: [PATCH 0129/1924] Automated g4 rollback of changelist 177353959 PiperOrigin-RevId: 177371177 --- tensorflow/python/client/tf_session.i | 43 -------------- tensorflow/python/client/tf_session_helper.cc | 19 ------- tensorflow/python/client/tf_session_helper.h | 14 ----- tensorflow/python/framework/ops.py | 57 ++++++------------- 4 files changed, 16 insertions(+), 117 deletions(-) diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index c286d5fe47..5fa1a7e8fc 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -532,49 +532,6 @@ def TF_Reset(target, containers=None, config=None): %unignore TF_GraphGetTensorShapeHelper; %ignore TF_GraphGetTensorShape; -// We use TF_GraphSetTensorShape_wrapper instead of -// TF_GraphSetTensorShape -%ignore TF_GraphSetTensorShape; -%unignore tensorflow; -%unignore TF_GraphSetTensorShape_wrapper; - -// $input is a Python list of ints to a vector for TF_GraphSetTensorShape_wrapper -%typemap(in) (const std::vector& dims) - (std::vector dims_local){ - if ($input != Py_None) { - if (!PyList_Check($input)) { - SWIG_exception_fail(SWIG_TypeError, tensorflow::strings::Printf( - "$symname: expected list but got %s ", Py_TYPE($input)->tp_name).c_str()); - } - size_t size = PyList_Size($input); - for (int i = 0; i < size; ++i) { - PyObject* item = PyList_GetItem($input, i); - dims_local.push_back(PyInt_AS_LONG(item)); - } - $1 = &dims_local; - } else { - $1 = nullptr; - } -} - -// We use TF_GraphGetTensorShape_wrapper instead of -// TF_GraphGetTensorShape -%ignore TF_GraphGetTensorShape; -%unignore tensorflow; -%unignore TF_GraphGetTensorShape_wrapper; - -// Build a Python list of ints and return it. -%typemap(out) std::vector tensorflow::TF_GraphGetTensorShape_wrapper { - $result = PyList_New($1.size()); - if (!$result) { - SWIG_exception_fail(SWIG_MemoryError, "$symname: couldn't create list"); - } - - for (size_t i = 0; i < $1.size(); ++i) { - PyList_SET_ITEM($result, i, PyInt_FromLong($1[i])); - } -} - %include "tensorflow/python/client/tf_session_helper.h" %unignoreall diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc index e4bf09a0ca..ad982e5dd8 100644 --- a/tensorflow/python/client/tf_session_helper.cc +++ b/tensorflow/python/client/tf_session_helper.cc @@ -407,23 +407,4 @@ TF_Function* TF_GraphToFunction_wrapper( opts, description, out_status); } -void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, - const std::vector& dims, - bool unknown_shape, TF_Status* status) { - if (unknown_shape) { - TF_GraphSetTensorShape(graph, output, nullptr, -1, status); - return; - } - TF_GraphSetTensorShape(graph, output, dims.data(), dims.size(), status); -} - -std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, - TF_Output output, - int num_dims, - TF_Status* status) { - std::vector dims(num_dims); - TF_GraphGetTensorShape(graph, output, dims.data(), num_dims, status); - return dims; -} - } // namespace tensorflow diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h index bb7171db31..6ed08d3a58 100644 --- a/tensorflow/python/client/tf_session_helper.h +++ b/tensorflow/python/client/tf_session_helper.h @@ -168,20 +168,6 @@ TF_Function* TF_GraphToFunction_wrapper( const std::vector& inputs, const std::vector& outputs, const NameVector& output_names, const TF_FunctionOptions* opts, const char* description, TF_Status* out_status); - -// Set the shape of output. If unknown is true, `num_dims` must be set to -// -1 and `dims` is set to nullptr. -void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, - const std::vector& dims, - bool unknown_shape, TF_Status* status); - -// Return the shape of output. `num_dims` should be the output of -// TF_GraphGetTensorNumDims. If `num_dims = -1`, this should not be called. -std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, - TF_Output output, - int num_dims, - TF_Status* status); - } // namespace tensorflow #endif // TENSORFLOW_PYTHON_CLIENT_TF_SESSION_HELPER_H_ diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 975a1c87ec..2217513966 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -374,19 +374,6 @@ class Tensor(_TensorLike): A `TensorShape` representing the shape of this tensor. """ - if _USE_C_API: - graph = self._op._graph._c_graph # pylint: disable=protected-access - with errors.raise_exception_on_not_ok_status() as status: - num_dims = c_api.TF_GraphGetTensorNumDims(graph, self._as_tf_output(), - status) - if num_dims == -1: - dim_list = None - else: - with errors.raise_exception_on_not_ok_status() as status: - dim_list = c_api.TF_GraphGetTensorShape_wrapper( - graph, self._as_tf_output(), num_dims, status) - dim_list = [None if i == -1 else i for i in dim_list] - return tensor_shape.TensorShape(dim_list) return self._shape def __iter__(self): @@ -406,8 +393,8 @@ class Tensor(_TensorLike): yield self[i] def _shape_as_list(self): - if self.shape.ndims is not None: - return [dim.value for dim in self.shape.dims] + if self._shape.ndims is not None: + return [dim.value for dim in self._shape.dims] else: return None @@ -423,7 +410,7 @@ class Tensor(_TensorLike): Returns: Integer rank or None """ - return self.shape.ndims + return self._shape.ndims def get_shape(self): """Alias of Tensor.shape.""" @@ -454,35 +441,14 @@ class Tensor(_TensorLike): ``` Args: - shape: A `TensorShape` representing the shape of this tensor, a - `TensorShapeProto`, a list, a tuple, or None. + shape: A `TensorShape` representing the shape of this tensor. Raises: ValueError: If `shape` is not compatible with the current shape of this tensor. """ - if not _USE_C_API: - self._shape = self._shape.merge_with(shape) # pylint: disable=protected-access - return - if not isinstance(shape, tensor_shape.TensorShape): - shape = tensor_shape.TensorShape(shape) - dim_list = [] - if shape.dims is None: - unknown_shape = True - else: - unknown_shape = False - for dim in shape.dims: - if dim.value is None: - dim_list.append(-1) - else: - dim_list.append(dim.value) - with errors.raise_exception_on_not_ok_status() as status: - c_api.TF_GraphSetTensorShape_wrapper( - self._op._graph._c_graph, # pylint: disable=protected-access - self._as_tf_output(), - dim_list, - unknown_shape, - status) + # TODO(skyewm): call C API + self._shape = self._shape.merge_with(shape) @property def value_index(self): @@ -4551,11 +4517,15 @@ def control_dependencies(control_inputs): See @{tf.Graph.control_dependencies} for more details. + When eager execution is enabled, any callable object in the `control_inputs` + list will be called. + Args: control_inputs: A list of `Operation` or `Tensor` objects which must be executed or computed before running the operations defined in the context. Can also be `None` to clear the control - dependencies. + dependencies. If eager execution is enabled, any callable object in the + `control_inputs` list will be called. Returns: A context manager that specifies control dependencies for all @@ -4564,6 +4534,11 @@ def control_dependencies(control_inputs): if context.in_graph_mode(): return get_default_graph().control_dependencies(control_inputs) else: + if control_inputs: + # Excute any pending callables. + for control in control_inputs: + if callable(control): + control() return _NullContextmanager() -- GitLab From 963a521e255d2a189e349fc5c24ebc2bc032be5b Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 29 Nov 2017 15:46:42 -0800 Subject: [PATCH 0130/1924] Using the C API in eager mode for graph functions. PiperOrigin-RevId: 177375237 --- tensorflow/python/eager/backprop.py | 2 +- tensorflow/python/eager/context.py | 15 ++ tensorflow/python/eager/function.py | 144 ++++++++++++------ tensorflow/python/eager/graph_callable.py | 18 ++- .../python/eager/graph_callable_test.py | 1 - tensorflow/python/framework/ops.py | 30 ++-- tensorflow/python/pywrap_tfe.i | 3 +- 7 files changed, 143 insertions(+), 70 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 0144f3b1e5..dc1142705a 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -540,7 +540,7 @@ def _ensure_unique_tensor_objects(parameter_positions, args): if i in parameter_positions: tid = ops.tensor_id(t) if tid in s: - args[i] = args[i]._dup() # pylint: disable=protected-access + args[i] = gen_array_ops.identity(args[i]) else: s.add(tid) return args diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 92f4e15c05..415416cfae 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -288,6 +288,21 @@ class Context(object): self._initialize_handle_and_devices() return self._num_gpus + def add_function(self, fn): + """Add a function definition to the context. + + Once added, the function (identified by its name) can be executed like any + other operation. + + Args: + fn: A wrapped TF_Function (returned from TF_GraphToFunction_wrapper). + """ + with errors.raise_exception_on_not_ok_status() as status: + pywrap_tensorflow.TFE_ContextAddFunction( + self._handle, # pylint: disable=protected-access + fn, + status) + def add_function_def(self, fdef): """Add a function definition to the context. diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 2f4b59e938..092b36ff20 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -25,15 +25,19 @@ import threading import numpy as np +from tensorflow.core.framework import function_pb2 +from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context from tensorflow.python.eager import execute from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder +from tensorflow.python.framework import c_api_util from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_module -from tensorflow.python.framework import graph_to_function_def +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -47,10 +51,41 @@ _scoped_captures = threading.local() _scoped_captures.tensors = None -def make_function_def(graph, operations, inputs, outputs): - """Makes function def from the given graph with the operations.""" - return graph_to_function_def.graph_to_function_def( - graph, operations, inputs, outputs) +def make_function_def(name, graph, operations, inputs, outputs): + """Makes FunctionDef proto and defined function. + + Args: + name: the function name + graph: the graph from which to build the function + operations: the operations in the function body + inputs: tensors to be used as function arguments + outputs: tensors to be returned from the function + + Returns: + fdef: a FunctionDef protocol buffer for the function + fn: a wrapped TF_Function for the function + """ + with errors.raise_exception_on_not_ok_status() as status: + fn = pywrap_tensorflow.TF_GraphToFunction_wrapper( + graph._c_graph, # pylint: disable=protected-access + compat.as_text(name), + False, + [o._c_op for o in operations], # pylint: disable=protected-access + [t._as_tf_output() for t in inputs], # pylint: disable=protected-access + [t._as_tf_output() for t in outputs], # pylint: disable=protected-access + [compat.as_text("%s" % i) for i in range(len(outputs))], + None, + compat.as_text(""), + status) + # TODO(apassos) avoid creating a FunctionDef (specially to grab the signature, + # but also in general it's nice not to depend on it. + with c_api_util.tf_buffer() as buffer_: + with errors.raise_exception_on_not_ok_status() as status: + pywrap_tensorflow.TF_FunctionToFunctionDef(fn, buffer_, status) + proto_data = pywrap_tensorflow.TF_GetBuffer(buffer_) + fdef = function_pb2.FunctionDef() + fdef.ParseFromString(compat.as_bytes(proto_data)) + return fdef, fn @contextlib.contextmanager @@ -115,6 +150,10 @@ class CapturingGraph(ops.Graph): # for resource tensors. self._last_op_using_resource_tensor = {} + # TODO(apassos) remove once the C API is used by default. + def _use_c_api_hack(self): + return True + def clear_resource_control_flow_state(self): self._last_op_using_resource_tensor = {} @@ -207,14 +246,20 @@ def _inference_name(n): return "__inference_%s_%s" % (n, ops.uid()) +# TODO(apassos) get rid of this by splitting framework.function._DefinedFunction +# so it doesn't have the definition-generating logic and is just a container for +# an already-defined function. class _DefinedFunction(object): """Mocks the interface of tf _DefinedFunction.""" - def __init__(self, fdef): + def __init__(self, fdef, fn): self.definition = fdef self.name = fdef.signature.name + self.signature = fdef.signature self.grad_func_name = None self.python_grad_func = None + self._c_func = fn + self._grad_func = None def _map_sequence_obj_to_idx(sequence): @@ -250,6 +295,7 @@ class GraphModeFunction(object): input_placeholders, extra_inputs, fdef, + fn, graph, operations, func_outputs, @@ -263,7 +309,7 @@ class GraphModeFunction(object): self._graph = graph self._has_backprop = False self._func_name = fdef.signature.name - self._fdef = _DefinedFunction(fdef) + self._fdef = _DefinedFunction(fdef, fn) self._num_outputs = len(fdef.signature.output_arg) self._ops = operations self._func_outputs = func_outputs @@ -283,38 +329,45 @@ class GraphModeFunction(object): with self._graph.as_default(), context.graph_mode(): c = _CapturingContext() with c: - filtered_outputs = [ - x for x in self._returns if x is not None - ] + filtered_outputs = [x for x in self._returns if x is not None] self._out_grad_placeholders = [ - graph_placeholder(x.dtype, x.shape) for x in filtered_outputs - ] + graph_placeholder(x.dtype, x.shape) for x in filtered_outputs] in_gradients = gradients_impl.gradients( filtered_outputs, self._input_placeholders, grad_ys=self._out_grad_placeholders) - shapes = [x.shape for x in in_gradients if x is not None] + shapes = tuple(x.shape for x in in_gradients if x is not None) captures = list(sorted(c.captured_tensors, key=lambda x: x.name)) - forward_function_def = make_function_def( - self._graph, self._ops, self._input_placeholders, + forward_name = _forward_name(self._func_name) + forward_function_def, forward_fn = make_function_def( + forward_name, self._graph, self._ops, self._input_placeholders, filtered_outputs + captures) - self._forward_fdef = _DefinedFunction(forward_function_def) - _register_with_name(_forward_name(self._func_name), forward_function_def) - backward_outputs = [x for x in in_gradients if x is not None] + self._forward_fdef = _DefinedFunction(forward_function_def, forward_fn) + _register(forward_fn) + backward_outputs = tuple(x for x in in_gradients if x is not None) all_inputs = self._out_grad_placeholders + captures - backward_function_def = make_function_def( - self._graph, [x.op for x in self._out_grad_placeholders - ] + list(sorted(c.known_ops, key=lambda x: x.name)), + # Excluding input ops from the body as we do not intend to execute these + # operations when the function is executed. + all_ignored_ops = frozenset(x.op for x in all_inputs) + # Enforce a deterministic order of operations in the generated graph. This + # means rerunning the function-defining code will always define the same + # function, which is useful if we serialize this etc. + fdef_ops = tuple(x for x in sorted(c.known_ops, key=lambda x: x.name) + if x not in all_ignored_ops) + bname = _backward_name(self._func_name) + backward_function_def, backward_fn = make_function_def( + bname, self._graph, fdef_ops, all_inputs, backward_outputs) - _register_with_name(_backward_name(self._func_name), backward_function_def) + _register(backward_fn) self._backward_function = GraphModeFunction( - all_inputs, [], backward_function_def, self._graph, c.known_ops, - in_gradients, _map_sequence_obj_to_idx(backward_outputs), shapes) + all_inputs, [], backward_function_def, backward_fn, self._graph, + c.known_ops, in_gradients, _map_sequence_obj_to_idx(backward_outputs), + shapes) def _backprop_call(self, args): """Calls the wrapped function and records the result on a tape.""" all_args = args + self._extra_inputs - signature = self._forward_fdef.definition.signature + signature = self._forward_fdef.signature ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() @@ -325,7 +378,7 @@ class GraphModeFunction(object): return ops.internal_convert_to_tensor(x, ctx=ctx) op = g.create_op( signature.name, [make_tensor(x) for x in all_args], - [dtypes_module.DType(x.type) for x in signature.output_arg], + tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) @@ -361,11 +414,8 @@ class GraphModeFunction(object): if v._trainable: # pylint: disable=protected-access tape.watch_variable(v) - tensor_inputs = [ - x for x in nest.flatten(args) - if isinstance(x, ops.Tensor) - ] - + tensor_inputs = [x for x in nest.flatten(args) + if isinstance(x, ops.Tensor)] if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if not self._has_backprop: @@ -384,7 +434,7 @@ class GraphModeFunction(object): args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.convert_to_tensor(x) for x in args], - [dtypes_module.DType(x.type) for x in signature.output_arg], + tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) @@ -469,29 +519,32 @@ def _defun_internal(name, func, args, kwds): extra_inputs = [] extra_placeholders = [] outputs_list = nest.flatten(func_outputs) - output_shapes = [x.shape for x in outputs_list if x is not None] + output_shapes = tuple(x.shape for x in outputs_list if x is not None) - flat_inputs = [ - x for x in nest.flatten(func_inputs) if isinstance(x, ops.Tensor) - ] + flat_inputs = [x for x in nest.flatten(func_inputs) + if isinstance(x, ops.Tensor)] all_inputs = flat_inputs + list(extra_placeholders) - + all_ignored_ops = frozenset(x.op for x in all_inputs) func_def_outputs = [x for x in outputs_list if x is not None] - inference_function_def = make_function_def( - tmp_graph, tmp_graph.get_operations(), all_inputs, func_def_outputs) + fname = _inference_name(name) + operations = tuple(x for x in tmp_graph.get_operations() + if x not in all_ignored_ops) + inference_function_def, fn = make_function_def( + fname, tmp_graph, operations, all_inputs, func_def_outputs) # Register any other functions defined in the graph # TODO(ashankar): Oh lord, forgive me for this lint travesty. for f in tmp_graph._functions.values(): # pylint: disable=protected-access # TODO(ashankar): What about the gradient registry? - _register_with_name(f.name, f.definition) - _register_with_name(_inference_name(name), inference_function_def) + _register(f._c_func) # pylint: disable=protected-access + _register(fn) return GraphModeFunction( all_inputs, extra_inputs, inference_function_def, + fn, tmp_graph, - tmp_graph.get_operations(), + operations, func_outputs, _map_sequence_obj_to_idx(func_def_outputs), output_shapes, @@ -517,10 +570,9 @@ def _cache_key(x): return x -def _register_with_name(name, fdef): - """Registers the function `fdef` with the name `name`.""" - fdef.signature.name = name - context.context().add_function_def(fdef) +def _register(fn): + """Registers the function `fn`.""" + context.context().add_function(fn) # TODO(apassos): better error messages for non-hashable arguments. diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index faf0ac88bc..3da100d800 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -318,7 +318,9 @@ def _graph_callable_internal(func, shape_and_dtypes): placeholder_inputs = flat_inputs+ list(extra_placeholders) func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)] - initializer_function_def = function.make_function_def( + initialization_name = function._inference_name(func.__name__) # pylint: disable=protected-access + initializer_function_def, initializer_fn = function.make_function_def( + initialization_name, tmp_graph, initializing_operations, placeholder_inputs, @@ -327,13 +329,13 @@ def _graph_callable_internal(func, shape_and_dtypes): # Also, what about the gradient registry of these functions? Those need to be # addressed as well. for f in tmp_graph._functions.values(): # pylint: disable=protected-access - function._register_with_name(f.name, f.definition) # pylint: disable=protected-access - function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access - initializer_function_def) + function._register(f._c_func) # pylint: disable=protected-access + function._register(initializer_fn) # pylint: disable=protected-access initializer_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, initializer_function_def, + initializer_fn, tmp_graph, initializing_operations, func_outputs, @@ -342,18 +344,20 @@ def _graph_callable_internal(func, shape_and_dtypes): capture_func_def_outputs = [ x for x in captured_outlist if isinstance(x, tf_ops.Tensor)] - captured_function_def = function.make_function_def( + captured_function_name = function._inference_name(func.__name__) # pylint: disable=protected-access + captured_function_def, capturing_fn = function.make_function_def( + captured_function_name, tmp_graph, capturing_operations, placeholder_inputs, capture_func_def_outputs) - function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access - captured_function_def) + function._register(capturing_fn) # pylint: disable=protected-access captured_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, captured_function_def, + capturing_fn, tmp_graph, capturing_operations, captured_outputs, diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py index 548e16a909..b9e6ca2a93 100644 --- a/tensorflow/python/eager/graph_callable_test.py +++ b/tensorflow/python/eager/graph_callable_test.py @@ -152,7 +152,6 @@ class GraphCallableTest(test.TestCase): self.assertAllEqual(5, f(constant_op.constant(2))) def testNestedFunction(self): - # TensorFlow function (which is what would be used in TensorFlow graph # construction). @function.Defun(dtypes.int32, dtypes.int32) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 2217513966..36daf59647 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -599,11 +599,6 @@ class Tensor(_TensorLike): """ return _eval_using_default_session(self, feed_dict, self.graph, session) - def _dup(self): - ret = copy.copy(self) - ret._id = uid() # pylint: disable=protected-access - return ret - # TODO(agarwal): consider getting rid of this. class _EagerTensorBase(Tensor): @@ -729,9 +724,6 @@ class _EagerTensorBase(Tensor): return new_tensor # pylint: enable=protected-access - def _dup(self): - return self._copy(device_name=self.device) - @property def shape(self): return tensor_shape.TensorShape(self._shape_tuple()) @@ -1794,7 +1786,7 @@ class Operation(object): c_api.SetRequestedDevice( self._graph._c_graph, # pylint: disable=protected-access self._c_op, # pylint: disable=protected-access - _device_string(device)) + compat.as_text(_device_string(device))) else: self._node_def.device = _device_string(device) @@ -2083,7 +2075,7 @@ class Operation(object): def _set_attr(self, attr_name, attr_value): """Private method used to set an attribute in the node_def.""" - if _USE_C_API: + if self._c_op: buf = c_api.TF_NewBufferFromString( compat.as_bytes(attr_value.SerializeToString())) try: @@ -2652,11 +2644,16 @@ class Graph(object): # TODO(skyewm): fold as much of the above as possible into the C # implementation - if _USE_C_API: + if _USE_C_API or self._use_c_api_hack(): self._scoped_c_graph = c_api_util.ScopedTFGraph() else: self._scoped_c_graph = None + # TODO(apassos) remove once the C API is used by default. + def _use_c_api_hack(self): + """Temporary hack; can be overridden to force C API usage.""" + return False + def _convert_stack(self, stack, include_func_start_lineno=False): """Converts a stack extracted using _extract_stack() to a traceback stack. @@ -2985,9 +2982,14 @@ class Graph(object): # Add function to graph # pylint: disable=protected-access if self._c_graph: - assert function._c_func, ( - "Cannot add function created without C API support to graph " - "created with C API support") + # Handle functions created without using the C API. TODO(apassos,skyewm) + # remove this when all functions are generated using the C API by default + # as this will be unnecessary. + if not function._c_func: + with errors.raise_exception_on_not_ok_status() as status: + serialized = function.definition.SerializeToString() + function._c_func = c_api.TF_FunctionImportFunctionDef( + serialized, status) with errors.raise_exception_on_not_ok_status() as status: gradient = function._grad_func._c_func if function._grad_func else None c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient, diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 82b154164e..82750e9e49 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -18,6 +18,7 @@ limitations under the License. %rename("%s") TFE_NewContext; %rename("%s") TFE_DeleteContext; %rename("%s") TFE_ContextListDevices; +%rename("%s") TFE_ContextAddFunction; %rename("%s") TFE_ContextAddFunctionDef; %rename("%s") TFE_OpNameGetAttrType; %rename("%s") TFE_Py_InitEagerTensor; @@ -149,7 +150,7 @@ limitations under the License. } $1 = &temp; $1->resize(PyInt_AsLong($input), nullptr); -} +} // Create new Status object. %typemap(in, numinputs=0) TF_Status *out_status { -- GitLab From bc7180f02002788a7b57b36b14ceb7a47d6c76f4 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Wed, 29 Nov 2017 16:12:31 -0800 Subject: [PATCH 0131/1924] Fix more clang-tidy warnings: - Parameter names consistent in function declarations and definitions - Class members naming PiperOrigin-RevId: 177379085 --- tensorflow/compiler/xla/literal_util.h | 6 +++--- tensorflow/compiler/xla/reference_util.h | 2 +- tensorflow/compiler/xla/shape_layout.cc | 8 ++++---- tensorflow/compiler/xla/shape_layout.h | 13 +++++++------ tensorflow/compiler/xla/shape_util.cc | 4 ++-- tensorflow/compiler/xla/shape_util.h | 2 +- tensorflow/compiler/xla/statusor_test.cc | 14 +++++++------- .../compiler/xla/tests/client_library_test_base.h | 2 +- tensorflow/compiler/xla/util.cc | 6 +++--- 9 files changed, 29 insertions(+), 28 deletions(-) diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index f37e529caf..069d1b33ca 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -285,11 +285,11 @@ class Literal { std::unique_ptr Relayout(const Layout& new_layout, const ShapeIndex& shape_index = {}) const; - // Creates a new literal by reshaping this literal to have 'shape'. Both the - // original shape and 'shape' must contain the same number of elements. The + // Creates a new literal by reshaping this literal to have the given + // dimensions. The total number of elements must not change; The // implementation currently only supports monotonic dim0-major layouts. StatusOr> Reshape( - tensorflow::gtl::ArraySlice shape) const; + tensorflow::gtl::ArraySlice dimensions) const; // Creates a new literal by reordering the dimensions of this literal. // The given `permutation` must be a permutation of the dimension numbers diff --git a/tensorflow/compiler/xla/reference_util.h b/tensorflow/compiler/xla/reference_util.h index ee244e9a66..58e1a84461 100644 --- a/tensorflow/compiler/xla/reference_util.h +++ b/tensorflow/compiler/xla/reference_util.h @@ -70,7 +70,7 @@ class ReferenceUtil { // dilation factors. static std::unique_ptr> ConvArray4DGeneralDimensionsDilated( const Array4D& lhs, const Array4D& rhs, - std::pair stride, Padding padding, + std::pair kernel_stride, Padding padding, std::pair lhs_dilation, std::pair rhs_dilation, ConvolutionDimensionNumbers dnums); diff --git a/tensorflow/compiler/xla/shape_layout.cc b/tensorflow/compiler/xla/shape_layout.cc index 5bf9842a6c..789eba5780 100644 --- a/tensorflow/compiler/xla/shape_layout.cc +++ b/tensorflow/compiler/xla/shape_layout.cc @@ -32,13 +32,13 @@ tensorflow::Status ShapeLayout::CopyLayoutFromShape(const Shape& other_shape) { return tensorflow::Status::OK(); } -tensorflow::Status ShapeLayout::AssignLayoutToShape(Shape* other_shape) const { - if (!ShapeUtil::Compatible(*other_shape, shape_)) { +tensorflow::Status ShapeLayout::AssignLayoutToShape(Shape* to_shape) const { + if (!ShapeUtil::Compatible(*to_shape, shape_)) { return InvalidArgument("Shape %s is not compatible with shape %s", - ShapeUtil::HumanString(*other_shape).c_str(), + ShapeUtil::HumanString(*to_shape).c_str(), ShapeUtil::HumanString(shape()).c_str()); } - *other_shape = shape_; + *to_shape = shape_; return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/shape_layout.h b/tensorflow/compiler/xla/shape_layout.h index 92564660f2..4c83750f3e 100644 --- a/tensorflow/compiler/xla/shape_layout.h +++ b/tensorflow/compiler/xla/shape_layout.h @@ -38,18 +38,19 @@ class ShapeLayout { explicit ShapeLayout(const Shape& shape) : shape_(shape) {} // Assigns the layouts in this ShapeLayout to the Layout fields of the given - // shape. 'shape' and the shape of the ShapeLayout object must be compatible. - tensorflow::Status AssignLayoutToShape(Shape* shape) const; + // shape. 'to_shape' and the shape of the ShapeLayout object must be + // compatible. + tensorflow::Status AssignLayoutToShape(Shape* to_shape) const; // Returns true if the Layouts in this ShapeLayout match the layouts in the // given shape. Returns false otherwise. If the given shape is not compatible // with the ShapeLayout's shape, then false is returned. bool MatchesLayoutInShape(const Shape& shape) const; - // Copies the layout from the given shape into this ShapeLayout. 'shape' must - // be compatible with the ShapeLayout's shape, and 'shape' must have a layout - // (LayoutUtil::HasLayout). - tensorflow::Status CopyLayoutFromShape(const Shape& shape); + // Copies the layout from the given shape into this ShapeLayout. 'other_shape' + // must be compatible with the ShapeLayout's shape, and 'other_shape' must + // have a layout (LayoutUtil::HasLayout). + tensorflow::Status CopyLayoutFromShape(const Shape& other_shape); // Clears (Layout::Clear) all the Layouts stored in this object. void Clear(); diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 74fa0b2f2e..9e3f06e527 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -694,9 +694,9 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { return LayoutUtil::ValidateLayoutInShape(shape); } -/* static */ Shape ShapeUtil::ChangeElementType(const Shape& shape, +/* static */ Shape ShapeUtil::ChangeElementType(const Shape& original, PrimitiveType type) { - Shape new_shape = shape; + Shape new_shape = original; new_shape.set_element_type(type); return new_shape; } diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index 2ea1bd95cb..df5b450438 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -170,7 +170,7 @@ class ShapeUtil { // As above, but for program shapes, returns a string for the form: // // (param_name: f32[42x12], ...) -> f32[24x42] - static string HumanString(const ProgramShape& shape); + static string HumanString(const ProgramShape& program_shape); // Parses a ShapeUtil::HumanString-format shape string back into a shape // object. diff --git a/tensorflow/compiler/xla/statusor_test.cc b/tensorflow/compiler/xla/statusor_test.cc index 5fa2211ac6..f9d25945bc 100644 --- a/tensorflow/compiler/xla/statusor_test.cc +++ b/tensorflow/compiler/xla/statusor_test.cc @@ -32,26 +32,26 @@ namespace { class Base1 { public: virtual ~Base1() {} - int pad; + int pad_; }; class Base2 { public: virtual ~Base2() {} - int yetotherpad; + int yetotherpad_; }; class Derived : public Base1, public Base2 { public: ~Derived() override {} - int evenmorepad; + int evenmorepad_; }; class CopyNoAssign { public: - explicit CopyNoAssign(int value) : foo(value) {} - CopyNoAssign(const CopyNoAssign& other) : foo(other.foo) {} - int foo; + explicit CopyNoAssign(int value) : foo_(value) {} + CopyNoAssign(const CopyNoAssign& other) : foo_(other.foo_) {} + int foo_; private: const CopyNoAssign& operator=(const CopyNoAssign&); @@ -253,7 +253,7 @@ TEST(StatusOr, TestCopyCtorNonAssignable) { StatusOr original(value); StatusOr copy(original); EXPECT_EQ(copy.status(), original.status()); - EXPECT_EQ(original.ValueOrDie().foo, copy.ValueOrDie().foo); + EXPECT_EQ(original.ValueOrDie().foo_, copy.ValueOrDie().foo_); } TEST(StatusOr, TestCopyCtorStatusOKConverting) { diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h index 1d27880fb1..d8fe12a72d 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.h +++ b/tensorflow/compiler/xla/tests/client_library_test_base.h @@ -194,7 +194,7 @@ class ClientLibraryTestBase : public ::testing::Test { tensorflow::gtl::ArraySlice arguments); void ComputeAndCompareTuple( ComputationBuilder* builder, const Literal& expected, - tensorflow::gtl::ArraySlice arguments, ErrorSpec abs_error); + tensorflow::gtl::ArraySlice arguments, ErrorSpec error); // Convenience method for running a built computation and comparing the result // with the HloEvaluator. diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc index e595df3052..fe5d29a6b6 100644 --- a/tensorflow/compiler/xla/util.cc +++ b/tensorflow/compiler/xla/util.cc @@ -191,9 +191,9 @@ std::vector ComposePermutations(tensorflow::gtl::ArraySlice p1, return output; } -bool IsIdentityPermutation(tensorflow::gtl::ArraySlice p) { - for (int64 i = 0; i < p.size(); ++i) { - if (p[i] != i) { +bool IsIdentityPermutation(tensorflow::gtl::ArraySlice permutation) { + for (int64 i = 0; i < permutation.size(); ++i) { + if (permutation[i] != i) { return false; } } -- GitLab From 4ada275eed7472ae32c67a1ec0b9b1dc8d80d1f0 Mon Sep 17 00:00:00 2001 From: "Joshua V. Dillon" Date: Wed, 29 Nov 2017 16:33:56 -0800 Subject: [PATCH 0132/1924] Change `tf.contrib.distributions` docstring examples to use `tfd` alias rather than `ds`, `bs`. PiperOrigin-RevId: 177381853 --- .../ops/bijectors/absolute_value_impl.py | 4 ++- .../bijectors/masked_autoregressive_impl.py | 28 +++++++++---------- .../python/ops/bijectors/permute_impl.py | 4 +-- .../python/ops/bijectors/reshape_impl.py | 4 +-- .../distributions/python/ops/cauchy.py | 22 ++++++++------- .../distributions/python/ops/deterministic.py | 6 ++-- .../distributions/python/ops/gumbel.py | 8 ++++-- .../distributions/python/ops/independent.py | 10 +++---- .../distributions/python/ops/inverse_gamma.py | 5 ++-- .../distributions/python/ops/logistic.py | 13 ++++----- .../distributions/python/ops/mixture.py | 10 +++---- .../python/ops/mixture_same_family.py | 16 +++++------ .../distributions/python/ops/mvn_diag.py | 8 +++--- .../python/ops/mvn_diag_plus_low_rank.py | 6 ++-- .../python/ops/mvn_full_covariance.py | 6 ++-- .../python/ops/mvn_linear_operator.py | 11 ++++---- .../distributions/python/ops/mvn_tril.py | 13 +++++---- .../python/ops/poisson_lognormal.py | 5 ++-- .../distributions/python/ops/sinh_arcsinh.py | 2 +- .../python/ops/vector_diffeomixture.py | 11 ++++---- .../python/ops/vector_exponential_diag.py | 7 ++--- .../ops/vector_exponential_linear_operator.py | 11 ++++---- .../python/ops/vector_laplace_diag.py | 8 +++--- .../ops/vector_laplace_linear_operator.py | 11 ++++---- .../python/ops/vector_sinh_arcsinh_diag.py | 2 +- .../python/ops/vector_student_t.py | 6 ++-- 26 files changed, 121 insertions(+), 116 deletions(-) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py index b84502003a..0fe9f6aa78 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py @@ -48,7 +48,9 @@ class AbsoluteValue(bijector.Bijector): ```python - abs = ds.bijectors.AbsoluteValue() + tfd = tf.contrib.distributions + + abs = tfd.bijectors.AbsoluteValue() abs.forward([-1., 0., 1.]) ==> [1., 0., 1.] diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py index ae14288393..f51c48d2dd 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py @@ -124,17 +124,17 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector): #### Example Use ```python - ds = tf.contrib.distributions - bs = tf.contrib.distributions.bijectors + tfd = tf.contrib.distributions + tfb = tfd.bijectors dims = 5 # A common choice for a normalizing flow is to use a Gaussian for the base # distribution. (However, any continuous distribution would work.) E.g., - maf = ds.TransformedDistribution( - distribution=ds.Normal(loc=0., scale=1.), - bijector=bs.MaskedAutoregressiveFlow( - shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + maf = tfd.TransformedDistribution( + distribution=tfd.Normal(loc=0., scale=1.), + bijector=tfb.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( hidden_layers=[512, 512])), event_shape=[dims]) @@ -143,10 +143,10 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector): maf.log_prob(0.) # Cheap; no `tf.while_loop` despite no Bijector caching. # [1] also describes an "Inverse Autoregressive Flow", e.g., - iaf = ds.TransformedDistribution( - distribution=ds.Normal(loc=0., scale=1.), - bijector=bs.Invert(bs.MaskedAutoregressiveFlow( - shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + iaf = tfd.TransformedDistribution( + distribution=tfd.Normal(loc=0., scale=1.), + bijector=tfb.Invert(tfb.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=tfb.masked_autoregressive_default_template( hidden_layers=[512, 512]))), event_shape=[dims]) @@ -158,10 +158,10 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector): # poor choice. Here's an example of using a "shift only" version and with a # different number/depth of hidden layers. shift_only = True - maf_no_scale_hidden2 = ds.TransformedDistribution( - distribution=ds.Normal(loc=0., scale=1.), - bijector=bs.MaskedAutoregressiveFlow( - bs.masked_autoregressive_default_template( + maf_no_scale_hidden2 = tfd.TransformedDistribution( + distribution=tfd.Normal(loc=0., scale=1.), + bijector=tfb.MaskedAutoregressiveFlow( + tfb.masked_autoregressive_default_template( hidden_layers=[32], shift_only=shift_only), is_constant_jacobian=shift_only), diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py index b1d8f2f41b..8654cc39d0 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py @@ -40,9 +40,9 @@ class Permute(bijector_lib.Bijector): """Permutes the rightmost dimension of a `Tensor`. ```python - bs = tf.contrib.distributions.bijectors + tfd = tf.contrib.distributions - reverse = bs.Permute(permutation=[2, 1, 0]) + reverse = tfd.bijectors.Permute(permutation=[2, 1, 0]) reverse.forward([-1., 0., 1.]) # ==> [1., 0., -1] diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py index 1eb8e74fda..55eca06312 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py @@ -63,9 +63,9 @@ class Reshape(bijector_lib.Bijector): Example usage: ```python - bs = tf.contrib.distributions.bijectors + tfd = tf.contrib.distributions - r = bs.Reshape(event_shape_out=[1, -1]) + r = tfd.bijectors.Reshape(event_shape_out=[1, -1]) r.forward([3., 4.]) # shape [2] # ==> [[3., 4.]] # shape [1, 2] diff --git a/tensorflow/contrib/distributions/python/ops/cauchy.py b/tensorflow/contrib/distributions/python/ops/cauchy.py index 8d59c1abfb..6f5d724a2a 100644 --- a/tensorflow/contrib/distributions/python/ops/cauchy.py +++ b/tensorflow/contrib/distributions/python/ops/cauchy.py @@ -43,16 +43,17 @@ class Cauchy(distribution.Distribution): The probability density function (pdf) is, ```none - pdf(x; loc, scale) = 1 / (pi * scale * (1 + ((x - loc) / scale)**2)) + pdf(x; loc, scale) = 1 / (pi scale (1 + z**2)) + z = (x - loc) / scale ``` where `loc` is the location, and `scale` is the scale. The Cauchy distribution is a member of the [location-scale family]( https://en.wikipedia.org/wiki/Location-scale_family), i.e. + `Y ~ Cauchy(loc, scale)` is equivalent to, ```none X ~ Cauchy(loc=0, scale=1) - Y ~ Cauchy(loc=loc, scale=scale) Y = loc + scale * X ``` @@ -61,14 +62,16 @@ class Cauchy(distribution.Distribution): Examples of initialization of one or a batch of distributions. ```python + tfd = tf.contrib.distributions + # Define a single scalar Cauchy distribution. - dist = Cauchy(loc=0., scale=3.) + dist = tfd.Cauchy(loc=0., scale=3.) # Evaluate the cdf at 1, returning a scalar. dist.cdf(1.) # Define a batch of two scalar valued Cauchy distributions. - dist = Cauchy(loc=[1, 2.], scale=[11, 22.]) + dist = tfd.Cauchy(loc=[1, 2.], scale=[11, 22.]) # Evaluate the pdf of the first distribution on 0, and the second on 1.5, # returning a length two tensor. @@ -76,18 +79,17 @@ class Cauchy(distribution.Distribution): # Get 3 samples, returning a 3 x 2 tensor. dist.sample([3]) - ``` - - Arguments are broadcast when possible. - ```python + # Arguments are broadcast when possible. # Define a batch of two scalar valued Cauchy distributions. # Both have median 1, but different scales. - dist = tf.contrib.distributions.Cauchy(loc=1., scale=[11, 22.]) + dist = tfd.Cauchy(loc=1., scale=[11, 22.]) + # Evaluate the pdf of both distributions on the same point, 3.0, # returning a length 2 tensor. - dist.prob(3.0) + dist.prob(3.) ``` + """ def __init__(self, diff --git a/tensorflow/contrib/distributions/python/ops/deterministic.py b/tensorflow/contrib/distributions/python/ops/deterministic.py index 850d08d1bd..8049522e9f 100644 --- a/tensorflow/contrib/distributions/python/ops/deterministic.py +++ b/tensorflow/contrib/distributions/python/ops/deterministic.py @@ -290,8 +290,10 @@ class VectorDeterministic(_BaseDeterministic): #### Examples ```python + tfd = tf.contrib.distributions + # Initialize a single VectorDeterministic supported at [0., 2.] in R^2. - constant = tf.contrib.distributions.Deterministic([0., 2.]) + constant = tfd.Deterministic([0., 2.]) constant.prob([0., 2.]) ==> 1. constant.prob([0., 3.]) @@ -299,7 +301,7 @@ class VectorDeterministic(_BaseDeterministic): # Initialize a [3] batch of constants on R^2. loc = [[0., 1.], [2., 3.], [4., 5.]] - constant = constant_lib.VectorDeterministic(loc) + constant = tfd.VectorDeterministic(loc) constant.prob([[0., 1.], [1.9, 3.], [3.99, 5.]]) ==> [1., 0., 0.] ``` diff --git a/tensorflow/contrib/distributions/python/ops/gumbel.py b/tensorflow/contrib/distributions/python/ops/gumbel.py index ba8d3c639b..d0efaefb8e 100644 --- a/tensorflow/contrib/distributions/python/ops/gumbel.py +++ b/tensorflow/contrib/distributions/python/ops/gumbel.py @@ -62,15 +62,17 @@ class _Gumbel(distribution.Distribution): Examples of initialization of one or a batch of distributions. ```python + tfd = tf.contrib.distributions + # Define a single scalar Gumbel distribution. - dist = tf.contrib.distributions.Gumbel(loc=0., scale=3.) + dist = tfd.Gumbel(loc=0., scale=3.) # Evaluate the cdf at 1, returning a scalar. dist.cdf(1.) # Define a batch of two scalar valued Gumbels. # The first has mean 1 and scale 11, the second 2 and 22. - dist = tf.contrib.distributions.Gumbel(loc=[1, 2.], scale=[11, 22.]) + dist = tfd.Gumbel(loc=[1, 2.], scale=[11, 22.]) # Evaluate the pdf of the first distribution on 0, and the second on 1.5, # returning a length two tensor. @@ -85,7 +87,7 @@ class _Gumbel(distribution.Distribution): ```python # Define a batch of two scalar valued Logistics. # Both have mean 1, but different scales. - dist = tf.contrib.distributions.Gumbel(loc=1., scale=[11, 22.]) + dist = tfd.Gumbel(loc=1., scale=[11, 22.]) # Evaluate the pdf of both distributions on the same point, 3.0, # returning a length 2 tensor. diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py index 6a74ca9a0a..cbce005013 100644 --- a/tensorflow/contrib/distributions/python/ops/independent.py +++ b/tensorflow/contrib/distributions/python/ops/independent.py @@ -68,11 +68,11 @@ class Independent(distribution_lib.Distribution): #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Make independent distribution from a 2-batch Normal. - ind = ds.Independent( - distribution=ds.Normal(loc=[-1., 1], scale=[0.1, 0.5]), + ind = tfd.Independent( + distribution=tfd.Normal(loc=[-1., 1], scale=[0.1, 0.5]), reinterpreted_batch_ndims=1) # All batch dims have been "absorbed" into event dims. @@ -80,8 +80,8 @@ class Independent(distribution_lib.Distribution): ind.event_shape # ==> [2] # Make independent distribution from a 2-batch bivariate Normal. - ind = ds.Independent( - distribution=ds.MultivariateNormalDiag( + ind = tfd.Independent( + distribution=tfd.MultivariateNormalDiag( loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]), reinterpreted_batch_ndims=1) diff --git a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py index 956dee38a3..ee4d86867d 100644 --- a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py +++ b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py @@ -88,8 +88,9 @@ class InverseGamma(distribution.Distribution): #### Examples ```python - dist = InverseGamma(concentration=3.0, rate=2.0) - dist2 = InverseGamma(concentration=[3.0, 4.0], rate=[2.0, 3.0]) + tfd = tf.contrib.distributions + dist = tfd.InverseGamma(concentration=3.0, rate=2.0) + dist2 = tfd.InverseGamma(concentration=[3.0, 4.0], rate=[2.0, 3.0]) ``` """ diff --git a/tensorflow/contrib/distributions/python/ops/logistic.py b/tensorflow/contrib/distributions/python/ops/logistic.py index 48794a4882..473677f8d9 100644 --- a/tensorflow/contrib/distributions/python/ops/logistic.py +++ b/tensorflow/contrib/distributions/python/ops/logistic.py @@ -60,15 +60,17 @@ class Logistic(distribution.Distribution): Examples of initialization of one or a batch of distributions. ```python + tfd = tf.contrib.distributions + # Define a single scalar Logistic distribution. - dist = tf.contrib.distributions.Logistic(loc=0., scale=3.) + dist = tfd.Logistic(loc=0., scale=3.) # Evaluate the cdf at 1, returning a scalar. dist.cdf(1.) # Define a batch of two scalar valued Logistics. # The first has mean 1 and scale 11, the second 2 and 22. - dist = tf.contrib.distributions.Logistic(loc=[1, 2.], scale=[11, 22.]) + dist = tfd.Logistic(loc=[1, 2.], scale=[11, 22.]) # Evaluate the pdf of the first distribution on 0, and the second on 1.5, # returning a length two tensor. @@ -76,14 +78,11 @@ class Logistic(distribution.Distribution): # Get 3 samples, returning a 3 x 2 tensor. dist.sample([3]) - ``` - Arguments are broadcast when possible. - - ```python + # Arguments are broadcast when possible. # Define a batch of two scalar valued Logistics. # Both have mean 1, but different scales. - dist = tf.contrib.distributions.Logistic(loc=1., scale=[11, 22.]) + dist = tfd.Logistic(loc=1., scale=[11, 22.]) # Evaluate the pdf of both distributions on the same point, 3.0, # returning a length 2 tensor. diff --git a/tensorflow/contrib/distributions/python/ops/mixture.py b/tensorflow/contrib/distributions/python/ops/mixture.py index e676931d91..f2d492f548 100644 --- a/tensorflow/contrib/distributions/python/ops/mixture.py +++ b/tensorflow/contrib/distributions/python/ops/mixture.py @@ -49,13 +49,13 @@ class Mixture(distribution.Distribution): ```python # Create a mixture of two Gaussians: - ds = tf.contrib.distributions + tfd = tf.contrib.distributions mix = 0.3 - bimix_gauss = ds.Mixture( - cat=ds.Categorical(probs=[mix, 1.-mix]), + bimix_gauss = tfd.Mixture( + cat=tfd.Categorical(probs=[mix, 1.-mix]), components=[ - ds.Normal(loc=-1., scale=0.1), - ds.Normal(loc=+1., scale=0.5), + tfd.Normal(loc=-1., scale=0.1), + tfd.Normal(loc=+1., scale=0.5), ]) # Plot the PDF. diff --git a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py index 5558ef0f25..5448918a50 100644 --- a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py +++ b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py @@ -43,15 +43,14 @@ class MixtureSameFamily(distribution.Distribution): #### Examples ```python - import matplotlib.pyplot as plt - ds = tf.contrib.distributions + tfd = tf.contrib.distributions ### Create a mixture of two scalar Gaussians: - gm = ds.MixtureSameFamily( - mixture_distribution=ds.Categorical( + gm = tfd.MixtureSameFamily( + mixture_distribution=tfd.Categorical( probs=[0.3, 0.7]), - components_distribution=ds.Normal( + components_distribution=tfd.Normal( loc=[-1., 1], # One for each component. scale=[0.1, 0.5])) # And same here. @@ -63,14 +62,15 @@ class MixtureSameFamily(distribution.Distribution): # Plot PDF. x = np.linspace(-2., 3., int(1e4), dtype=np.float32) + import matplotlib.pyplot as plt plt.plot(x, gm.prob(x).eval()); ### Create a mixture of two Bivariate Gaussians: - gm = ds.MixtureSameFamily( - mixture_distribution=ds.Categorical( + gm = tfd.MixtureSameFamily( + mixture_distribution=tfd.Categorical( probs=[0.3, 0.7]), - components_distribution=ds.MultivariateNormalDiag( + components_distribution=tfd.MultivariateNormalDiag( loc=[[-1., 1], # component 1 [1, -1]], # component 2 scale_identity_multiplier=[.3, .6])) diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag.py b/tensorflow/contrib/distributions/python/ops/mvn_diag.py index 163cf75d99..e862552880 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_diag.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_diag.py @@ -84,10 +84,10 @@ class MultivariateNormalDiag( #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 2-variate Gaussian. - mvn = ds.MultivariateNormalDiag( + mvn = tfd.MultivariateNormalDiag( loc=[1., -1], scale_diag=[1, 2.]) @@ -101,7 +101,7 @@ class MultivariateNormalDiag( mvn.prob([-1., 0]).eval() # shape: [] # Initialize a 3-batch, 2-variate scaled-identity Gaussian. - mvn = ds.MultivariateNormalDiag( + mvn = tfd.MultivariateNormalDiag( loc=[1., -1], scale_identity_multiplier=[1, 2., 3]) @@ -119,7 +119,7 @@ class MultivariateNormalDiag( mvn.prob([-1., 0]).eval() # shape: [3] # Initialize a 2-batch of 3-variate Gaussians. - mvn = ds.MultivariateNormalDiag( + mvn = tfd.MultivariateNormalDiag( loc=[[1., 2, 3], [11, 22, 33]] # shape: [2, 3] scale_diag=[[1., 2, 3], diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py index 040bc23072..413e88f03a 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py @@ -86,7 +86,7 @@ class MultivariateNormalDiagPlusLowRank( #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 3-variate Gaussian with covariance `cov = S @ S.T`, # `S = diag(d) + U @ diag(m) @ U.T`. The perturbation, `U @ diag(m) @ U.T`, is @@ -97,7 +97,7 @@ class MultivariateNormalDiagPlusLowRank( [-1, 1], [2, -0.5]] # shape: [3, 2] m = [4., 5] # shape: [2] - mvn = ds.MultivariateNormalDiagPlusLowRank( + mvn = tfd.MultivariateNormalDiagPlusLowRank( loc=mu scale_diag=d scale_perturb_factor=U, @@ -118,7 +118,7 @@ class MultivariateNormalDiagPlusLowRank( m = [[0.1, 0.2], [0.4, 0.5]] # shape: [b, r] = [2, 2] - mvn = ds.MultivariateNormalDiagPlusLowRank( + mvn = tfd.MultivariateNormalDiagPlusLowRank( loc=mu, scale_perturb_factor=U, scale_perturb_diag=m) diff --git a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py index f9952b2069..8e69dadfb4 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py @@ -73,14 +73,14 @@ class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL): #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 3-variate Gaussian. mu = [1., 2, 3] cov = [[ 0.36, 0.12, 0.06], [ 0.12, 0.29, -0.13], [ 0.06, -0.13, 0.26]] - mvn = ds.MultivariateNormalFullCovariance( + mvn = tfd.MultivariateNormalFullCovariance( loc=mu, covariance_matrix=cov) @@ -100,7 +100,7 @@ class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL): mu = [[1., 2, 3], [11, 22, 33]] # shape: [2, 3] covariance_matrix = ... # shape: [2, 3, 3], symmetric, positive definite. - mvn = ds.MultivariateNormalFullCovariance( + mvn = tfd.MultivariateNormalFullCovariance( loc=mu, covariance=covariance_matrix) diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py index 300bdd5f60..a739979289 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py @@ -90,8 +90,7 @@ class MultivariateNormalLinearOperator( #### Examples ```python - ds = tf.contrib.distributions - la = tf.linalg + tfd = tf.contrib.distributions # Initialize a single 3-variate Gaussian. mu = [1., 2, 3] @@ -103,9 +102,9 @@ class MultivariateNormalLinearOperator( # [ 0.2, 0.5, 0. ], # [ 0.1, -0.3, 0.4]]) - mvn = ds.MultivariateNormalLinearOperator( + mvn = tfd.MultivariateNormalLinearOperator( loc=mu, - scale=la.LinearOperatorLowerTriangular(scale)) + scale=tf.linalg.LinearOperatorLowerTriangular(scale)) # Covariance agrees with cholesky(cov) parameterization. mvn.covariance().eval() @@ -122,9 +121,9 @@ class MultivariateNormalLinearOperator( scale_diag = [[1., 2, 3], [0.5, 1, 1.5]] # shape: [2, 3] - mvn = ds.MultivariateNormalLinearOperator( + mvn = tfd.MultivariateNormalLinearOperator( loc=mu, - scale=la.LinearOperatorDiag(scale_diag)) + scale=tf.linalg.LinearOperatorDiag(scale_diag)) # Compute the pdf of two `R^3` observations; return a length-2 vector. x = [[-0.9, 0, 0.1], diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py index 260dcc18f5..6c7dc4ca7a 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py @@ -76,12 +76,13 @@ class MultivariateNormalTriL( ``` Trainable (batch) lower-triangular matrices can be created with - `ds.matrix_diag_transform()` and/or `ds.fill_triangular()` + `tf.contrib.distributions.matrix_diag_transform()` and/or + `tf.contrib.distributions.fill_triangular()` #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 3-variate Gaussian. mu = [1., 2, 3] @@ -92,7 +93,7 @@ class MultivariateNormalTriL( # ==> [[ 0.6, 0. , 0. ], # [ 0.2, 0.5, 0. ], # [ 0.1, -0.3, 0.4]]) - mvn = ds.MultivariateNormalTriL( + mvn = tfd.MultivariateNormalTriL( loc=mu, scale_tril=scale) @@ -112,7 +113,7 @@ class MultivariateNormalTriL( mu = [[1., 2, 3], [11, 22, 33]] # shape: [2, 3] tril = ... # shape: [2, 3, 3], lower triangular, non-zero diagonal. - mvn = ds.MultivariateNormalTriL( + mvn = tfd.MultivariateNormalTriL( loc=mu, scale_tril=tril) @@ -124,9 +125,9 @@ class MultivariateNormalTriL( # Instantiate a "learnable" MVN. dims = 4 with tf.variable_scope("model"): - mvn = ds.MultivariateNormalTriL( + mvn = tfd.MultivariateNormalTriL( loc=tf.get_variable(shape=[dims], dtype=tf.float32, name="mu"), - scale_tril=ds.fill_triangular( + scale_tril=tfd.fill_triangular( tf.get_variable(shape=[dims * (dims + 1) / 2], dtype=tf.float32, name="chol_Sigma"))) ``` diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 8a95038a3c..96dff85665 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -107,10 +107,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions + # Create two batches of PoissonLogNormalQuadratureCompounds, one with # prior `loc = 0.` and another with `loc = 1.` In both cases `scale = 1.` - pln = ds.PoissonLogNormalQuadratureCompound( + pln = tfd.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., quadrature_grid_and_probs=( diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py index b05f15771a..c4b8f055b7 100644 --- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py +++ b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py @@ -115,7 +115,7 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution): tailweight: Tailweight parameter. Default is `1.0` (unchanged tailweight) distribution: `tf.Distribution`-like instance. Distribution that is transformed to produce this distribution. - Default is `ds.Normal(0., 1.)`. + Default is `tf.distributions.Normal(0., 1.)`. Must be a scalar-batch, scalar-event distribution. Typically `distribution.reparameterization_type = FULLY_REPARAMETERIZED` or it is a function of non-trainable parameters. WARNING: If you backprop through diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 92043d6a08..904724af42 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -188,8 +188,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): #### Examples ```python - ds = tf.contrib.distributions - la = tf.linalg + tfd = tf.contrib.distributions # Create two batches of VectorDiffeomixtures, one with mix_loc=[0.] and # another with mix_loc=[1]. In both cases, `K=2` and the affine @@ -197,20 +196,20 @@ class VectorDiffeomixture(distribution_lib.Distribution): # k=0: loc=zeros(dims) scale=LinearOperatorScaledIdentity # k=1: loc=[2.]*dims scale=LinOpDiag dims = 5 - vdm = ds.VectorDiffeomixture( + vdm = tfd.VectorDiffeomixture( mix_loc=[[0.], [1]], mix_scale=[1.], - distribution=ds.Normal(loc=0., scale=1.), + distribution=tfd.Normal(loc=0., scale=1.), loc=[ None, # Equivalent to `np.zeros(dims, dtype=np.float32)`. np.float32([2.]*dims), ], scale=[ - la.LinearOperatorScaledIdentity( + tf.linalg.LinearOperatorScaledIdentity( num_rows=dims, multiplier=np.float32(1.1), is_positive_definite=True), - la.LinearOperatorDiag( + tf.linalg.LinearOperatorDiag( diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), is_positive_definite=True), ], diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py index 356d78b67a..526fe2d39a 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py +++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py @@ -89,14 +89,13 @@ class VectorExponentialDiag( #### Examples ```python - ds = tf.contrib.distributions - la = tf.linalg + tfd = tf.contrib.distributions # Initialize a single 2-variate VectorExponential, supported on # {(x, y) in R^2 : x > 0, y > 0}. # The first component has pdf exp{-x}, the second 0.5 exp{-x / 2} - vex = ds.VectorExponentialDiag(scale_diag=[1., 2.]) + vex = tfd.VectorExponentialDiag(scale_diag=[1., 2.]) # Compute the pdf of an`R^2` observation; return a scalar. vex.prob([3., 4.]).eval() # shape: [] @@ -107,7 +106,7 @@ class VectorExponentialDiag( scale_diag = [[1., 2, 3], [0.5, 1, 1.5]] # shape: [2, 3] - vex = ds.VectorExponentialDiag(loc, scale_diag) + vex = tfd.VectorExponentialDiag(loc, scale_diag) # Compute the pdf of two `R^3` observations; return a length-2 vector. x = [[1.9, 2.2, 3.1], diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py index b313a851b3..9d5fd9ac41 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py @@ -107,16 +107,15 @@ class VectorExponentialLinearOperator( #### Examples ```python - ds = tf.contrib.distributions - la = tf.linalg + tfd = tf.contrib.distributions # Initialize a single 2-variate VectorExponential, supported on # {(x, y) in R^2 : x > 0, y > 0}. mat = [[1.0, 0.1], [0.1, 1.0]] - vex = ds.VectorExponentialLinearOperator( - scale=la.LinearOperatorFullMatrix(mat)) + vex = tfd.VectorExponentialLinearOperator( + scale=tf.linalg.LinearOperatorFullMatrix(mat)) # Compute the pdf of an`R^2` observation; return a scalar. vex.prob([1., 2.]).eval() # shape: [] @@ -127,9 +126,9 @@ class VectorExponentialLinearOperator( scale_diag = [[1., 2, 3], [0.5, 1, 1.5]] # shape: [2, 3] - vex = ds.VectorExponentialLinearOperator( + vex = tfd.VectorExponentialLinearOperator( loc=mu, - scale=la.LinearOperatorDiag(scale_diag)) + scale=tf.linalg.LinearOperatorDiag(scale_diag)) # Compute the pdf of two `R^3` observations; return a length-2 vector. x = [[1.9, 2.2, 3.1], diff --git a/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py b/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py index 0e3867809a..8dd983b750 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py +++ b/tensorflow/contrib/distributions/python/ops/vector_laplace_diag.py @@ -101,10 +101,10 @@ class VectorLaplaceDiag( #### Examples ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 2-variate VectorLaplace. - vla = ds.VectorLaplaceDiag( + vla = tfd.VectorLaplaceDiag( loc=[1., -1], scale_diag=[1, 2.]) @@ -118,7 +118,7 @@ class VectorLaplaceDiag( vla.prob([-1., 0]).eval() # shape: [] # Initialize a 3-batch, 2-variate scaled-identity VectorLaplace. - vla = ds.VectorLaplaceDiag( + vla = tfd.VectorLaplaceDiag( loc=[1., -1], scale_identity_multiplier=[1, 2., 3]) @@ -136,7 +136,7 @@ class VectorLaplaceDiag( vla.prob([-1., 0]).eval() # shape: [3] # Initialize a 2-batch of 3-variate VectorLaplace's. - vla = ds.VectorLaplaceDiag( + vla = tfd.VectorLaplaceDiag( loc=[[1., 2, 3], [11, 22, 33]] # shape: [2, 3] scale_diag=[[1., 2, 3], diff --git a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py index c7abdbb4ca..ec485c95c1 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py @@ -109,8 +109,7 @@ class VectorLaplaceLinearOperator( #### Examples ```python - ds = tf.contrib.distributions - la = tf.linalg + tfd = tf.contrib.distributions # Initialize a single 3-variate VectorLaplace with some desired covariance. mu = [1., 2, 3] @@ -124,9 +123,9 @@ class VectorLaplaceLinearOperator( # [ 0.1, -0.3, 0.4]]) # Divide scale by sqrt(2) so that the final covariance will be what we want. - vla = ds.VectorLaplaceLinearOperator( + vla = tfd.VectorLaplaceLinearOperator( loc=mu, - scale=la.LinearOperatorLowerTriangular(scale / tf.sqrt(2))) + scale=tf.linalg.LinearOperatorLowerTriangular(scale / tf.sqrt(2.))) # Covariance agrees with cholesky(cov) parameterization. vla.covariance().eval() @@ -143,9 +142,9 @@ class VectorLaplaceLinearOperator( scale_diag = [[1., 2, 3], [0.5, 1, 1.5]] # shape: [2, 3] - vla = ds.VectorLaplaceLinearOperator( + vla = tfd.VectorLaplaceLinearOperator( loc=mu, - scale=la.LinearOperatorDiag(scale_diag)) + scale=tf.linalg.LinearOperatorDiag(scale_diag)) # Compute the pdf of two `R^3` observations; return a length-2 vector. x = [[-0.9, 0, 0.1], diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py index 544a871070..e1ccf11645 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py +++ b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py @@ -143,7 +143,7 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution): broadcastable with `event_shape`. distribution: `tf.Distribution`-like instance. Distribution from which `k` iid samples are used as input to transformation `F`. Default is - `ds.Normal(0., 1.)`. + `tf.distributions.Normal(loc=0., scale=1.)`. Must be a scalar-batch, scalar-event distribution. Typically `distribution.reparameterization_type = FULLY_REPARAMETERIZED` or it is a function of non-trainable parameters. WARNING: If you backprop through diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py index 29d41ab81c..8c67647a61 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py +++ b/tensorflow/contrib/distributions/python/ops/vector_student_t.py @@ -91,14 +91,14 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution): Extra leading dimensions, if provided, allow for batches. ```python - ds = tf.contrib.distributions + tfd = tf.contrib.distributions # Initialize a single 3-variate vector Student's t-distribution. mu = [1., 2, 3] chol = [[1., 0, 0.], [1, 3, 0], [1, 2, 3]] - vt = ds.VectorStudentT(df=2, loc=mu, scale_tril=chol) + vt = tfd.VectorStudentT(df=2, loc=mu, scale_tril=chol) # Evaluate this on an observation in R^3, returning a scalar. vt.prob([-1., 0, 1]) @@ -107,7 +107,7 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution): mu = [[1., 2, 3], [11, 22, 33]] chol = ... # shape 2 x 3 x 3, lower triangular, positive diagonal. - vt = ds.VectorStudentT(loc=mu, scale_tril=chol) + vt = tfd.VectorStudentT(loc=mu, scale_tril=chol) # Evaluate this on a two observations, each in R^3, returning a length two # tensor. -- GitLab From cb4ef362e4a18b3c42a2c90bdad8754d5ead4caf Mon Sep 17 00:00:00 2001 From: Yangzihao Wang Date: Wed, 29 Nov 2017 16:38:32 -0800 Subject: [PATCH 0133/1924] Add native dilated support for conv2d and its gradients in cudnn v>=6. PiperOrigin-RevId: 177382431 --- .../compiler/tf2xla/kernels/conv_ops.cc | 57 +++ .../fused_conv2d_bias_activation_op.cc | 2 + .../fused_conv/kernels/fused_conv_ops_gpu.h | 9 +- .../ops/fused_conv2d_bias_activation_op.cc | 6 + .../api_def/base_api/api_def_Conv2D.pbtxt | 12 +- .../api_def_Conv2DBackpropFilter.pbtxt | 10 + .../api_def_Conv2DBackpropInput.pbtxt | 10 + .../api_def/base_api/api_def_Conv3D.pbtxt | 10 + .../api_def_Conv3DBackpropFilterV2.pbtxt | 10 + .../api_def_Conv3DBackpropInputV2.pbtxt | 10 + .../api_def_DepthwiseConv2dNative.pbtxt | 10 + ..._DepthwiseConv2dNativeBackpropFilter.pbtxt | 10 + ...f_DepthwiseConv2dNativeBackpropInput.pbtxt | 10 + .../base_api/api_def_QuantizedConv2D.pbtxt | 10 + tensorflow/core/framework/common_shape_fns.cc | 23 +- .../core/framework/common_shape_fns_test.cc | 106 ++++- .../core/kernels/conv_grad_filter_ops.cc | 93 +++- .../core/kernels/conv_grad_input_ops.cc | 97 ++++- tensorflow/core/kernels/conv_grad_ops.h | 16 +- tensorflow/core/kernels/conv_grad_ops_3d.cc | 4 + tensorflow/core/kernels/conv_ops.cc | 113 +++-- tensorflow/core/kernels/conv_ops.h | 10 +- tensorflow/core/kernels/conv_ops_3d.cc | 3 + tensorflow/core/kernels/conv_ops_gpu.h | 12 +- tensorflow/core/kernels/conv_ops_test.cc | 4 + tensorflow/core/kernels/depthwise_conv_op.cc | 5 +- tensorflow/core/kernels/quantized_conv_ops.cc | 13 + tensorflow/core/ops/nn_ops.cc | 80 +++- .../conv2d_backprop_filter_grad_test.py | 54 ++- .../python/kernel_tests/conv_ops_test.py | 407 +++++++++++++++++- tensorflow/python/ops/nn_grad.py | 90 ++-- tensorflow/python/ops/nn_ops.py | 28 +- .../tools/api/golden/tensorflow.nn.pbtxt | 18 +- 33 files changed, 1181 insertions(+), 171 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc index c5017704e2..c150394c07 100644 --- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc @@ -121,6 +121,7 @@ class ConvOp : public XlaOpKernel { : XlaOpKernel(ctx), num_spatial_dims_(num_spatial_dims), depthwise_(depthwise) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("dilations", &dilations_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("strides", &strides_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("padding", &padding_)); @@ -144,6 +145,23 @@ class ConvOp : public XlaOpKernel { errors::Unimplemented("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(ctx, dilations_.size() == num_dims(), + errors::InvalidArgument("Dilations field must " + "specify ", + num_dims(), " dimensions")); + OP_REQUIRES( + ctx, dilations_[batch_dim] == 1 && dilations_[feature_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + for (int i = 0; i < num_spatial_dims_; ++i) { + int input_dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); + OP_REQUIRES( + ctx, dilations_[input_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the ", + i, "th spatial dimension.")); + } + const TensorShape input_shape = ctx->InputShape(0); // Input filter is of the following dimensions: // [ filter_rows, filter_cols, ..., in_depth, out_depth] @@ -204,6 +222,7 @@ class ConvOp : public XlaOpKernel { protected: const int num_spatial_dims_; const bool depthwise_; + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_ = FORMAT_NHWC; @@ -241,6 +260,7 @@ class ConvBackpropInputOp : public XlaOpKernel { : XlaOpKernel(ctx), num_spatial_dims_(num_spatial_dims), depthwise_(depthwise) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("dilations", &dilations_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("strides", &strides_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("padding", &padding_)); string data_format; @@ -263,6 +283,23 @@ class ConvBackpropInputOp : public XlaOpKernel { errors::Unimplemented("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(ctx, dilations_.size() == num_dims(), + errors::InvalidArgument("Dilations field must " + "specify ", + num_dims(), " dimensions")); + OP_REQUIRES( + ctx, dilations_[batch_dim] == 1 && dilations_[feature_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + for (int i = 0; i < num_spatial_dims_; ++i) { + int input_dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); + OP_REQUIRES( + ctx, dilations_[input_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the ", + i, "th spatial dimension.")); + } + TensorShape input_shape; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsShape(0, &input_shape)); @@ -336,6 +373,7 @@ class ConvBackpropInputOp : public XlaOpKernel { protected: const int num_spatial_dims_; const bool depthwise_; + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_ = FORMAT_NHWC; @@ -373,6 +411,7 @@ class ConvBackpropFilterOp : public XlaOpKernel { : XlaOpKernel(ctx), num_spatial_dims_(num_spatial_dims), depthwise_(depthwise) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("dilations", &dilations_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("strides", &strides_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("padding", &padding_)); string data_format; @@ -392,6 +431,23 @@ class ConvBackpropFilterOp : public XlaOpKernel { errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(ctx, dilations_.size() == num_dims(), + errors::InvalidArgument("Dilations field must " + "specify ", + num_dims(), " dimensions")); + OP_REQUIRES( + ctx, dilations_[n_dim] == 1 && dilations_[c_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + for (int i = 0; i < num_spatial_dims_; ++i) { + int input_dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); + OP_REQUIRES( + ctx, dilations_[input_dim] == 1, + errors::Unimplemented("Current implementation does not yet support " + "dilations in the ", + i, "th spatial dimension.")); + } + const TensorShape activations_shape = ctx->InputShape(0); TensorShape filter_shape; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsShape(1, &filter_shape)); @@ -526,6 +582,7 @@ class ConvBackpropFilterOp : public XlaOpKernel { protected: const int num_spatial_dims_; const bool depthwise_; + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_ = FORMAT_NHWC; diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc index 88306094ab..5fec69ea43 100644 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc +++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc @@ -493,6 +493,8 @@ void LaunchFusedConv2DBiasActivationOp:: {{conv_input_rows, conv_input_cols}}, output_depth, {{filter_rows, filter_cols}}, + // TODO(yangzihao): Add support for arbitrary dilations for fused conv. + {{1, 1}}, // dilation_rows, dilation_cols {{row_stride, col_stride}}, {{padding_rows, padding_cols}}, conv_input->dtype(), diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h b/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h index dc43af1158..fa7a3c03aa 100644 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h +++ b/tensorflow/contrib/fused_conv/kernels/fused_conv_ops_gpu.h @@ -30,11 +30,12 @@ class FusedConvParameters : public ConvParameters { public: FusedConvParameters(int64 batch, int64 in_depths, const SpatialArray& in, int64 out_depths, const SpatialArray& filter, - const SpatialArray& stride, const SpatialArray& padding, - DataType dtype, int device_id, bool has_side_input, + const SpatialArray& dilation, const SpatialArray& stride, + const SpatialArray& padding, DataType dtype, + int device_id, bool has_side_input, ActivationMode activation_mode) - : ConvParameters(batch, in_depths, in, out_depths, filter, stride, - padding, dtype, device_id), + : ConvParameters(batch, in_depths, in, out_depths, filter, dilation, + stride, padding, dtype, device_id), activation_mode_(activation_mode), has_side_input_(has_side_input) { hash_code_ = Hash64Combine(hash_code_, has_side_input); diff --git a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc index 887ebc5a6c..6a56237f67 100644 --- a/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc +++ b/tensorflow/contrib/fused_conv/ops/fused_conv2d_bias_activation_op.cc @@ -52,6 +52,7 @@ REGISTER_OP("FusedConv2DBiasActivation") .Attr("data_format: {'NHWC', 'NCHW', 'NCHW_VECT_C'} = 'NHWC'") .Attr("filter_format: {'HWIO', 'OIHW', 'OIHW_VECT_I'} = 'HWIO'") .Attr("activation_mode: {'Relu'} = 'Relu'") + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](shape_inference::InferenceContext* c) { using shape_inference::ShapeHandle; using shape_inference::DimensionHandle; @@ -151,6 +152,11 @@ REGISTER_OP("FusedConv2DBiasActivation") kernel_height, kernel_width, input_channels % 4 ]` activation_mode: The activation applied to the output. Currently must be "Relu". + dilations: 1-D tensor of length 4. The dilation factor for each dimension + of `input`. If set to k > 1, there will be k-1 skipped cells between + each filter element on that dimension. The dimension order is determined + by the value of `data_format`, see above for details. Dilations in the + batch and depth dimensions must be 1. )doc"); } // namespace tensorflow diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt index 6522ce976f..070d6adb97 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv2D.pbtxt @@ -26,7 +26,7 @@ END description: < 1, there will be k-1 skipped cells between each +filter element on that dimension. The dimension order is determined by the +value of `data_format`, see above for details. Dilations in the batch and +depth dimensions must be 1. END } summary: "Computes a 2-D convolution given 4-D `input` and `filter` tensors." diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt index 4ea3374dbb..ff2d9d71db 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropFilter.pbtxt @@ -51,6 +51,16 @@ default format "NHWC", the data is stored in the order of: [batch, in_height, in_width, in_channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, in_channels, in_height, in_width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each filter +element on that dimension. The dimension order is determined by the value of +`data_format`, see above for details. Dilations in the batch and depth +dimensions must be 1. END } summary: "Computes the gradients of convolution with respect to the filter." diff --git a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt index 4420073e38..2de38b4263 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv2DBackpropInput.pbtxt @@ -50,6 +50,16 @@ default format "NHWC", the data is stored in the order of: [batch, in_height, in_width, in_channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, in_channels, in_height, in_width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each filter +element on that dimension. The dimension order is determined by the value of +`data_format`, see above for details. Dilations in the batch and depth +dimensions must be 1. END } summary: "Computes the gradients of convolution with respect to the input." diff --git a/tensorflow/core/api_def/base_api/api_def_Conv3D.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv3D.pbtxt index 8f3cd4493c..d26564097e 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv3D.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv3D.pbtxt @@ -34,6 +34,16 @@ default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each +filter element on that dimension. The dimension order is determined by the +value of `data_format`, see above for details. Dilations in the batch and +depth dimensions must be 1. END } summary: "Computes a 3-D convolution given 5-D `input` and `filter` tensors." diff --git a/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropFilterV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropFilterV2.pbtxt index 6f9b917237..937c9c8ead 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropFilterV2.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropFilterV2.pbtxt @@ -43,6 +43,16 @@ default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each +filter element on that dimension. The dimension order is determined by the +value of `data_format`, see above for details. Dilations in the batch and +depth dimensions must be 1. END } summary: "Computes the gradients of 3-D convolution with respect to the filter." diff --git a/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropInputV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropInputV2.pbtxt index 19aba156d5..414e418dc5 100644 --- a/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropInputV2.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_Conv3DBackpropInputV2.pbtxt @@ -43,6 +43,16 @@ default format "NDHWC", the data is stored in the order of: [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each +filter element on that dimension. The dimension order is determined by the +value of `data_format`, see above for details. Dilations in the batch and +depth dimensions must be 1. END } summary: "Computes the gradients of 3-D convolution with respect to the input." diff --git a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNative.pbtxt b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNative.pbtxt index cc10ebe923..3c313f7be6 100644 --- a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNative.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNative.pbtxt @@ -21,6 +21,16 @@ default format "NHWC", the data is stored in the order of: [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each filter +element on that dimension. The dimension order is determined by the value of +`data_format`, see above for details. Dilations in the batch and depth +dimensions must be 1. END } summary: "Computes a 2-D depthwise convolution given 4-D `input` and `filter` tensors." diff --git a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt index 9126be2afa..e66aa3b707 100644 --- a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropFilter.pbtxt @@ -54,6 +54,16 @@ default format "NHWC", the data is stored in the order of: [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each filter +element on that dimension. The dimension order is determined by the value of +`data_format`, see above for details. Dilations in the batch and depth +dimensions must be 1. END } summary: "Computes the gradients of depthwise convolution with respect to the filter." diff --git a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt index f1d16858db..f501ad21b3 100644 --- a/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_DepthwiseConv2dNativeBackpropInput.pbtxt @@ -54,6 +54,16 @@ default format "NHWC", the data is stored in the order of: [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +END + } + attr { + name: "dilations" + description: < 1, there will be k-1 skipped cells between each filter +element on that dimension. The dimension order is determined by the value of +`data_format`, see above for details. Dilations in the batch and depth +dimensions must be 1. END } summary: "Computes the gradients of depthwise convolution with respect to the input." diff --git a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2D.pbtxt b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2D.pbtxt index b19bbeab12..d18bafdce9 100644 --- a/tensorflow/core/api_def/base_api/api_def_QuantizedConv2D.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_QuantizedConv2D.pbtxt @@ -53,6 +53,16 @@ END name: "padding" description: < 1, there will be k-1 skipped cells between each +filter element on that dimension. The dimension order is determined by the +value of `data_format`, see above for details. Dilations in the batch and +depth dimensions must be 1. END } summary: "Computes a 2D convolution given quantized 4D input and filter tensors." diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index be7f2e2808..036e3473b1 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -397,6 +397,15 @@ Status Conv2DShape(shape_inference::InferenceContext* c) { TF_RETURN_IF_ERROR( CheckFormatConstraintsOnShape(data_format, filter_shape, "filter", c)); + std::vector dilations; + TF_RETURN_IF_ERROR(c->GetAttr("dilations", &dilations)); + + if (dilations.size() != 4) { + return errors::InvalidArgument( + "Conv2D requires the dilation attribute to contain 4 values, but got: ", + dilations.size()); + } + std::vector strides; TF_RETURN_IF_ERROR(c->GetAttr("strides", &strides)); @@ -410,6 +419,8 @@ Status Conv2DShape(shape_inference::InferenceContext* c) { const int32 stride_rows = GetTensorDim(strides, data_format, 'H'); const int32 stride_cols = GetTensorDim(strides, data_format, 'W'); + const int32 dilation_rows = GetTensorDim(dilations, data_format, 'H'); + const int32 dilation_cols = GetTensorDim(dilations, data_format, 'W'); DimensionHandle batch_size_dim; DimensionHandle input_depth_dim; @@ -447,12 +458,12 @@ Status Conv2DShape(shape_inference::InferenceContext* c) { TF_RETURN_IF_ERROR(c->GetAttr("padding", &padding)); DimensionHandle output_rows, output_cols; - TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDims(c, input_spatial_dims[0], - filter_rows_dim, stride_rows, - padding, &output_rows)); - TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDims(c, input_spatial_dims[1], - filter_cols_dim, stride_cols, - padding, &output_cols)); + TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2( + c, input_spatial_dims[0], filter_rows_dim, dilation_rows, stride_rows, + padding, &output_rows)); + TF_RETURN_IF_ERROR(GetWindowedOutputSizeFromDimsV2( + c, input_spatial_dims[1], filter_cols_dim, dilation_cols, stride_cols, + padding, &output_cols)); ShapeHandle output_shape; TF_RETURN_IF_ERROR( diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc index ec9746b2af..5f3e5ad457 100644 --- a/tensorflow/core/framework/common_shape_fns_test.cc +++ b/tensorflow/core/framework/common_shape_fns_test.cc @@ -423,6 +423,15 @@ TEST(CommonShapeFnsTest, Conv2DShapeTest) { .Finalize(&op.node_def)); }; + // Invalid rank for input + INFER_ERROR("must be rank 4", op, "[4,4];[2,1,1,1]"); + // Invalid rank for filter + INFER_ERROR("must be rank 4", op, "[1,4,4,1];[2,1,1]"); + + // Invalid value for strides + set_op({{1, 1, 0, 1}}, "VALID", "NHWC", "HWIO"); + INFER_ERROR("must be > 0", op, "[1,2,2,1];[1,1,1,1]"); + // 1x1 filter set_op({{1, 1, 1, 1}}, "VALID", "NHWC", "HWIO"); INFER_OK(op, "[1,2,2,1];[1,1,1,1]", "[d0_0,2,2,d1_3]"); @@ -443,11 +452,6 @@ TEST(CommonShapeFnsTest, Conv2DShapeTest) { set_op({{1, 1, 2, 1}}, "VALID", "NHWC", "HWIO"); INFER_OK(op, "[1,4,4,1];[2,1,1,1]", "[d0_0,3,2,d1_3]"); - // Invalid rank for input - INFER_ERROR("must be rank 4", op, "[4,4];[2,1,1,1]"); - // Invalid rank for filter - INFER_ERROR("must be rank 4", op, "[1,4,4,1];[2,1,1]"); - // Unknown dims in the critical fields lead to partial inference. INFER_OK(op, "[1,4,4,1];[2,1,1,1]", "[d0_0,3,2,d1_3]"); INFER_OK(op, "[1,?,4,1];[2,1,1,1]", "[d0_0,?,2,d1_3]"); @@ -538,6 +542,98 @@ TEST(CommonShapeFnsTest, Conv2DShapeTest) { INFER_OK(op, "[1,4,4,?];[?,?,?,?]", "[d0_0,2,2,d1_3]"); } +TEST(CommonShapeFnsTest, Conv2DDilatedShapeTest) { + ShapeInferenceTestOp op("Conv2D"); + auto set_op = [&op](const std::vector& dilations, + const std::vector& strides, const string& padding, + const string& data_format) { + TF_CHECK_OK(NodeDefBuilder("test", "Conv2D") + .Input("input", 0, DT_FLOAT) + .Input("filter", 0, DT_FLOAT) + .Attr("dilations", dilations) + .Attr("strides", strides) + .Attr("padding", padding) + .Attr("data_format", data_format) + .Finalize(&op.node_def)); + }; + + // Invalid rank for dilation + set_op({{1, 2, 1}}, {{1, 1, 1, 1}}, "VALID", "NHWC"); + INFER_ERROR("contain 4 values", op, "[1,2,2,1];[1,1,1,1]"); + + // Invalid value for dilation + set_op({{1, 0, 1, 1}}, {{1, 1, 1, 1}}, "VALID", "NHWC"); + INFER_ERROR("must be >= 1", op, "[1,2,2,1];[1,1,1,1]"); + + // Tests for NHWC + // 1x1 filter, 2x1 dilations, 1x1 strides + set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,2,2,1];[1,1,1,1]", "[d0_0,2,2,d1_3]"); + + // 1x1 filter, 2x1 dilations, 2x1 strides + set_op({{1, 2, 1, 1}}, {{1, 2, 1, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,2,4,d1_3]"); + + // 1x1 filter, 2x1 dilations, 2x2 strides + set_op({{1, 2, 1, 1}}, {{1, 2, 2, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,2,2,d1_3]"); + + // 3x3 filter, 2x1 dilations, 1x1 strides + set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,5,5,1];[3,3,1,1]", "[d0_0,1,3,d1_3]"); + + // 3x3 filter, 2x1 dilations, 2x1 strides + set_op({{1, 2, 1, 1}}, {{1, 2, 1, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,5,5,1];[3,3,1,1]", "[d0_0,1,3,d1_3]"); + + // 3x3 filter, 1x2 dilations, 2x2 strides + set_op({{1, 1, 2, 1}}, {{1, 2, 2, 1}}, "VALID", "NHWC"); + INFER_OK(op, "[1,5,5,1];[3,3,1,1]", "[d0_0,2,1,d1_3]"); + + // Tests for NCHW + // 1x1 filter, 2x1 dilations, 1x1 strides + set_op({{1, 1, 2, 1}}, {{1, 1, 1, 1}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,2,2];[1,1,1,1]", "[d0_0,d1_3,2,2]"); + + // 1x1 filter, 2x1 dilations, 2x1 strides + set_op({{1, 1, 2, 1}}, {{1, 1, 2, 1}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,4,4];[1,1,1,1]", "[d0_0,d1_3,2,4]"); + + // 1x1 filter, 2x1 dilations, 2x2 strides + set_op({{1, 1, 2, 1}}, {{1, 1, 2, 2}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,4,4];[1,1,1,1]", "[d0_0,d1_3,2,2]"); + + // 3x3 filter, 2x1 dilations, 1x1 strides + set_op({{1, 1, 2, 1}}, {{1, 1, 1, 1}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,5,5];[3,3,1,1]", "[d0_0,d1_3,1,3]"); + + // 3x3 filter, 2x1 dilations, 2x1 strides + set_op({{1, 1, 2, 1}}, {{1, 1, 2, 1}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,5,5];[3,3,1,1]", "[d0_0,d1_3,1,3]"); + + // 3x3 filter, 1x2 dilations, 2x2 strides + set_op({{1, 1, 1, 2}}, {{1, 1, 2, 2}}, "VALID", "NCHW"); + INFER_OK(op, "[1,1,5,5];[3,3,1,1]", "[d0_0,d1_3,2,1]"); + + // Some tests for "SAME" padding + + // 4x4 input, 1x1 filter, 2x1 dilations, 1x1 stride + set_op({{1, 2, 1, 1}}, {{1, 1, 1, 1}}, "SAME", "NHWC"); + INFER_OK(op, "[1,4,4,1];[1,1,1,1]", "[d0_0,d0_1,d0_2,d1_3]"); + + // 3x3 input, 2x2 filter, 2x2 dilations, 1x1 stride + set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "SAME", "NHWC"); + INFER_OK(op, "[1,3,3,1];[2,2,1,1]", "[d0_0,d0_1,d0_2,d1_3]"); + + // 4x4 input, 2x2 filter, 1x2 dilations, 2x2 stride + set_op({{1, 1, 2, 1}}, {{1, 2, 2, 1}}, "SAME", "NHWC"); + INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,2,2,d1_3]"); + + // 4x4 input, 2x2 filter, 2x2 dilations, 1x1 stride + set_op({{1, 2, 2, 1}}, {{1, 1, 1, 1}}, "SAME", "NHWC"); + INFER_OK(op, "[1,4,4,1];[2,2,1,1]", "[d0_0,d0_1,d0_2,d1_3]"); +} + TEST(CommonShapeFnsTest, Conv3DShapeTest) { ShapeInferenceTestOp op("Conv3D"); auto set_op = [&op](const std::vector& strides, diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc index 3d2bb57aff..1791c51096 100644 --- a/tensorflow/core/kernels/conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc @@ -194,7 +194,23 @@ class Conv2DFastBackpropFilterOp : public OpKernel { context, (strides_[0] == 1 && strides_[3] == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + OP_REQUIRES(context, (dilations_[0] == 1 && dilations_[3] == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + // TODO(yangzihao): Add a CPU implementation for dilated convolution. + OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1), + errors::InvalidArgument( + "Current Eigen and libxsmm implementations do not " + "yet support dilation rates larger than 1.")); } void Compute(OpKernelContext* context) override { @@ -262,6 +278,7 @@ class Conv2DFastBackpropFilterOp : public OpKernel { } private: + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_; @@ -290,7 +307,23 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { context, (strides_[0] == 1 && strides_[3] == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + OP_REQUIRES(context, (dilations_[0] == 1 && dilations_[3] == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + // TODO(yangzihao): Add a CPU implementation for dilated convolution. + OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1), + errors::InvalidArgument( + "Current libxsmm and customized CPU implementations do " + "not yet support dilation rates larger than 1.")); } void Compute(OpKernelContext* context) override { @@ -459,6 +492,7 @@ class Conv2DCustomBackpropFilterOp : public OpKernel { } private: + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_; @@ -510,10 +544,30 @@ class Conv2DSlowBackpropFilterOp : public OpKernel { OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_)); int stride_n = GetTensorDim(strides_, data_format_, 'N'); int stride_c = GetTensorDim(strides_, data_format_, 'C'); + int stride_h = GetTensorDim(strides_, data_format_, 'H'); + int stride_w = GetTensorDim(strides_, data_format_, 'W'); OP_REQUIRES( context, (stride_n == 1 && stride_c == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, stride_h > 0 && stride_w > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + int dilation_n = GetTensorDim(dilations_, data_format_, 'N'); + int dilation_c = GetTensorDim(dilations_, data_format_, 'C'); + int dilation_h = GetTensorDim(dilations_, data_format_, 'H'); + int dilation_w = GetTensorDim(dilations_, data_format_, 'W'); + OP_REQUIRES(context, dilation_n == 1 && dilation_c == 1, + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + OP_REQUIRES( + context, dilation_h > 0 && dilation_w > 0, + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("use_cudnn_on_gpu", &use_cudnn_)); use_cudnn_ &= CanUseCudnn(); cudnn_use_autotune_ = CudnnUseAutotune(); @@ -546,13 +600,16 @@ class Conv2DSlowBackpropFilterOp : public OpKernel { // do not support striding on the batch or depth dimension). const int stride_rows = GetTensorDim(strides_, data_format_, 'H'); const int stride_cols = GetTensorDim(strides_, data_format_, 'W'); + const int dilation_rows = GetTensorDim(dilations_, data_format_, 'H'); + const int dilation_cols = GetTensorDim(dilations_, data_format_, 'W'); launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, input, - stride_rows, stride_cols, padding_, filter_backprop, - data_format_); + dilation_rows, dilation_cols, stride_rows, stride_cols, padding_, + filter_backprop, data_format_); } private: + std::vector dilations_; std::vector strides_; Padding padding_; bool use_cudnn_; @@ -566,38 +623,46 @@ class Conv2DSlowBackpropFilterOp : public OpKernel { template void LaunchConv2DBackpropFilterOp::operator()( OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& out_backprop, const Tensor& input, int row_stride, - int col_stride, const Padding& padding, Tensor* filter_backprop, - TensorFormat data_format) { + const Tensor& out_backprop, const Tensor& input, int row_dilation, + int col_dilation, int row_stride, int col_stride, const Padding& padding, + Tensor* filter_backprop, TensorFormat data_format) { using perftools::gputools::dnn::AlgorithmConfig; using perftools::gputools::dnn::AlgorithmDesc; using perftools::gputools::dnn::ProfileResult; + std::vector dilations(4, 1); + dilations[GetTensorDimIndex(data_format, 'H')] = row_dilation; + dilations[GetTensorDimIndex(data_format, 'W')] = col_dilation; + std::vector strides(4, 1); strides[GetTensorDimIndex(data_format, 'H')] = row_stride; strides[GetTensorDimIndex(data_format, 'W')] = col_stride; TensorShape filter_shape = filter_backprop->shape(); ConvBackpropDimensions dims; - OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensions( + OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2( "Conv2DSlowBackpropFilter", /*num_spatial_dims=*/2, input.shape(), filter_shape, out_backprop.shape(), - strides, padding, data_format, &dims)); + dilations, strides, padding, data_format, &dims)); + // TODO(yangzihao): The padding computations should be done in + // GetWindowedOutputSize() functions. const int padding_rows = (padding == VALID) ? 0 : std::max(0, (dims.spatial_dims[0].output_size - 1) * dims.spatial_dims[0].stride + - dims.spatial_dims[0].filter_size - - dims.spatial_dims[0].input_size); + (dims.spatial_dims[0].filter_size - 1) * + dims.spatial_dims[0].dilation + + 1 - dims.spatial_dims[0].input_size); const int padding_cols = (padding == VALID) ? 0 : std::max(0, (dims.spatial_dims[1].output_size - 1) * dims.spatial_dims[1].stride + - dims.spatial_dims[1].filter_size - - dims.spatial_dims[1].input_size); + (dims.spatial_dims[1].filter_size - 1) * + dims.spatial_dims[1].dilation + + 1 - dims.spatial_dims[1].input_size); // TODO(zhengxq): cuDNN only supports equal padding on both sides, so only // calling it when that is true. Remove this check when (if?) cuDNN starts @@ -730,7 +795,9 @@ void LaunchConv2DBackpropFilterOp::operator()( .set_input_feature_map_count(dims.in_depth) .set_output_feature_map_count(dims.out_depth); perftools::gputools::dnn::ConvolutionDescriptor conv_desc; - conv_desc.set_vertical_filter_stride(dims.spatial_dims[0].stride) + conv_desc.set_vertical_dilation_rate(dims.spatial_dims[0].dilation) + .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation) + .set_vertical_filter_stride(dims.spatial_dims[0].stride) .set_horizontal_filter_stride(dims.spatial_dims[1].stride) .set_zero_padding_height(padding_rows / 2) .set_zero_padding_width(padding_cols / 2); @@ -821,6 +888,8 @@ void LaunchConv2DBackpropFilterOp::operator()( dims.out_depth, // out_depths {{dims.spatial_dims[0].filter_size, // filter_rows dims.spatial_dims[1].filter_size}}, // filter_cols + {{dims.spatial_dims[0].dilation, // dilation_rows + dims.spatial_dims[1].dilation}}, // dilation_cols {{dims.spatial_dims[0].stride, // stride_rows dims.spatial_dims[1].stride}}, // stride_cols {{padding_rows, // padding_rows diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc index d28f6b4d10..736241a029 100644 --- a/tensorflow/core/kernels/conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/conv_grad_input_ops.cc @@ -198,7 +198,23 @@ class Conv2DFastBackpropInputOp : public OpKernel { context, (strides_[0] == 1 && strides_[3] == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + OP_REQUIRES(context, (dilations_[0] && dilations_[3]), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + // TODO(yangzihao): Add a CPU implementation for dilated convolution. + OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1), + errors::InvalidArgument( + "Current Eigen and libxsmm implementations do not " + "yet support dilation rates larger than 1.")); } void Compute(OpKernelContext* context) override { @@ -268,6 +284,7 @@ class Conv2DFastBackpropInputOp : public OpKernel { } private: + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_; @@ -296,7 +313,23 @@ class Conv2DCustomBackpropInputOp : public OpKernel { context, (strides_[0] == 1 && strides_[3] == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, strides_[1] > 0 && strides_[2] > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + OP_REQUIRES(context, (dilations_[0] == 1 && dilations_[3] == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + // TODO(yangzihao): Add a CPU implementation for dilated convolution. + OP_REQUIRES(context, (dilations_[1] == 1 && dilations_[2] == 1), + errors::InvalidArgument( + "Current libxsmm and customized CPU implementations do " + "not yet support dilation rates larger than 1.")); } void Compute(OpKernelContext* context) override { @@ -532,6 +565,7 @@ class Conv2DCustomBackpropInputOp : public OpKernel { } private: + std::vector dilations_; std::vector strides_; Padding padding_; TensorFormat data_format_; @@ -586,10 +620,30 @@ class Conv2DSlowBackpropInputOp : public OpKernel { "specify 4 dimensions")); int stride_n = GetTensorDim(strides_, data_format_, 'N'); int stride_c = GetTensorDim(strides_, data_format_, 'C'); + int stride_h = GetTensorDim(strides_, data_format_, 'H'); + int stride_w = GetTensorDim(strides_, data_format_, 'W'); OP_REQUIRES( context, (stride_n == 1 && stride_c == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, stride_h > 0 && stride_w > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); + int dilation_n = GetTensorDim(dilations_, data_format_, 'N'); + int dilation_c = GetTensorDim(dilations_, data_format_, 'C'); + int dilation_h = GetTensorDim(dilations_, data_format_, 'H'); + int dilation_w = GetTensorDim(dilations_, data_format_, 'W'); + OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + OP_REQUIRES( + context, dilation_h > 0 && dilation_w > 0, + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("use_cudnn_on_gpu", &use_cudnn_)); use_cudnn_ &= CanUseCudnn(); cudnn_use_autotune_ = CudnnUseAutotune(); @@ -622,12 +676,16 @@ class Conv2DSlowBackpropInputOp : public OpKernel { // do not support striding on the batch or depth dimension). const int stride_rows = GetTensorDim(strides_, data_format_, 'H'); const int stride_cols = GetTensorDim(strides_, data_format_, 'W'); + const int dilation_rows = GetTensorDim(dilations_, data_format_, 'H'); + const int dilation_cols = GetTensorDim(dilations_, data_format_, 'W'); launcher_(context, use_cudnn_, cudnn_use_autotune_, out_backprop, filter, - stride_rows, stride_cols, padding_, in_backprop, data_format_); + dilation_rows, dilation_cols, stride_rows, stride_cols, padding_, + in_backprop, data_format_); } private: + std::vector dilations_; std::vector strides_; Padding padding_; bool use_cudnn_; @@ -641,39 +699,48 @@ class Conv2DSlowBackpropInputOp : public OpKernel { template void LaunchConv2DBackpropInputOp::operator()( OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& out_backprop, const Tensor& filter, int row_stride, - int col_stride, const Padding& padding, Tensor* in_backprop, - TensorFormat data_format) { + const Tensor& out_backprop, const Tensor& filter, int row_dilation, + int col_dilation, int row_stride, int col_stride, const Padding& padding, + Tensor* in_backprop, TensorFormat data_format) { using perftools::gputools::dnn::AlgorithmConfig; using perftools::gputools::dnn::AlgorithmDesc; using perftools::gputools::dnn::ProfileResult; std::vector strides(4, 1); - strides[GetTensorDimIndex(data_format, 'H')] = row_stride; - strides[GetTensorDimIndex(data_format, 'W')] = col_stride; + std::vector dilations(4, 1); + auto input_h = GetTensorDimIndex(data_format, 'H'); + auto input_w = GetTensorDimIndex(data_format, 'W'); + strides[input_h] = row_stride; + strides[input_w] = col_stride; + dilations[input_h] = row_dilation; + dilations[input_w] = col_dilation; TensorShape input_shape = in_backprop->shape(); const TensorShape& filter_shape = filter.shape(); ConvBackpropDimensions dims; - OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensions( + OP_REQUIRES_OK(ctx, ConvBackpropComputeDimensionsV2( "Conv2DSlowBackpropInput", /*num_spatial_dims=*/2, input_shape, filter_shape, out_backprop.shape(), - strides, padding, data_format, &dims)); + dilations, strides, padding, data_format, &dims)); + // TODO(yangzihao): The padding computations should be done in + // GetWindowedOutputSize() functions. const int padding_rows = (padding == VALID) ? 0 : std::max(0, (dims.spatial_dims[0].output_size - 1) * dims.spatial_dims[0].stride + - dims.spatial_dims[0].filter_size - - dims.spatial_dims[0].input_size); + (dims.spatial_dims[0].filter_size - 1) * + dims.spatial_dims[0].dilation + + 1 - dims.spatial_dims[0].input_size); const int padding_cols = (padding == VALID) ? 0 : std::max(0, (dims.spatial_dims[1].output_size - 1) * dims.spatial_dims[1].stride + - dims.spatial_dims[1].filter_size - - dims.spatial_dims[1].input_size); + (dims.spatial_dims[1].filter_size - 1) * + dims.spatial_dims[1].dilation + + 1 - dims.spatial_dims[1].input_size); // TODO(keveman): cuDNN only supports equal padding on both sides, so only // calling it when that is true. Remove this check when (if?) cuDNN starts @@ -789,7 +856,9 @@ void LaunchConv2DBackpropInputOp::operator()( .set_input_feature_map_count(dims.in_depth) .set_output_feature_map_count(dims.out_depth); perftools::gputools::dnn::ConvolutionDescriptor conv_desc; - conv_desc.set_vertical_filter_stride(dims.spatial_dims[0].stride) + conv_desc.set_vertical_dilation_rate(dims.spatial_dims[0].dilation) + .set_horizontal_dilation_rate(dims.spatial_dims[1].dilation) + .set_vertical_filter_stride(dims.spatial_dims[0].stride) .set_horizontal_filter_stride(dims.spatial_dims[1].stride) .set_zero_padding_height(padding_rows / 2) .set_zero_padding_width(padding_cols / 2); @@ -875,6 +944,8 @@ void LaunchConv2DBackpropInputOp::operator()( dims.out_depth, // out_depths {{dims.spatial_dims[0].filter_size, // filter_rows dims.spatial_dims[1].filter_size}}, // filter_cols + {{dims.spatial_dims[0].dilation, // dilation_rows + dims.spatial_dims[1].dilation}}, // dilation_cols {{dims.spatial_dims[0].stride, // stride_rows dims.spatial_dims[1].stride}}, // stride_cols {{padding_rows, // padding_rows diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h index e068fb8684..535586d53a 100644 --- a/tensorflow/core/kernels/conv_grad_ops.h +++ b/tensorflow/core/kernels/conv_grad_ops.h @@ -175,15 +175,17 @@ template struct LaunchConv2DBackpropInputOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, const Tensor& out_backprop, const Tensor& filter, - int row_stride, int col_stride, const Padding& padding, - Tensor* in_backprop, TensorFormat data_format); + int row_dilation, int col_dilation, int row_stride, + int col_stride, const Padding& padding, Tensor* in_backprop, + TensorFormat data_format); }; template struct LaunchConv2DBackpropFilterOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, const Tensor& out_backprop, const Tensor& input, - int row_stride, int col_stride, const Padding& padding, + int row_dilation, int col_dilation, int row_stride, + int col_stride, const Padding& padding, Tensor* filter_backprop, TensorFormat data_format); }; @@ -191,8 +193,9 @@ struct LaunchConv2DBackpropFilterOp { template struct LaunchConv2DBackpropInputOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& input, const Tensor& filter, int row_stride, - int col_stride, const Padding& padding, Tensor* output, + const Tensor& input, const Tensor& filter, int row_dilation, + int col_dilation, int row_stride, int col_stride, + const Padding& padding, Tensor* output, TensorFormat data_format); }; @@ -200,7 +203,8 @@ template struct LaunchConv2DBackpropFilterOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, const Tensor& out_backprop, const Tensor& input, - int row_stride, int col_stride, const Padding& padding, + int row_dilation, int col_dilation, int row_stride, + int col_stride, const Padding& padding, Tensor* filter_backprop, TensorFormat data_format); }; #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc index c2d24d1f12..4d0f1ab317 100644 --- a/tensorflow/core/kernels/conv_grad_ops_3d.cc +++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc @@ -645,6 +645,9 @@ class Conv3DBackpropInputOp : public OpKernel { {{input_size[0], input_size[1], input_size[2]}}, out_depth, {{filter_size[0], filter_size[1], filter_size[2]}}, + // TODO(yangzihao): Send in arbitrary dilation rates after the dilated + // conv is supported. + /*dilations=*/{{1, 1, 1}}, {{strides[0], strides[1], strides[2]}}, {{padding_planes, padding_rows, padding_cols}}, dtype, @@ -1011,6 +1014,7 @@ class Conv3DBackpropFilterOp : public OpKernel { {{input_size[0], input_size[1], input_size[2]}}, out_depth, {{filter_size[0], filter_size[1], filter_size[2]}}, + {{1, 1, 1}}, {{strides[0], strides[1], strides[2]}}, {{padding_planes, padding_rows, padding_cols}}, dtype, diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc index bb67113fb0..ba40c428e4 100644 --- a/tensorflow/core/kernels/conv_ops.cc +++ b/tensorflow/core/kernels/conv_ops.cc @@ -112,7 +112,8 @@ struct LaunchGeneric { template struct LaunchConv2DOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& input, const Tensor& filter, int row_stride, + const Tensor& input, const Tensor& filter, + int /*row_dilation*/, int /*col_dilation*/, int row_stride, int col_stride, const Padding& padding, Tensor* output, TensorFormat data_format) { if (data_format != FORMAT_NHWC) { @@ -133,8 +134,10 @@ class LaunchDeepConvOp { const Tensor& filter, int batch, int input_rows, int input_cols, int in_depth, int filter_rows, int filter_cols, int pad_rows, int pad_cols, int out_rows, - int out_cols, int out_depth, int stride_rows, int stride_cols, - Tensor* output, TensorFormat data_format) { + int /*out_cols*/, int /*out_depth*/, int /*dilation_rows*/, + int /*dilation_cols*/, int /*stride_rows*/, + int /*stride_cols*/, Tensor* /*output*/, + TensorFormat /*data_format*/) { return false; } }; @@ -147,9 +150,11 @@ class LaunchDeepConvOp { const Tensor& filter, int batch, int input_rows, int input_cols, int in_depth, int filter_rows, int filter_cols, int pad_rows, int pad_cols, int out_rows, - int out_cols, int out_depth, int stride_rows, int stride_cols, + int out_cols, int out_depth, int dilation_rows, + int dilation_cols, int stride_rows, int stride_cols, Tensor* output, TensorFormat data_format) { - if (data_format != FORMAT_NHWC || + if (data_format != FORMAT_NHWC || dilation_rows != 1 || + dilation_cols != 1 || !CanUseDeepConv2D(stride_rows, stride_cols, filter_rows, filter_cols, in_depth, out_depth, out_rows, out_cols)) { return false; @@ -187,7 +192,8 @@ class LaunchXsmmConvOp { int input_cols, int in_depth, int filter_rows, int filter_cols, int pad_rows, int pad_cols, int out_rows, int out_cols, int out_depth, int stride_rows, int stride_cols, - Tensor* output, TensorFormat data_format) { + int dilation_rows, int dilation_cols, Tensor* output, + TensorFormat data_format) { return false; } }; @@ -199,7 +205,8 @@ class LaunchXsmmConvOp { const Tensor& filter, int batch, int input_rows, int input_cols, int in_depth, int filter_rows, int filter_cols, int pad_rows, int pad_cols, int out_rows, - int out_cols, int out_depth, int stride_rows, int stride_cols, + int out_cols, int out_depth, int dilation_rows, + int dilation_cols, int stride_rows, int stride_cols, Tensor* output, TensorFormat data_format) { auto num_threads = ctx->device()->tensorflow_cpu_worker_threads()->num_threads; @@ -228,11 +235,8 @@ class LaunchXsmmConvOp { desc.options = LIBXSMM_DNN_CONV_OPTION_WU_EXT_FILTER_REDUCE_OVERWRITE; desc.datatype = LIBXSMM_DNN_DATATYPE_F32; - if (!CanUseXsmmConv2D(desc, data_format)) { - return false; - } - - if (!CanUseXsmmConv2D(desc, data_format)) { + if (dilation_rows != 1 || dilation_cols != 1 || + !CanUseXsmmConv2D(desc, data_format)) { return false; } @@ -251,6 +255,7 @@ template class Conv2DOp : public BinaryOp { public: explicit Conv2DOp(OpKernelConstruction* context) : BinaryOp(context) { + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_)); OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_)); string data_format; OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format)); @@ -259,15 +264,35 @@ class Conv2DOp : public BinaryOp { OP_REQUIRES_OK(context, context->GetAttr("use_cudnn_on_gpu", &use_cudnn_)); use_cudnn_ &= CanUseCudnn(); cudnn_use_autotune_ = CudnnUseAutotune(); + OP_REQUIRES(context, dilations_.size() == 4, + errors::InvalidArgument("Sliding window dilations field must " + "specify 4 dimensions")); OP_REQUIRES(context, strides_.size() == 4, errors::InvalidArgument("Sliding window strides field must " "specify 4 dimensions")); const int64 stride_n = GetTensorDim(strides_, data_format_, 'N'); const int64 stride_c = GetTensorDim(strides_, data_format_, 'C'); + const int64 stride_h = GetTensorDim(strides_, data_format_, 'H'); + const int64 stride_w = GetTensorDim(strides_, data_format_, 'W'); OP_REQUIRES( context, stride_n == 1 && stride_c == 1, errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + OP_REQUIRES(context, stride_h > 0 && stride_w > 0, + errors::InvalidArgument( + "Row and column strides should be larger than 0.")); + + const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N'); + const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C'); + const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H'); + const int64 dilation_w = GetTensorDim(dilations_, data_format_, 'W'); + OP_REQUIRES(context, dilation_n == 1 && dilation_c == 1, + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); + OP_REQUIRES( + context, dilation_h > 0 && dilation_w > 0, + errors::InvalidArgument("Dilated rates should be larger than 0.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); } @@ -334,18 +359,22 @@ class Conv2DOp : public BinaryOp { errors::InvalidArgument("batch is too large")); const int batch = static_cast(batch_raw); - // For now we take the stride from the second and third dimensions only (we - // do not support striding on the batch or depth dimension). + // For now we take the stride and dilation from the second and third + // dimensions only (we do not support striding or dilation on the batch or + // depth dimension). const int stride_rows = GetTensorDim(strides_, data_format_, 'H'); const int stride_cols = GetTensorDim(strides_, data_format_, 'W'); + const int dilation_rows = GetTensorDim(dilations_, data_format_, 'H'); + const int dilation_cols = GetTensorDim(dilations_, data_format_, 'W'); + int64 out_rows = 0, out_cols = 0, pad_rows = 0, pad_cols = 0; - OP_REQUIRES_OK(context, - GetWindowedOutputSize(input_rows, filter_rows, stride_rows, - padding_, &out_rows, &pad_rows)); - OP_REQUIRES_OK(context, - GetWindowedOutputSize(input_cols, filter_cols, stride_cols, - padding_, &out_cols, &pad_cols)); + OP_REQUIRES_OK(context, GetWindowedOutputSizeV2( + input_rows, filter_rows, dilation_rows, + stride_rows, padding_, &out_rows, &pad_rows)); + OP_REQUIRES_OK(context, GetWindowedOutputSizeV2( + input_cols, filter_cols, dilation_cols, + stride_cols, padding_, &out_cols, &pad_cols)); TensorShape out_shape = ShapeFromFormat(data_format_, batch, out_rows, out_cols, out_depth); @@ -361,6 +390,8 @@ class Conv2DOp : public BinaryOp { << ", filter_rows = " << filter_rows << ", stride_rows = " << stride_rows << ", stride_cols = " << stride_cols + << ", dilation_rows = " << dilation_rows + << ", dilation_cols = " << dilation_cols << ", out_depth = " << out_depth; // If there is nothing to compute, return. @@ -372,7 +403,8 @@ class Conv2DOp : public BinaryOp { if (LaunchXsmmConvOp::Run( context, input, filter, batch, input_rows, input_cols, in_depth, filter_rows, filter_cols, pad_rows, pad_cols, out_rows, out_cols, - out_depth, stride_rows, stride_cols, output, data_format_)) { + out_depth, dilation_rows, dilation_cols, stride_rows, stride_cols, + output, data_format_)) { return; } #endif @@ -380,15 +412,18 @@ class Conv2DOp : public BinaryOp { if (LaunchDeepConvOp::Run( context, input, filter, batch, input_rows, input_cols, in_depth, filter_rows, filter_cols, pad_rows, pad_cols, out_rows, out_cols, - out_depth, stride_rows, stride_cols, output, data_format_)) { + out_depth, dilation_rows, dilation_cols, stride_rows, stride_cols, + output, data_format_)) { return; } launcher_(context, use_cudnn_, cudnn_use_autotune_, input, filter, - stride_rows, stride_cols, padding_, output, data_format_); + dilation_rows, dilation_cols, stride_rows, stride_cols, padding_, + output, data_format_); } private: + std::vector dilations_; std::vector strides_; bool use_cudnn_; Padding padding_; @@ -443,9 +478,9 @@ typedef AutoTuneSingleton void LaunchConv2DOp::operator()( OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& input_param, const Tensor& filter, int row_stride, - int col_stride, const Padding& padding, Tensor* output, - TensorFormat data_format) { + const Tensor& input_param, const Tensor& filter, int row_dilation, + int col_dilation, int row_stride, int col_stride, const Padding& padding, + Tensor* output, TensorFormat data_format) { using perftools::gputools::dnn::AlgorithmConfig; using perftools::gputools::dnn::AlgorithmDesc; using perftools::gputools::dnn::ProfileResult; @@ -461,8 +496,9 @@ void LaunchConv2DOp::operator()( Tensor input = input_param; - if (filter.dim_size(0) == 1 && filter.dim_size(1) == 1 && row_stride == 1 && - col_stride == 1 && data_format == FORMAT_NHWC) { + if (filter.dim_size(0) == 1 && filter.dim_size(1) == 1 && row_dilation == 1 && + col_dilation == 1 && row_stride == 1 && col_stride == 1 && + data_format == FORMAT_NHWC) { // 1x1 filter, so call cublas directly. const uint64 m = input.dim_size(0) * input.dim_size(1) * input.dim_size(2); const uint64 k = filter.dim_size(2); @@ -487,7 +523,8 @@ void LaunchConv2DOp::operator()( } return; } else if (filter.dim_size(0) == input.dim_size(1) && - filter.dim_size(1) == input.dim_size(2) && padding == VALID && + filter.dim_size(1) == input.dim_size(2) && row_dilation == 1 && + col_dilation == 1 && padding == VALID && data_format == FORMAT_NHWC) { // The input data and filter have the same height/width, so call cublas // directly. @@ -530,17 +567,19 @@ void LaunchConv2DOp::operator()( const int64 patch_cols = filter.dim_size(1); if (padding == SAME) { // Total padding on rows and cols is - // Pr = (R' - 1) * S + Kr - R - // Pc = (C' - 1) * S + Kc - C + // Pr = (R' - 1) * S + (Kr - 1) * Dr + 1 - R + // Pc = (C' - 1) * S + (Kc - 1) * Dc + 1 - C // where (R', C') are output dimensions, (R, C) are input dimensions, S - // is stride, (Kr, Kc) are filter dimensions. + // is stride, (Dr, Dc) are dilations, (Kr, Kc) are filter dimensions. // We pad Pr/2 on the left and Pr - Pr/2 on the right, Pc/2 on the top // and Pc - Pc/2 on the bottom. When Pr or Pc is odd, this means // we pad more on the right and bottom than on the top and left. padding_rows = - std::max(0, (out_rows - 1) * row_stride + patch_rows - in_rows); + std::max(0, (out_rows - 1) * row_stride + + (patch_rows - 1) * row_dilation + 1 - in_rows); padding_cols = - std::max(0, (out_cols - 1) * col_stride + patch_cols - in_cols); + std::max(0, (out_cols - 1) * col_stride + + (patch_cols - 1) * col_dilation + 1 - in_cols); const bool rows_odd = (padding_rows % 2 != 0); const bool cols_odd = (padding_cols % 2 != 0); if (rows_odd || cols_odd) { @@ -605,7 +644,9 @@ void LaunchConv2DOp::operator()( .set_input_feature_map_count(filter.dim_size(2)) .set_output_feature_map_count(filter.dim_size(3)); perftools::gputools::dnn::ConvolutionDescriptor conv_desc; - conv_desc.set_vertical_filter_stride(row_stride) + conv_desc.set_vertical_dilation_rate(row_dilation) + .set_horizontal_dilation_rate(col_dilation) + .set_vertical_filter_stride(row_stride) .set_horizontal_filter_stride(col_stride) .set_zero_padding_height(padding_rows / 2) .set_zero_padding_width(padding_cols / 2); @@ -652,6 +693,8 @@ void LaunchConv2DOp::operator()( out_depths, // out_depths {{patch_rows, // filter_rows patch_cols}}, // filter_cols + {{row_dilation, // dilation_rows + col_dilation}}, // dilation_cols {{row_stride, // stride_rows col_stride}}, // stride_cols {{padding_rows, // padding_rows diff --git a/tensorflow/core/kernels/conv_ops.h b/tensorflow/core/kernels/conv_ops.h index e29271dff2..09a3b78776 100644 --- a/tensorflow/core/kernels/conv_ops.h +++ b/tensorflow/core/kernels/conv_ops.h @@ -34,8 +34,9 @@ class OpKernelContext; template struct LaunchConv2DOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& input, const Tensor& filter, int row_stride, - int col_stride, const Padding& padding, Tensor* output, + const Tensor& input, const Tensor& filter, int row_dilation, + int col_dilation, int row_stride, int col_stride, + const Padding& padding, Tensor* output, TensorFormat data_format); }; @@ -43,8 +44,9 @@ struct LaunchConv2DOp { template struct LaunchConv2DOp { void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune, - const Tensor& input, const Tensor& filter, int row_stride, - int col_stride, const Padding& padding, Tensor* output, + const Tensor& input, const Tensor& filter, int row_dilation, + int col_dilation, int row_stride, int col_stride, + const Padding& padding, Tensor* output, TensorFormat data_format); }; #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc index 37cb67bc51..39202d7334 100644 --- a/tensorflow/core/kernels/conv_ops_3d.cc +++ b/tensorflow/core/kernels/conv_ops_3d.cc @@ -377,6 +377,9 @@ struct LaunchConvOp { {{in_planes, in_rows, in_cols}}, out_depth, {{filter_planes, filter_rows, filter_cols}}, + // TODO(yangzihao): Send in arbitrary dilation rates after the dilated + // conv is supported. + /*dilations=*/{{1, 1, 1}}, {{strides[0], strides[1], strides[2]}}, {{pad_planes, pad_rows, pad_cols}}, dtype, diff --git a/tensorflow/core/kernels/conv_ops_gpu.h b/tensorflow/core/kernels/conv_ops_gpu.h index c852dc9991..6f82698596 100644 --- a/tensorflow/core/kernels/conv_ops_gpu.h +++ b/tensorflow/core/kernels/conv_ops_gpu.h @@ -91,13 +91,14 @@ class ConvParameters { using SpatialArray = gtl::InlinedVector; ConvParameters(int64 batch, int64 in_depths, const SpatialArray& in, int64 out_depths, const SpatialArray& filter, - const SpatialArray& stride, const SpatialArray& padding, - DataType dtype, int device_id) + const SpatialArray& dilation, const SpatialArray& stride, + const SpatialArray& padding, DataType dtype, int device_id) : batch_(batch), in_depths_(in_depths), out_depths_(out_depths), in_(in), filter_(filter), + dilation_(dilation), stride_(stride), padding_(padding), dtype_(dtype), @@ -107,6 +108,7 @@ class ConvParameters { for (int64 val : in) hash_code_ = Hash64Combine(hash_code_, val); hash_code_ = Hash64Combine(hash_code_, out_depths); for (int64 val : filter) hash_code_ = Hash64Combine(hash_code_, val); + for (int64 val : dilation) hash_code_ = Hash64Combine(hash_code_, val); for (int64 val : stride) hash_code_ = Hash64Combine(hash_code_, val); for (int64 val : padding) hash_code_ = Hash64Combine(hash_code_, val); hash_code_ = Hash64Combine(hash_code_, dtype); @@ -128,6 +130,7 @@ class ConvParameters { "(", str_util::Join(in_, ", "), "), ", out_depths_, ", ", "(", str_util::Join(filter_, ", "), "), ", + "(", str_util::Join(dilation_, ", "), "), ", "(", str_util::Join(stride_, ", "), "), ", "(", str_util::Join(padding_, ", "), "), ", dtype_, ", ", @@ -154,11 +157,11 @@ class ConvParameters { protected: using ParameterDataType = std::tuple; + SpatialArray, SpatialArray, DataType, int>; ParameterDataType get_data_as_tuple() const { return std::make_tuple(batch_, in_depths_, in_, out_depths_, filter_, - stride_, padding_, dtype_, device_id_); + dilation_, stride_, padding_, dtype_, device_id_); } uint64 hash_code_; @@ -169,6 +172,7 @@ class ConvParameters { int64 out_depths_; SpatialArray in_; SpatialArray filter_; + SpatialArray dilation_; SpatialArray stride_; SpatialArray padding_; DataType dtype_; diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc index ea54d6cf6c..666bca265c 100644 --- a/tensorflow/core/kernels/conv_ops_test.cc +++ b/tensorflow/core/kernels/conv_ops_test.cc @@ -43,6 +43,8 @@ TEST(ConvParameters, WinogradNonfusedAlgoSize) { 128, // out_depths {{3, // filter_rows 3}}, // filter_cols + {{1, // dilation_rows + 1}}, // dilation_cols {{1, // stride_rows 1}}, // stride_cols {{0, // padding_rows @@ -60,6 +62,8 @@ TEST(ConvParameters, WinogradNonfusedAlgoSize) { 768, // out_depths {{3, // filter_rows 3}}, // filter_cols + {{1, // dilation_rows + 1}}, // dilation_cols {{1, // stride_rows 1}}, // stride_cols {{0, // padding_rows diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc index 7c43dcb670..02da64ce98 100644 --- a/tensorflow/core/kernels/depthwise_conv_op.cc +++ b/tensorflow/core/kernels/depthwise_conv_op.cc @@ -373,8 +373,11 @@ class DepthwiseConv2dNativeOp : public BinaryOp { // If in_depth==1, this operation is just a standard convolution, so // invoke that op. if (std::is_same::value && in_depth == 1) { + // TODO(yangzihao): Send in arbitrary dilation rates after the dilated + // conv is supported. launcher_(context, use_cudnn_, cudnn_use_autotune_, input, filter, - stride_, stride_, padding_, output, data_format_); + /*row_dilation=*/1, /*col_dilation=*/1, stride_, stride_, + padding_, output, data_format_); return; } diff --git a/tensorflow/core/kernels/quantized_conv_ops.cc b/tensorflow/core/kernels/quantized_conv_ops.cc index 3b0764bb9b..f83998e0c1 100644 --- a/tensorflow/core/kernels/quantized_conv_ops.cc +++ b/tensorflow/core/kernels/quantized_conv_ops.cc @@ -457,6 +457,19 @@ class QuantizedConv2DOp : public OpKernel { context, (strides_[0] == 1 && strides_[3] == 1), errors::InvalidArgument("Current implementation does not yet support " "strides in the batch and depth dimensions.")); + std::vector dilations; + OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations)); + OP_REQUIRES(context, dilations.size() == 4, + errors::InvalidArgument("Dilations field must " + "specify 4 dimensions")); + OP_REQUIRES(context, dilations[1] == 1 && dilations[2] == 1, + errors::InvalidArgument( + "Current implementation only supports dilated rate as 1 " + "in the row and column dimensions.")); + OP_REQUIRES(context, (dilations[0] == 1 && dilations[3] == 1), + errors::InvalidArgument( + "Current implementation does not yet support " + "dilations in the batch and depth dimensions.")); OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_)); } diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 654e890b57..59c4642e4d 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -513,6 +513,7 @@ REGISTER_OP("Conv2D") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn(shape_inference::Conv2DShape) .Doc(R"doc( Computes a 2-D convolution given 4-D `input` and `filter` tensors. @@ -546,7 +547,7 @@ filter: A 4-D tensor of shape output: A 4-D tensor. The dimension order is determined by the value of `data_format`, see below for details. strides: 1-D tensor of length 4. The stride of the sliding window for each - dimension of `input`. The dimension order is determined by the value of + dimension of `input`. The dimension order is determined by the value of `data_format`, see below for details. padding: The type of padding algorithm to use. data_format: Specify the data format of the input and output data. With the @@ -554,6 +555,11 @@ data_format: Specify the data format of the input and output data. With the [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. )doc"); REGISTER_OP("Conv2DBackpropInput") @@ -566,6 +572,7 @@ REGISTER_OP("Conv2DBackpropInput") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -589,10 +596,15 @@ padding: The type of padding algorithm to use. output: 4-D with shape `[batch, in_height, in_width, in_channels]`. Gradient w.r.t. the input of the convolution. data_format: Specify the data format of the input and output data. With the - default format "NHWC", the data is stored in the order of: - [batch, in_height, in_width, in_channels]. - Alternatively, the format could be "NCHW", the data storage order of: - [batch, in_channels, in_height, in_width]. + default format "NHWC", the data is stored in the order of: + [batch, in_height, in_width, in_channels]. + Alternatively, the format could be "NCHW", the data storage order of: + [batch, in_channels, in_height, in_width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each filter + element on that dimension. The dimension order is determined by the value of + `data_format`, see above for details. Dilations in the batch and depth + dimensions must be 1. )doc"); // TODO(jeff): Instead of 'use_cudnn_for_gpu', maybe we should have a @@ -608,6 +620,7 @@ REGISTER_OP("Conv2DBackpropFilter") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s)); @@ -632,10 +645,15 @@ output: 4-D with shape `[filter_height, filter_width, in_channels, out_channels]`. Gradient w.r.t. the `filter` input of the convolution. data_format: Specify the data format of the input and output data. With the - default format "NHWC", the data is stored in the order of: - [batch, in_height, in_width, in_channels]. - Alternatively, the format could be "NCHW", the data storage order of: - [batch, in_channels, in_height, in_width]. + default format "NHWC", the data is stored in the order of: + [batch, in_height, in_width, in_channels]. + Alternatively, the format could be "NCHW", the data storage order of: + [batch, in_channels, in_height, in_width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each filter + element on that dimension. The dimension order is determined by the value of + `data_format`, see above for details. Dilations in the batch and depth + dimensions must be 1. )doc"); namespace { @@ -823,6 +841,7 @@ REGISTER_OP("DepthwiseConv2dNative") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn(shape_inference::DepthwiseConv2DNativeShape) .Doc(R"doc( Computes a 2-D depthwise convolution given 4-D `input` and `filter` tensors. @@ -845,7 +864,6 @@ for k in 0..in_channels-1 Must have `strides[0] = strides[3] = 1`. For the most common case of the same horizontal and vertices strides, `strides = [1, stride, stride, 1]`. - strides: 1-D of length 4. The stride of the sliding window for each dimension of `input`. padding: The type of padding algorithm to use. @@ -854,6 +872,11 @@ data_format: Specify the data format of the input and output data. With the [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each filter + element on that dimension. The dimension order is determined by the value of + `data_format`, see above for details. Dilations in the batch and depth + dimensions must be 1. )doc"); REGISTER_OP("DepthwiseConv2dNativeBackpropInput") @@ -865,6 +888,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropInput") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -892,6 +916,11 @@ data_format: Specify the data format of the input and output data. With the [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each filter + element on that dimension. The dimension order is determined by the value of + `data_format`, see above for details. Dilations in the batch and depth + dimensions must be 1. output: 4-D with shape according to `data_format`. For example, if `data_format` is 'NHWC', output shape is `[batch, in_height, in_width, in_channels]`. Gradient w.r.t. the input of the @@ -907,6 +936,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropFilter") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s)); @@ -935,6 +965,11 @@ data_format: Specify the data format of the input and output data. With the [batch, height, width, channels]. Alternatively, the format could be "NCHW", the data storage order of: [batch, channels, height, width]. +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each filter + element on that dimension. The dimension order is determined by the value of + `data_format`, see above for details. Dilations in the batch and depth + dimensions must be 1. output: 4-D with shape `[filter_height, filter_width, in_channels, out_channels]`. Gradient w.r.t. the `filter` input of the convolution. @@ -949,6 +984,7 @@ REGISTER_OP("Conv3D") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1, 1]") .SetShapeFn(shape_inference::Conv3DShape) .Doc(R"doc( Computes a 3-D convolution given 5-D `input` and `filter` tensors. @@ -970,6 +1006,11 @@ data_format: The data format of the input and output data. With the [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +dilations: 1-D tensor of length 5. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. )doc"); REGISTER_OP("Conv3DBackpropInput") @@ -1036,6 +1077,7 @@ REGISTER_OP("Conv3DBackpropInputV2") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -1061,6 +1103,11 @@ data_format: The data format of the input and output data. With the [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +dilations: 1-D tensor of length 5. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. )doc"); @@ -1073,6 +1120,7 @@ REGISTER_OP("Conv3DBackpropFilterV2") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s)); @@ -1098,6 +1146,11 @@ data_format: The data format of the input and output data. With the [batch, in_depth, in_height, in_width, in_channels]. Alternatively, the format could be "NCDHW", the data storage order is: [batch, in_channels, in_depth, in_height, in_width]. +dilations: 1-D tensor of length 5. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. )doc"); @@ -2613,6 +2666,7 @@ REGISTER_OP("QuantizedConv2D") .Attr("out_type: quantizedtype = DT_QINT32") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) + .Attr("dilations: list(int) = [1, 1, 1, 1]") .SetShapeFn([](InferenceContext* c) { TF_RETURN_IF_ERROR(shape_inference::Conv2DShape(c)); ShapeHandle unused; @@ -2641,7 +2695,11 @@ min_filter: The float value that the lowest quantized filter value represents. max_filter: The float value that the highest quantized filter value represents. min_output: The float value that the lowest quantized output value represents. max_output: The float value that the highest quantized output value represents. - +dilations: 1-D tensor of length 4. The dilation factor for each dimension of + `input`. If set to k > 1, there will be k-1 skipped cells between each + filter element on that dimension. The dimension order is determined by the + value of `data_format`, see above for details. Dilations in the batch and + depth dimensions must be 1. )doc"); REGISTER_OP("QuantizedMaxPool") diff --git a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py index 1679857bd5..be299beee4 100644 --- a/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py +++ b/tensorflow/python/kernel_tests/conv2d_backprop_filter_grad_test.py @@ -42,17 +42,21 @@ class Conv2DBackpropFilterGradTest(test.TestCase): filter_shape = [3, 3, 4, 6] # Make a convolution op with the current settings, just to easily get # the shape of the output. - conv_out = nn_ops.conv2d(in_val, - array_ops.zeros(filter_shape), - [1, stride, stride, 1], padding) + conv_out = nn_ops.conv2d( + in_val, + array_ops.zeros(filter_shape), + strides=[1, stride, stride, 1], + padding=padding) out_backprop_shape = conv_out.get_shape().as_list() out_backprop_val = constant_op.constant( 2 * np.random.random_sample(out_backprop_shape) - 1, dtype=dtypes.float32) - output = nn_ops.conv2d_backprop_filter(in_val, filter_shape, - out_backprop_val, - [1, stride, stride, 1], - padding) + output = nn_ops.conv2d_backprop_filter( + in_val, + filter_shape, + out_backprop_val, + strides=[1, stride, stride, 1], + padding=padding) err = gradient_checker.compute_gradient_error( [in_val, out_backprop_val], [in_shape, out_backprop_shape], output, filter_shape) @@ -60,6 +64,42 @@ class Conv2DBackpropFilterGradTest(test.TestCase): err_tolerance = 2e-3 self.assertLess(err, err_tolerance) + def testGradientDilatedConv(self): + if test.is_gpu_available(cuda_only=True): + with self.test_session(use_gpu=True): + for padding in ["SAME", "VALID"]: + for stride in [1, 2]: + np.random.seed(1) + in_shape = [5, 8, 6, 4] + in_val = constant_op.constant( + 2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32) + filter_shape = [3, 3, 4, 6] + # Make a convolution op with the current settings, + # just to easily get the shape of the output. + conv_out = nn_ops.conv2d( + in_val, + array_ops.zeros(filter_shape), + dilations=[1, 2, 2, 1], + strides=[1, stride, stride, 1], + padding=padding) + out_backprop_shape = conv_out.get_shape().as_list() + out_backprop_val = constant_op.constant( + 2 * np.random.random_sample(out_backprop_shape) - 1, + dtype=dtypes.float32) + output = nn_ops.conv2d_backprop_filter( + in_val, + filter_shape, + out_backprop_val, + dilations=[1, 2, 2, 1], + strides=[1, stride, stride, 1], + padding=padding) + err = gradient_checker.compute_gradient_error( + [in_val, out_backprop_val], [in_shape, out_backprop_shape], + output, filter_shape) + print("conv2d_backprop_filter gradient err = %g " % err) + err_tolerance = 2e-3 + self.assertLess(err, err_tolerance) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py index 22e5400c37..bf7245a2ae 100644 --- a/tensorflow/python/kernel_tests/conv_ops_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_test.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections import os import time @@ -32,6 +33,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_impl from tensorflow.python.ops import nn_ops from tensorflow.python.ops import random_ops @@ -240,6 +242,77 @@ class Conv2DTest(test.TestCase): for i in range(1, len(values)): self.assertAllClose(values[0], values[i], rtol=1e-5, atol=1e-5) + def _ComputeReferenceDilatedConv(self, tensor_in_sizes, filter_in_sizes, + stride, dilation, padding, data_format, + use_gpu): + total_size_1 = 1 + total_size_2 = 1 + for s in tensor_in_sizes: + total_size_1 *= s + for s in filter_in_sizes: + total_size_2 *= s + + # Initializes the input tensor with array containing incrementing + # numbers from 1. + x1 = [f * 1.0 for f in range(1, total_size_1 + 1)] + x2 = [f * 1.0 for f in range(1, total_size_2 + 1)] + with test_util.device(use_gpu): + t1 = constant_op.constant(x1, shape=tensor_in_sizes) + t2 = constant_op.constant(x2, shape=filter_in_sizes) + if isinstance(stride, collections.Iterable): + strides = list(stride) + else: + strides = [stride, stride] + if data_format == "NCHW": + t1 = test_util.NHWCToNCHW(t1) + full_strides = [1, 1] + strides + full_dilation = [1, 1] + dilation + else: + full_strides = [1] + strides + [1] + full_dilation = [1] + dilation + [1] + expected = nn_ops.convolution( + t1, + t2, + padding=padding, + strides=strides, + dilation_rate=dilation, + data_format=data_format) + computed = nn_ops.conv2d( + t1, + t2, + strides=full_strides, + dilations=full_dilation, + padding=padding, + data_format=data_format) + if data_format == "NCHW": + expected = test_util.NCHWToNHWC(expected) + computed = test_util.NCHWToNHWC(computed) + return expected, computed + + def _VerifyDilatedConvValues(self, tensor_in_sizes, filter_in_sizes, strides, + padding, dilations): + expected_results = [] + computed_results = [] + default_dilations = (dilations[0] == 1 and dilations[1] == 1) + for data_format, use_gpu in GetTestConfigs(): + # If any dilation rate is larger than 1, only do test on the GPU + # because we currently do not have a CPU implementation for arbitrary + # dilation rates. + if default_dilations or use_gpu: + expected, computed = self._ComputeReferenceDilatedConv( + tensor_in_sizes, filter_in_sizes, strides, dilations, padding, + data_format, use_gpu) + expected_results.append(expected) + computed_results.append(computed) + tolerance = 1e-2 if use_gpu else 1e-5 + expected_values = self.evaluate(expected_results) + computed_values = self.evaluate(computed_results) + for e_value, c_value in zip(expected_values, computed_values): + print("expected = ", e_value) + print("actual = ", c_value) + self.assertAllClose( + e_value.flatten(), c_value.flatten(), atol=tolerance, rtol=1e-6) + def _VerifyValues(self, tensor_in_sizes, filter_in_sizes, strides, padding, expected): tensors = [] @@ -279,6 +352,16 @@ class Conv2DTest(test.TestCase): padding="VALID", expected=expected_output) + @test_util.run_in_graph_and_eager_modes() + def testConv2D2x2Filter2x1Dilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 4, 4, 1], + filter_in_sizes=[2, 2, 1, 1], + strides=[1, 1], + dilations=[2, 1], + padding="VALID") + @test_util.run_in_graph_and_eager_modes() def testConv2DEmpty(self): expected_output = [] @@ -289,6 +372,16 @@ class Conv2DTest(test.TestCase): padding="VALID", expected=expected_output) + @test_util.run_in_graph_and_eager_modes() + def testConv2DEmptyDilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[0, 2, 3, 3], + filter_in_sizes=[1, 1, 3, 3], + strides=[1, 1], + dilations=[2, 1], + padding="VALID") + @test_util.run_in_graph_and_eager_modes() def testConv2D2x2Filter(self): # The outputs are computed using third_party/py/IPython/notebook. @@ -300,6 +393,16 @@ class Conv2DTest(test.TestCase): padding="VALID", expected=expected_output) + @test_util.run_in_graph_and_eager_modes() + def testConv2D2x2FilterDilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 2, 3, 3], + filter_in_sizes=[2, 2, 3, 3], + strides=[1, 1], + dilations=[1, 2], + padding="VALID") + @test_util.run_in_graph_and_eager_modes() def testConv2D1x2Filter(self): # The outputs are computed using third_party/py/IPython/notebook. @@ -314,6 +417,16 @@ class Conv2DTest(test.TestCase): padding="VALID", expected=expected_output) + @test_util.run_in_graph_and_eager_modes() + def testConv2D1x2FilterDilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 2, 3, 3], + filter_in_sizes=[1, 2, 3, 3], + strides=[1, 1], + dilations=[2, 1], + padding="VALID") + @test_util.run_in_graph_and_eager_modes() def testConv2D2x2FilterStride2(self): expected_output = [2271.0, 2367.0, 2463.0] @@ -386,13 +499,23 @@ class Conv2DTest(test.TestCase): padding="VALID", expected=[50, 60]) - # TODO this currently fails. - # self._VerifyValues(tensor_in_sizes=[1, 8, 8, 1], - # filter_in_sizes=[2, 2, 1, 1], - # strides=[4, 4], padding="SAME", - # expected=[72, 112, 392, 432]) + @test_util.run_in_graph_and_eager_modes() + def testConv2DKernelSizeMatchesInputSizeDilation(self): + if test.is_gpu_available(cuda_only=True): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 3, 3, 1], + filter_in_sizes=[2, 2, 1, 2], + strides=[1, 1], + dilations=[2, 2], + padding="VALID") + + # TODO this currently fails. + # self._VerifyValues(tensor_in_sizes=[1, 8, 8, 1], + # filter_in_sizes=[2, 2, 1, 1], + # strides=[4, 4], padding="SAME", + # expected=[72, 112, 392, 432]) - # Testing for backprops + # Testing for backprops def _RunAndVerifyBackpropInput(self, input_sizes, filter_sizes, output_sizes, strides, padding, expected, data_format, use_gpu, err): @@ -724,6 +847,255 @@ class Conv2DTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + # Testing for backprops + def _RunAndVerifyBackpropInputDilation(self, input_sizes, filter_sizes, + output_sizes, strides, dilations, + padding, data_format, use_gpu, err): + total_input_size = 1 + total_filter_size = 1 + for s in input_sizes: + total_input_size *= s + for s in filter_sizes: + total_filter_size *= s + # Initializes the input tensor with array containing incrementing + # numbers from 1. + x1 = [f * 1.0 for f in range(1, total_input_size + 1)] + x2 = [f * 1.0 for f in range(1, total_filter_size + 1)] + default_dilations = (dilations[0] == 1 and dilations[1] == 1) + if default_dilations or use_gpu: + with self.test_session(use_gpu=use_gpu) as sess: + if data_format == "NCHW": + input_sizes = test_util.NHWCToNCHW(input_sizes) + t1 = constant_op.constant(x1, shape=input_sizes) + t2 = constant_op.constant(x2, shape=filter_sizes) + full_strides = [1] + strides + [1] + full_dilations = [1] + dilations + [1] + if data_format == "NCHW": + full_strides = test_util.NHWCToNCHW(full_strides) + full_dilations = test_util.NHWCToNCHW(full_dilations) + conv_forward = nn_ops.conv2d( + t1, + t2, + strides=full_strides, + dilations=full_dilations, + padding=padding, + data_format=data_format) + conv_forward_2 = nn_ops.convolution( + t1, + t2, + padding=padding, + strides=strides, + dilation_rate=dilations, + data_format=data_format) + if data_format == "NCHW": + conv_forward = test_util.NCHWToNHWC(conv_forward) + conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2) + conv = gradients_impl.gradients(conv_forward, t1)[0] + conv_2 = gradients_impl.gradients(conv_forward_2, t1)[0] + # "values" consists of two tensors for two backprops + value = sess.run(conv) + value_2 = sess.run(conv_2) + self.assertShapeEqual(value, conv) + self.assertShapeEqual(value_2, conv_2) + print("expected = ", value_2) + print("actual = ", value) + self.assertArrayNear(value_2.flatten(), value.flatten(), err) + + # Testing for backprops + def _RunAndVerifyBackpropFilterDilation(self, input_sizes, filter_sizes, + output_sizes, strides, dilations, + padding, data_format, use_gpu, err): + total_input_size = 1 + total_filter_size = 1 + for s in input_sizes: + total_input_size *= s + for s in filter_sizes: + total_filter_size *= s + # Initializes the input tensor with array containing incrementing + # numbers from 1. + x1 = [f * 1.0 for f in range(1, total_input_size + 1)] + x2 = [f * 1.0 for f in range(1, total_filter_size + 1)] + default_dilations = (dilations[0] == 1 and dilations[1] == 1) + if default_dilations or use_gpu: + with self.test_session(use_gpu=use_gpu) as sess: + if data_format == "NCHW": + input_sizes = test_util.NHWCToNCHW(input_sizes) + t1 = constant_op.constant(x1, shape=input_sizes) + t2 = constant_op.constant(x2, shape=filter_sizes) + full_strides = [1] + strides + [1] + full_dilations = [1] + dilations + [1] + if data_format == "NCHW": + full_strides = test_util.NHWCToNCHW(full_strides) + full_dilations = test_util.NHWCToNCHW(full_dilations) + conv_forward = nn_ops.conv2d( + t1, + t2, + strides=full_strides, + dilations=full_dilations, + padding=padding, + data_format=data_format) + conv_forward_2 = nn_ops.convolution( + t1, + t2, + padding=padding, + strides=strides, + dilation_rate=dilations, + data_format=data_format) + if data_format == "NCHW": + conv_forward = test_util.NCHWToNHWC(conv_forward) + conv_forward_2 = test_util.NCHWToNHWC(conv_forward_2) + conv = gradients_impl.gradients(conv_forward, t2)[0] + conv_2 = gradients_impl.gradients(conv_forward, t2)[0] + value = sess.run(conv) + value_2 = sess.run(conv_2) + self.assertShapeEqual(value, conv) + self.assertShapeEqual(value_2, conv_2) + print("expected = ", value_2) + print("actual = ", value) + self.assertArrayNear(value_2.flatten(), value.flatten(), err) + + def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropFilterDilation( + input_sizes=[1, 3, 6, 1], + filter_sizes=[2, 2, 1, 1], + output_sizes=[1, 1, 5, 1], + strides=[1, 1], + dilations=[2, 1], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2D2x2Depth1ValidBackpropFilterDilation1x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropFilterDilation( + input_sizes=[1, 2, 3, 1], + filter_sizes=[2, 2, 1, 1], + output_sizes=[1, 1, 2, 1], + strides=[1, 1], + dilations=[1, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2DEmptyBackpropFilterDilation1x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropFilterDilation( + input_sizes=[1, 2, 3, 1], + filter_sizes=[2, 2, 1, 0], + output_sizes=[1, 1, 2, 0], + strides=[1, 1], + dilations=[1, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2D2x2Depth3ValidBackpropFilterDilation2x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropFilterDilation( + input_sizes=[1, 3, 4, 3], + filter_sizes=[2, 2, 3, 3], + output_sizes=[1, 1, 2, 3], + strides=[1, 1], + dilations=[2, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2DKernelSizeMatchesInputSizeBackpropFilterDilation2x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropFilterDilation( + input_sizes=[1, 3, 3, 1], + filter_sizes=[2, 2, 1, 2], + output_sizes=[1, 1, 1, 2], + strides=[1, 1], + dilations=[2, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2D2x2Depth3ValidBackpropInputStride1x1Dilation2x1(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropInputDilation( + input_sizes=[1, 3, 6, 1], + filter_sizes=[2, 2, 1, 1], + output_sizes=[1, 1, 5, 1], + strides=[1, 1], + dilations=[2, 1], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2D2x2Depth1ValidBackpropInputDilation1x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropInputDilation( + input_sizes=[1, 2, 3, 1], + filter_sizes=[2, 2, 1, 1], + output_sizes=[1, 1, 2, 1], + strides=[1, 1], + dilations=[1, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2DEmptyBackpropInputDilation1x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropInputDilation( + input_sizes=[0, 2, 3, 1], + filter_sizes=[2, 2, 1, 1], + output_sizes=[0, 1, 2, 1], + strides=[1, 1], + dilations=[1, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + + def testConv2D2x2Depth3ValidBackpropInputDilation2x1(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + # The GPU version of this test is not very stable. So adjusting the + # error threshold to 1e-4. + self._RunAndVerifyBackpropInputDilation( + input_sizes=[1, 3, 2, 3], + filter_sizes=[2, 2, 3, 3], + output_sizes=[1, 1, 2, 3], + strides=[1, 1], + dilations=[2, 1], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-4) + + def testConv2DKernelSizeMatchesInputSizeBackpropInputDilation2x2(self): + if test.is_gpu_available(cuda_only=True): + for (data_format, use_gpu) in GetTestConfigs(): + self._RunAndVerifyBackpropInputDilation( + input_sizes=[1, 3, 3, 1], + filter_sizes=[2, 2, 1, 2], + output_sizes=[1, 1, 1, 2], + strides=[1, 1], + dilations=[2, 2], + padding="VALID", + data_format=data_format, + use_gpu=use_gpu, + err=1e-5) + # Gradient checkers def ConstructAndTestGradient(self, batch, input_rows, input_cols, filter_rows, filter_cols, in_depth, out_depth, stride_rows, @@ -1457,6 +1829,22 @@ def GetInceptionFwdTest(input_size, filter_size, stride, padding, return Test +def GetInceptionFwdDilatedConvTest(input_size, filter_size, stride, padding): + + def Test(self): + if test.is_gpu_available(cuda_only=True) and stride == 1: + tf_logging.info("Testing InceptionFwd with dilations %s", + (input_size, filter_size, stride, padding)) + self._VerifyDilatedConvValues( + tensor_in_sizes=input_size, + filter_in_sizes=filter_size, + strides=[stride, stride], + dilations=[2, 2], + padding=padding) + + return Test + + def GetInceptionBackInputTest(input_size, filter_size, output_size, stride, padding, gpu_only=False): @@ -1497,6 +1885,10 @@ if __name__ == "__main__": test_util.run_in_graph_and_eager_modes()( GetInceptionFwdTest(input_size_, filter_size_, stride_, padding_))) + setattr( + Conv2DTest, "testInceptionFwdDilatedConv_" + str(index), + test_util.run_in_graph_and_eager_modes()(GetInceptionFwdDilatedConvTest( + input_size_, filter_size_, stride_, padding_))) setattr(Conv2DTest, "testInceptionBackInput_" + str(index), test_util.run_in_graph_and_eager_modes()( GetInceptionBackInputTest(input_size_, filter_size_, @@ -1519,6 +1911,9 @@ if __name__ == "__main__": setattr(Conv2DTest, "testInceptionFwd_No_Winograd_Nonfused", test_util.run_in_graph_and_eager_modes()( GetInceptionFwdTest(ishape, fshape, 1, "SAME", gpu_only=True))) + setattr(Conv2DTest, "testInceptionFwdDilatedConv_No_Winograd_Nonfused", + test_util.run_in_graph_and_eager_modes()( + GetInceptionFwdDilatedConvTest(ishape, fshape, 1, "SAME"))) setattr(Conv2DTest, "testInceptionBackInput_No_Winograd_Nonfused", test_util.run_in_graph_and_eager_modes()( GetInceptionBackInputTest(ishape, fshape, oshape, 1, "SAME", diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index 4b406ba840..8cd535aa0b 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -41,33 +41,48 @@ def _Conv2DBackpropInputGrad(op, grad): Returns: the gradients w.r.t. the input and the filter """ - return [None, - nn_ops.conv2d_backprop_filter(grad, array_ops.shape(op.inputs[1]), - op.inputs[2], op.get_attr("strides"), - op.get_attr("padding"), - op.get_attr("use_cudnn_on_gpu"), - op.get_attr("data_format")), - nn_ops.conv2d(grad, op.inputs[1], op.get_attr("strides"), - op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"), - op.get_attr("data_format"))] + return [ + None, + nn_ops.conv2d_backprop_filter( + grad, + array_ops.shape(op.inputs[1]), + op.inputs[2], + dilations=op.get_attr("dilations"), + strides=op.get_attr("strides"), + padding=op.get_attr("padding"), + use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), + data_format=op.get_attr("data_format")), + nn_ops.conv2d( + grad, + op.inputs[1], + dilations=op.get_attr("dilations"), + strides=op.get_attr("strides"), + padding=op.get_attr("padding"), + use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), + data_format=op.get_attr("data_format")) + ] @ops.RegisterGradient("Conv2DBackpropFilter") def _Conv2DBackpropFilterGrad(op, grad): return [ nn_ops.conv2d_backprop_input( - array_ops.shape(op.inputs[0]), grad, op.inputs[2], - op.get_attr("strides"), - op.get_attr("padding"), - op.get_attr("use_cudnn_on_gpu"), - op.get_attr("data_format")), - None, + array_ops.shape(op.inputs[0]), + grad, + op.inputs[2], + dilations=op.get_attr("dilations"), + strides=op.get_attr("strides"), + padding=op.get_attr("padding"), + use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), + data_format=op.get_attr("data_format")), None, nn_ops.conv2d( - op.inputs[0], grad, - op.get_attr("strides"), - op.get_attr("padding"), - op.get_attr("use_cudnn_on_gpu"), - op.get_attr("data_format")) + op.inputs[0], + grad, + dilations=op.get_attr("dilations"), + strides=op.get_attr("strides"), + padding=op.get_attr("padding"), + use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), + data_format=op.get_attr("data_format")) ] @@ -466,25 +481,32 @@ def _SparseSoftmaxCrossEntropyWithLogitsGrad(op, grad_0, _): @ops.RegisterGradient("Conv2D") def _Conv2DGrad(op, grad): + dilations = op.get_attr("dilations") strides = op.get_attr("strides") padding = op.get_attr("padding") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") data_format = op.get_attr("data_format") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) - return [nn_ops.conv2d_backprop_input(shape_0, - op.inputs[1], - grad, - strides, - padding, - use_cudnn_on_gpu, - data_format), - nn_ops.conv2d_backprop_filter(op.inputs[0], - shape_1, - grad, - strides, - padding, - use_cudnn_on_gpu, - data_format)] + return [ + nn_ops.conv2d_backprop_input( + shape_0, + op.inputs[1], + grad, + dilations=dilations, + strides=strides, + padding=padding, + use_cudnn_on_gpu=use_cudnn_on_gpu, + data_format=data_format), + nn_ops.conv2d_backprop_filter( + op.inputs[0], + shape_1, + grad, + dilations=dilations, + strides=strides, + padding=padding, + use_cudnn_on_gpu=use_cudnn_on_gpu, + data_format=data_format) + ] @ops.RegisterGradient("DepthwiseConv2dNative") diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index ec7b9372ca..b3c0a22efc 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -1205,13 +1205,14 @@ def conv2d_transpose(value, raise ValueError("padding must be either VALID or SAME:" " {}".format(padding)) - return gen_nn_ops.conv2d_backprop_input(input_sizes=output_shape_, - filter=filter, - out_backprop=value, - strides=strides, - padding=padding, - data_format=data_format, - name=name) + return gen_nn_ops.conv2d_backprop_input( + input_sizes=output_shape_, + filter=filter, + out_backprop=value, + strides=strides, + padding=padding, + data_format=data_format, + name=name) def atrous_conv2d_transpose(value, @@ -1343,12 +1344,13 @@ def atrous_conv2d_transpose(value, (in_width + pad_right_extra) // rate, output_shape[3]] - value = gen_nn_ops.conv2d_backprop_input(input_sizes=input_sizes, - filter=filters, - out_backprop=value, - strides=[1, 1, 1, 1], - padding="VALID", - data_format="NHWC") + value = gen_nn_ops.conv2d_backprop_input( + input_sizes=input_sizes, + filter=filters, + out_backprop=value, + strides=[1, 1, 1, 1], + padding="VALID", + data_format="NHWC") # The crops argument to batch_to_space includes both padding components. batch_to_space_crop = [[pad_top, pad_bottom + pad_bottom_extra], diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt index ebd9c079b5..d920fef770 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt @@ -54,15 +54,15 @@ tf_module { } member_method { name: "conv2d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv2d_backprop_filter" - argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv2d_backprop_input" - argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'None\'], " + argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'use_cudnn_on_gpu\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'True\', \'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv2d_transpose" @@ -70,11 +70,11 @@ tf_module { } member_method { name: "conv3d" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv3d_backprop_filter_v2" - argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NDHWC\', \'[1, 1, 1, 1, 1]\', \'None\'], " } member_method { name: "conv3d_transpose" @@ -106,15 +106,15 @@ tf_module { } member_method { name: "depthwise_conv2d_native" - argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "depthwise_conv2d_native_backprop_filter" - argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], " + argspec: "args=[\'input\', \'filter_sizes\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "depthwise_conv2d_native_backprop_input" - argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'None\'], " + argspec: "args=[\'input_sizes\', \'filter\', \'out_backprop\', \'strides\', \'padding\', \'data_format\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\'NHWC\', \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "dilation2d" @@ -234,7 +234,7 @@ tf_module { } member_method { name: "quantized_conv2d" - argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'strides\', \'padding\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " + argspec: "args=[\'input\', \'filter\', \'min_input\', \'max_input\', \'min_filter\', \'max_filter\', \'strides\', \'padding\', \'out_type\', \'dilations\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'[1, 1, 1, 1]\', \'None\'], " } member_method { name: "quantized_max_pool" -- GitLab From b97585f5d2157b1e0273a4b20a568635fb58ad57 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 29 Nov 2017 17:55:53 -0800 Subject: [PATCH 0134/1924] Always leverage shapes inference now that it can handle fed nodes conservatively. PiperOrigin-RevId: 177391746 --- .../grappler/optimizers/constant_folding.cc | 88 ++++++++++--------- .../grappler/optimizers/constant_folding.h | 6 +- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 03eaa4a84a..b5172a4833 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -190,6 +190,14 @@ Status ConvertShapeToConstant(const string& op, const DataType& type, return Status::OK(); } +bool ConstantFolding::IsReallyConstant(const NodeDef& node) const { + if (!IsConstant(node)) { + return false; + } + // If the node is fed it's not constant anymore. + return feed_nodes_.find(node.name()) == feed_nodes_.end(); +} + Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { // We may add some nodes to the graph to encode control dependencies: there is // no need to process these, so only iterate over the nodes of the input @@ -327,9 +335,9 @@ Status ConstantFolding::MaterializeBroadcastGradientArgs( const NodeDef* shape_node1 = node_map_->GetNode(node.input(0)); const NodeDef* shape_node2 = node_map_->GetNode(node.input(1)); if (shape_node1 == nullptr || - (shape_node1->op() != "Shape" && shape_node1->op() != "Const") || + (shape_node1->op() != "Shape" && !IsReallyConstant(*shape_node1)) || shape_node2 == nullptr || - (shape_node2->op() != "Shape" && shape_node2->op() != "Const")) { + (shape_node2->op() != "Shape" && !IsReallyConstant(*shape_node2))) { return Status::OK(); } int64 min_id = 0; @@ -409,7 +417,7 @@ Status ConstantFolding::MaterializeReductionIndices( return Status::OK(); } const NodeDef* indices = node_map_->GetNode(node->input(1)); - if (!indices || IsConstant(*indices)) { + if (!indices || IsReallyConstant(*indices)) { // The reduction indices are already constant, there's nothing to do. return Status::OK(); } @@ -506,24 +514,23 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const { if (node.input().empty()) { return false; } - // Skips nodes that must be preserved except whitelisted nodes. if (nodes_to_preserve_.find(node.name()) != nodes_to_preserve_.end() && nodes_whitelist_.find(node.name()) == nodes_whitelist_.end()) { return false; } - - // Skips ops that don't benefit from folding. - const string& op = node.op(); - // Skip constants, they're already folded - if (op == "Const") { + // Skip control flow nodes, they can't be folded + if (ModifiesFrameInfo(node)) { return false; } - // Skip constrol flow nodes, they can't be folded - if (op == "Enter" || op == "RefEnter" || op == "Exit" || op == "RefExit" || - op == "NextIteration" || op == "RefNextIteration") { + // Skip constants, they're already folded + if (IsConstant(node)) { return false; } + + // Skips ops that don't benefit from folding. + const string& op = node.op(); + if (op.find("Placeholder") == 0) { return false; } @@ -577,7 +584,7 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const { if (!input_node) { return false; } - bool is_const = IsConstant(*input_node); + bool is_const = IsReallyConstant(*input_node); if (!is_const && !is_merge) { return false; } @@ -703,7 +710,7 @@ Status ConstantFolding::EvaluateOneFoldable(const NodeDef& node, break; } const NodeDef* input_node = node_map_->GetNode(input); - if (!IsConstant(*input_node)) { + if (!IsReallyConstant(*input_node)) { return Status(error::INVALID_ARGUMENT, strings::StrCat("Can't fold ", node.name(), ", its ", input, " isn't constant")); @@ -757,7 +764,7 @@ Status ConstantFolding::FoldNode(NodeDef* node, GraphDef* output_graph) { continue; } NodeDef* input_node = node_map_->GetNode(input); - if (!IsConstant(*input_node)) { + if (!IsReallyConstant(*input_node)) { continue; } bool valid_input = true; @@ -999,7 +1006,7 @@ bool ConstantFolding::IsSimplifiableReduction(const NodeDef& node) const { if (IsReduction(node)) { CHECK_LE(2, node.input_size()); const NodeDef* reductions_indices = node_map_->GetNode(node.input(1)); - if (IsConstant(*reductions_indices)) { + if (IsReallyConstant(*reductions_indices)) { TensorVector output; Status s = EvaluateNode(*reductions_indices, TensorVector(), &output); if (!s.ok()) { @@ -1023,7 +1030,7 @@ bool ConstantFolding::IsSimplifiableReshape( } CHECK_LE(2, node.input_size()); const NodeDef* new_shape = node_map_->GetNode(node.input(1)); - if (!IsConstant(*new_shape)) { + if (!IsReallyConstant(*new_shape)) { return false; } TensorVector outputs; @@ -1074,7 +1081,8 @@ bool ConstantFolding::IsSimplifiableReshape( } Status ConstantFolding::SimplifyGraph(GraphDef* output, - const GraphProperties& properties) { + const GraphProperties& properties, + bool use_shape_info) { for (auto& node : *output->mutable_node()) { if (IsSimplifiableReduction(node)) { // Replace the reduction node with an identity node, that can be further @@ -1099,10 +1107,10 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output, *node.add_input() = input; } } - // It's possible to feed a placeholder with a tensor that doesn't have the - // proper shape, and reshape this tensor later on. Therefore only remove - // reshapes in graphs that don't have placeholders. - if (IsSimplifiableReshape(node, properties)) { + const bool safe_to_use_shapes = + use_shape_info && + (feed_nodes_.empty() || opt_level_ == RewriterConfig::AGGRESSIVE); + if (safe_to_use_shapes && IsSimplifiableReshape(node, properties)) { const NodeDef* new_shape = node_map_->GetNode(node.input(1)); DataType output_type = node.attr().at("T").type(); node.set_op("Identity"); @@ -1141,36 +1149,34 @@ Status ConstantFolding::RunOptimizationPass(Cluster* cluster, } GraphProperties properties(item); - const bool has_feed = !item.feed.empty(); - bool needs_shapes = !has_feed || opt_level_ == RewriterConfig::AGGRESSIVE; - Status s = errors::Unknown( - "The graph properties are needed but were not initialized"); - if (needs_shapes) { - s = properties.InferStatically(false); - } - - if (!has_feed && s.ok()) { - // Only use static shape information when there is no feed in the - // graph. That's because it's possible to feed a placeholder with a tensor - // of any shape, which could make the static information inconsistent with - // the shapes actually fed. + // It's possible to feed a placeholder with a tensor of any shape: make sure + // that the shape inference deals with this conservatively unless we're in + // aggressive mode. + const bool assume_valid_feeds = opt_level_ == RewriterConfig::AGGRESSIVE; + Status s = properties.InferStatically(assume_valid_feeds); + const bool can_use_shape_info = s.ok(); + + if (can_use_shape_info) { TF_RETURN_IF_ERROR(MaterializeShapes(properties)); - } - if (opt_level_ == RewriterConfig::AGGRESSIVE && s.ok()) { - TF_RETURN_IF_ERROR(MaterializeConstants(properties)); + + if (opt_level_ == RewriterConfig::AGGRESSIVE) { + TF_RETURN_IF_ERROR(MaterializeConstants(properties)); + } } TF_RETURN_IF_ERROR(FoldGraph(output)); - if (!has_feed && s.ok()) { - TF_RETURN_IF_ERROR(SimplifyGraph(output, properties)); - } + TF_RETURN_IF_ERROR(SimplifyGraph(output, properties, can_use_shape_info)); + return Status::OK(); } Status ConstantFolding::Optimize(Cluster* cluster, const GrapplerItem& item, GraphDef* output) { nodes_to_preserve_ = item.NodesToPreserve(); + for (const auto& feed : item.feed) { + feed_nodes_.insert(NodeName(feed.first)); + } if (cpu_device_ == nullptr) { owned_device_.reset(new DeviceSimple()); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h index 7c5db2a70f..8af5b5fbe6 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.h +++ b/tensorflow/core/grappler/optimizers/constant_folding.h @@ -51,6 +51,8 @@ class ConstantFolding : public GraphOptimizer { const GraphDef& optimize_output, double result) override; private: + bool IsReallyConstant(const NodeDef& node) const; + Status MaterializeShapes(const GraphProperties& properties); Status MaterializeBroadcastGradientArgs(const NodeDef& node, @@ -75,7 +77,8 @@ class ConstantFolding : public GraphOptimizer { bool IsSimplifiableReduction(const NodeDef& node) const; bool IsSimplifiableReshape(const NodeDef& node, const GraphProperties& properties) const; - Status SimplifyGraph(GraphDef* output, const GraphProperties& properties); + Status SimplifyGraph(GraphDef* output, const GraphProperties& properties, + bool use_shape_info); Status RunOptimizationPass(Cluster* cluster, const GrapplerItem& item, GraphDef* output); @@ -90,6 +93,7 @@ class ConstantFolding : public GraphOptimizer { std::unique_ptr node_map_; std::unordered_set nodes_to_preserve_; std::unordered_set nodes_whitelist_; + std::unordered_set feed_nodes_; bool has_fetch_; }; -- GitLab From 65778d86a898d2aa73038837ab6c589b0a345d76 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 30 Nov 2017 02:24:29 +0000 Subject: [PATCH 0135/1924] Add `AWS_REGION` env for S3 in TensorFlow This fix tries to address the issue raised in 14951 where the region can only be specified with non-common `S3_REGION` environment variables. This fix adds the support of `AWS_REGION` which takes precedence over `S3_REGION`. This fix fixes 14951. Signed-off-by: Yong Tang --- tensorflow/core/platform/s3/s3_file_system.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/platform/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc index 234f3c3aed..682ad97eec 100644 --- a/tensorflow/core/platform/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -12,9 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/platform/s3/s3_file_system.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/s3/s3_file_system.h" #include "tensorflow/core/platform/s3/s3_crypto.h" #include @@ -49,9 +49,15 @@ Aws::Client::ClientConfiguration& GetDefaultClientConfig() { if (endpoint) { cfg.endpointOverride = Aws::String(endpoint); } - const char* region = getenv("S3_REGION"); + const char* region = getenv("AWS_REGION"); if (region) { cfg.region = Aws::String(region); + } else { + // TODO (yongtang): `S3_REGION` should be deprecated after 2.0. + const char* region = getenv("S3_REGION"); + if (region) { + cfg.region = Aws::String(region); + } } const char* use_https = getenv("S3_USE_HTTPS"); if (use_https) { -- GitLab From d8d43898b972a1224db50035a771e82985f60035 Mon Sep 17 00:00:00 2001 From: FredZhang <654496915@qq.com> Date: Thu, 30 Nov 2017 12:40:49 +0800 Subject: [PATCH 0136/1924] wrong code in programmer's guide in Variable Section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In Programmer's guide Variable section 
the assignment variable is a `tf.Tensor` and should use `assignment.op.run()` instead of `assignment.run()` Otherwise, this code would produce an error: ``` AttributeError: 'Tensor' object has no attribute 'run' ``` Or we can use sess.run(assignment) to finish this assignment operation --- tensorflow/docs_src/programmers_guide/variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/docs_src/programmers_guide/variables.md b/tensorflow/docs_src/programmers_guide/variables.md index 16753c931f..bac385c02c 100644 --- a/tensorflow/docs_src/programmers_guide/variables.md +++ b/tensorflow/docs_src/programmers_guide/variables.md @@ -205,7 +205,7 @@ methods: v = tf.get_variable("v", shape=(), initializer=tf.zeros_initializer()) assignment = v.assign_add(1) tf.global_variables_initializer().run() -assignment.run() +sess.run(assignment) # or assignment.op.run() ``` Most TensorFlow optimizers have specialized ops that efficiently update the -- GitLab From 4422aaa61338c3af8ce80034d92693a1bd33b09d Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 30 Nov 2017 00:18:03 -0800 Subject: [PATCH 0137/1924] Automated g4 rollback of changelist 177375237 PiperOrigin-RevId: 177418947 --- tensorflow/python/eager/backprop.py | 2 +- tensorflow/python/eager/context.py | 15 -- tensorflow/python/eager/function.py | 144 ++++++------------ tensorflow/python/eager/graph_callable.py | 18 +-- .../python/eager/graph_callable_test.py | 1 + tensorflow/python/framework/ops.py | 30 ++-- tensorflow/python/pywrap_tfe.i | 3 +- 7 files changed, 70 insertions(+), 143 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index dc1142705a..0144f3b1e5 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -540,7 +540,7 @@ def _ensure_unique_tensor_objects(parameter_positions, args): if i in parameter_positions: tid = ops.tensor_id(t) if tid in s: - args[i] = gen_array_ops.identity(args[i]) + args[i] = args[i]._dup() # pylint: disable=protected-access else: s.add(tid) return args diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 415416cfae..92f4e15c05 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -288,21 +288,6 @@ class Context(object): self._initialize_handle_and_devices() return self._num_gpus - def add_function(self, fn): - """Add a function definition to the context. - - Once added, the function (identified by its name) can be executed like any - other operation. - - Args: - fn: A wrapped TF_Function (returned from TF_GraphToFunction_wrapper). - """ - with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.TFE_ContextAddFunction( - self._handle, # pylint: disable=protected-access - fn, - status) - def add_function_def(self, fdef): """Add a function definition to the context. diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 092b36ff20..2f4b59e938 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -25,19 +25,15 @@ import threading import numpy as np -from tensorflow.core.framework import function_pb2 -from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context from tensorflow.python.eager import execute from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder -from tensorflow.python.framework import c_api_util from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_module -from tensorflow.python.framework import errors +from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.ops import gradients_impl -from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -51,41 +47,10 @@ _scoped_captures = threading.local() _scoped_captures.tensors = None -def make_function_def(name, graph, operations, inputs, outputs): - """Makes FunctionDef proto and defined function. - - Args: - name: the function name - graph: the graph from which to build the function - operations: the operations in the function body - inputs: tensors to be used as function arguments - outputs: tensors to be returned from the function - - Returns: - fdef: a FunctionDef protocol buffer for the function - fn: a wrapped TF_Function for the function - """ - with errors.raise_exception_on_not_ok_status() as status: - fn = pywrap_tensorflow.TF_GraphToFunction_wrapper( - graph._c_graph, # pylint: disable=protected-access - compat.as_text(name), - False, - [o._c_op for o in operations], # pylint: disable=protected-access - [t._as_tf_output() for t in inputs], # pylint: disable=protected-access - [t._as_tf_output() for t in outputs], # pylint: disable=protected-access - [compat.as_text("%s" % i) for i in range(len(outputs))], - None, - compat.as_text(""), - status) - # TODO(apassos) avoid creating a FunctionDef (specially to grab the signature, - # but also in general it's nice not to depend on it. - with c_api_util.tf_buffer() as buffer_: - with errors.raise_exception_on_not_ok_status() as status: - pywrap_tensorflow.TF_FunctionToFunctionDef(fn, buffer_, status) - proto_data = pywrap_tensorflow.TF_GetBuffer(buffer_) - fdef = function_pb2.FunctionDef() - fdef.ParseFromString(compat.as_bytes(proto_data)) - return fdef, fn +def make_function_def(graph, operations, inputs, outputs): + """Makes function def from the given graph with the operations.""" + return graph_to_function_def.graph_to_function_def( + graph, operations, inputs, outputs) @contextlib.contextmanager @@ -150,10 +115,6 @@ class CapturingGraph(ops.Graph): # for resource tensors. self._last_op_using_resource_tensor = {} - # TODO(apassos) remove once the C API is used by default. - def _use_c_api_hack(self): - return True - def clear_resource_control_flow_state(self): self._last_op_using_resource_tensor = {} @@ -246,20 +207,14 @@ def _inference_name(n): return "__inference_%s_%s" % (n, ops.uid()) -# TODO(apassos) get rid of this by splitting framework.function._DefinedFunction -# so it doesn't have the definition-generating logic and is just a container for -# an already-defined function. class _DefinedFunction(object): """Mocks the interface of tf _DefinedFunction.""" - def __init__(self, fdef, fn): + def __init__(self, fdef): self.definition = fdef self.name = fdef.signature.name - self.signature = fdef.signature self.grad_func_name = None self.python_grad_func = None - self._c_func = fn - self._grad_func = None def _map_sequence_obj_to_idx(sequence): @@ -295,7 +250,6 @@ class GraphModeFunction(object): input_placeholders, extra_inputs, fdef, - fn, graph, operations, func_outputs, @@ -309,7 +263,7 @@ class GraphModeFunction(object): self._graph = graph self._has_backprop = False self._func_name = fdef.signature.name - self._fdef = _DefinedFunction(fdef, fn) + self._fdef = _DefinedFunction(fdef) self._num_outputs = len(fdef.signature.output_arg) self._ops = operations self._func_outputs = func_outputs @@ -329,45 +283,38 @@ class GraphModeFunction(object): with self._graph.as_default(), context.graph_mode(): c = _CapturingContext() with c: - filtered_outputs = [x for x in self._returns if x is not None] + filtered_outputs = [ + x for x in self._returns if x is not None + ] self._out_grad_placeholders = [ - graph_placeholder(x.dtype, x.shape) for x in filtered_outputs] + graph_placeholder(x.dtype, x.shape) for x in filtered_outputs + ] in_gradients = gradients_impl.gradients( filtered_outputs, self._input_placeholders, grad_ys=self._out_grad_placeholders) - shapes = tuple(x.shape for x in in_gradients if x is not None) + shapes = [x.shape for x in in_gradients if x is not None] captures = list(sorted(c.captured_tensors, key=lambda x: x.name)) - forward_name = _forward_name(self._func_name) - forward_function_def, forward_fn = make_function_def( - forward_name, self._graph, self._ops, self._input_placeholders, + forward_function_def = make_function_def( + self._graph, self._ops, self._input_placeholders, filtered_outputs + captures) - self._forward_fdef = _DefinedFunction(forward_function_def, forward_fn) - _register(forward_fn) - backward_outputs = tuple(x for x in in_gradients if x is not None) + self._forward_fdef = _DefinedFunction(forward_function_def) + _register_with_name(_forward_name(self._func_name), forward_function_def) + backward_outputs = [x for x in in_gradients if x is not None] all_inputs = self._out_grad_placeholders + captures - # Excluding input ops from the body as we do not intend to execute these - # operations when the function is executed. - all_ignored_ops = frozenset(x.op for x in all_inputs) - # Enforce a deterministic order of operations in the generated graph. This - # means rerunning the function-defining code will always define the same - # function, which is useful if we serialize this etc. - fdef_ops = tuple(x for x in sorted(c.known_ops, key=lambda x: x.name) - if x not in all_ignored_ops) - bname = _backward_name(self._func_name) - backward_function_def, backward_fn = make_function_def( - bname, self._graph, fdef_ops, + backward_function_def = make_function_def( + self._graph, [x.op for x in self._out_grad_placeholders + ] + list(sorted(c.known_ops, key=lambda x: x.name)), all_inputs, backward_outputs) - _register(backward_fn) + _register_with_name(_backward_name(self._func_name), backward_function_def) self._backward_function = GraphModeFunction( - all_inputs, [], backward_function_def, backward_fn, self._graph, - c.known_ops, in_gradients, _map_sequence_obj_to_idx(backward_outputs), - shapes) + all_inputs, [], backward_function_def, self._graph, c.known_ops, + in_gradients, _map_sequence_obj_to_idx(backward_outputs), shapes) def _backprop_call(self, args): """Calls the wrapped function and records the result on a tape.""" all_args = args + self._extra_inputs - signature = self._forward_fdef.signature + signature = self._forward_fdef.definition.signature ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() @@ -378,7 +325,7 @@ class GraphModeFunction(object): return ops.internal_convert_to_tensor(x, ctx=ctx) op = g.create_op( signature.name, [make_tensor(x) for x in all_args], - tuple(dtypes_module.DType(x.type) for x in signature.output_arg), + [dtypes_module.DType(x.type) for x in signature.output_arg], op_def=signature, name="FunctionCall", compute_shapes=False) @@ -414,8 +361,11 @@ class GraphModeFunction(object): if v._trainable: # pylint: disable=protected-access tape.watch_variable(v) - tensor_inputs = [x for x in nest.flatten(args) - if isinstance(x, ops.Tensor)] + tensor_inputs = [ + x for x in nest.flatten(args) + if isinstance(x, ops.Tensor) + ] + if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if not self._has_backprop: @@ -434,7 +384,7 @@ class GraphModeFunction(object): args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.convert_to_tensor(x) for x in args], - tuple(dtypes_module.DType(x.type) for x in signature.output_arg), + [dtypes_module.DType(x.type) for x in signature.output_arg], op_def=signature, name="FunctionCall", compute_shapes=False) @@ -519,32 +469,29 @@ def _defun_internal(name, func, args, kwds): extra_inputs = [] extra_placeholders = [] outputs_list = nest.flatten(func_outputs) - output_shapes = tuple(x.shape for x in outputs_list if x is not None) + output_shapes = [x.shape for x in outputs_list if x is not None] - flat_inputs = [x for x in nest.flatten(func_inputs) - if isinstance(x, ops.Tensor)] + flat_inputs = [ + x for x in nest.flatten(func_inputs) if isinstance(x, ops.Tensor) + ] all_inputs = flat_inputs + list(extra_placeholders) - all_ignored_ops = frozenset(x.op for x in all_inputs) + func_def_outputs = [x for x in outputs_list if x is not None] - fname = _inference_name(name) - operations = tuple(x for x in tmp_graph.get_operations() - if x not in all_ignored_ops) - inference_function_def, fn = make_function_def( - fname, tmp_graph, operations, all_inputs, func_def_outputs) + inference_function_def = make_function_def( + tmp_graph, tmp_graph.get_operations(), all_inputs, func_def_outputs) # Register any other functions defined in the graph # TODO(ashankar): Oh lord, forgive me for this lint travesty. for f in tmp_graph._functions.values(): # pylint: disable=protected-access # TODO(ashankar): What about the gradient registry? - _register(f._c_func) # pylint: disable=protected-access - _register(fn) + _register_with_name(f.name, f.definition) + _register_with_name(_inference_name(name), inference_function_def) return GraphModeFunction( all_inputs, extra_inputs, inference_function_def, - fn, tmp_graph, - operations, + tmp_graph.get_operations(), func_outputs, _map_sequence_obj_to_idx(func_def_outputs), output_shapes, @@ -570,9 +517,10 @@ def _cache_key(x): return x -def _register(fn): - """Registers the function `fn`.""" - context.context().add_function(fn) +def _register_with_name(name, fdef): + """Registers the function `fdef` with the name `name`.""" + fdef.signature.name = name + context.context().add_function_def(fdef) # TODO(apassos): better error messages for non-hashable arguments. diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 3da100d800..faf0ac88bc 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -318,9 +318,7 @@ def _graph_callable_internal(func, shape_and_dtypes): placeholder_inputs = flat_inputs+ list(extra_placeholders) func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)] - initialization_name = function._inference_name(func.__name__) # pylint: disable=protected-access - initializer_function_def, initializer_fn = function.make_function_def( - initialization_name, + initializer_function_def = function.make_function_def( tmp_graph, initializing_operations, placeholder_inputs, @@ -329,13 +327,13 @@ def _graph_callable_internal(func, shape_and_dtypes): # Also, what about the gradient registry of these functions? Those need to be # addressed as well. for f in tmp_graph._functions.values(): # pylint: disable=protected-access - function._register(f._c_func) # pylint: disable=protected-access - function._register(initializer_fn) # pylint: disable=protected-access + function._register_with_name(f.name, f.definition) # pylint: disable=protected-access + function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access + initializer_function_def) initializer_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, initializer_function_def, - initializer_fn, tmp_graph, initializing_operations, func_outputs, @@ -344,20 +342,18 @@ def _graph_callable_internal(func, shape_and_dtypes): capture_func_def_outputs = [ x for x in captured_outlist if isinstance(x, tf_ops.Tensor)] - captured_function_name = function._inference_name(func.__name__) # pylint: disable=protected-access - captured_function_def, capturing_fn = function.make_function_def( - captured_function_name, + captured_function_def = function.make_function_def( tmp_graph, capturing_operations, placeholder_inputs, capture_func_def_outputs) - function._register(capturing_fn) # pylint: disable=protected-access + function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access + captured_function_def) captured_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, captured_function_def, - capturing_fn, tmp_graph, capturing_operations, captured_outputs, diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py index b9e6ca2a93..548e16a909 100644 --- a/tensorflow/python/eager/graph_callable_test.py +++ b/tensorflow/python/eager/graph_callable_test.py @@ -152,6 +152,7 @@ class GraphCallableTest(test.TestCase): self.assertAllEqual(5, f(constant_op.constant(2))) def testNestedFunction(self): + # TensorFlow function (which is what would be used in TensorFlow graph # construction). @function.Defun(dtypes.int32, dtypes.int32) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 36daf59647..2217513966 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -599,6 +599,11 @@ class Tensor(_TensorLike): """ return _eval_using_default_session(self, feed_dict, self.graph, session) + def _dup(self): + ret = copy.copy(self) + ret._id = uid() # pylint: disable=protected-access + return ret + # TODO(agarwal): consider getting rid of this. class _EagerTensorBase(Tensor): @@ -724,6 +729,9 @@ class _EagerTensorBase(Tensor): return new_tensor # pylint: enable=protected-access + def _dup(self): + return self._copy(device_name=self.device) + @property def shape(self): return tensor_shape.TensorShape(self._shape_tuple()) @@ -1786,7 +1794,7 @@ class Operation(object): c_api.SetRequestedDevice( self._graph._c_graph, # pylint: disable=protected-access self._c_op, # pylint: disable=protected-access - compat.as_text(_device_string(device))) + _device_string(device)) else: self._node_def.device = _device_string(device) @@ -2075,7 +2083,7 @@ class Operation(object): def _set_attr(self, attr_name, attr_value): """Private method used to set an attribute in the node_def.""" - if self._c_op: + if _USE_C_API: buf = c_api.TF_NewBufferFromString( compat.as_bytes(attr_value.SerializeToString())) try: @@ -2644,16 +2652,11 @@ class Graph(object): # TODO(skyewm): fold as much of the above as possible into the C # implementation - if _USE_C_API or self._use_c_api_hack(): + if _USE_C_API: self._scoped_c_graph = c_api_util.ScopedTFGraph() else: self._scoped_c_graph = None - # TODO(apassos) remove once the C API is used by default. - def _use_c_api_hack(self): - """Temporary hack; can be overridden to force C API usage.""" - return False - def _convert_stack(self, stack, include_func_start_lineno=False): """Converts a stack extracted using _extract_stack() to a traceback stack. @@ -2982,14 +2985,9 @@ class Graph(object): # Add function to graph # pylint: disable=protected-access if self._c_graph: - # Handle functions created without using the C API. TODO(apassos,skyewm) - # remove this when all functions are generated using the C API by default - # as this will be unnecessary. - if not function._c_func: - with errors.raise_exception_on_not_ok_status() as status: - serialized = function.definition.SerializeToString() - function._c_func = c_api.TF_FunctionImportFunctionDef( - serialized, status) + assert function._c_func, ( + "Cannot add function created without C API support to graph " + "created with C API support") with errors.raise_exception_on_not_ok_status() as status: gradient = function._grad_func._c_func if function._grad_func else None c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient, diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 82750e9e49..82b154164e 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -18,7 +18,6 @@ limitations under the License. %rename("%s") TFE_NewContext; %rename("%s") TFE_DeleteContext; %rename("%s") TFE_ContextListDevices; -%rename("%s") TFE_ContextAddFunction; %rename("%s") TFE_ContextAddFunctionDef; %rename("%s") TFE_OpNameGetAttrType; %rename("%s") TFE_Py_InitEagerTensor; @@ -150,7 +149,7 @@ limitations under the License. } $1 = &temp; $1->resize(PyInt_AsLong($input), nullptr); -} +} // Create new Status object. %typemap(in, numinputs=0) TF_Status *out_status { -- GitLab From bec3d96c1f9973c22136f6fd33388edbd78f0824 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 00:31:02 -0800 Subject: [PATCH 0138/1924] Automated g4 rollback of changelist 177362829 PiperOrigin-RevId: 177419730 --- tensorflow/compiler/xla/tests/slice_test.cc | 88 ++++++--------------- 1 file changed, 23 insertions(+), 65 deletions(-) diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc index 981d075089..c21124750a 100644 --- a/tensorflow/compiler/xla/tests/slice_test.cc +++ b/tensorflow/compiler/xla/tests/slice_test.cc @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/core/lib/gtl/array_slice.h" -#include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -212,13 +211,6 @@ class SliceR1Test : public ClientLibraryTestBase, } }; -string SliceR1TestDataToString(const ::testing::TestParamInfo& data) { - const R1Spec& spec = data.param; - return ::tensorflow::strings::Printf("%lld_%lld_%lld_%lld", spec.input_dim0, - spec.slice_start, spec.slice_limit, - spec.slice_stride); -} - XLA_TEST_P(SliceR1Test, DoIt_F32) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_F64) { Run(GetParam()); } @@ -231,64 +223,30 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_S64) { Run(GetParam()); } -// Tests for R1 slice ops. -// The format for each testcase is {input size, start, limit, stride}. -// clang-format off -INSTANTIATE_TEST_CASE_P( - SliceR1TestInstantiation, - SliceR1Test, - ::testing::Values( - R1Spec{10, 0, 0, 1}, - R1Spec{10, 7, 7, 1}, - R1Spec{10, 0, 5, 1}, - R1Spec{10, 3, 5, 1}, - R1Spec{10, 0, 10, 1}, - R1Spec{1024, 0, 5, 1}, - R1Spec{1024, 3, 5, 1}, - R1Spec{1024 + 17, 0, 5, 1}, - R1Spec{1024 + 17, 3, 5, 1}, - R1Spec{1024 + 17, 1024, 1024 + 6, 1}, - R1Spec{1024 + 17, 1024 + 1, 1024 + 6, 1}, - R1Spec{1024, 1024 - 4, 1024, 1}, - R1Spec{4 * 1024, 7, 7 + 1024, 1}, - R1Spec{4 * 1024, 0, 4 * 1024, 1}, - R1Spec{4 * 1024, 1, 4 * 1024 - 1, 1}, - R1Spec{4 * 1024, 1024, 3 * 1024, 1}, - R1Spec{4 * 1024, 1024 + 1, 3 * 1024 - 1, 1}, - R1Spec{16 * 1024, 0, 5, 1}, - R1Spec{16 * 1024, 3, 5, 1}, - R1Spec{16 * 1024 + 17, 0, 5, 1}, - R1Spec{16 * 1024 + 17, 3, 5, 1}, - R1Spec{16 * 1024 + 17, 16 * 1024, 16 * 1024 + 6, 1}, - R1Spec{16 * 1024 + 17, 16 * 1024 + 1, 16 * 1024 + 6, 1}, - R1Spec{64 * 1024, 0, 64 * 1024, 1}, - R1Spec{64 * 1024, 1, 64 * 1024 - 1, 1}, - R1Spec{64 * 1024, 1024, 63 * 1024, 1}, - R1Spec{64 * 1024, 1024 + 1, 63 * 1024 - 1, 1}, - R1Spec{64 * 1024, 32 * 1024, 33 * 1024, 1}, - R1Spec{64 * 1024, 32 * 1024 + 1, 33 * 1024 - 1, 1}, -// TODO(b/69425338): This uses too much memory on GPU. -#ifndef XLA_TEST_BACKEND_GPU - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1}, - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1}, - R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1}, -#endif - R1Spec{10, 2, 4, 2}, - R1Spec{10, 0, 10, 2}, - R1Spec{10, 0, 10, 3}, - R1Spec{10, 0, 10, 4}, - R1Spec{10, 0, 10, 5}, - R1Spec{10, 0, 10, 10}, - R1Spec{500, 200, 400, 7}, - R1Spec{4096, 1, 4095, 3}, - R1Spec{2047, 1024 - 24, 1024 + 160, 31}, - R1Spec{2047, 1, 2046, 3 * 128}, - R1Spec{4096, 1024 + 3, 4095, 500}, - R1Spec{8192, 0, 8192, 1024 * 3 + 400} - ), - SliceR1TestDataToString +INSTANTIATE_TEST_CASE_P( // + SliceR1TestInstantiation, // + SliceR1Test, // + ::testing::Values( // + R1Spec{10, 0, 0, 1}, // + R1Spec{10, 7, 7, 1}, // + R1Spec{10, 2, 4, 1}, // + R1Spec{10, 2, 4, 2}, // + R1Spec{10, 0, 10, 1}, // + R1Spec{1024, 1024 - 4, 1024, 1}, // + R1Spec{4096, 7, 7 + 1024, 1}, // + R1Spec{10, 0, 10, 2}, // + R1Spec{10, 0, 10, 3}, // + R1Spec{10, 0, 10, 4}, // + R1Spec{10, 0, 10, 5}, // + R1Spec{10, 0, 10, 10}, // + R1Spec{500, 200, 400, 7}, // + R1Spec{4096, 1, 4095, 3}, // + R1Spec{2047, 1024 - 24, 1024 + 160, 31}, // + R1Spec{2047, 1, 2046, 3 * 128}, // + R1Spec{4096, 1024 + 3, 4095, 500}, // + R1Spec{8192, 0, 8192, 1024 * 3 + 400} // + ) // ); -// clang-format on struct R2Spec { int64 input_dim0; -- GitLab From ad3213bc53d9905c788509948412ad9703fa976b Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 30 Nov 2017 01:21:49 -0800 Subject: [PATCH 0139/1924] Disable baseline_test in asan. PiperOrigin-RevId: 177423981 --- tensorflow/python/estimator/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index 8e6945b0f3..e062e1fbfe 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -215,6 +215,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_pip", + "noasan", # test flakily times out in asan mode. "notsan", # b/67510291 ], deps = [ -- GitLab From f5491b8305d056083f090ce90968cc7e7f19d3e1 Mon Sep 17 00:00:00 2001 From: Ishant Mrinal Haloi Date: Thu, 30 Nov 2017 16:11:01 +0530 Subject: [PATCH 0140/1924] fix clip weights tests --- .../contrib/gan/python/features/python/clip_weights_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/gan/python/features/python/clip_weights_test.py b/tensorflow/contrib/gan/python/features/python/clip_weights_test.py index 030e37ec67..8427f91368 100644 --- a/tensorflow/contrib/gan/python/features/python/clip_weights_test.py +++ b/tensorflow/contrib/gan/python/features/python/clip_weights_test.py @@ -36,7 +36,7 @@ class ClipWeightsTest(test.TestCase): 'VarTuple', ['discriminator_variables'])(self.variables) def _test_weight_clipping_helper(self, use_tuple): - loss = self.variables[0] * 2.0 + loss = self.variables[0] opt = training.GradientDescentOptimizer(1.0) if use_tuple: opt_clip = clip_weights.weight_clip(opt, self.variables, 0.1) -- GitLab From 8a98563eb6d552f0bd0931f83837640481c1f938 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 30 Nov 2017 06:13:53 -0800 Subject: [PATCH 0141/1924] Add support for int32 output types to the Multinomial op. PiperOrigin-RevId: 177444775 --- .../compiler/tests/categorical_op_test.py | 38 ++++++++------ tensorflow/core/kernels/multinomial_op.cc | 51 ++++++++++++------- tensorflow/core/kernels/multinomial_op.h | 2 +- .../core/kernels/multinomial_op_gpu.cu.cc | 30 +++++++---- tensorflow/core/ops/random_ops.cc | 3 +- .../random/multinomial_op_test.py | 14 ++--- tensorflow/python/ops/random_ops.py | 5 +- tensorflow/tools/api/golden/tensorflow.pbtxt | 2 +- 8 files changed, 90 insertions(+), 55 deletions(-) diff --git a/tensorflow/compiler/tests/categorical_op_test.py b/tensorflow/compiler/tests/categorical_op_test.py index 5e06f9a724..035cdea178 100644 --- a/tensorflow/compiler/tests/categorical_op_test.py +++ b/tensorflow/compiler/tests/categorical_op_test.py @@ -35,6 +35,9 @@ from tensorflow.python.platform import googletest class CategoricalTest(XLATestCase): """Test cases for random-number generating operators.""" + def output_dtypes(self): + return set(self.int_types).intersection([np.int32, np.int64]) + def _chi2(self, expected, actual): """Returns Chi2 GOF statistic.""" actual = np.asarray(actual) @@ -55,7 +58,8 @@ class CategoricalTest(XLATestCase): """ with self.test_session() as sess, self.test_scope(): random_seed.set_random_seed(1618) - op = random_ops.multinomial(logits, num_samples) + op = random_ops.multinomial(logits, num_samples, + output_dtype=dtypes.int32) d = sess.run(op) batch_size, num_classes = logits.shape @@ -73,11 +77,11 @@ class CategoricalTest(XLATestCase): return freqs_mat - def _testRngIsNotConstant(self, rng, dtype): + def _testRngIsNotConstant(self, rng, dtype, output_dtype): # Tests that 'rng' does not always return the same value. with self.test_session() as sess: with self.test_scope(): - x = rng(dtype) + x = rng(dtype, output_dtype) # The random-number generator, if working correctly, should produce the # same output multiple times with low probability. @@ -92,21 +96,25 @@ class CategoricalTest(XLATestCase): (not np.array_equal(y, w))) def testCategoricalIsNotConstant(self): - def rng(unused_dtype): - return random_ops.multinomial([[1., 1., 1.]], 10) + def rng(dtype, output_dtype): + return random_ops.multinomial(np.array([[1., 1., 1.]], dtype=dtype), 10, + output_dtype=output_dtype) - dtype = dtypes.float32 - self._testRngIsNotConstant(rng, dtype) + dtype = np.float32 + for output_dtype in self.output_dtypes(): + self._testRngIsNotConstant(rng, dtype, output_dtype) def testCategoricalIsInRange(self): - for dtype in [dtypes.float32, dtypes.float64]: - with self.test_session() as sess: - with self.test_scope(): - x = random_ops.multinomial( - array_ops.ones(shape=[1, 20], dtype=dtype), 1000) - y = sess.run(x) - self.assertTrue((y >= 0).sum() == 1000) - self.assertTrue((y < 20).sum() == 1000) + for dtype in self.float_types: + for output_dtype in self.output_dtypes(): + with self.test_session() as sess: + with self.test_scope(): + x = random_ops.multinomial( + array_ops.ones(shape=[1, 20], dtype=dtype), 1000, + output_dtype=output_dtype) + y = sess.run(x) + self.assertTrue((y >= 0).sum() == 1000) + self.assertTrue((y < 20).sum() == 1000) def testSamplingCorrectness(self): np.random.seed(1618) # Make it reproducible. diff --git a/tensorflow/core/kernels/multinomial_op.cc b/tensorflow/core/kernels/multinomial_op.cc index 8c0109f5c8..d086abb247 100644 --- a/tensorflow/core/kernels/multinomial_op.cc +++ b/tensorflow/core/kernels/multinomial_op.cc @@ -40,7 +40,7 @@ typedef Eigen::GpuDevice GPUDevice; namespace functor { -template +template struct MultinomialFunctor { void operator()(OpKernelContext* ctx, const Device& d, typename TTypes::ConstMatrix logits, @@ -49,11 +49,11 @@ struct MultinomialFunctor { typename TTypes::Flat scratch, int batch_size, int num_classes, int num_samples, const random::PhiloxRandom& gen, - typename TTypes::Matrix output); + typename TTypes::Matrix output); }; -template -struct MultinomialFunctor { +template +struct MultinomialFunctor { void operator()(OpKernelContext* ctx, const CPUDevice& d, typename TTypes::ConstMatrix logits, typename TTypes::Flat /* noises */, @@ -61,7 +61,7 @@ struct MultinomialFunctor { typename TTypes::Flat /* scratch */, int batch_size, int num_classes, int num_samples, const random::PhiloxRandom& gen, - typename TTypes::Matrix output) { + typename TTypes::Matrix output) { auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); // The implementation only parallelizes by batch. @@ -128,7 +128,7 @@ struct MultinomialFunctor { } // namespace functor // Samples from a multinomial distribution. -template +template class MultinomialOp : public OpKernel { public: explicit MultinomialOp(OpKernelConstruction* context) : OpKernel(context) { @@ -195,11 +195,11 @@ class MultinomialOp : public OpKernel { if (std::is_same::value) num_samples_ceil_4 *= 2; auto rng = generator_.ReserveRandomOutputs(batch_size * num_samples_ceil_4, 256); - functor::MultinomialFunctor()( + functor::MultinomialFunctor()( ctx, ctx->eigen_device(), logits_t.matrix(), noises.flat(), scores.flat(), scratch.flat(), batch_size, num_classes, num_samples, rng, - samples_t->matrix()); + samples_t->matrix()); } } @@ -209,10 +209,17 @@ class MultinomialOp : public OpKernel { TF_DISALLOW_COPY_AND_ASSIGN(MultinomialOp); }; -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("Multinomial").Device(DEVICE_CPU).TypeConstraint("T"), \ - MultinomialOp); +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT32), \ + MultinomialOp); \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT64), \ + MultinomialOp); TF_CALL_half(REGISTER); TF_CALL_float(REGISTER); @@ -220,12 +227,20 @@ TF_CALL_double(REGISTER); #undef REGISTER #if GOOGLE_CUDA -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER(Name("Multinomial") \ - .Device(DEVICE_GPU) \ - .HostMemory("num_samples") \ - .TypeConstraint("T"), \ - MultinomialOp) +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_GPU) \ + .HostMemory("num_samples") \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT32), \ + MultinomialOp) \ + REGISTER_KERNEL_BUILDER(Name("Multinomial") \ + .Device(DEVICE_GPU) \ + .HostMemory("num_samples") \ + .TypeConstraint("T") \ + .TypeConstraint("output_dtype", DT_INT64), \ + MultinomialOp) + TF_CALL_half(REGISTER); TF_CALL_float(REGISTER); TF_CALL_double(REGISTER); diff --git a/tensorflow/core/kernels/multinomial_op.h b/tensorflow/core/kernels/multinomial_op.h index af5e81f219..6e41060aa4 100644 --- a/tensorflow/core/kernels/multinomial_op.h +++ b/tensorflow/core/kernels/multinomial_op.h @@ -21,7 +21,7 @@ namespace tensorflow { namespace functor { // Generic helper functor for the Multinomial Op. -template +template struct MultinomialFunctor; } // namespace functor diff --git a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc index 19b4f3ca55..5cc5877cce 100644 --- a/tensorflow/core/kernels/multinomial_op_gpu.cu.cc +++ b/tensorflow/core/kernels/multinomial_op_gpu.cu.cc @@ -37,20 +37,22 @@ using GPUDevice = Eigen::GpuDevice; // Kernel for Multinomial op. Data is interpreted to have the following shapes: // scores: [B, S, C]; maxima: [B, S]; output: [B, S]. +template __global__ void MultinomialKernel(int32 nthreads, const int32 num_classes, const int32 num_samples, const float* scores, - const float* maxima, int64* output) { + const float* maxima, OutputType* output) { CUDA_1D_KERNEL_LOOP(index, nthreads) { const int maxima_idx = index / num_classes; if (ldg(maxima + maxima_idx) == ldg(scores + index)) { - CudaAtomicMax(reinterpret_cast(output + maxima_idx), - static_cast(index % num_classes)); + using UnsignedOutputType = typename std::make_unsigned::type; + CudaAtomicMax(reinterpret_cast(output + maxima_idx), + static_cast(index % num_classes)); } } } -template -struct MultinomialFunctor { +template +struct MultinomialFunctor { void operator()(OpKernelContext* ctx, const GPUDevice& d, typename TTypes::ConstMatrix logits, typename TTypes::Flat noises, @@ -58,7 +60,7 @@ struct MultinomialFunctor { typename TTypes::Flat maxima, int batch_size, int num_classes, int num_samples, const random::PhiloxRandom& gen, - typename TTypes::Matrix output) { + typename TTypes::Matrix output) { // Uniform, [0, 1). typedef random::UniformDistribution Dist; functor::FillPhiloxRandom()(ctx, d, gen, noises.data(), @@ -111,11 +113,17 @@ struct MultinomialFunctor { }; // Explicit instantiation of the GPU functors. -template struct MultinomialFunctor; -template struct MultinomialFunctor; -template struct MultinomialFunctor; -template struct MultinomialFunctor; -template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; + +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; +template struct MultinomialFunctor; } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/ops/random_ops.cc b/tensorflow/core/ops/random_ops.cc index 2429171fa9..5a436fb93e 100644 --- a/tensorflow/core/ops/random_ops.cc +++ b/tensorflow/core/ops/random_ops.cc @@ -201,10 +201,11 @@ REGISTER_OP("Multinomial") .SetIsStateful() .Input("logits: T") .Input("num_samples: int32") - .Output("output: int64") + .Output("output: output_dtype") .Attr("seed: int = 0") .Attr("seed2: int = 0") .Attr("T: realnumbertype") + .Attr("output_dtype: {int32, int64} = DT_INT64") .SetShapeFn([](InferenceContext* c) { ShapeHandle logits_shape; ShapeHandle unused; diff --git a/tensorflow/python/kernel_tests/random/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py index ca48ba6cad..a9dc7b7de0 100644 --- a/tensorflow/python/kernel_tests/random/multinomial_op_test.py +++ b/tensorflow/python/kernel_tests/random/multinomial_op_test.py @@ -57,12 +57,14 @@ class MultinomialTest(test.TestCase): @test_util.run_in_graph_and_eager_modes() def testSmallEntropy(self): random_seed.set_random_seed(1618) - with test_util.device(use_gpu=True): - # A logit value of -10 corresponds to a probability of ~5e-5. - logits = constant_op.constant([[-10., 10., -10.], [-10., -10., 10.]]) - num_samples = 1000 - samples = self.evaluate(random_ops.multinomial(logits, num_samples)) - self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples) + for output_dtype in [np.int32, np.int64]: + with test_util.device(use_gpu=True): + # A logit value of -10 corresponds to a probability of ~5e-5. + logits = constant_op.constant([[-10., 10., -10.], [-10., -10., 10.]]) + num_samples = 1000 + samples = self.evaluate(random_ops.multinomial( + logits, num_samples, output_dtype=output_dtype)) + self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples) def testOneOpMultipleStepsIndependent(self): with self.test_session(use_gpu=True) as sess: diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py index 52fb5131cf..afaff8ca41 100644 --- a/tensorflow/python/ops/random_ops.py +++ b/tensorflow/python/ops/random_ops.py @@ -316,7 +316,7 @@ def random_crop(value, size, seed=None, name=None): return array_ops.slice(value, offset, size, name=name) -def multinomial(logits, num_samples, seed=None, name=None): +def multinomial(logits, num_samples, seed=None, name=None, output_dtype=None): """Draws samples from a multinomial distribution. Example: @@ -336,6 +336,7 @@ def multinomial(logits, num_samples, seed=None, name=None): @{tf.set_random_seed} for behavior. name: Optional name for the operation. + output_dtype: integer type to use for the output. Defaults to int64. Returns: The drawn samples of shape `[batch_size, num_samples]`. @@ -344,7 +345,7 @@ def multinomial(logits, num_samples, seed=None, name=None): logits = ops.convert_to_tensor(logits, name="logits") seed1, seed2 = random_seed.get_seed(seed) return gen_random_ops.multinomial( - logits, num_samples, seed=seed1, seed2=seed2) + logits, num_samples, seed=seed1, seed2=seed2, output_dtype=output_dtype) ops.NotDifferentiable("Multinomial") diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index 0edd4153d7..57573d5024 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -1394,7 +1394,7 @@ tf_module { } member_method { name: "multinomial" - argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " + argspec: "args=[\'logits\', \'num_samples\', \'seed\', \'name\', \'output_dtype\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } member_method { name: "multiply" -- GitLab From 976049bb0bcdebe10d0a67f6c843f2b51eb1348c Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Thu, 30 Nov 2017 09:09:16 -0800 Subject: [PATCH 0142/1924] Implement Python-specific device and colocation logic in import_graph_def with C API enabled. PiperOrigin-RevId: 177462553 --- tensorflow/python/framework/importer.py | 63 ++++++++++- tensorflow/python/framework/importer_test.py | 108 ++++++++----------- tensorflow/python/framework/ops.py | 6 +- 3 files changed, 107 insertions(+), 70 deletions(-) diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py index 73c35de578..ada8c30fab 100644 --- a/tensorflow/python/framework/importer.py +++ b/tensorflow/python/framework/importer.py @@ -251,10 +251,67 @@ def _PopulateTFImportGraphDefOptions(options, prefix, input_map, def _ProcessNewOps(graph): """Processes the newly-added TF_Operations in `graph`.""" - for c_op in c_api_util.new_tf_operations(graph): - graph._create_op_from_tf_operation(c_op) # pylint: disable=protected-access + # Maps from a node to the names of the ops it's colocated with, if colocation + # is specified in the attributes. + colocation_pairs = {} - # TODO(skyewm): colocation logic + for c_op in c_api_util.new_tf_operations(graph): + # pylint: disable=protected-access + new_op = graph._create_op_from_tf_operation(c_op, compute_device=False) + # pylint: enable=protected-access + + colocation_names = _GetColocationNames(new_op) + if colocation_names: + colocation_pairs[new_op] = colocation_names + # Don't apply this op's device function, since colocation constraints + # override device functions. Note that this op's device may still be set + # by the loop below. + else: + with _MaybeDevice(new_op.device): + graph._apply_device_functions(new_op) # pylint: disable=protected-access + + # The following loop populates the device field of ops that are colocated + # with another op. This is implied by the colocation attribute, but we + # propagate the device field for completeness. + for op, coloc_op_list in colocation_pairs.items(): + coloc_device = None + # Find any device in the list of colocated ops that have a device, if it + # exists. We assume that if multiple ops have devices, they refer to the + # same device. Otherwise, a runtime error will occur since the colocation + # property cannot be guaranteed. + # + # One possible improvement is to try to check for compatibility of all + # devices in this list at import time here, which would require + # implementing a compatibility function for device specs in python. + for coloc_op_name in coloc_op_list: + try: + coloc_op = graph._get_operation_by_name_unsafe(coloc_op_name) # pylint: disable=protected-access + except KeyError: + raise ValueError('Specified colocation to an op that ' + 'does not exist during import: %s in %s' % ( + coloc_op_name, op.name)) + if coloc_op.device: + coloc_device = pydev.DeviceSpec.from_string(coloc_op.device) + break + if coloc_device: + op._set_device(coloc_device) # pylint: disable=protected-access + + +def _GetColocationNames(op): + """Returns names of the ops that `op` should be colocated with.""" + colocation_names = [] + try: + class_values = op.get_attr('_class') + except ValueError: + # No _class attr + return + for val in class_values: + val = compat.as_str(val) + if val.startswith('loc:@'): + colocation_node_name = val[len('loc:@'):] + if colocation_node_name != op.name: + colocation_names.append(colocation_node_name) + return colocation_names def _GatherReturnElements(requested_return_elements, graph, results): diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py index 000a88bc09..4a215abd2e 100644 --- a/tensorflow/python/framework/importer_test.py +++ b/tensorflow/python/framework/importer_test.py @@ -642,8 +642,6 @@ class ImportGraphDefTest(test.TestCase): b.node_def.attr["_class"]) def testColocationWithDeviceFn(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - original_graph_def = self._MakeGraphDef(""" node { name: 'A' op: 'None' attr { key: '_class' @@ -665,23 +663,17 @@ class ImportGraphDefTest(test.TestCase): with ops.Graph().as_default(): with ops.device(CustomDeviceFn): - b, = importer.import_graph_def( - original_graph_def, return_elements=["B"], name="imported_graph") - - self.assertProtoEqualsVersion(""" - node { name: 'imported_graph/A' op: 'None' device: "/device:A:0" - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } - } - node { name: 'imported_graph/B' op: 'None' device: "/device:A:0" - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } }""", b.graph.as_graph_def()) - - # Test a scenario where 'A' doesn't get a device; 'A' should - # not have a device, but during runtime will get colocated with - # 'B' because of the colocation attribute. + a, b = importer.import_graph_def(original_graph_def, + return_elements=["A", "B"], + name="imported_graph") + self.assertEqual(a.device, "/device:A:0") + self.assertEqual(b.device, "/device:A:0") + self.assertEqual(a.colocation_groups(), [b"loc:@imported_graph/A"]) + self.assertEqual(b.colocation_groups(), [b"loc:@imported_graph/A"]) + + # Test a scenario where 'A' doesn't get a device; 'A' should not have a + # device, but during runtime will get colocated with 'B' because of the + # colocation attribute. B's device function is still overridden by A. def BDeviceFn(op): if "B" in op.name: return "/device:B:0" @@ -689,19 +681,13 @@ class ImportGraphDefTest(test.TestCase): with ops.Graph().as_default(): with ops.device(BDeviceFn): - b, = importer.import_graph_def( - original_graph_def, return_elements=["B"], name="imported_graph") - - self.assertProtoEqualsVersion(""" - node { name: 'imported_graph/A' op: 'None' - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } - } - node { name: 'imported_graph/B' op: 'None' - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } }""", b.graph.as_graph_def()) + a, b = importer.import_graph_def(original_graph_def, + return_elements=["A", "B"], + name="imported_graph") + self.assertEqual(a.device, "") + self.assertEqual(b.device, "") + self.assertEqual(a.colocation_groups(), [b"loc:@imported_graph/A"]) + self.assertEqual(b.colocation_groups(), [b"loc:@imported_graph/A"]) # Only A gets a device, so B inherits it implicitly. def ADeviceFn(op): @@ -711,23 +697,15 @@ class ImportGraphDefTest(test.TestCase): with ops.Graph().as_default(): with ops.device(ADeviceFn): - b, = importer.import_graph_def( - original_graph_def, return_elements=["B"], name="imported_graph") - - self.assertProtoEqualsVersion(""" - node { name: 'imported_graph/A' op: 'None' device: "/device:A:0" - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } - } - node { name: 'imported_graph/B' op: 'None' device: "/device:A:0" - attr { - key: '_class' value { list { s: 'loc:@imported_graph/A' } } - } }""", b.graph.as_graph_def()) + a, b = importer.import_graph_def(original_graph_def, + return_elements=["A", "B"], + name="imported_graph") + self.assertEqual(a.device, "/device:A:0") + self.assertEqual(b.device, "/device:A:0") + self.assertEqual(a.colocation_groups(), [b"loc:@imported_graph/A"]) + self.assertEqual(b.colocation_groups(), [b"loc:@imported_graph/A"]) def testMultipleColocationWithDeviceFn(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - original_graph_def = self._MakeGraphDef(""" node { name: 'A' op: 'None'} node { name: 'B' op: 'None'} @@ -748,23 +726,19 @@ class ImportGraphDefTest(test.TestCase): with ops.Graph().as_default(): with ops.device(CustomDeviceFn): - c, = importer.import_graph_def( - original_graph_def, return_elements=["C"], name="imported_graph") - - self.assertProtoEqualsVersion(""" - node { name: 'imported_graph/A' op: 'None' } - node { name: 'imported_graph/B' op: 'None' device: "/device:B:0" } - node { name: 'imported_graph/C' op: 'None' device: "/device:B:0" - attr { - key: '_class' value { - list { s: 'loc:@imported_graph/A' - s: 'loc:@imported_graph/B' } - } - } - }""", c.graph.as_graph_def()) + a, b, c = importer.import_graph_def(original_graph_def, + return_elements=["A", "B", "C"], + name="imported_graph") + self.assertEqual(a.device, "") + self.assertEqual(b.device, "/device:B:0") + self.assertEqual(c.device, "/device:B:0") + self.assertEqual(a.colocation_groups(), [b"loc:@imported_graph/A"]) + self.assertEqual(b.colocation_groups(), [b"loc:@imported_graph/B"]) + self.assertEqual(c.colocation_groups(), + [b"loc:@imported_graph/A", b"loc:@imported_graph/B"]) def testNamePrefixColocationAttrsMultipleImport(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API + if ops._USE_C_API: return # TODO(skyewm): set uniquify_names original_graph_def = self._MakeGraphDef(""" node { name: 'A' op: 'None' } @@ -791,15 +765,19 @@ class ImportGraphDefTest(test.TestCase): } }""", b.graph.as_graph_def()) def testNamePrefixColocationAttrsNotFound(self): - if ops._USE_C_API: return # TODO(skyewm): make this work with C API - original_graph_def = self._MakeGraphDef(""" node { name: 'B' op: 'None' attr { key: '_class' value { list { s: 'loc:@A' } } } }""") + + if ops._USE_C_API: + error_msg = "Node 'B' expects to be colocated with unknown node 'A'" + else: + error_msg = "does not exist during import" + with ops.Graph().as_default(): - with self.assertRaisesRegexp(ValueError, "does not exist during import"): + with self.assertRaisesRegexp(ValueError, error_msg): importer.import_graph_def( original_graph_def, return_elements=["B"], name="imported_graph") diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 2217513966..a616b15cf7 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -3102,7 +3102,7 @@ class Graph(object): compute_device=compute_device) return ret - def _create_op_from_tf_operation(self, c_op): + def _create_op_from_tf_operation(self, c_op, compute_device=True): """Creates an `Operation` in this graph from the supplied TF_Operation. This method is like create_op() except the new Operation is constructed @@ -3112,6 +3112,8 @@ class Graph(object): Args: c_op: a wrapped TF_Operation + compute_device: (Optional.) If True, device functions will be executed + to compute the device property of the Operation. Returns: An `Operation` object. @@ -3122,7 +3124,7 @@ class Graph(object): for output in tf_outputs) control_inputs = self._control_dependencies_for_inputs(input_ops) ret = Operation(c_op, self, control_inputs=control_inputs) - self._create_op_helper(ret) + self._create_op_helper(ret, compute_device=compute_device) return ret def _create_op_helper(self, op, compute_shapes=True, compute_device=True): -- GitLab From 5d52b95279be57076a794c2f334c150a26566360 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Abrams Date: Wed, 29 Nov 2017 22:25:37 -0800 Subject: [PATCH 0143/1924] Adds Operations() method to Graph There is currently no way to list all of the operations in a graph from the go api. This patch ads an Operations() method to retrieve the list using the existing TF_GraphNextOperation c api. The graph_test was modified to include testing this new method. Signed-off-by: Vishvananda Ishaya Abrams --- tensorflow/go/graph.go | 14 ++++++++++++++ tensorflow/go/graph_test.go | 22 +++++++++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tensorflow/go/graph.go b/tensorflow/go/graph.go index 46c600eab1..a40aded3bf 100644 --- a/tensorflow/go/graph.go +++ b/tensorflow/go/graph.go @@ -114,6 +114,20 @@ func (g *Graph) Operation(name string) *Operation { return &Operation{cop, g} } +// Operations returns a list of all operations in the graph +func (g *Graph) Operations() []Operation { + var pos C.size_t = 0 + ops := []Operation{} + for { + cop := C.TF_GraphNextOperation(g.c, &pos) + if cop == nil { + break + } + ops = append(ops, Operation{cop, g}) + } + return ops +} + // OpSpec is the specification of an Operation to be added to a Graph // (using Graph.AddOperation). type OpSpec struct { diff --git a/tensorflow/go/graph_test.go b/tensorflow/go/graph_test.go index c3120bc720..b8d65c54f6 100644 --- a/tensorflow/go/graph_test.go +++ b/tensorflow/go/graph_test.go @@ -29,10 +29,26 @@ func hasOperations(g *Graph, ops ...string) error { missing = append(missing, op) } } - if len(missing) == 0 { - return nil + if len(missing) != 0 { + return fmt.Errorf("Graph does not have the operations %v", missing) } - return fmt.Errorf("Graph does not have the operations %v", missing) + + inList := map[string]bool{} + for _, op := range g.Operations() { + inList[op.Name()] = true + } + + for _, op := range ops { + if !inList[op] { + missing = append(missing, op) + } + } + + if len(missing) != 0 { + return fmt.Errorf("Operations %v are missing from graph.Operations()", missing) + } + + return nil } func TestGraphWriteToAndImport(t *testing.T) { -- GitLab From 1c4810141e71289d71bfd94a74434bd09ee6b20f Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Thu, 30 Nov 2017 09:27:16 -0800 Subject: [PATCH 0144/1924] Hoist function input placeholders out of any control flow context. Prior to this change, functions that closed over external tensors in a while loop would cause a segfault at runtime. This is because the external tensors are temporarily represented as placeholders in the function body before being replaced by input parameters, and the placeholders would be created directly in the while loop body. This would eventually lead to using the input tensor in the while loop body without an enter node. This wasn't caught by the runtime check because it isn't applied to function bodies. This change adds tests for capturing tensors in a while loop body and in a cond context. Note that the cond test passed without this fix. PiperOrigin-RevId: 177464541 --- tensorflow/python/framework/function.py | 5 ++- tensorflow/python/framework/function_test.py | 32 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index 29cf223724..366025a0d8 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -692,7 +692,10 @@ class _FuncGraph(ops.Graph): else: # Substitute with a placeholder. self.extra_inputs.append(x) - ph = array_ops.placeholder(x.dtype, shape=x.get_shape()) + # Hoist the new input placeholder out of any control flow context + # we're currently in. + with ops.control_dependencies(None): + ph = array_ops.placeholder(x.dtype, shape=x.get_shape()) # pylint: disable=protected-access ph._handle_data = x._handle_data # pylint: enable=protected-access diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index ba43e9199b..11f343c579 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -724,6 +724,38 @@ class FunctionTest(test.TestCase): # NOTE: We still do not support capturing control deps. _ = Foo(x) + def testCaptureInWhileLoop(self): + g = ops.Graph() + with g.as_default(): + x = constant_op.constant(1) + + @function.Defun() + def Foo(): + return control_flow_ops.while_loop(lambda i: i < 10, + lambda i: i + x, + [0]) + y = Foo() + + with self.test_session(graph=g) as sess: + self.assertEqual(sess.run(y), 10) + + def testCaptureInCond(self): + g = ops.Graph() + with g.as_default(): + x = constant_op.constant(1) + + @function.Defun(dtypes.bool) + def Foo(pred): + return control_flow_ops.cond(pred, + lambda: x, + lambda: x + 1) + y = Foo(True) + z = Foo(False) + + with self.test_session(graph=g) as sess: + self.assertEqual(sess.run(y), 1) + self.assertEqual(sess.run(z), 2) + def testStableName(self): @function.Defun() -- GitLab From 9308470197bcc068dca9fe227d0ab144157950e1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 09:52:28 -0800 Subject: [PATCH 0145/1924] Rename tests. PiperOrigin-RevId: 177467740 --- ...se_am_model_test.cc => speech_asr_am_model_test.cc} | 10 +++++----- ...se_lm_model_test.cc => speech_asr_lm_model_test.cc} | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) rename tensorflow/contrib/lite/models/{speech_terse_am_model_test.cc => speech_asr_am_model_test.cc} (93%) rename tensorflow/contrib/lite/models/{speech_terse_lm_model_test.cc => speech_asr_lm_model_test.cc} (94%) diff --git a/tensorflow/contrib/lite/models/speech_terse_am_model_test.cc b/tensorflow/contrib/lite/models/speech_asr_am_model_test.cc similarity index 93% rename from tensorflow/contrib/lite/models/speech_terse_am_model_test.cc rename to tensorflow/contrib/lite/models/speech_asr_am_model_test.cc index 30d89a1354..bf95b313f3 100644 --- a/tensorflow/contrib/lite/models/speech_terse_am_model_test.cc +++ b/tensorflow/contrib/lite/models/speech_asr_am_model_test.cc @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Unit test for speech TERSE AM model using TFLite Ops. +// Unit test for speech ASR AM model using TFLite Ops. #include @@ -45,10 +45,10 @@ constexpr int kLstmLayer5OutputStateTensor = 103; constexpr int kLstmLayer5CellStateTensor = 104; constexpr int kModelOutputTensor = 109; -TEST(SpeechTerseAm, RandomIOTest) { +TEST(SpeechAsrAm, RandomIOTest) { // Read the model. string tflite_file_path = - file::JoinPath(TestDataPath(), "speech_terse_am_model.tflite"); + file::JoinPath(TestDataPath(), "speech_asr_am_model.tflite"); auto model = FlatBufferModel::BuildFromFile(tflite_file_path.c_str()); CHECK(model) << "Failed to mmap model " << tflite_file_path; @@ -62,13 +62,13 @@ TEST(SpeechTerseAm, RandomIOTest) { // Load the input frames. Frames input_frames; const string input_file_path = - file::JoinPath(TestDataPath(), "speech_terse_am_model_in.csv"); + file::JoinPath(TestDataPath(), "speech_asr_am_model_in.csv"); ReadFrames(input_file_path, &input_frames); // Load the golden output results. Frames output_frames; const string output_file_path = - file::JoinPath(TestDataPath(), "speech_terse_am_model_out.csv"); + file::JoinPath(TestDataPath(), "speech_asr_am_model_out.csv"); ReadFrames(output_file_path, &output_frames); const int speech_batch_size = diff --git a/tensorflow/contrib/lite/models/speech_terse_lm_model_test.cc b/tensorflow/contrib/lite/models/speech_asr_lm_model_test.cc similarity index 94% rename from tensorflow/contrib/lite/models/speech_terse_lm_model_test.cc rename to tensorflow/contrib/lite/models/speech_asr_lm_model_test.cc index 04c54ffb22..53f2b66da4 100644 --- a/tensorflow/contrib/lite/models/speech_terse_lm_model_test.cc +++ b/tensorflow/contrib/lite/models/speech_asr_lm_model_test.cc @@ -59,10 +59,10 @@ static void ClearLstmStates(Interpreter* interpreter) { interpreter->tensor(kLstmLayer3CellStateTensor)->bytes); } -TEST(SpeechTerseLm, EndToEndTest) { +TEST(SpeechAsrLm, EndToEndTest) { // Read the model. string tflite_file_path = - file::JoinPath(TestDataPath(), "speech_terse_lm_model.tflite"); + file::JoinPath(TestDataPath(), "speech_asr_lm_model.tflite"); auto model = FlatBufferModel::BuildFromFile(tflite_file_path.c_str()); CHECK(model) << "Failed to mmap model " << tflite_file_path; @@ -76,13 +76,13 @@ TEST(SpeechTerseLm, EndToEndTest) { // Load the input frames. Frames input_frames; const string input_file_path = - file::JoinPath(TestDataPath(), "speech_terse_lm_model_in.csv"); + file::JoinPath(TestDataPath(), "speech_asr_lm_model_in.csv"); ReadFrames(input_file_path, &input_frames); // Load the golden output results. Frames output_frames; const string output_file_path = - file::JoinPath(TestDataPath(), "speech_terse_lm_model_out.csv"); + file::JoinPath(TestDataPath(), "speech_asr_lm_model_out.csv"); ReadFrames(output_file_path, &output_frames); CHECK_EQ(interpreter->tensor(kModelInput1Tensor)->dims->size, 1); -- GitLab From f283173062f3ff9b6f69e8fc8a77421dcfdaa8f2 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 30 Nov 2017 09:56:07 -0800 Subject: [PATCH 0146/1924] [TF:XLA] Add support for the V2 variants of the FusedBatchNorm operators, which support mixed precision training. Until the necessary support for mixed precision fused batch norm is added to XLA, implement by casting to a common type. PiperOrigin-RevId: 177468202 --- .../compiler/tf2xla/kernels/batch_norm_op.cc | 74 ++++++++++++++----- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc index 248e9d111e..468af34aab 100644 --- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc @@ -14,7 +14,7 @@ limitations under the License. ==============================================================================*/ // XLA implementation of BatchNorm operations. -#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" @@ -42,27 +42,44 @@ class FusedBatchNormOp : public XlaOpKernel { } void Compile(XlaOpKernelContext* ctx) override { + xla::PrimitiveType input_type; + OP_REQUIRES_OK(ctx, + DataTypeToPrimitiveType(ctx->input_type(0), &input_type)); + xla::PrimitiveType stats_type; + OP_REQUIRES_OK(ctx, + DataTypeToPrimitiveType(ctx->input_type(1), &stats_type)); + + xla::ComputationBuilder* builder = ctx->builder(); + + xla::ComputationDataHandle input = ctx->Input(0); + + // TODO(b/69928690): support mixed precision in the XLA batch normalization + // operators. As a workaround, cast everything to the statistics type (which + // may be more precise than the input type). + input = builder->ConvertElementType(input, stats_type); + if (is_training_) { - xla::ComputationDataHandle output = ctx->builder()->BatchNormTraining( - ctx->Input(0), ctx->Input(1), ctx->Input(2), epsilon_, - feature_index_); + xla::ComputationDataHandle output = builder->BatchNormTraining( + input, ctx->Input(1), ctx->Input(2), epsilon_, feature_index_); // In training mode, outputs the normalized value as well as the // calculated mean and variance. - for (int i = 0; i < 3; i++) { - ctx->SetOutput(i, ctx->builder()->GetTupleElement(output, i)); - } + ctx->SetOutput(0, builder->ConvertElementType( + builder->GetTupleElement(output, 0), input_type)); + ctx->SetOutput(1, builder->GetTupleElement(output, 1)); + ctx->SetOutput(2, builder->GetTupleElement(output, 2)); + // Output 3 and 4 for "FusedBatchNorm" are currently marked as "reserved // space 1 & 2". They are used to pass the per-batch mean and // variance to the gradient. Here we maintain the same behavior by setting // them to the mean and variance calculated by BatchNormTraining. - ctx->SetOutput(3, ctx->builder()->GetTupleElement(output, 1)); - ctx->SetOutput(4, ctx->builder()->GetTupleElement(output, 2)); + ctx->SetOutput(3, builder->GetTupleElement(output, 1)); + ctx->SetOutput(4, builder->GetTupleElement(output, 2)); } else { - xla::ComputationDataHandle output = ctx->builder()->BatchNormInference( - ctx->Input(0), ctx->Input(1), ctx->Input(2), ctx->Input(3), - ctx->Input(4), epsilon_, feature_index_); - ctx->SetOutput(0, output); + xla::ComputationDataHandle output = builder->BatchNormInference( + input, ctx->Input(1), ctx->Input(2), ctx->Input(3), ctx->Input(4), + epsilon_, feature_index_); + ctx->SetOutput(0, builder->ConvertElementType(output, input_type)); // Directly send input to output as mean and variance in inference mode. ctx->SetOutput(1, ctx->Input(3)); ctx->SetOutput(2, ctx->Input(4)); @@ -78,6 +95,7 @@ class FusedBatchNormOp : public XlaOpKernel { }; REGISTER_XLA_OP(Name("FusedBatchNorm"), FusedBatchNormOp); +REGISTER_XLA_OP(Name("FusedBatchNormV2"), FusedBatchNormOp); class FusedBatchNormGradOp : public XlaOpKernel { public: @@ -101,19 +119,36 @@ class FusedBatchNormGradOp : public XlaOpKernel { } void Compile(XlaOpKernelContext* ctx) override { + xla::ComputationBuilder* builder = ctx->builder(); + auto grad_output = ctx->Input(0); auto activation = ctx->Input(1); auto scale = ctx->Input(2); auto mean = ctx->Input(3); auto var = ctx->Input(4); - xla::ComputationDataHandle output = ctx->builder()->BatchNormGrad( + + xla::PrimitiveType input_type; + OP_REQUIRES_OK(ctx, + DataTypeToPrimitiveType(ctx->input_type(0), &input_type)); + xla::PrimitiveType stats_type; + OP_REQUIRES_OK(ctx, + DataTypeToPrimitiveType(ctx->input_type(3), &stats_type)); + + // TODO(b/69928690): support mixed precision in the XLA batch normalization + // operators. As a workaround, cast everything to the statistics type (which + // may be more precise than the input type). + grad_output = builder->ConvertElementType(grad_output, stats_type); + activation = builder->ConvertElementType(activation, stats_type); + + xla::ComputationDataHandle output = builder->BatchNormGrad( activation, scale, mean, var, grad_output, epsilon_, feature_index_); - for (int i = 0; i < 3; i++) { - ctx->SetOutput(i, ctx->builder()->GetTupleElement(output, i)); - } - ctx->SetOutput(3, ctx->builder()->GetTupleElement(output, 1)); - ctx->SetOutput(4, ctx->builder()->GetTupleElement(output, 2)); + ctx->SetOutput(0, builder->ConvertElementType( + builder->GetTupleElement(output, 0), input_type)); + ctx->SetOutput(1, builder->GetTupleElement(output, 1)); + ctx->SetOutput(2, builder->GetTupleElement(output, 2)); + ctx->SetOutput(3, builder->GetTupleElement(output, 1)); + ctx->SetOutput(4, builder->GetTupleElement(output, 2)); } private: @@ -122,6 +157,7 @@ class FusedBatchNormGradOp : public XlaOpKernel { }; REGISTER_XLA_OP(Name("FusedBatchNormGrad"), FusedBatchNormGradOp); +REGISTER_XLA_OP(Name("FusedBatchNormGradV2"), FusedBatchNormGradOp); } // namespace } // namespace tensorflow -- GitLab From 12976748822cdb3885f37dbda42ce8674afa6f91 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Thu, 30 Nov 2017 10:09:52 -0800 Subject: [PATCH 0147/1924] Uses C API for eager functions. Rolls back the rollback with some swiggery to get python3 to work. PiperOrigin-RevId: 177470328 --- tensorflow/python/eager/backprop.py | 2 +- tensorflow/python/eager/context.py | 15 ++ tensorflow/python/eager/function.py | 144 ++++++++++++------ tensorflow/python/eager/graph_callable.py | 18 ++- .../python/eager/graph_callable_test.py | 1 - tensorflow/python/framework/ops.py | 30 ++-- tensorflow/python/pywrap_tfe.i | 3 +- 7 files changed, 143 insertions(+), 70 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 0144f3b1e5..dc1142705a 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -540,7 +540,7 @@ def _ensure_unique_tensor_objects(parameter_positions, args): if i in parameter_positions: tid = ops.tensor_id(t) if tid in s: - args[i] = args[i]._dup() # pylint: disable=protected-access + args[i] = gen_array_ops.identity(args[i]) else: s.add(tid) return args diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 92f4e15c05..415416cfae 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -288,6 +288,21 @@ class Context(object): self._initialize_handle_and_devices() return self._num_gpus + def add_function(self, fn): + """Add a function definition to the context. + + Once added, the function (identified by its name) can be executed like any + other operation. + + Args: + fn: A wrapped TF_Function (returned from TF_GraphToFunction_wrapper). + """ + with errors.raise_exception_on_not_ok_status() as status: + pywrap_tensorflow.TFE_ContextAddFunction( + self._handle, # pylint: disable=protected-access + fn, + status) + def add_function_def(self, fdef): """Add a function definition to the context. diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 2f4b59e938..cadabb3a24 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -25,15 +25,19 @@ import threading import numpy as np +from tensorflow.core.framework import function_pb2 +from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context from tensorflow.python.eager import execute from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder +from tensorflow.python.framework import c_api_util from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_module -from tensorflow.python.framework import graph_to_function_def +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -47,10 +51,41 @@ _scoped_captures = threading.local() _scoped_captures.tensors = None -def make_function_def(graph, operations, inputs, outputs): - """Makes function def from the given graph with the operations.""" - return graph_to_function_def.graph_to_function_def( - graph, operations, inputs, outputs) +def make_function_def(name, graph, operations, inputs, outputs): + """Makes FunctionDef proto and defined function. + + Args: + name: the function name + graph: the graph from which to build the function + operations: the operations in the function body + inputs: tensors to be used as function arguments + outputs: tensors to be returned from the function + + Returns: + fdef: a FunctionDef protocol buffer for the function + fn: a wrapped TF_Function for the function + """ + with errors.raise_exception_on_not_ok_status() as status: + fn = pywrap_tensorflow.TF_GraphToFunction_wrapper( + graph._c_graph, # pylint: disable=protected-access + compat.as_str(name), + False, + [o._c_op for o in operations], # pylint: disable=protected-access + [t._as_tf_output() for t in inputs], # pylint: disable=protected-access + [t._as_tf_output() for t in outputs], # pylint: disable=protected-access + [], + None, + compat.as_str(""), + status) + # TODO(apassos) avoid creating a FunctionDef (specially to grab the signature, + # but also in general it's nice not to depend on it. + with c_api_util.tf_buffer() as buffer_: + with errors.raise_exception_on_not_ok_status() as status: + pywrap_tensorflow.TF_FunctionToFunctionDef(fn, buffer_, status) + proto_data = pywrap_tensorflow.TF_GetBuffer(buffer_) + fdef = function_pb2.FunctionDef() + fdef.ParseFromString(compat.as_bytes(proto_data)) + return fdef, fn @contextlib.contextmanager @@ -115,6 +150,10 @@ class CapturingGraph(ops.Graph): # for resource tensors. self._last_op_using_resource_tensor = {} + # TODO(apassos) remove once the C API is used by default. + def _use_c_api_hack(self): + return True + def clear_resource_control_flow_state(self): self._last_op_using_resource_tensor = {} @@ -207,14 +246,20 @@ def _inference_name(n): return "__inference_%s_%s" % (n, ops.uid()) +# TODO(apassos) get rid of this by splitting framework.function._DefinedFunction +# so it doesn't have the definition-generating logic and is just a container for +# an already-defined function. class _DefinedFunction(object): """Mocks the interface of tf _DefinedFunction.""" - def __init__(self, fdef): + def __init__(self, fdef, fn): self.definition = fdef self.name = fdef.signature.name + self.signature = fdef.signature self.grad_func_name = None self.python_grad_func = None + self._c_func = fn + self._grad_func = None def _map_sequence_obj_to_idx(sequence): @@ -250,6 +295,7 @@ class GraphModeFunction(object): input_placeholders, extra_inputs, fdef, + fn, graph, operations, func_outputs, @@ -263,7 +309,7 @@ class GraphModeFunction(object): self._graph = graph self._has_backprop = False self._func_name = fdef.signature.name - self._fdef = _DefinedFunction(fdef) + self._fdef = _DefinedFunction(fdef, fn) self._num_outputs = len(fdef.signature.output_arg) self._ops = operations self._func_outputs = func_outputs @@ -283,38 +329,45 @@ class GraphModeFunction(object): with self._graph.as_default(), context.graph_mode(): c = _CapturingContext() with c: - filtered_outputs = [ - x for x in self._returns if x is not None - ] + filtered_outputs = [x for x in self._returns if x is not None] self._out_grad_placeholders = [ - graph_placeholder(x.dtype, x.shape) for x in filtered_outputs - ] + graph_placeholder(x.dtype, x.shape) for x in filtered_outputs] in_gradients = gradients_impl.gradients( filtered_outputs, self._input_placeholders, grad_ys=self._out_grad_placeholders) - shapes = [x.shape for x in in_gradients if x is not None] + shapes = tuple(x.shape for x in in_gradients if x is not None) captures = list(sorted(c.captured_tensors, key=lambda x: x.name)) - forward_function_def = make_function_def( - self._graph, self._ops, self._input_placeholders, + forward_name = _forward_name(self._func_name) + forward_function_def, forward_fn = make_function_def( + forward_name, self._graph, self._ops, self._input_placeholders, filtered_outputs + captures) - self._forward_fdef = _DefinedFunction(forward_function_def) - _register_with_name(_forward_name(self._func_name), forward_function_def) - backward_outputs = [x for x in in_gradients if x is not None] + self._forward_fdef = _DefinedFunction(forward_function_def, forward_fn) + _register(forward_fn) + backward_outputs = tuple(x for x in in_gradients if x is not None) all_inputs = self._out_grad_placeholders + captures - backward_function_def = make_function_def( - self._graph, [x.op for x in self._out_grad_placeholders - ] + list(sorted(c.known_ops, key=lambda x: x.name)), + # Excluding input ops from the body as we do not intend to execute these + # operations when the function is executed. + all_ignored_ops = frozenset(x.op for x in all_inputs) + # Enforce a deterministic order of operations in the generated graph. This + # means rerunning the function-defining code will always define the same + # function, which is useful if we serialize this etc. + fdef_ops = tuple(x for x in sorted(c.known_ops, key=lambda x: x.name) + if x not in all_ignored_ops) + bname = _backward_name(self._func_name) + backward_function_def, backward_fn = make_function_def( + bname, self._graph, fdef_ops, all_inputs, backward_outputs) - _register_with_name(_backward_name(self._func_name), backward_function_def) + _register(backward_fn) self._backward_function = GraphModeFunction( - all_inputs, [], backward_function_def, self._graph, c.known_ops, - in_gradients, _map_sequence_obj_to_idx(backward_outputs), shapes) + all_inputs, [], backward_function_def, backward_fn, self._graph, + c.known_ops, in_gradients, _map_sequence_obj_to_idx(backward_outputs), + shapes) def _backprop_call(self, args): """Calls the wrapped function and records the result on a tape.""" all_args = args + self._extra_inputs - signature = self._forward_fdef.definition.signature + signature = self._forward_fdef.signature ctx = context.context() if ctx.in_graph_mode(): g = ops.get_default_graph() @@ -325,7 +378,7 @@ class GraphModeFunction(object): return ops.internal_convert_to_tensor(x, ctx=ctx) op = g.create_op( signature.name, [make_tensor(x) for x in all_args], - [dtypes_module.DType(x.type) for x in signature.output_arg], + tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) @@ -361,11 +414,8 @@ class GraphModeFunction(object): if v._trainable: # pylint: disable=protected-access tape.watch_variable(v) - tensor_inputs = [ - x for x in nest.flatten(args) - if isinstance(x, ops.Tensor) - ] - + tensor_inputs = [x for x in nest.flatten(args) + if isinstance(x, ops.Tensor)] if tape.should_record(tensor_inputs) or tape.should_record( self._extra_inputs): if not self._has_backprop: @@ -384,7 +434,7 @@ class GraphModeFunction(object): args = list(tensor_inputs) + self._extra_inputs op = g.create_op( signature.name, [ops.convert_to_tensor(x) for x in args], - [dtypes_module.DType(x.type) for x in signature.output_arg], + tuple(dtypes_module.DType(x.type) for x in signature.output_arg), op_def=signature, name="FunctionCall", compute_shapes=False) @@ -469,29 +519,32 @@ def _defun_internal(name, func, args, kwds): extra_inputs = [] extra_placeholders = [] outputs_list = nest.flatten(func_outputs) - output_shapes = [x.shape for x in outputs_list if x is not None] + output_shapes = tuple(x.shape for x in outputs_list if x is not None) - flat_inputs = [ - x for x in nest.flatten(func_inputs) if isinstance(x, ops.Tensor) - ] + flat_inputs = [x for x in nest.flatten(func_inputs) + if isinstance(x, ops.Tensor)] all_inputs = flat_inputs + list(extra_placeholders) - + all_ignored_ops = frozenset(x.op for x in all_inputs) func_def_outputs = [x for x in outputs_list if x is not None] - inference_function_def = make_function_def( - tmp_graph, tmp_graph.get_operations(), all_inputs, func_def_outputs) + fname = _inference_name(name) + operations = tuple(x for x in tmp_graph.get_operations() + if x not in all_ignored_ops) + inference_function_def, fn = make_function_def( + fname, tmp_graph, operations, all_inputs, func_def_outputs) # Register any other functions defined in the graph # TODO(ashankar): Oh lord, forgive me for this lint travesty. for f in tmp_graph._functions.values(): # pylint: disable=protected-access # TODO(ashankar): What about the gradient registry? - _register_with_name(f.name, f.definition) - _register_with_name(_inference_name(name), inference_function_def) + _register(f._c_func) # pylint: disable=protected-access + _register(fn) return GraphModeFunction( all_inputs, extra_inputs, inference_function_def, + fn, tmp_graph, - tmp_graph.get_operations(), + operations, func_outputs, _map_sequence_obj_to_idx(func_def_outputs), output_shapes, @@ -517,10 +570,9 @@ def _cache_key(x): return x -def _register_with_name(name, fdef): - """Registers the function `fdef` with the name `name`.""" - fdef.signature.name = name - context.context().add_function_def(fdef) +def _register(fn): + """Registers the function `fn`.""" + context.context().add_function(fn) # TODO(apassos): better error messages for non-hashable arguments. diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index faf0ac88bc..3da100d800 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -318,7 +318,9 @@ def _graph_callable_internal(func, shape_and_dtypes): placeholder_inputs = flat_inputs+ list(extra_placeholders) func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)] - initializer_function_def = function.make_function_def( + initialization_name = function._inference_name(func.__name__) # pylint: disable=protected-access + initializer_function_def, initializer_fn = function.make_function_def( + initialization_name, tmp_graph, initializing_operations, placeholder_inputs, @@ -327,13 +329,13 @@ def _graph_callable_internal(func, shape_and_dtypes): # Also, what about the gradient registry of these functions? Those need to be # addressed as well. for f in tmp_graph._functions.values(): # pylint: disable=protected-access - function._register_with_name(f.name, f.definition) # pylint: disable=protected-access - function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access - initializer_function_def) + function._register(f._c_func) # pylint: disable=protected-access + function._register(initializer_fn) # pylint: disable=protected-access initializer_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, initializer_function_def, + initializer_fn, tmp_graph, initializing_operations, func_outputs, @@ -342,18 +344,20 @@ def _graph_callable_internal(func, shape_and_dtypes): capture_func_def_outputs = [ x for x in captured_outlist if isinstance(x, tf_ops.Tensor)] - captured_function_def = function.make_function_def( + captured_function_name = function._inference_name(func.__name__) # pylint: disable=protected-access + captured_function_def, capturing_fn = function.make_function_def( + captured_function_name, tmp_graph, capturing_operations, placeholder_inputs, capture_func_def_outputs) - function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access - captured_function_def) + function._register(capturing_fn) # pylint: disable=protected-access captured_function = function.GraphModeFunction( placeholder_inputs, extra_inputs, captured_function_def, + capturing_fn, tmp_graph, capturing_operations, captured_outputs, diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py index 548e16a909..b9e6ca2a93 100644 --- a/tensorflow/python/eager/graph_callable_test.py +++ b/tensorflow/python/eager/graph_callable_test.py @@ -152,7 +152,6 @@ class GraphCallableTest(test.TestCase): self.assertAllEqual(5, f(constant_op.constant(2))) def testNestedFunction(self): - # TensorFlow function (which is what would be used in TensorFlow graph # construction). @function.Defun(dtypes.int32, dtypes.int32) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index a616b15cf7..5f945ac133 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -599,11 +599,6 @@ class Tensor(_TensorLike): """ return _eval_using_default_session(self, feed_dict, self.graph, session) - def _dup(self): - ret = copy.copy(self) - ret._id = uid() # pylint: disable=protected-access - return ret - # TODO(agarwal): consider getting rid of this. class _EagerTensorBase(Tensor): @@ -729,9 +724,6 @@ class _EagerTensorBase(Tensor): return new_tensor # pylint: enable=protected-access - def _dup(self): - return self._copy(device_name=self.device) - @property def shape(self): return tensor_shape.TensorShape(self._shape_tuple()) @@ -1794,7 +1786,7 @@ class Operation(object): c_api.SetRequestedDevice( self._graph._c_graph, # pylint: disable=protected-access self._c_op, # pylint: disable=protected-access - _device_string(device)) + compat.as_str(_device_string(device))) else: self._node_def.device = _device_string(device) @@ -2083,7 +2075,7 @@ class Operation(object): def _set_attr(self, attr_name, attr_value): """Private method used to set an attribute in the node_def.""" - if _USE_C_API: + if self._c_op: buf = c_api.TF_NewBufferFromString( compat.as_bytes(attr_value.SerializeToString())) try: @@ -2652,11 +2644,16 @@ class Graph(object): # TODO(skyewm): fold as much of the above as possible into the C # implementation - if _USE_C_API: + if _USE_C_API or self._use_c_api_hack(): self._scoped_c_graph = c_api_util.ScopedTFGraph() else: self._scoped_c_graph = None + # TODO(apassos) remove once the C API is used by default. + def _use_c_api_hack(self): + """Temporary hack; can be overridden to force C API usage.""" + return False + def _convert_stack(self, stack, include_func_start_lineno=False): """Converts a stack extracted using _extract_stack() to a traceback stack. @@ -2985,9 +2982,14 @@ class Graph(object): # Add function to graph # pylint: disable=protected-access if self._c_graph: - assert function._c_func, ( - "Cannot add function created without C API support to graph " - "created with C API support") + # Handle functions created without using the C API. TODO(apassos,skyewm) + # remove this when all functions are generated using the C API by default + # as this will be unnecessary. + if not function._c_func: + with errors.raise_exception_on_not_ok_status() as status: + serialized = function.definition.SerializeToString() + function._c_func = c_api.TF_FunctionImportFunctionDef( + serialized, status) with errors.raise_exception_on_not_ok_status() as status: gradient = function._grad_func._c_func if function._grad_func else None c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient, diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 82b154164e..82750e9e49 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -18,6 +18,7 @@ limitations under the License. %rename("%s") TFE_NewContext; %rename("%s") TFE_DeleteContext; %rename("%s") TFE_ContextListDevices; +%rename("%s") TFE_ContextAddFunction; %rename("%s") TFE_ContextAddFunctionDef; %rename("%s") TFE_OpNameGetAttrType; %rename("%s") TFE_Py_InitEagerTensor; @@ -149,7 +150,7 @@ limitations under the License. } $1 = &temp; $1->resize(PyInt_AsLong($input), nullptr); -} +} // Create new Status object. %typemap(in, numinputs=0) TF_Status *out_status { -- GitLab From b8c9b75bbb75def92b2cae2406ba88d20630b66a Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 30 Nov 2017 10:33:27 -0800 Subject: [PATCH 0148/1924] Change "Datasets" to "`tf.data`" in the "Reading Data" API guide. PiperOrigin-RevId: 177473833 --- .../api_guides/python/reading_data.md | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md index b3ebaa0f0a..4594887349 100644 --- a/tensorflow/docs_src/api_guides/python/reading_data.md +++ b/tensorflow/docs_src/api_guides/python/reading_data.md @@ -1,11 +1,11 @@ # Reading data Note: The preferred way to feed data into a tensorflow program is using the -@{$datasets$Datasets API}. +@{$datasets$`tf.data` API}. There are four methods of getting data into a TensorFlow program: -* `Dataset` API: Easily construct a complex input pipeline. (preferred method) +* `tf.data` API: Easily construct a complex input pipeline. (preferred method) * Feeding: Python code provides the data when running each step. * `QueueRunner`: a queue-based input pipeline reads the data from files at the beginning of a TensorFlow graph. @@ -14,26 +14,27 @@ There are four methods of getting data into a TensorFlow program: [TOC] -## Dataset API +## `tf.data` API See the @{$datasets$programmer's guide} for an in-depth explanation of -@{tf.data.Dataset}. The `Dataset` API allows you to extract and preprocess data -from different input/file formats, and apply transformations such as batch, -shuffle, and map to the dataset. This is an improved version of the old input -methods, feeding and `QueueRunner`. +@{tf.data.Dataset}. The `tf.data` API enables you to extract and preprocess data +from different input/file formats, and apply transformations such as batching, +shuffling, and mapping functions over the dataset. This is an improved version +of the old input methods---feeding and `QueueRunner`---which are described +below for historical purposes. ## Feeding +Warning: "Feeding" is the least efficient way to feed data into a TensorFlow +program and should only be used for small experiments and debugging. + TensorFlow's feed mechanism lets you inject data into any Tensor in a -computation graph. A python computation can thus feed data directly into the +computation graph. A Python computation can thus feed data directly into the graph. Supply feed data through the `feed_dict` argument to a run() or eval() call that initiates computation. -Warning: "Feeding" is the least efficient way to feed data into a tensorflow -program and should only be used for small experiments and debugging. - ```python with tf.Session(): input = tf.placeholder(tf.float32) @@ -55,6 +56,10 @@ and is described in the @{$mechanics$MNIST tutorial}. ## `QueueRunner` +Warning: This section discusses implementing input pipelines using the +queue-based APIs which can be cleanly replaced by the @{$datasets$`tf.data` +API}. + A typical queue-based pipeline for reading records from files has the following stages: 1. The list of filenames @@ -66,9 +71,6 @@ A typical queue-based pipeline for reading records from files has the following 7. *Optional* preprocessing 8. Example queue -Warning: This section discusses implementing input pipelines using the -queue-based APIs which can be cleanly replaced by the @{$datasets$Datasets API}. - ### Filenames, shuffling, and epoch limits For the list of filenames, use either a constant string Tensor (like @@ -499,7 +501,7 @@ You can have the train and eval in the same graph in the same process, and share their trained variables or layers. See @{$variables$the shared variables tutorial}. To support the single-graph approach -@{$programmers_guide/datasets$Datasets} also supplies +@{$programmers_guide/datasets$`tf.data`} also supplies @{$programmers_guide/datasets#creating_an_iterator$advanced iterator types} that that allow the user to change the input pipeline without rebuilding the graph or session. -- GitLab From 3b9a26d04544ba6e13181a7df07bb693769b7d7c Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Thu, 30 Nov 2017 10:40:46 -0800 Subject: [PATCH 0149/1924] Turned a verbose log into a vlog PiperOrigin-RevId: 177474943 --- tensorflow/core/grappler/grappler_item_builder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc index a186e9a181..da99777bbc 100644 --- a/tensorflow/core/grappler/grappler_item_builder.cc +++ b/tensorflow/core/grappler/grappler_item_builder.cc @@ -188,7 +188,7 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( << ", skipping this input"; return nullptr; } - LOG(INFO) << "Will use fetch node " << name; + VLOG(1) << "Will use fetch node " << name; new_item->fetch.push_back(name); } } -- GitLab From 0369392dfcb569a2b8c55fb7a5d3dc08b6cb6ef8 Mon Sep 17 00:00:00 2001 From: Russell Power Date: Thu, 30 Nov 2017 11:00:11 -0800 Subject: [PATCH 0150/1924] Internal testing change. PiperOrigin-RevId: 177478003 --- tensorflow/contrib/tpu/BUILD | 15 - .../contrib/tpu/python/tpu/test_util.py | 296 ------------------ 2 files changed, 311 deletions(-) delete mode 100644 tensorflow/contrib/tpu/python/tpu/test_util.py diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index f542d94139..a34c7f91f2 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -31,21 +31,6 @@ cc_library( ], ) -py_library( - name = "tpu_test_util", - srcs = ["python/tpu/test_util.py"], - srcs_version = "PY2AND3", - deps = [ - ":tpu_lib", - ":tpu_py", - "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", - "//tensorflow/python:framework_test_lib", - "//tensorflow/python:session", - "//tensorflow/python:variables", - ], -) - py_library( name = "tpu_estimator", srcs = [ diff --git a/tensorflow/contrib/tpu/python/tpu/test_util.py b/tensorflow/contrib/tpu/python/tpu/test_util.py deleted file mode 100644 index a5d4ff9722..0000000000 --- a/tensorflow/contrib/tpu/python/tpu/test_util.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# =================================================================== -"""Utilities to ease testing on TPU devices.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path -import pickle -import tempfile - -import numpy as np - -from tensorflow.contrib.tpu.python.tpu import tpu -from tensorflow.contrib.tpu.python.tpu import tpu_config -from tensorflow.contrib.tpu.python.tpu import tpu_estimator -from tensorflow.core.protobuf import config_pb2 -from tensorflow.python.client import session as tf_session -from tensorflow.python.estimator import model_fn as model_fn_lib -from tensorflow.python.framework import errors -from tensorflow.python.framework import ops -from tensorflow.python.framework import random_seed -from tensorflow.python.framework import test_util -from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import variables -from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.training import saver as tf_saver - - -def has_tpu(): - """Check if a TPU device is available. - - Device enumeration via `device_lib` currently fails for TPU systems. - (http://b/68333779). To work around this, we determine the existence of a - TPU by a successful call to `initialize_system`. - - Returns: - boolean, True if a TPU device is available, otherwise False. - """ - - def _check(): - with tf_session.Session() as sess: - sess.run(tpu.initialize_system()) - sess.run(tpu.shutdown_system()) - - try: - _check() - return True - except errors.OpError as _: - return False - - -def _available_devices(): - devices = ["cpu"] - if not test_util.gpu_device_name(): - devices.append("gpu") - - if has_tpu(): - devices.append("tpu") - - return tuple(devices) - - -def copy_dir(src, tgt): - """Copy src to tgt.""" - gfile.MakeDirs(tgt) - seen_dirs = set() - for dirname, _, files in gfile.Walk(src): - for f in files: - src_f = os.path.join(dirname, f) - tgt_f = src_f.replace(src, tgt) - tgt_d = os.path.dirname(tgt_f) - if tgt_d not in seen_dirs: - gfile.MkDir(tgt_d) - seen_dirs.add(tgt_d) - gfile.Copy(src_f, tgt_f, overwrite=True) - - -def compare_model(model_fn, - input_fn, - params, - master="local", - temp_dir=None, - num_shards=2, - tolerance=1e-4): - """Compare the results of running `model_fn` on the TPU and CPU.""" - if not temp_dir: - temp_dir = tempfile.mkdtemp() - - cpu_model_dir = "%s/cpu-model" % temp_dir - tpu_model_dir = "%s/tpu-model" % temp_dir - initial_model_dir = "%s/initial-model" % temp_dir - - logging.info("Checkpoints and weights will be written to %s", temp_dir) - - num_steps = 1 - - def _model_adapter(features, labels, mode, params): - """Run users model function with random seeds fixed to known values.""" - random_seed.set_random_seed(0) - np.random.seed(0) - return model_fn(features, labels, mode, params) - - def _input_adapter(params): - random_seed.set_random_seed(0) - np.random.seed(0) - return input_fn(params) - - def _make_run_config(model_dir): - return tpu_config.RunConfig( - master=master, - model_dir=model_dir, - save_checkpoints_secs=10000, - session_config=config_pb2.ConfigProto( - allow_soft_placement=True, log_device_placement=False), - tpu_config=tpu_config.TPUConfig( - iterations_per_loop=num_steps, - num_shards=num_shards, - ), - ) - - def _make_estimator(use_tpu, model_dir): - return tpu_estimator.TPUEstimator( - model_fn=_model_adapter, - use_tpu=use_tpu, - config=_make_run_config(model_dir), - train_batch_size=num_shards, - params=dict(params, use_tpu=use_tpu), - ) - - def _extract_weights(checkpoint): - """Extract model weights from the given checkpoint file.""" - weights = {} - graph = ops.Graph() - with graph.as_default(): - features, labels = _input_adapter(dict(params, batch_size=num_shards)) - model_fn( - features, labels, - params=dict(params, use_tpu=False), - mode=model_fn_lib.ModeKeys.TRAIN) - saver = tf_saver.Saver() - with tf_session.Session(graph=graph) as sess: - saver.restore(sess, checkpoint) - all_vars = [] - all_vars.extend(graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) - all_vars.extend(graph.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) - all_vars.extend(graph.get_collection(ops.GraphKeys.MODEL_VARIABLES)) - - for var in all_vars: - weights[var.name] = sess.run(var) - return weights - - def _run_step(use_tpu, model_dir): - """Create an estimator and run a single step on the given device.""" - tf_session.Session.reset(target=master) - - logging.info("Running step. TPU=%d. model_dir=%s", use_tpu, model_dir) - est = _make_estimator(use_tpu=use_tpu, model_dir=model_dir) - est.train(input_fn=_input_adapter, steps=num_steps) - weights = _extract_weights(est.latest_checkpoint()) - with gfile.Open(os.path.join(temp_dir, "tpu-%d.weights" % use_tpu), - "wb") as f: - f.write(pickle.dumps(weights)) - return weights - - # initialize models to the same weights by running a single step on the CPU - _run_step(use_tpu=False, model_dir=initial_model_dir) - - copy_dir(initial_model_dir, cpu_model_dir) - copy_dir(initial_model_dir, tpu_model_dir) - - cpu_weights = _run_step(use_tpu=False, model_dir=cpu_model_dir) - tpu_weights = _run_step(use_tpu=True, model_dir=tpu_model_dir) - - bad_weights = False - for k in cpu_weights: - if k not in tpu_weights: - raise KeyError("Missing weight %s from TPU checkpoint.", k) - - if not np.allclose( - cpu_weights[k], tpu_weights[k], rtol=tolerance, atol=tolerance): - bad_weights = True - logging.error("Weights for layer %s have diverged.", k) - - if bad_weights: - raise ValueError("Some weights have diverged. Output pickle files have " - "been written to %s for inspection." % temp_dir) - - -class TPUTestCase(test_util.TensorFlowTestCase): - """Adds helpers for testing on TPU devices to `TensorFlowTestCase`. - - Example usage: - - ``` - def model_fn(features): - return tf.reduce_sum(features * 2) - - class ModelTests(test_util.TPUTestCase): - def test_sum(self): - v = np.random.randn(10, 10).astype("float32") - self.assert_device_output(model_fn, [v], (v*2).sum(), - devices=("cpu", "tpu")) - ``` - """ - - def __init__(self, methodName="runTest"): # pylint: disable=invalid-name - super(TPUTestCase, self).__init__(methodName) - self._available_devices = _available_devices() - - def run_on_device(self, model_fn, model_inputs, device): - """Runs `model_fn` on the given device. - - Raises an exception if no such device is available. `model_fn` should - return one or more tensors as a list or tuple. - - Args: - model_fn: Function returning one or more tensors. - model_inputs: An iterable of Numpy arrays or scalars. - These will be passed as arguments to `model_fn`. - device: Device to run on. One of ("tpu", "gpu", "cpu"). - - Returns: - Output from the model function. - """ - - def _make_placeholders(): - return dict([(gen_array_ops.placeholder_with_default(v, v.shape), v) - for v in model_inputs]) - - if device == "tpu": - with self.test_session(graph=ops.Graph()) as sess: - placeholders = _make_placeholders() - tpu_computation = tpu.rewrite(model_fn, placeholders.keys()) - sess.run(tpu.initialize_system()) - sess.run(variables.global_variables_initializer()) - result = sess.run(tpu_computation, placeholders) - sess.run(tpu.shutdown_system()) - # TODO(b/36891278): supports non-flat returns lists in tpu.rewrite(). - if len(result) == 1: - return result[0] - return result - elif device == "gpu": - with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: - placeholders = _make_placeholders() - sess.run(variables.global_variables_initializer()) - return sess.run(model_fn(placeholders.keys()), placeholders) - elif device == "cpu": - # TODO(power) -- will this interact poorly with cached GPU sessions? - with self.test_session(graph=ops.Graph(), use_gpu=False) as sess: - placeholders = _make_placeholders() - sess.run(variables.global_variables_initializer()) - return sess.run(model_fn(placeholders.keys()), placeholders) - - def _compare_values(self, actual_outputs, expected_outputs): - if isinstance(expected_outputs, (list, tuple)): - for a, b in zip(actual_outputs, expected_outputs): - self.assertAllCloseAccordingToType(a, b) - else: - self.assertAllCloseAccordingToType(actual_outputs, expected_outputs) - - def assert_device_output(self, - model_fn, - model_inputs, - expected_outputs, - devices=("cpu", "gpu", "tpu")): - """Run `model_fn` on the given devices. - - Results are compared via `assertAllCloseAccordingToType`. - - Args: - model_fn: Function returning one or more tensors - model_inputs: Numpy arrays or scalars passed as arguments to model_fn - expected_outputs: Numpy arrays or scalars to compare against. - devices: Set of devices to run on. If a device is not available, tests - will be skipped for that device. - """ - devices = set(devices).intersection(self._available_devices) - - for device in devices: - device_out = self.run_on_device(model_fn, model_inputs, device=device) - self._compare_values(device_out, expected_outputs) -- GitLab From 4e8301be75a234d53b08bec577ac0069fc40bea3 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 30 Nov 2017 11:00:54 -0800 Subject: [PATCH 0151/1924] Disable dnn_linear_combined_test PiperOrigin-RevId: 177478106 --- tensorflow/contrib/estimator/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index e4d51aa148..706a174efb 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -93,6 +93,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_pip", + "notap", # b/62204861 "notsan", ], deps = [ -- GitLab From ea1c29552b01f3404e27999a27a1919b3accc594 Mon Sep 17 00:00:00 2001 From: Blake Hechtman Date: Thu, 30 Nov 2017 11:13:19 -0800 Subject: [PATCH 0152/1924] Change depthwise convolution filter expansion and contraction with algebraic manipulation instead of slices and pads that are more difficult to fuse. PiperOrigin-RevId: 177480353 --- .../compiler/tf2xla/kernels/conv_ops.cc | 166 ++++++++++++------ 1 file changed, 112 insertions(+), 54 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc index c150394c07..61f4d1993a 100644 --- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc @@ -46,72 +46,130 @@ TensorShape ExpandedFilterShapeForDepthwiseConvolution( return expanded_shape; } +// Broadcast zeros to ExpandedFilterShapeForDepthwiseConvolution. +xla::ComputationDataHandle CreateExpandedZero( + const TensorShape& filter_shape, DataType dtype, + xla::ComputationBuilder* builder) { + TensorShape expanded_filter_shape = + ExpandedFilterShapeForDepthwiseConvolution(filter_shape); + return builder->Broadcast(XlaHelpers::Zero(builder, dtype), + expanded_filter_shape.dim_sizes()); +} + +// Create a mask for depthwise convolution that will make a normal convolution +// produce the same results as a depthwise convolution. For a [2, 2, 3, 2] +// depthwise filter this returns a [2, 2, 3, 6] tesnsor +// 1 1 0 0 0 0 1 1 0 0 0 0 +// 0 0 1 1 0 0 0 0 1 1 0 0 +// 0 0 0 0 1 1 0 0 0 0 1 1 +// +// 1 1 0 0 0 0 1 1 0 0 0 0 +// 0 0 1 1 0 0 0 0 1 1 0 0 +// 0 0 0 0 1 1 0 0 0 0 1 1 +// +// The first step is to create a one tensor, A, that is [3] +// 0 1 2 +// +// and another tensor, B, that is [3 * 2] +// 0 1 2 3 4 5 +// +// and divide B it by 2 to get +// 0 0 1 1 2 2 +// +// then we broadcast the B to [2, 2, 3, 3 * 2] +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// 0 0 1 1 2 2 0 0 1 1 2 2 +// +// Finally compare A and broadcasted B in dimension 2 amd return the result at +// the beginning of the comment. +xla::ComputationDataHandle CreateExpandedFilterMask( + const TensorShape& filter_shape, xla::ComputationBuilder* builder) { + TensorShape expanded_filter_shape = + ExpandedFilterShapeForDepthwiseConvolution(filter_shape); + int64 depthwise_multiplier = filter_shape.dim_size(filter_shape.dims() - 1); + int64 input_feature = filter_shape.dim_size(filter_shape.dims() - 2); + + // Create a M sized linspace and an M*N sized linspace that will be + // broadcasted into perpendicular dimensions and compared. + xla::ComputationDataHandle input_feature_iota; + // DT_INT32 Iota will always return status::OK(). + TF_CHECK_OK(XlaHelpers::Iota(builder, DataType::DT_INT32, input_feature, + &input_feature_iota)); + xla::ComputationDataHandle expanded_feature_iota; + TF_CHECK_OK(XlaHelpers::Iota(builder, DataType::DT_INT32, + input_feature * depthwise_multiplier, + &expanded_feature_iota)); + + // Divide the M*N sized linspace by the depthwise_multiplier to create + // [0 0 1 1 2 2] in the example in the function comment. + expanded_feature_iota = + builder->Div(expanded_feature_iota, + XlaHelpers::IntegerLiteral(builder, DataType::DT_INT32, + depthwise_multiplier)); + + // Broadcast the N*M linspace to [H, W, ..., M, M*N]. + auto expanded_feature_broadcast_dims = expanded_filter_shape.dim_sizes(); + expanded_feature_broadcast_dims.pop_back(); + auto broadcasted_expanded_feature_iota = builder->Broadcast( + expanded_feature_iota, expanded_feature_broadcast_dims); + + // Compare the broadcasted linspace to the input feature linspace in the + // input feature dimension to create a diagonal predicate. + return builder->Eq(broadcasted_expanded_feature_iota, input_feature_iota, + {expanded_filter_shape.dims() - 2}); +} + // Expands a filter of shape [H, W, ..., M, N] to [H, W, ..., M, M*N] by adding // zeros for the cross-depth filters. Used to build a depthwise convolution. xla::ComputationDataHandle ExpandFilterForDepthwiseConvolution( const TensorShape& filter_shape, DataType dtype, const xla::ComputationDataHandle& filter, xla::ComputationBuilder* builder) { - // Filter has shape [H, W, ..., M, N] - // Dilate to [H, W, ..., M*M, N] using M inter-element padding, and then - // reshape to [H, W, ..., M, M*N]. - int num_spatial_dims = filter_shape.dims() - 2; - const int64 in_depth = filter_shape.dim_size(num_spatial_dims); - xla::PaddingConfig padding = xla::MakeNoPaddingConfig(filter_shape.dims()); - padding.mutable_dimensions(num_spatial_dims)->set_interior_padding(in_depth); - auto dilated_filter = - builder->Pad(filter, XlaHelpers::Zero(builder, dtype), padding); - + int64 depthwise_multiplier = filter_shape.dim_size(filter_shape.dims() - 1); + int64 input_feature = filter_shape.dim_size(filter_shape.dims() - 2); TensorShape expanded_filter_shape = ExpandedFilterShapeForDepthwiseConvolution(filter_shape); - return builder->Reshape(dilated_filter, expanded_filter_shape.dim_sizes()); + + // Create a [H, W, ..., 1, N*M] reshape of the filter. + TensorShape implicit_broadcast_filter_shape = expanded_filter_shape; + implicit_broadcast_filter_shape.set_dim( + implicit_broadcast_filter_shape.dims() - 2, 1); + implicit_broadcast_filter_shape.set_dim( + implicit_broadcast_filter_shape.dims() - 1, + depthwise_multiplier * input_feature); + auto implicit_broadcast_filter = + builder->Reshape(filter, implicit_broadcast_filter_shape.dim_sizes()); + + // Broadcast the filter to [H, W, ..., M, M*N]. + auto expanded_zero = CreateExpandedZero(filter_shape, dtype, builder); + auto expanded_filter = builder->Add(implicit_broadcast_filter, expanded_zero); + + // If the filter mask is set, choose the broadcasted filter, othwerwise, + // choose zero. + return builder->Select(CreateExpandedFilterMask(filter_shape, builder), + expanded_filter, expanded_zero); } // Inverse of ExpandFilterForDepthwiseConvolution. xla::ComputationDataHandle ContractFilterForDepthwiseBackprop( - const TensorShape& filter_shape, DataType dtype, + XlaOpKernelContext* ctx, const TensorShape& filter_shape, DataType dtype, const xla::ComputationDataHandle& filter_backprop, xla::ComputationBuilder* builder) { - int num_spatial_dims = filter_shape.dims() - 2; - - // Reshape to [H, W, ..., M*M, N] - TensorShape shape = filter_shape; - int64 in_depth = filter_shape.dim_size(num_spatial_dims); - shape.set_dim(num_spatial_dims, in_depth * in_depth); - auto reshaped = builder->Reshape(filter_backprop, shape.dim_sizes()); - - std::vector zeros(filter_shape.dims()); - std::vector strides(filter_shape.dims(), 1LL); - strides[num_spatial_dims] = in_depth + 1; - return builder->Slice(reshaped, zeros, shape.dim_sizes(), strides); - - // Alternate implementation for backends without strided Slice() support. - // TODO(phawkins): Remove when all backends support strided slice. - // // Pad [..., M * (M + 1), N] - // xla::PaddingConfig config = - // xla::MakeNoPaddingConfig(filter_shape.dims()); - // config.mutable_dimensions(num_spatial_dims) - // ->set_edge_padding_high(in_depth); - // auto zero = XlaHelpers::Zero(builder, dtype); - // auto padded = builder->Pad(reshaped, zero, config); - // - // // Reshape to [..., M, M + 1, N] - // shape = filter_shape; - // shape.set_dim(num_spatial_dims, in_depth); - // shape.set_dim(num_spatial_dims + 1, in_depth + 1); - // int64 out_depth = filter_shape.dim_size(num_spatial_dims + 1); - // shape.AddDim(out_depth); - // reshaped = builder->Reshape(padded, shape.dim_sizes()); - // - // // Slice to [..., M, 1, N] - // std::vector zeros(shape.dims()); - // std::vector strides(shape.dims(), 1LL); - // shape.set_dim(num_spatial_dims + 1, 1); - // auto sliced = builder->Slice(reshaped, zeros, shape.dim_sizes(), - // strides); - // - // // Reshape to [..., M, N] - // return builder->Reshape(sliced, filter_shape.dim_sizes()); + TensorShape expanded_filter_shape = + ExpandedFilterShapeForDepthwiseConvolution(filter_shape); + auto masked_expanded_filter = builder->Select( + CreateExpandedFilterMask(filter_shape, builder), filter_backprop, + CreateExpandedZero(filter_shape, dtype, builder)); + return builder->Reshape( + builder->Reduce(masked_expanded_filter, XlaHelpers::Zero(builder, dtype), + *ctx->GetOrCreateAdd(dtype), + {expanded_filter_shape.dims() - 2}), + filter_shape.dim_sizes()); } class ConvOp : public XlaOpKernel { @@ -202,7 +260,7 @@ class ConvOp : public XlaOpKernel { dims.set_input_feature_dimension(feature_dim); dims.set_output_feature_dimension(feature_dim); for (int i = 0; i < num_spatial_dims_; ++i) { - int64 dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); + const int64 dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); dims.add_input_spatial_dimensions(dim); dims.add_kernel_spatial_dimensions(i); dims.add_output_spatial_dimensions(dim); @@ -574,7 +632,7 @@ class ConvBackpropFilterOp : public XlaOpKernel { if (depthwise_) { filter_backprop_reshaped = ContractFilterForDepthwiseBackprop( - filter_shape, ctx->input_type(0), filter_backprop_reshaped, b); + ctx, filter_shape, ctx->input_type(0), filter_backprop_reshaped, b); } ctx->SetOutput(0, filter_backprop_reshaped); } -- GitLab From 4146ff1259c0b4ada8afbbad11a7b37d8373d1b9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 11:18:54 -0800 Subject: [PATCH 0153/1924] [XLA] Adds Dot with DotDimensionNumbers proto for specifying arbitrary contracting and batch dimensions. PiperOrigin-RevId: 177481231 --- .../xla/client/computation_builder.cc | 36 +++- .../compiler/xla/client/computation_builder.h | 5 + .../xla/service/algebraic_simplifier.cc | 17 +- .../xla/service/algebraic_simplifier_test.cc | 6 +- .../xla/service/buffer_assignment_test.cc | 11 +- .../cpu/cpu_instruction_fusion_test.cc | 36 ++-- .../compiler/xla/service/cpu/ir_emitter.cc | 5 + .../xla/service/gpu/ir_emitter_unnested.cc | 5 + .../compiler/xla/service/graphviz_example.cc | 5 +- .../xla/service/heap_simulator_test.cc | 24 ++- tensorflow/compiler/xla/service/hlo.proto | 3 + .../compiler/xla/service/hlo_cost_analysis.cc | 5 +- .../xla/service/hlo_evaluator_test.cc | 21 ++- .../compiler/xla/service/hlo_instruction.cc | 56 +++++- .../compiler/xla/service/hlo_instruction.h | 18 ++ .../xla/service/hlo_instruction_test.cc | 12 +- .../compiler/xla/service/hlo_verifier.cc | 6 +- .../xla/service/liveness_util_test.cc | 10 +- tensorflow/compiler/xla/service/service.cc | 3 + .../compiler/xla/service/shape_inference.cc | 156 ++++++++++++--- .../compiler/xla/service/shape_inference.h | 6 +- .../xla/service/shape_inference_test.cc | 177 ++++++++++++++++-- .../xla/service/transpose_folding_test.cc | 27 ++- .../compiler/xla/service/user_computation.cc | 57 +++++- .../compiler/xla/service/user_computation.h | 4 + .../xla/service/user_computation_test.cc | 45 ----- .../compiler/xla/tests/dot_operation_test.cc | 20 ++ .../xla/tests/multioutput_fusion_test.cc | 12 +- tensorflow/compiler/xla/xla_data.proto | 23 ++- .../performance/xla/operation_semantics.md | 81 ++++++++ 30 files changed, 736 insertions(+), 156 deletions(-) diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc index cce9310003..9febea8dcf 100644 --- a/tensorflow/compiler/xla/client/computation_builder.cc +++ b/tensorflow/compiler/xla/client/computation_builder.cc @@ -625,7 +625,41 @@ ComputationDataHandle ComputationBuilder::Lt( ComputationDataHandle ComputationBuilder::Dot( const ComputationDataHandle& lhs, const ComputationDataHandle& rhs) { - return BinaryOp(BINOP_DOT, lhs, rhs, /*broadcast_dimensions=*/{}); + StatusOr> lhs_shape_or_status = GetShape(lhs); + if (!lhs_shape_or_status.ok()) { + NoteError(lhs_shape_or_status.status()); + return ComputationDataHandle(); + } + std::unique_ptr lhs_shape = lhs_shape_or_status.ConsumeValueOrDie(); + + DotDimensionNumbers dimension_numbers; + dimension_numbers.add_lhs_contracting_dimensions( + lhs_shape->dimensions_size() == 1 ? 0 : 1); + dimension_numbers.add_rhs_contracting_dimensions(0); + return DotGeneral(lhs, rhs, dimension_numbers); +} + +ComputationDataHandle ComputationBuilder::DotGeneral( + const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, + const DotDimensionNumbers& dimension_numbers) { + if (!first_error_.ok() || !PrepareComputation().ok()) { + return ComputationDataHandle(); + } + + DotRequest request; + *request.mutable_lhs() = lhs; + *request.mutable_rhs() = rhs; + *request.mutable_dimension_numbers() = dimension_numbers; + + OpRequest op_request; + *op_request.mutable_computation() = computation_.handle(); + *op_request.mutable_dot_request() = request; + AddCommonFieldsToOpRequest(&op_request); + OpResponse response; + + VLOG(2) << "making Dot request"; + Status s = client_->stub()->Op(&op_request, &response); + return ParseOpResponse(s, &response); } ComputationDataHandle ComputationBuilder::Conv( diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index d2dbbbbebb..531b98cfb9 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -393,6 +393,11 @@ class ComputationBuilder { ComputationDataHandle Dot(const ComputationDataHandle& lhs, const ComputationDataHandle& rhs); + // Enqueues a general dot instruction onto the computation. + ComputationDataHandle DotGeneral( + const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, + const DotDimensionNumbers& dimension_numbers); + // Default dimension numbers used for a 2D convolution. static constexpr int64 kConvBatchDimension = 0; static constexpr int64 kConvFeatureDimension = 1; diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 71491218aa..b1d0345e70 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -597,9 +597,13 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) { // Simplify dot(transpose(a), transpose(b)) to transpose(dot(b,a)). if (lhs->IsRank2Transpose() && rhs->IsRank2Transpose()) { - auto new_dot = computation_->AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::PermuteDimensions({1, 0}, dot->shape()), HloOpcode::kDot, - rhs->mutable_operand(0), lhs->mutable_operand(0))); + DotDimensionNumbers dot_dimension_numbers; + dot_dimension_numbers.add_lhs_contracting_dimensions(1); + dot_dimension_numbers.add_rhs_contracting_dimensions(0); + auto new_dot = computation_->AddInstruction(HloInstruction::CreateDot( + ShapeUtil::PermuteDimensions({1, 0}, dot->shape()), + rhs->mutable_operand(0), lhs->mutable_operand(0), + dot_dimension_numbers)); return ReplaceWithNewInstruction( dot, HloInstruction::CreateTranspose(dot->shape(), new_dot, {1, 0})); } @@ -1616,8 +1620,11 @@ Status AlgebraicSimplifierVisitor::HandleConvolution( auto new_lhs = add_bitcast(new_input_shape, lhs); auto new_rhs = add_bitcast(new_filter_shape, rhs); - auto dot = computation_->AddInstruction(HloInstruction::CreateBinary( - dot_output_shape, HloOpcode::kDot, new_lhs, new_rhs)); + DotDimensionNumbers dot_dimension_numbers; + dot_dimension_numbers.add_lhs_contracting_dimensions(1); + dot_dimension_numbers.add_rhs_contracting_dimensions(0); + auto dot = computation_->AddInstruction(HloInstruction::CreateDot( + dot_output_shape, new_lhs, new_rhs, dot_dimension_numbers)); return ReplaceInstruction(convolution, add_bitcast(convolution_shape, dot)); } diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 56dfb1cf0b..3d70505f6e 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -2138,8 +2138,10 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) { builder.AddInstruction(HloInstruction::CreateParameter(0, r1f32, "x")); HloInstruction* y = builder.AddInstruction(HloInstruction::CreateParameter(1, r1f32, "y")); - builder.AddInstruction( - HloInstruction::CreateBinary(r1f32, HloOpcode::kDot, x, y)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + builder.AddInstruction(HloInstruction::CreateDot(r1f32, x, y, dot_dnums)); std::unique_ptr dot_computation(builder.Build()); HloComputation::Builder call_builder(TestName() + ".Call"); diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index 4d4c5b953e..75c71dfeb1 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -1360,10 +1360,13 @@ TEST_F(BufferAssignmentTest, OneTempAllocation) { HloInstruction::CreateParameter(1, shape_3x4, "param_b")); auto param_c = builder.AddInstruction( HloInstruction::CreateParameter(2, shape_4x4, "param_c")); - auto dot_ab = builder.AddInstruction(HloInstruction::CreateBinary( - shape_2x4, HloOpcode::kDot, param_a, param_b)); - auto dot_bc = builder.AddInstruction(HloInstruction::CreateBinary( - shape_3x4, HloOpcode::kDot, param_b, param_c)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + auto dot_ab = builder.AddInstruction( + HloInstruction::CreateDot(shape_2x4, param_a, param_b, dot_dnums)); + auto dot_bc = builder.AddInstruction( + HloInstruction::CreateDot(shape_3x4, param_b, param_c, dot_dnums)); builder.AddInstruction( HloInstruction::CreateConcatenate(shape_5x4, {dot_ab, dot_bc}, 1)); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc index b9e4d006d7..1c04c9835e 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion_test.cc @@ -31,6 +31,14 @@ namespace { using InstructionFusionTest = HloTestBase; +std::unique_ptr MakeDot(const Shape& shape, HloInstruction* lhs, + HloInstruction* rhs) { + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + return HloInstruction::CreateDot(shape, lhs, rhs, dot_dnums); +} + TEST_F(InstructionFusionTest, DotOperationFusion_Basic_0) { HloComputation::Builder builder(TestName()); HloInstruction* arg0 = builder.AddInstruction(HloInstruction::CreateParameter( @@ -40,8 +48,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_Basic_0) { HloInstruction* exp0 = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(S32, {1024, 256}), HloOpcode::kExp, arg0)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1024, 1}), HloOpcode::kDot, exp0, arg1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1024, 1}), exp0, arg1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -59,8 +67,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_Basic_1) { HloInstruction* exp1 = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(S32, {256, 1024}), HloOpcode::kExp, arg1)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1, 1024}), HloOpcode::kDot, arg0, exp1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1, 1024}), arg0, exp1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -80,8 +88,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_Bitcast) { ShapeUtil::MakeShape(S32, {2, 512, 2, 128}), HloOpcode::kExp, arg0)); HloInstruction* bitcast0 = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(S32, {1024, 256}), HloOpcode::kBitcast, exp0)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1024, 1}), HloOpcode::kDot, bitcast0, arg1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1024, 1}), bitcast0, arg1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -102,8 +110,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_Reshape) { HloInstruction* reshape0 = builder.AddInstruction(HloInstruction::CreateReshape( ShapeUtil::MakeShape(S32, {1024, 256}), exp0)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1024, 1}), HloOpcode::kDot, reshape0, arg1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1024, 1}), reshape0, arg1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -121,8 +129,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_TooLarge) { HloInstruction* exp1 = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(S32, {256, 32 * 1024}), HloOpcode::kExp, arg1)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1, 32 * 1024}), HloOpcode::kDot, arg0, exp1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1, 32 * 1024}), arg0, exp1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -140,8 +148,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_ElementReuse) { HloInstruction* exp1 = builder.AddInstruction(HloInstruction::CreateUnary( ShapeUtil::MakeShape(S32, {256, 1024}), HloOpcode::kExp, arg1)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {2, 1024}), HloOpcode::kDot, arg0, exp1)); + HloInstruction* dot = builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {2, 1024}), arg0, exp1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); @@ -162,8 +170,8 @@ TEST_F(InstructionFusionTest, DotOperationFusion_TransposeFusion) { HloInstruction* transpose1 = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(S32, {256, 1024}), exp1, {1, 0})); - builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1, 1024}), HloOpcode::kDot, arg0, transpose1)); + builder.AddInstruction( + MakeDot(ShapeUtil::MakeShape(F32, {1, 1024}), arg0, transpose1)); auto module = CreateNewModule(); auto computation = module->AddEntryComputation(builder.Build()); diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 3f991c03e9..f242e0acb8 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -796,6 +796,11 @@ Status IrEmitter::HandleDot(HloInstruction* dot) { TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*dot, /*operands=*/{lhs, rhs}, /*supported_types=*/{F32, F64, C64})); + const DotDimensionNumbers& dnums = dot->dot_dimension_numbers(); + if (dnums.lhs_batch_dimensions_size() > 0 || + dnums.rhs_batch_dimensions_size() > 0) { + return Unimplemented("Dot with batch dimensions not implemented."); + } llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs)); llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs)); diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 1b863c9e3c..abc739d181 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -246,6 +246,11 @@ Status IrEmitterUnnested::DefaultAction(HloInstruction* hlo) { } Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { + const DotDimensionNumbers& dnums = dot->dot_dimension_numbers(); + if (dnums.lhs_batch_dimensions_size() > 0 || + dnums.rhs_batch_dimensions_size() > 0) { + return Unimplemented("Dot with batch dimensions not implemented."); + } if (ImplementedAsGemm(*dot)) { thunk_sequence_->emplace_back(BuildGemmThunk(dot)); return Status::OK(); diff --git a/tensorflow/compiler/xla/service/graphviz_example.cc b/tensorflow/compiler/xla/service/graphviz_example.cc index 049e8d80d8..05017008e2 100644 --- a/tensorflow/compiler/xla/service/graphviz_example.cc +++ b/tensorflow/compiler/xla/service/graphviz_example.cc @@ -108,8 +108,11 @@ std::unique_ptr MakeBigGraph() { HloInstruction::CreateUnary(vshape, HloOpcode::kCopy, param_v0)); auto clamp = builder.AddInstruction(HloInstruction::CreateTernary( vshape, HloOpcode::kClamp, copy, param_v1, param_v2)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot = builder.AddInstruction( - HloInstruction::CreateBinary(vshape, HloOpcode::kDot, clamp, param_v0)); + HloInstruction::CreateDot(vshape, clamp, param_v0, dot_dnums)); auto tuple = builder.AddInstruction( HloInstruction::CreateTuple({dot, param_s, clamp})); auto scalar = builder.AddInstruction( diff --git a/tensorflow/compiler/xla/service/heap_simulator_test.cc b/tensorflow/compiler/xla/service/heap_simulator_test.cc index 17b926c874..387b649a73 100644 --- a/tensorflow/compiler/xla/service/heap_simulator_test.cc +++ b/tensorflow/compiler/xla/service/heap_simulator_test.cc @@ -259,8 +259,11 @@ TEST_F(HeapSimulatorTest, MultiplyDot) { HloInstruction::CreateParameter(2, f32scalar_, "paramY")); auto mul = builder.AddInstruction(HloInstruction::CreateBinary( f32vec4_, HloOpcode::kMultiply, paramA, paramX)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, mul, paramY)); + HloInstruction::CreateDot(f32vec4_, mul, paramY, dot_dnums)); // The buffer for dot is the output, and it cannot be shared with the buffer // for mul, since dot isn't elementwise. @@ -292,8 +295,11 @@ TEST_F(HeapSimulatorTest, MultiplyDotAdd) { HloInstruction::CreateParameter(2, f32scalar_, "paramY")); auto mul = builder.AddInstruction(HloInstruction::CreateBinary( f32vec4_, HloOpcode::kMultiply, paramA, paramX)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, mul, paramY)); + HloInstruction::CreateDot(f32vec4_, mul, paramY, dot_dnums)); auto add = builder.AddInstruction( HloInstruction::CreateBinary(f32vec4_, HloOpcode::kAdd, dot, paramA)); @@ -327,10 +333,13 @@ TEST_F(HeapSimulatorTest, MultiplyDotDot) { HloInstruction::CreateParameter(2, f32scalar_, "paramY")); auto mul = builder.AddInstruction(HloInstruction::CreateBinary( f32vec4_, HloOpcode::kMultiply, paramA, paramX)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot0 = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, mul, paramY)); + HloInstruction::CreateDot(f32vec4_, mul, paramY, dot_dnums)); auto dot1 = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, dot0, paramY)); + HloInstruction::CreateDot(f32vec4_, dot0, paramY, dot_dnums)); // The buffer for dot1 is the output. No buffers can be shared. The buffer // for mul is freed before the end, since it's no longer used after dot0 @@ -365,10 +374,13 @@ TEST_F(HeapSimulatorTest, MultiplyDotDotTuple) { HloInstruction::CreateParameter(2, f32scalar_, "paramY")); auto mul = builder.AddInstruction(HloInstruction::CreateBinary( f32vec4_, HloOpcode::kMultiply, paramA, paramX)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot0 = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, mul, paramY)); + HloInstruction::CreateDot(f32vec4_, mul, paramY, dot_dnums)); auto dot1 = builder.AddInstruction( - HloInstruction::CreateBinary(f32vec4_, HloOpcode::kDot, dot0, paramY)); + HloInstruction::CreateDot(f32vec4_, dot0, paramY, dot_dnums)); auto tuple = builder.AddInstruction(HloInstruction::CreateTuple({dot0, dot1})); diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index e984bdb5f7..5d0cfba1fc 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -118,6 +118,9 @@ message HloInstructionProto { // Shape of outfeed request. xla.Shape outfeed_shape = 29; + + // Describes the dimension numbers used for a dot operation + xla.DotDimensionNumbers dot_dimension_numbers = 30; } // Serialization of HloComputation. diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index 6fcc01dd64..0ed64e6779 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -201,10 +201,11 @@ Status HloCostAnalysis::HandleCopy(const HloInstruction*) { Status HloCostAnalysis::HandleDot(const HloInstruction* dot) { const Shape& lhs_shape = dot->operand(0)->shape(); const Shape& rhs_shape = dot->operand(1)->shape(); + const DotDimensionNumbers& dnums = dot->dot_dimension_numbers(); // Count of elements along the reduction dimension (last dimension for the // rhs). - int64 reduction_width = lhs_shape.dimensions(ShapeUtil::Rank(lhs_shape) - 1); - + int64 reduction_width = + lhs_shape.dimensions(dnums.lhs_contracting_dimensions(0)); // First divide by reduction width before multiplying by rhs elements to avoid // overflow. int64 fma_count; diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc index b2c4351896..a5d39fe086 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc @@ -621,8 +621,11 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank1) { b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_literal))); Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); - b.AddInstruction(HloInstruction::CreateBinary( - shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + b.AddInstruction(HloInstruction::CreateDot(shape, lhs_instruction, + rhs_instruction, dot_dnums)); auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = @@ -664,8 +667,11 @@ TEST_F(HloEvaluatorTest, DotRank1AndRank2) { b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_literal))); Shape shape = ShapeUtil::MakeShape(F32, {2}); - b.AddInstruction(HloInstruction::CreateBinary( - shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(0); + dot_dnums.add_rhs_contracting_dimensions(0); + b.AddInstruction(HloInstruction::CreateDot(shape, lhs_instruction, + rhs_instruction, dot_dnums)); auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = @@ -705,8 +711,11 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank2) { b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_literal))); Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); - b.AddInstruction(HloInstruction::CreateBinary( - shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + b.AddInstruction(HloInstruction::CreateDot(shape, lhs_instruction, + rhs_instruction, dot_dnums)); auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index c30c432654..b4bac18bcd 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -118,6 +118,10 @@ StatusOr> HloInstruction::CreateFromProto( MakeUnique( proto.convolution_dimension_numbers()); } + if (proto.has_dot_dimension_numbers()) { + instruction->dot_dimension_numbers_ = + MakeUnique(proto.dot_dimension_numbers()); + } for (const HloInstructionProto::SliceDimensions& slice_dimensions : proto.slice_dimensions()) { instruction->slice_starts_.push_back(slice_dimensions.start()); @@ -332,6 +336,17 @@ HloInstruction::CreateGetTupleElement(const Shape& shape, return instruction; } +/* static */ std::unique_ptr HloInstruction::CreateDot( + const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, + const DotDimensionNumbers& dimension_numbers) { + auto instruction = WrapUnique(new HloInstruction(HloOpcode::kDot, shape)); + instruction->AppendOperand(lhs); + instruction->AppendOperand(rhs); + instruction->dot_dimension_numbers_ = + MakeUnique(dimension_numbers); + return instruction; +} + /* static */ std::unique_ptr HloInstruction::CreateReducePrecision(const Shape& shape, HloInstruction* operand, @@ -1086,7 +1101,6 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kLe: case HloOpcode::kLt: case HloOpcode::kNe: - case HloOpcode::kDot: case HloOpcode::kMaximum: case HloOpcode::kMinimum: case HloOpcode::kPower: @@ -1138,6 +1152,11 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( clone = CreateConvolve(shape, new_operands[0], new_operands[1], *window_, *convolution_dimension_numbers_); break; + case HloOpcode::kDot: + CHECK_EQ(new_operands.size(), 2); + clone = CreateDot(shape, new_operands[0], new_operands[1], + *dot_dimension_numbers_); + break; case HloOpcode::kCrossReplicaSum: CHECK_EQ(new_operands.size(), 1); clone = CreateCrossReplicaSum(shape, new_operands[0]); @@ -1509,7 +1528,6 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kCos: case HloOpcode::kCrossReplicaSum: case HloOpcode::kDivide: - case HloOpcode::kDot: case HloOpcode::kEq: case HloOpcode::kExp: case HloOpcode::kFloor: @@ -1582,6 +1600,10 @@ bool HloInstruction::IdenticalSlowPath( protobuf_util::ProtobufEquals( convolution_dimension_numbers(), other.convolution_dimension_numbers()); + // Check dot dimension numbers. + case HloOpcode::kDot: + return protobuf_util::ProtobufEquals(dot_dimension_numbers(), + other.dot_dimension_numbers()); // Reduction results are determined by the reduction dimension and the // reduction computation. @@ -1990,6 +2012,9 @@ std::vector HloInstruction::ExtraAttributesToString() const { if (convolution_dimension_numbers_ != nullptr) { extra.push_back(ConvolutionDimensionNumbersToString()); } + if (dot_dimension_numbers_ != nullptr) { + extra.push_back(DotDimensionNumbersToString()); + } if (opcode() == HloOpcode::kWhile) { extra.push_back(StrCat("condition=%", while_condition()->name())); @@ -2086,6 +2111,9 @@ HloInstructionProto HloInstruction::ToProto() const { *proto.mutable_convolution_dimension_numbers() = *convolution_dimension_numbers_; } + if (dot_dimension_numbers_ != nullptr) { + *proto.mutable_dot_dimension_numbers() = *dot_dimension_numbers_; + } for (int i = 0; i < slice_starts_.size(); ++i) { auto* slice_dimension = proto.add_slice_dimensions(); slice_dimension->set_start(slice_starts_[i]); @@ -3051,6 +3079,30 @@ string HloInstruction::ConvolutionDimensionNumbersToString() const { return result; } +string HloInstruction::DotDimensionNumbersToString() const { + string result; + if (dot_dimension_numbers_ == nullptr) { + return result; + } + const DotDimensionNumbers& dnums = *dot_dimension_numbers_; + if (!dnums.lhs_batch_dimensions().empty()) { + result += "lhs_batch_dims="; + StrAppend(&result, Join(dnums.lhs_batch_dimensions(), ",")); + } + result += "lhs_contracting_dims="; + StrAppend(&result, Join(dnums.lhs_contracting_dimensions(), ",")); + + result += ","; + if (!dnums.rhs_batch_dimensions().empty()) { + result += "rhs_batch_dims="; + StrAppend(&result, Join(dnums.rhs_batch_dimensions(), ",")); + } + result += "rhs_contracting_dims="; + StrAppend(&result, Join(dnums.rhs_contracting_dimensions(), ",")); + + return result; +} + bool HloInstruction::CouldBeBitcast() const { switch (opcode_) { case HloOpcode::kTranspose: diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index cda8b07c61..768c027a42 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -160,6 +160,12 @@ class HloInstruction { const Window& window, const ConvolutionDimensionNumbers& dimension_numbers); + // Creates a dot op with operands 'lhs' and 'rhs' with contracting and batch + // dimensions specified in 'dimension_numbers'. + static std::unique_ptr CreateDot( + const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, + const DotDimensionNumbers& dimension_numbers); + // Creates a reduce-precision op, where operand is the data to reduce in // precision, and exponent_bits and mantissa_bits describe the precision to // reduce it to. @@ -915,6 +921,15 @@ class HloInstruction { // Returns the dump string of the convolution dimension numbers. string ConvolutionDimensionNumbersToString() const; + // Returns data on the dimension numbers used for a dot operation. + const DotDimensionNumbers& dot_dimension_numbers() const { + CHECK(dot_dimension_numbers_ != nullptr); + return *dot_dimension_numbers_; + } + + // Returns the dump string of the dot dimension numbers. + string DotDimensionNumbersToString() const; + // Returns the random distribution for this rng node. // // Precondition: opcode() == HloOpcode::kRng @@ -1173,6 +1188,9 @@ class HloInstruction { // Describes the dimension numbers used for a convolution. std::unique_ptr convolution_dimension_numbers_; + // Describes the dimension numbers used for a dot. + std::unique_ptr dot_dimension_numbers_; + // Describes the [begin, end) index range for a slice. std::vector slice_starts_; std::vector slice_limits_; diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index 76b12fc8d3..11420cae63 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -1068,8 +1068,11 @@ TEST_F(HloInstructionTest, CloneOfFusionPreservesShape) { builder.AddInstruction(HloInstruction::CreateParameter(1, s2, "y")); HloInstruction* reshape = builder.AddInstruction(HloInstruction::CreateTranspose(s2t, y, {1, 0})); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); HloInstruction* dot = builder.AddInstruction( - HloInstruction::CreateBinary(sout, HloOpcode::kDot, x, reshape)); + HloInstruction::CreateDot(sout, x, reshape, dot_dnums)); HloModule module(TestName()); auto* computation = module.AddEntryComputation(builder.Build()); @@ -1182,12 +1185,15 @@ TEST_F(HloInstructionTest, Stringification) { builder.AddInstruction(HloInstruction::CreateParameter(1, s2, "y")); HloInstruction* reshape = builder.AddInstruction(HloInstruction::CreateTranspose(s2t, y, {1, 0})); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); HloInstruction* dot = builder.AddInstruction( - HloInstruction::CreateBinary(sout, HloOpcode::kDot, x, reshape)); + HloInstruction::CreateDot(sout, x, reshape, dot_dnums)); EXPECT_EQ(dot->ToString(false, false), "%dot = f32[5,20]{1,0} dot(f32[5,10]{1,0} %x, f32[10,20]{1,0} " - "%transpose)"); + "%transpose), lhs_contracting_dims=1,rhs_contracting_dims=0"); HloModule module(TestName()); auto* computation = module.AddEntryComputation(builder.Build()); diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 2c09d2defb..ea7775b18a 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -75,7 +75,11 @@ class ShapeVerifier : public DfsHloVisitor { } Status HandleDot(HloInstruction* dot) override { - return CheckBinaryShape(dot); + TF_ASSIGN_OR_RETURN(const Shape expected, + ShapeInference::InferDotOpShape( + dot->operand(0)->shape(), dot->operand(1)->shape(), + dot->dot_dimension_numbers())); + return CheckShape(dot, expected); } Status HandleConvolution(HloInstruction* convolution) override { diff --git a/tensorflow/compiler/xla/service/liveness_util_test.cc b/tensorflow/compiler/xla/service/liveness_util_test.cc index 476e86fa72..2c2a02f637 100644 --- a/tensorflow/compiler/xla/service/liveness_util_test.cc +++ b/tensorflow/compiler/xla/service/liveness_util_test.cc @@ -277,8 +277,11 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedDotAdd) { auto b = builder.AddInstruction(HloInstruction::CreateConstant( Literal::CreateR2({{2.0, 2.0}, {2.0, 2.0}}))); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot = builder.AddInstruction( - HloInstruction::CreateBinary(data_shape, HloOpcode::kDot, a, b)); + HloInstruction::CreateDot(data_shape, a, b, dot_dnums)); auto one = builder.AddInstruction( HloInstruction::CreateConstant(Literal::CreateR0(1.0))); @@ -312,8 +315,11 @@ TEST_F(CanShareOperandBufferWithUserTest, FusedTransposeDotAdd) { auto b_t = builder.AddInstruction( HloInstruction::CreateTranspose(data_shape, b, {1, 0})); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto dot = builder.AddInstruction( - HloInstruction::CreateBinary(data_shape, HloOpcode::kDot, a, b_t)); + HloInstruction::CreateDot(data_shape, a, b_t, dot_dnums)); auto one = builder.AddInstruction( HloInstruction::CreateConstant(Literal::CreateR0(1.0))); diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index d997cab83f..fa62080be4 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -1381,6 +1381,9 @@ tensorflow::Status Service::Op(const OpRequest* arg, OpResponse* result) { handle_status = computation->AddCustomCallInstruction(arg->custom_call_request()); break; + case OpRequest::kDotRequest: + handle_status = computation->AddDotInstruction(arg->dot_request()); + break; case OpRequest::kDynamicSliceRequest: handle_status = computation->AddDynamicSliceInstruction(arg->dynamic_slice_request()); diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index 3df1911d07..7178eb40dd 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/gtl/flatset.h" #include "tensorflow/core/lib/math/math_util.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -90,8 +91,6 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) { return BINOP_ATAN2; case HloOpcode::kComplex: return BINOP_COMPLEX; - case HloOpcode::kDot: - return BINOP_DOT; case HloOpcode::kMultiply: return BINOP_MUL; case HloOpcode::kAdd: @@ -549,8 +548,98 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, return ShapeUtil::MakeShape(operand_shape.element_type(), dimensions); } -/* static */ StatusOr ShapeInference::InferDotOpShape(const Shape& lhs, - const Shape& rhs) { +// Current DotDimensionNumbers Requirements: +// +// Contracting Dimensions: +// *) Exactly one contracting dimension on both lhs and rhs. +// *) Contracting dimension size must be the same on both lhs and rhs. +// *) Contracting dimension numbers do not need to be the same (i.e. transposes +// are passed on to emitter implementations). +// +// Batch Dimensions: +// *) Same number of batch dimensions on both lhs and rhs. +// *) Same batch dimension numbers (and sizes) on both lhs and rhs. +// +// Non-Contracting-Non-Batch Dimensions: +// *) Can be 0 (matrix-vector) or 1 (matrix-matrix). +// + +namespace { + +Status ValidateDotDimensionNumbers( + const Shape& lhs, const Shape& rhs, + const DotDimensionNumbers& dimension_numbers) { + // Check that dimension numbers are in range. + auto dims_in_range = + [](const int64 rank, tensorflow::gtl::ArraySlice contracting_dims, + tensorflow::gtl::ArraySlice batch_dims) -> bool { + auto in_range = [&rank](int64 i) -> bool { return 0 <= i && i < rank; }; + return std::all_of(contracting_dims.begin(), contracting_dims.end(), + in_range) && + std::all_of(batch_dims.begin(), batch_dims.end(), in_range); + }; + + tensorflow::gtl::ArraySlice lhs_contracting_dimensions = + AsInt64Slice(dimension_numbers.lhs_contracting_dimensions()); + tensorflow::gtl::ArraySlice rhs_contracting_dimensions = + AsInt64Slice(dimension_numbers.rhs_contracting_dimensions()); + tensorflow::gtl::ArraySlice lhs_batch_dimensions = + AsInt64Slice(dimension_numbers.lhs_batch_dimensions()); + tensorflow::gtl::ArraySlice rhs_batch_dimensions = + AsInt64Slice(dimension_numbers.rhs_batch_dimensions()); + + if (!dims_in_range(ShapeUtil::Rank(lhs), lhs_contracting_dimensions, + lhs_batch_dimensions) || + !dims_in_range(ShapeUtil::Rank(rhs), rhs_contracting_dimensions, + rhs_batch_dimensions)) { + return InvalidArgument("A dimension number is out of range in dot: %s", + dimension_numbers.DebugString().c_str()); + } + + // Check that dimension numbers are unique. + auto dims_unique = [](tensorflow::gtl::ArraySlice contracting_dims, + tensorflow::gtl::ArraySlice batch_dims) -> bool { + tensorflow::gtl::FlatSet dim_set; + auto is_unique = [&dim_set](int64 i) -> bool { + return dim_set.insert(i).second; + }; + return std::all_of(contracting_dims.begin(), contracting_dims.end(), + is_unique) && + std::all_of(batch_dims.begin(), batch_dims.end(), is_unique); + }; + + if (!dims_unique(lhs_contracting_dimensions, lhs_batch_dimensions) || + !dims_unique(rhs_contracting_dimensions, rhs_batch_dimensions)) { + return InvalidArgument("A dimension number is not unique in dot: %s", + dimension_numbers.DebugString().c_str()); + } + + // Check that the count of non-contracting-non-batch dimensions is in {0, 1}. + const int64 lhs_non_contracting_non_batch_dims = + ShapeUtil::Rank(lhs) - + dimension_numbers.lhs_contracting_dimensions_size() - + dimension_numbers.lhs_batch_dimensions_size(); + const int64 rhs_non_contracting_non_batch_dims = + ShapeUtil::Rank(rhs) - + dimension_numbers.rhs_contracting_dimensions_size() - + dimension_numbers.rhs_batch_dimensions_size(); + if (lhs_non_contracting_non_batch_dims < 0 || + lhs_non_contracting_non_batch_dims > 1 || + rhs_non_contracting_non_batch_dims < 0 || + rhs_non_contracting_non_batch_dims > 1) { + return InvalidArgument( + "batch and contracting dimension number mismatch " + "with rank "); + } + + return Status::OK(); +} + +} // namespace + +/* static */ StatusOr ShapeInference::InferDotOpShape( + const Shape& lhs, const Shape& rhs, + const DotDimensionNumbers& dimension_numbers) { TF_RETURN_IF_ERROR(ExpectNotTupleOrOpaque(lhs, "lhs of dot")); TF_RETURN_IF_ERROR(ExpectNotTupleOrOpaque(rhs, "rhs of dot")); @@ -570,37 +659,62 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, return fail("element types do not match"); } - if (ShapeUtil::Rank(lhs) < 1 || ShapeUtil::Rank(lhs) > 2 || - ShapeUtil::Rank(rhs) < 1 || ShapeUtil::Rank(rhs) > 2) { - return fail("dot only supports rank 1 or 2"); + if ((ShapeUtil::Rank(lhs) < 1) || (ShapeUtil::Rank(rhs) < 1)) { + return fail("dot only supports rank 1 or above."); } - // Determine the index of the contracted dimensions for input tensors. - // dimensions -1 of lhs and dimension 0 of rhs are contracted. - int64 lhs_contracted_dimension = ShapeUtil::GetDimensionNumber(lhs, -1); - int64 rhs_contracted_dimension = 0; + // Validate basic properties of dot dimension numbers. + TF_RETURN_IF_ERROR(ValidateDotDimensionNumbers(lhs, rhs, dimension_numbers)); + + // Check that there is only one contracting dimension for both lhs and rhs. + if (dimension_numbers.lhs_contracting_dimensions_size() != + dimension_numbers.rhs_contracting_dimensions_size() || + dimension_numbers.lhs_contracting_dimensions_size() != 1) { + return fail("must specify one contracting dimension for both lhs and rhs."); + } - // Check if the contracted dimension sizes are the same. - if ((lhs_contracted_dimension < ShapeUtil::Rank(lhs) && - rhs_contracted_dimension < ShapeUtil::Rank(rhs)) && - lhs.dimensions(lhs_contracted_dimension) != - rhs.dimensions(rhs_contracted_dimension)) { - return fail("contracted dimensions mismatch"); + // Check that contracting dimension sizes match. + const int64 lhs_contracting_dimension = + dimension_numbers.lhs_contracting_dimensions(0); + const int64 rhs_contracting_dimension = + dimension_numbers.rhs_contracting_dimensions(0); + if (lhs.dimensions(lhs_contracting_dimension) != + rhs.dimensions(rhs_contracting_dimension)) { + return fail("contracting dimension sizes do not match."); + } + + // Check that number of batch dimensions match. + if (dimension_numbers.lhs_batch_dimensions_size() != + dimension_numbers.rhs_batch_dimensions_size()) { + return fail("must the same number of batch dimensions for lhs and rhs."); + } + + // Check that batch dimension numbers and sizes match. + for (int64 i = 0; i < dimension_numbers.lhs_batch_dimensions_size(); ++i) { + if (dimension_numbers.lhs_batch_dimensions(i) != + dimension_numbers.rhs_batch_dimensions(i) || + lhs.dimensions(dimension_numbers.lhs_batch_dimensions(i)) != + rhs.dimensions(dimension_numbers.rhs_batch_dimensions(i))) { + return fail("batch dimension numbers and sizes must match for lhs/rhs."); + } } // The ranks of lhs and rhs are decremented by 1 respectively due to the // contraction, and added for the rank of the result. When an input tensor is // a scalar, its contribution to the rank of the result is 0. // Generate the result dimensions in order, rhs dimensions followed by lhs - // dimensions except the contracted dimensions. + // dimensions except the contracted and batch dimensions. std::vector dimensions; + std::unordered_set rhs_batch_dims( + dimension_numbers.rhs_batch_dimensions().begin(), + dimension_numbers.rhs_batch_dimensions().end()); for (int64 i = 0; i < ShapeUtil::Rank(lhs); i++) { - if (i != lhs_contracted_dimension) { + if (i != lhs_contracting_dimension) { dimensions.push_back(lhs.dimensions(i)); } } for (int64 i = 0; i < ShapeUtil::Rank(rhs); i++) { - if (i != rhs_contracted_dimension) { + if (i != rhs_contracting_dimension && rhs_batch_dims.count(i) == 0) { dimensions.push_back(rhs.dimensions(i)); } } @@ -816,8 +930,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( rhs, tensorflow::strings::StrCat("rhs of binary operation ", BinaryOperation_Name(operation)))); switch (operation) { - case BINOP_DOT: - return InferDotOpShape(lhs, rhs); case BINOP_MAX: case BINOP_MIN: case BINOP_SUB: diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h index 0aadb98a40..382c4f8abc 100644 --- a/tensorflow/compiler/xla/service/shape_inference.h +++ b/tensorflow/compiler/xla/service/shape_inference.h @@ -229,11 +229,13 @@ class ShapeInference { tensorflow::gtl::ArraySlice arg_shapes, const ProgramShape& to_apply); - private: // Helper that infers the shape produced by performing a dot operation with // the given LHS and RHS shapes. - static StatusOr InferDotOpShape(const Shape& lhs, const Shape& rhs); + static StatusOr InferDotOpShape( + const Shape& lhs, const Shape& rhs, + const DotDimensionNumbers& dimension_numbers); + private: // Helper that infers the shape produced by performing an element-wise binary // operation with the given LHS and RHS shapes. // Note: By "element-wise" we mean operations that look at a single element in diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc index be93c879c0..6e53d2d609 100644 --- a/tensorflow/compiler/xla/service/shape_inference_test.cc +++ b/tensorflow/compiler/xla/service/shape_inference_test.cc @@ -898,8 +898,11 @@ TEST_F(ShapeInferenceTest, BroadcastScalar) { // scalar vector: error TEST_F(ShapeInferenceTest, ScalarDotVector) { + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); auto inferred_status = - ShapeInference::InferBinaryOpShape(BINOP_DOT, f32_, vector_32_, {}); + ShapeInference::InferDotOpShape(f32_, vector_32_, dot_dnums); ASSERT_FALSE(inferred_status.ok()); ASSERT_THAT(inferred_status.status().error_message(), HasSubstr("dot only supports rank")); @@ -907,61 +910,199 @@ TEST_F(ShapeInferenceTest, ScalarDotVector) { // 3D 2D: error TEST_F(ShapeInferenceTest, DotWithRankHigherThanTwo) { - auto inferred_status = ShapeInference::InferBinaryOpShape( - BINOP_DOT, ShapeUtil::MakeShape(F32, {32, 32, 32}), matrix_32_64_, {}); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + auto inferred_status = ShapeInference::InferDotOpShape( + ShapeUtil::MakeShape(F32, {32, 32, 32}), matrix_32_64_, dot_dnums); ASSERT_FALSE(inferred_status.ok()); ASSERT_THAT(inferred_status.status().error_message(), - HasSubstr("dot only supports rank")); + HasSubstr("batch and contracting dimension number mismatch")); } // vector vector -> scalar TEST_F(ShapeInferenceTest, VectorDotVector) { + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(0); + dot_dnums.add_rhs_contracting_dimensions(0); auto inferred_status = - ShapeInference::InferBinaryOpShape(BINOP_DOT, vector_64_, vector_64_, {}); + ShapeInference::InferDotOpShape(vector_64_, vector_64_, dot_dnums); ASSERT_IS_OK(inferred_status.status()); ASSERT_TRUE(ShapeUtil::Equal(f32_, inferred_status.ValueOrDie())); auto inferred_status_mismatch = - ShapeInference::InferBinaryOpShape(BINOP_DOT, vector_64_, vector_32_, {}); + ShapeInference::InferDotOpShape(vector_64_, vector_32_, dot_dnums); ASSERT_FALSE(inferred_status_mismatch.ok()); } // matrix vector -> vector TEST_F(ShapeInferenceTest, MatrixDotVector) { - auto inferred_status = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, matrix_32_64_, vector_64_, {}); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + auto inferred_status = + ShapeInference::InferDotOpShape(matrix_32_64_, vector_64_, dot_dnums); ASSERT_IS_OK(inferred_status.status()); ASSERT_TRUE(ShapeUtil::Equal(inferred_status.ValueOrDie(), vector_32_)); - auto inferred_status_mismatch = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, matrix_32_64_, vector_32_, {}); + auto inferred_status_mismatch = + ShapeInference::InferDotOpShape(matrix_32_64_, vector_32_, dot_dnums); ASSERT_FALSE(inferred_status_mismatch.ok()); } // vector matrix -> vector TEST_F(ShapeInferenceTest, VectorDotMatrix) { - auto inferred_status = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, vector_32_, matrix_32_64_, {}); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(0); + dot_dnums.add_rhs_contracting_dimensions(0); + auto inferred_status = + ShapeInference::InferDotOpShape(vector_32_, matrix_32_64_, dot_dnums); ASSERT_IS_OK(inferred_status.status()); ASSERT_TRUE(ShapeUtil::Equal(inferred_status.ValueOrDie(), vector_64_)); - auto inferred_status_mismatch = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, vector_64_, matrix_32_64_, {}); + auto inferred_status_mismatch = + ShapeInference::InferDotOpShape(vector_64_, matrix_32_64_, dot_dnums); ASSERT_FALSE(inferred_status_mismatch.ok()); } // matrix matrix -> matrix TEST_F(ShapeInferenceTest, MatrixDotMatrix) { - auto inferred_status_match = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, matrix_32_64_, matrix_64_48_, {}); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + auto inferred_status_match = + ShapeInference::InferDotOpShape(matrix_32_64_, matrix_64_48_, dot_dnums); ASSERT_IS_OK(inferred_status_match.status()); ASSERT_TRUE( ShapeUtil::Equal(inferred_status_match.ValueOrDie(), matrix_32_48_)) << "inferred: " << ShapeUtil::HumanString(inferred_status_match.ValueOrDie()) << " expected: " << ShapeUtil::HumanString(matrix_64_48_); - auto inferred_status_mismatch = ShapeInference::InferBinaryOpShape( - BinaryOperation::BINOP_DOT, matrix_32_64_, matrix_32_64_, {}); + auto inferred_status_mismatch = + ShapeInference::InferDotOpShape(matrix_32_64_, matrix_32_64_, dot_dnums); ASSERT_FALSE(inferred_status_mismatch.ok()); } +// BatchMatMul with two batch dimensions and one contracting dimension. +TEST_F(ShapeInferenceTest, DotGeneral) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {5, 2, 11, 3}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {5, 2, 3, 14}); + Shape output_shape = ShapeUtil::MakeShape(F32, {5, 2, 11, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(3); + dot_dnums.add_lhs_batch_dimensions(0); + dot_dnums.add_lhs_batch_dimensions(1); + + dot_dnums.add_rhs_contracting_dimensions(2); + dot_dnums.add_rhs_batch_dimensions(0); + dot_dnums.add_rhs_batch_dimensions(1); + + auto inferred_status_match = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_IS_OK(inferred_status_match.status()); + ASSERT_TRUE( + ShapeUtil::Equal(inferred_status_match.ValueOrDie(), output_shape)) + << "inferred: " + << ShapeUtil::HumanString(inferred_status_match.ValueOrDie()) + << " expected: " << ShapeUtil::HumanString(output_shape); +} + +// BatchMatMul with two contracting dimensions fails. +TEST_F(ShapeInferenceTest, DotWithTwoContractingDimsFails) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {2, 11, 3, 2}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {2, 3, 14}); + Shape output_shape = ShapeUtil::MakeShape(F32, {2, 11, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(2); + dot_dnums.add_lhs_contracting_dimensions(3); + dot_dnums.add_lhs_batch_dimensions(0); + + dot_dnums.add_rhs_contracting_dimensions(1); + dot_dnums.add_rhs_batch_dimensions(0); + + auto inferred_status = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_FALSE(inferred_status.ok()); + ASSERT_THAT(inferred_status.status().error_message(), + HasSubstr("must specify one contracting dimension for both " + "lhs and rhs")); +} + +// BatchMatMul with different batch dimension sizes fails. +TEST_F(ShapeInferenceTest, DotWithMisatchedBatchDimSizesFails) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {2, 11, 3}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {3, 3, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(2); + dot_dnums.add_lhs_batch_dimensions(0); + + dot_dnums.add_rhs_contracting_dimensions(1); + dot_dnums.add_rhs_batch_dimensions(0); + + auto inferred_status = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_FALSE(inferred_status.ok()); + ASSERT_THAT(inferred_status.status().error_message(), + HasSubstr("batch dimension numbers and sizes must match")); +} + +// BatchMatMul with different batch dimension numbers fails. +TEST_F(ShapeInferenceTest, DotWithMisatchedBatchDimNumbersFails) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {2, 11, 3}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {3, 2, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(2); + dot_dnums.add_lhs_batch_dimensions(0); + + dot_dnums.add_rhs_contracting_dimensions(0); + dot_dnums.add_rhs_batch_dimensions(1); + + auto inferred_status = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_FALSE(inferred_status.ok()); + ASSERT_THAT(inferred_status.status().error_message(), + HasSubstr("batch dimension numbers and sizes must match")); +} + +// BatchMatMul with out-of-range dimension numbers fails. +TEST_F(ShapeInferenceTest, DotWithContractingDimNumberOutOfRange) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {2, 11, 3}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {2, 3, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(3); + dot_dnums.add_lhs_batch_dimensions(0); + + dot_dnums.add_rhs_contracting_dimensions(0); + dot_dnums.add_rhs_batch_dimensions(1); + + auto inferred_status = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_FALSE(inferred_status.ok()); + ASSERT_THAT(inferred_status.status().error_message(), + HasSubstr("A dimension number is out of range")); +} + +// BatchMatMul with non-unique dimension numbers fails. +TEST_F(ShapeInferenceTest, DotWithContractingNonUniqueDimNumber) { + Shape lhs_shape = ShapeUtil::MakeShape(F32, {2, 11, 3}); + Shape rhs_shape = ShapeUtil::MakeShape(F32, {2, 3, 14}); + + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(0); + dot_dnums.add_lhs_batch_dimensions(0); + + dot_dnums.add_rhs_contracting_dimensions(0); + dot_dnums.add_rhs_batch_dimensions(1); + + auto inferred_status = + ShapeInference::InferDotOpShape(lhs_shape, rhs_shape, dot_dnums); + ASSERT_FALSE(inferred_status.ok()); + ASSERT_THAT(inferred_status.status().error_message(), + HasSubstr("A dimension number is not unique")); +} + TEST_F(ShapeInferenceTest, BinOpBroadcastMatrixVector) { // Test variations of broadcasting a vector for a binary add with a // matrix. diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc index ba99852905..caa1a111ad 100644 --- a/tensorflow/compiler/xla/service/transpose_folding_test.cc +++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc @@ -64,9 +64,12 @@ TEST_F(TransposeFoldingTest, FoldDotTranspose) { HloInstruction* transpose_y = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(F32, {3, 2}), y, {1, 0})); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {2, 2}), /*opcode=*/HloOpcode::kDot, - /*lhs=*/x, /*rhs=*/transpose_y)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + HloInstruction* dot = builder.AddInstruction( + HloInstruction::CreateDot(ShapeUtil::MakeShape(F32, {2, 2}), /*lhs=*/x, + /*rhs=*/transpose_y, dot_dnums)); HloModule module("test_module"); HloComputation* entry_computation = @@ -104,9 +107,12 @@ TEST_F(TransposeFoldingTest, FoldDotTransposeConstant) { HloInstruction* transpose1 = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(F32, {2, 3}), const1, {1, 0})); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1, 3}), /*opcode=*/HloOpcode::kDot, - /*lhs=*/transpose0, /*rhs=*/transpose1)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateDot( + ShapeUtil::MakeShape(F32, {1, 3}), + /*lhs=*/transpose0, /*rhs=*/transpose1, dot_dnums)); HloModule module("test_module"); HloComputation* entry_computation = @@ -169,9 +175,12 @@ TEST_F(TransposeFoldingTest, FoldDotTransposeInWhile) { HloInstruction* transpose_y = builder.AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(F32, {3, 2}), y, {1, 0})); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {2, 2}), /*opcode=*/HloOpcode::kDot, - /*lhs=*/x, /*rhs=*/transpose_y)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); + HloInstruction* dot = builder.AddInstruction( + HloInstruction::CreateDot(ShapeUtil::MakeShape(F32, {2, 2}), /*lhs=*/x, + /*rhs=*/transpose_y, dot_dnums)); HloModule module("test_module"); HloComputation* entry_computation = diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index 4e90491b55..6d0d367981 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -88,8 +88,6 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) { return HloOpcode::kAtan2; case BINOP_COMPLEX: return HloOpcode::kComplex; - case BINOP_DOT: - return HloOpcode::kDot; case BINOP_MUL: return HloOpcode::kMultiply; case BINOP_ADD: @@ -1207,6 +1205,33 @@ StatusOr UserComputation::AddCustomCallInstruction( return handle; } +StatusOr UserComputation::AddDotInstruction( + const DotRequest& dot_request) { + tensorflow::mutex_lock lock(mutex_); + + TF_ASSIGN_OR_RETURN(const OperationRequest* lhs, + LookUpRequest(dot_request.lhs())); + TF_ASSIGN_OR_RETURN(const OperationRequest* rhs, + LookUpRequest(dot_request.rhs())); + + TF_ASSIGN_OR_RETURN(Shape shape, ShapeInference::InferDotOpShape( + lhs->output_shape(), rhs->output_shape(), + dot_request.dimension_numbers())); + + const ComputationDataHandle handle = CreateComputationDataHandle(); + + OperationRequest& request = + (*session_computation_.mutable_requests())[handle.handle()]; + *request.mutable_output_handle() = handle; + *request.mutable_output_shape() = shape; + *request.mutable_request()->mutable_dot_request() = dot_request; + + VLOG(1) << "AddDotInstruction (" << GetVersionedHandleInternal() + << "), data handle " << handle.handle() << ": " + << dot_request.ShortDebugString(); + return handle; +} + StatusOr UserComputation::AddUnaryInstruction( const UnaryOpRequest& unary_request) { tensorflow::mutex_lock lock(mutex_); @@ -1629,6 +1654,15 @@ void PureFunctionalVisitor(const SessionComputation& session_computation, break; } + case OpRequest::kDotRequest: { + const DotRequest& dot_request = request.request().dot_request(); + PureFunctionalVisitor(session_computation, dot_request.lhs(), + num_parameters, visited, is_functional); + PureFunctionalVisitor(session_computation, dot_request.rhs(), + num_parameters, visited, is_functional); + break; + } + case OpRequest::kSendRequest: { *is_functional = false; break; @@ -2453,6 +2487,13 @@ static void ForEachOperand( break; } + case OpRequest::kDotRequest: { + const DotRequest& dot_request = request.request().dot_request(); + apply(dot_request.rhs()); + apply(dot_request.lhs()); + break; + } + case OpRequest::kUnaryOpRequest: { const UnaryOpRequest& unary_op_request = request.request().unary_op_request(); @@ -2732,6 +2773,15 @@ void ComputationLowerer::Visit( break; } + case OpRequest::kDotRequest: { + const DotRequest& dot_request = request.request().dot_request(); + HloInstruction* lhs = lookup_instruction(dot_request.lhs()); + HloInstruction* rhs = lookup_instruction(dot_request.rhs()); + hlo_instruction = add_instruction(HloInstruction::CreateDot( + request.output_shape(), lhs, rhs, dot_request.dimension_numbers())); + break; + } + case OpRequest::kCrossReplicaSumRequest: { const CrossReplicaSumRequest& cross_replica_sum_request = request.request().cross_replica_sum_request(); @@ -3151,8 +3201,7 @@ void ComputationLowerer::Visit( lhs = (lhs == operand_to_broadcast) ? broadcasted_operand : lhs; rhs = (rhs == operand_to_broadcast) ? broadcasted_operand : rhs; } - if (debug_options_.xla_eliminate_hlo_implicit_broadcast() && - binary_op_request.binop() != BINOP_DOT) { + if (debug_options_.xla_eliminate_hlo_implicit_broadcast()) { if (!ShapeUtil::SameDimensions(request.output_shape(), lhs->shape())) { // lhs side is being implicitly broadcast. Change to explicit. lhs = diff --git a/tensorflow/compiler/xla/service/user_computation.h b/tensorflow/compiler/xla/service/user_computation.h index 317c631dca..b6686c3f1a 100644 --- a/tensorflow/compiler/xla/service/user_computation.h +++ b/tensorflow/compiler/xla/service/user_computation.h @@ -153,6 +153,10 @@ class UserComputation { StatusOr AddCustomCallInstruction( const CustomCallRequest& custom_call_request); + // Enqueues a dot instruction onto this user computation. + StatusOr AddDotInstruction( + const DotRequest& dot_request); + // Enqueues a broadcast instruction onto this user computation. StatusOr AddBroadcastInstruction( const BroadcastRequest& broadcast_request); diff --git a/tensorflow/compiler/xla/service/user_computation_test.cc b/tensorflow/compiler/xla/service/user_computation_test.cc index 5afaf226ae..e45673300b 100644 --- a/tensorflow/compiler/xla/service/user_computation_test.cc +++ b/tensorflow/compiler/xla/service/user_computation_test.cc @@ -334,50 +334,5 @@ TEST_F(UserComputationTest, EliminateDegenerateBroadcastAfterIndimBroadcast) { operands[1]->opcode() == HloOpcode::kBroadcast); } -TEST_F(UserComputationTest, SkipDotInEliminatingImplicitBroadcast) { - auto debug_options = DebugOptions(); - debug_options.set_xla_eliminate_hlo_implicit_broadcast(true); - - // %a = Param({1, 3}); - // %b = Param({3, 1}); - // %dot = Dot(%a, %b); - ComputationHandle handle; - handle.set_handle(123); - UserComputation computation("TheComputation", handle); - - ParameterRequest a_request; - *a_request.mutable_shape() = ShapeUtil::MakeShape(F32, {1, 3}); - a_request.set_name("a"); - a_request.set_parameter(0); - TF_ASSERT_OK_AND_ASSIGN(ComputationDataHandle a_handle, - computation.AddParameterInstruction(a_request)); - - ParameterRequest b_request; - *b_request.mutable_shape() = ShapeUtil::MakeShape(F32, {3, 1}); - b_request.set_name("b"); - b_request.set_parameter(1); - TF_ASSERT_OK_AND_ASSIGN(ComputationDataHandle b_handle, - computation.AddParameterInstruction(b_request)); - - BinaryOpRequest dot; - dot.set_binop(BINOP_DOT); - *dot.mutable_lhs() = a_handle; - *dot.mutable_rhs() = b_handle; - TF_ASSERT_OK(computation.AddBinaryInstruction(dot).status()); - - auto hlo_resolver = [](const VersionedComputationHandle& handle) { - return nullptr; - }; - VersionedComputationHandle latest_version = computation.GetVersionedHandle(); - - // Build the HLO computation. - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr hlo_computation, - computation.BuildHloComputation(latest_version.version, hlo_resolver, - debug_options)); - - EXPECT_EQ(3, hlo_computation->instruction_count()); -} - } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc index bfb04fd9f9..680d790b57 100644 --- a/tensorflow/compiler/xla/tests/dot_operation_test.cc +++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc @@ -561,5 +561,25 @@ TEST_F(DotOperationTest, TransposeFolding) { } } +XLA_TEST_F(DotOperationTest, DotGeneralUnimplemented) { + ComputationBuilder builder(client_, TestName()); + auto lhs = builder.ConstantR3FromArray3D( + {{{1.0, 2.0}, {3.0, 4.0}}, {{5.0, 6.0}, {7.0, 8.0}}}); + auto rhs = builder.ConstantR3FromArray3D( + {{{1.0, 0.0}, {0.0, 1.0}}, {{0.0, 1.0}, {1.0, 0.0}}}); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(2); + dot_dnums.add_rhs_contracting_dimensions(1); + dot_dnums.add_lhs_batch_dimensions(0); + dot_dnums.add_rhs_batch_dimensions(0); + builder.DotGeneral(lhs, rhs, dot_dnums); + + auto status = Execute(&builder, {}).status(); + EXPECT_FALSE(status.ok()); + EXPECT_THAT( + status.error_message(), + ::testing::HasSubstr("Dot with batch dimensions not implemented.")); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc b/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc index 22d2b917a1..89fa6ed9f7 100644 --- a/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc +++ b/tensorflow/compiler/xla/tests/multioutput_fusion_test.cc @@ -76,8 +76,11 @@ class MultiOutputFusionTest : public HloTestBase { elem_shape2, HloOpcode::kAdd, broadcast, param1)); HloInstruction* sub = builder.AddInstruction(HloInstruction::CreateBinary( elem_shape2, HloOpcode::kSubtract, param1, broadcast)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(1); + dot_dnums.add_rhs_contracting_dimensions(0); HloInstruction* dot = builder.AddInstruction( - HloInstruction::CreateBinary(elem_shape2, HloOpcode::kDot, sub, add2)); + HloInstruction::CreateDot(elem_shape2, sub, add2, dot_dnums)); auto computation = hlo_module->AddEntryComputation(builder.Build(dot)); if (manual_fusion) { @@ -133,8 +136,11 @@ class MultiOutputFusionTest : public HloTestBase { HloInstruction* reshape = builder.AddInstruction(HloInstruction::CreateReshape( ShapeUtil::MakeShape(F32, {size, 1}), add)); - HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateBinary( - ShapeUtil::MakeShape(F32, {1}), HloOpcode::kDot, sub, reshape)); + DotDimensionNumbers dot_dnums; + dot_dnums.add_lhs_contracting_dimensions(0); + dot_dnums.add_rhs_contracting_dimensions(0); + HloInstruction* dot = builder.AddInstruction(HloInstruction::CreateDot( + ShapeUtil::MakeShape(F32, {1}), sub, reshape, dot_dnums)); auto computation = hlo_module->AddEntryComputation(builder.Build(dot)); if (manual_fusion) { diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index b560354050..7efdf8552e 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -498,6 +498,23 @@ message CustomCallRequest { Shape shape = 4; } +message DotDimensionNumbers { + // The dimension numbers that represent the 'lhs' contracting dimensions. + repeated int64 lhs_contracting_dimensions = 1; + // The dimension numbers that represent the 'rhs' contracting dimensions. + repeated int64 rhs_contracting_dimensions = 2; + // The dimension numbers that represent the 'lhs' batch dimensions. + repeated int64 lhs_batch_dimensions = 3; + // The dimension numbers that represent the 'rhs' batch dimensions. + repeated int64 rhs_batch_dimensions = 4; +}; + +message DotRequest { + ComputationDataHandle lhs = 2; + ComputationDataHandle rhs = 3; + DotDimensionNumbers dimension_numbers = 4; +} + message MapRequest { repeated ComputationDataHandle operands = 2; ComputationHandle to_apply = 3; @@ -732,9 +749,6 @@ enum BinaryOperation { BINOP_LT = 9; BINOP_NE = 10; - // Dot product, matrix multiply. - BINOP_DOT = 12; - // Element-wise maximum. BINOP_MAX = 14; @@ -885,6 +899,7 @@ message OpRequest { ConvolveRequest convolve_request = 8; CrossReplicaSumRequest cross_replica_sum_request = 9; CustomCallRequest custom_call_request = 10; + DotRequest dot_request = 43; DynamicSliceRequest dynamic_slice_request = 11; DynamicUpdateSliceRequest dynamic_update_slice_request = 12; GetTupleElementRequest get_tuple_element_request = 13; @@ -914,7 +929,7 @@ message OpRequest { BatchNormInferenceRequest batch_norm_inference_request = 38; FftRequest fft_request = 41; ConvertRequest bitcast_convert_request = 42; - // Next: 43 + // Next: 44 } } diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 8831b3d0fd..4333f94486 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -511,6 +511,87 @@ contracted dimensions of `lhs` and `rhs` must be of the same size. In practice, it can be used to perform dot products between vectors, vector/matrix multiplications or matrix/matrix multiplications. +## DotGeneral + +See also +[`ComputationBuilder::DotGeneral`](https://www.tensorflow.org/code/tensorflow/compiler/xla/client/computation_builder.h). + + `DotGeneral(lhs, rhs, dimension_numbers)` + +| Arguments | Type | Semantics +| --------- | ----------------------- | --------------- +| `lhs` | `ComputationDataHandle` | array of type T +| `rhs` | `ComputationDataHandle` | array of type T +| `dimension_numbers` | `DotDimensionNumbers` | array of type T + +As Dot, but allows contracting and batch dimension numbers to be specified for +both the 'lhs' and 'rhs'. + +| DotDimensionNumbers Fields | Type | Semantics +| --------- | ----------------------- | --------------- +| 'lhs_contracting_dimensions' | repeated int64 | 'lhs' contracting dimension numbers | +| 'rhs_contracting_dimensions' | repeated int64 | 'rhs' contracting dimension numbers | +| 'lhs_batch_dimensions' | repeated int64 | 'lhs' batch dimension numbers | +| 'rhs_batch_dimensions' | repeated int64 | 'rhs' batch dimension numbers | + +DotGeneral performs the sum of products over contracting dimensions specified +in 'dimension_numbers'. + +Associated contracting dimension numbers from the 'lhs' and 'rhs' do not need +to be the same, but must be listed in the same order in both +'lhs/rhs_contracting_dimensions' arrays and have the same dimension sizes. + +Example with contracting dimension numbers: + +``` +lhs = { {1.0, 2.0, 3.0}, + {4.0, 5.0, 6.0} } + +rhs = { {1.0, 1.0, 1.0}, + {2.0, 2.0, 2.0} } + +DotDimensionNumbers dnums; +dnums.add_lhs_contracting_dimensions(1); +dnums.add_rhs_contracting_dimensions(1); + +DotGeneral(lhs, rhs, dnums) -> { {6.0, 12.0}, + {15.0, 30.0} } +``` + +Associated batch dimension numbers from the 'lhs' and 'rhs' must have the same +dimension number, must be listed in the same order in both arrays, and must +have the same dimension sizes. + +Example with batch dimension numbers (batch size 2, 2x2 matrices): + +``` +lhs = { { {1.0, 2.0}, + {3.0, 4.0} }, + { {5.0, 6.0}, + {7.0, 8.0} } } + +rhs = { { {1.0, 0.0}, + {0.0, 1.0} }, + { {1.0, 0.0}, + {0.0, 1.0} } } + +DotDimensionNumbers dnums; +dnums.add_lhs_contracting_dimensions(2); +dnums.add_rhs_contracting_dimensions(1); +dnums.add_lhs_batch_dimensions(0); +dnums.add_rhs_batch_dimensions(0); + +DotGeneral(lhs, rhs, dnums) -> { { {1.0, 2.0}, + {3.0, 4.0} }, + { {5.0, 6.0}, + {7.0, 8.0} } } +``` + +| Input | Output | Semantics | +| ----------------------------------- | ----------------- | ---------------- | +| [b0, m, k] `dot` [b0, k, n] | [b0, m, n] | batch matmul | +| [b0, b1, m, k] `dot` [b0, b1, k, n] | [b0, b1, m, n] | batch matmul | + ## Element-wise binary arithmetic operations See also -- GitLab From eafa8efc55fb9989a679e36b030742c6d87b0310 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 30 Nov 2017 11:23:25 -0800 Subject: [PATCH 0154/1924] [XLA:CPU] Add Hlo profiling support to XlaJitCompiledCpuFunction Some of the functionality has bled into the generic XlaCompiledCpuFunction, but there still remains a fair amount of work to do before the AOT side of things start working. This CL also fixes a bug I introduced in a previous CL -- when I changed IrEmitter::hlo_to_profile_idx_ to a value, I changed the signature of the generated function to always have the "profile_counters" argument when the AOT client code expects the signature to not have that argument. In practice this wasn't an issue for the standard x86 calling convention, but it could easily have been problematic on other architectures and calling conventions. After this change the mismatch is no longer present. PiperOrigin-RevId: 177481998 --- tensorflow/compiler/aot/codegen.cc | 8 +-- tensorflow/compiler/aot/codegen_test_h.golden | 8 +-- .../compiler/aot/tests/tfcompile_test.cc | 4 +- .../tf2xla/xla_compiled_cpu_function.cc | 15 +++++- .../tf2xla/xla_compiled_cpu_function.h | 53 ++++++++++++++----- .../tf2xla/xla_jit_compiled_cpu_function.cc | 25 ++++----- .../compiler/xla/service/cpu/ir_function.cc | 2 +- .../xla/service/hlo_execution_profile.cc | 3 +- .../xla/service/hlo_profile_printer.h | 8 ++- 9 files changed, 82 insertions(+), 44 deletions(-) diff --git a/tensorflow/compiler/aot/codegen.cc b/tensorflow/compiler/aot/codegen.cc index ae22f7edc4..28ac40df18 100644 --- a/tensorflow/compiler/aot/codegen.cc +++ b/tensorflow/compiler/aot/codegen.cc @@ -418,7 +418,7 @@ namespace xla { class ExecutableRunOptions; } // (Implementation detail) Entry point to the function in the object file. extern "C" void {{ENTRY}}( void* result, const xla::ExecutableRunOptions* run_options, - const void** args, void** temps); + const void** args, void** temps, tensorflow::int64* profile_counters); {{NS_START}} // {{CLASS}} represents a computation previously specified in a @@ -483,7 +483,7 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction { return *kStaticData; } - {{CLASS}}(AllocMode alloc_mode = AllocMode::ARGS_RESULTS_AND_TEMPS) + {{CLASS}}(AllocMode alloc_mode = AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS) : XlaCompiledCpuFunction(StaticData(), alloc_mode) {} {{CLASS}}(const {{CLASS}}&) = delete; @@ -496,8 +496,8 @@ class {{CLASS}} : public tensorflow::XlaCompiledCpuFunction { // void set_argN_data(void* data) // Sets the buffer of type T for positional argument N. May be called in // any AllocMode. Must be called before Run to have an affect. Must be - // called in AllocMode::RESULTS_AND_TEMPS_ONLY for each positional argument, - // to set the argument buffers. + // called in AllocMode::RESULTS_PROFILES_AND_TEMPS_ONLY for each positional + // argument, to set the argument buffers. // // T* argN_data() // Returns the buffer of type T for positional argument N. diff --git a/tensorflow/compiler/aot/codegen_test_h.golden b/tensorflow/compiler/aot/codegen_test_h.golden index 65f342ce27..cf01bee325 100644 --- a/tensorflow/compiler/aot/codegen_test_h.golden +++ b/tensorflow/compiler/aot/codegen_test_h.golden @@ -19,7 +19,7 @@ namespace xla { class ExecutableRunOptions; } // (Implementation detail) Entry point to the function in the object file. extern "C" void entry_point( void* result, const xla::ExecutableRunOptions* run_options, - const void** args, void** temps); + const void** args, void** temps, tensorflow::int64* profile_counters); namespace foo { namespace bar { @@ -86,7 +86,7 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { return *kStaticData; } - MyClass(AllocMode alloc_mode = AllocMode::ARGS_RESULTS_AND_TEMPS) + MyClass(AllocMode alloc_mode = AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS) : XlaCompiledCpuFunction(StaticData(), alloc_mode) {} MyClass(const MyClass&) = delete; @@ -99,8 +99,8 @@ class MyClass : public tensorflow::XlaCompiledCpuFunction { // void set_argN_data(void* data) // Sets the buffer of type T for positional argument N. May be called in // any AllocMode. Must be called before Run to have an affect. Must be - // called in AllocMode::RESULTS_AND_TEMPS_ONLY for each positional argument, - // to set the argument buffers. + // called in AllocMode::RESULTS_PROFILES_AND_TEMPS_ONLY for each positional + // argument, to set the argument buffers. // // T* argN_data() // Returns the buffer of type T for positional argument N. diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc index 6b037f276a..413efd9cea 100644 --- a/tensorflow/compiler/aot/tests/tfcompile_test.cc +++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc @@ -70,7 +70,7 @@ TEST(TFCompileTest, Add) { // Run tests that use set_argN_data separately, to avoid accidentally re-using // non-existent buffers. TEST(TFCompileTest, Add_SetArg) { - AddComp add(AddComp::AllocMode::RESULTS_AND_TEMPS_ONLY); + AddComp add(AddComp::AllocMode::RESULTS_PROFILES_AND_TEMPS_ONLY); int32 arg_x = 10; int32 arg_y = 32; @@ -258,7 +258,7 @@ TEST(TFCompileTest, MatMul2_SetArg) { Eigen::ThreadPoolDevice device(&tp, tp.NumThreads()); foo::bar::MatMulComp matmul( - foo::bar::MatMulComp::AllocMode::RESULTS_AND_TEMPS_ONLY); + foo::bar::MatMulComp::AllocMode::RESULTS_PROFILES_AND_TEMPS_ONLY); matmul.set_thread_pool(&device); // Test using the set_argN_data() methods. diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.cc index b5c17c5273..43d0e17c2c 100644 --- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.cc +++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.cc @@ -28,9 +28,10 @@ XlaCompiledCpuFunction::XlaCompiledCpuFunction(const StaticData& static_data, temps_(new void*[static_data.num_temps]), arg_names_(static_data.arg_names), result_names_(static_data.result_names), - program_shape_(static_data.program_shape) { + program_shape_(static_data.program_shape), + hlo_profile_printer_(static_data.hlo_profile_printer) { // Allocate arg and temp buffers. - if (alloc_mode == AllocMode::ARGS_RESULTS_AND_TEMPS) { + if (alloc_mode == AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS) { alloc_args_ = tensorflow::tfcompile::runtime::MallocContiguousBuffers( static_data.arg_sizes, static_data.num_args, args_, /*annotate_initialized=*/false); @@ -43,6 +44,15 @@ XlaCompiledCpuFunction::XlaCompiledCpuFunction(const StaticData& static_data, if (static_data.requires_runtime_context) { args_[static_data.num_args - 1] = &context_; } + + // If Hlo profiling is enabled the generated code expects an appropriately + // sized buffer to be passed in as the last argument. If Hlo profiling is + // disabled the last function argument is still present in the function + // signature, but it is ignored by the generated code and we pass in null for + // it. + if (hlo_profiling_enabled()) { + profile_counters_ = new int64[static_data.profile_counters_size](); + } } XlaCompiledCpuFunction::~XlaCompiledCpuFunction() { @@ -50,6 +60,7 @@ XlaCompiledCpuFunction::~XlaCompiledCpuFunction() { tensorflow::tfcompile::runtime::FreeContiguous(alloc_temps_); delete[] args_; delete[] temps_; + delete[] profile_counters_; } namespace { diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h index f49a788922..3c4314d498 100644 --- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h +++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h @@ -16,7 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_TF2XLA_XLA_COMPILED_CPU_FUNCTION_H_ #define TENSORFLOW_COMPILER_TF2XLA_XLA_COMPILED_CPU_FUNCTION_H_ -#include +#include #include #include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h" @@ -27,6 +27,7 @@ limitations under the License. // never use this functionality. namespace xla { class ProgramShape; +class HloProfilePrinter; } namespace tensorflow { @@ -48,12 +49,10 @@ namespace tensorflow { class XlaCompiledCpuFunction { public: // Type of the raw function, produced by either JIT or AOT. - // - // TODO(toddw): Add support for hlo profiling, and replace std::function with - // a raw function pointer, for some codesize savings. - using RawFunction = std::function; + using RawFunction = void (*)(void* result, + const xla::ExecutableRunOptions* run_options, + const void** args, void** temps, + int64* profile_counters); // StaticData represents the state necessary to run an XLA-compiled // function. For JIT this is backed by data in XlaJitCompiledCpuFunction; for @@ -81,21 +80,29 @@ class XlaCompiledCpuFunction { // [Optional] Arg and result shapes. const xla::ProgramShape* program_shape = nullptr; + + // [Optional] Profile printer. Null if profiling is disabled. + const xla::HloProfilePrinter* hlo_profile_printer = nullptr; + + // [Optional] The number of profile counters expected in the profile counter + // buffer by the generated code and hlo_profile_printer. 0 if profiling is + // disabled. + int64 profile_counters_size = 0; }; // AllocMode controls the buffer allocation mode. enum class AllocMode { - // Allocate all buffers - args, results and temps. - ARGS_RESULTS_AND_TEMPS, + // Allocate all buffers - args, results, profile and temps. + ARGS_RESULTS_PROFILES_AND_TEMPS, - // Only allocate result and temp buffers. + // Only allocate result, profile and temp buffers. // Use set_arg_data to set argument buffers before Run is called. - RESULTS_AND_TEMPS_ONLY, + RESULTS_PROFILES_AND_TEMPS_ONLY, }; XlaCompiledCpuFunction( const StaticData& static_data, - AllocMode alloc_mode = AllocMode::ARGS_RESULTS_AND_TEMPS); + AllocMode alloc_mode = AllocMode::ARGS_RESULTS_PROFILES_AND_TEMPS); virtual ~XlaCompiledCpuFunction(); XlaCompiledCpuFunction(const XlaCompiledCpuFunction&) = delete; @@ -113,7 +120,7 @@ class XlaCompiledCpuFunction { context_.error = false; context_.error_msg.clear(); raw_function_(temps_[result_index_], &run_options_, - const_cast(args_), temps_); + const_cast(args_), temps_, profile_counters_); return !context_.error; } @@ -162,6 +169,16 @@ class XlaCompiledCpuFunction { return static_cast(temps_[result_index_]); } + // Profile counters for this XLA computation. + // + // When Hlo profiling is enabled (`hlo_profiling_enabled()` return true in + // this case) these counters are non-null and are automatically populated by + // `Run`. The counters can then be pretty-printed using + // `hlo_profile_printer()`. + // + // When Hlo profiling is disabled, this accessor returns null. + const int64* profile_counters() const { return profile_counters_; } + // Returns the buffer for the positional result at the given `index`. void* result_data(size_t index) { return results()[index]; } const void* result_data(size_t index) const { return results()[index]; } @@ -195,6 +212,12 @@ class XlaCompiledCpuFunction { // program shape isn't available. const xla::ProgramShape* ProgramShape() const { return program_shape_; } + bool hlo_profiling_enabled() const { return hlo_profile_printer_ != nullptr; } + const xla::HloProfilePrinter& hlo_profile_printer() const { + assert(hlo_profiling_enabled()); + return *hlo_profile_printer_; + } + private: const RawFunction raw_function_; const size_t result_index_; @@ -208,6 +231,9 @@ class XlaCompiledCpuFunction { void* alloc_args_ = nullptr; void* alloc_temps_ = nullptr; + // Backing memory for profiling counters. + int64* profile_counters_ = nullptr; + // Options and context passed to the compiled function. xla::ExecutableRunOptions run_options_; tensorflow::XlaLocalRuntimeContext context_; @@ -216,6 +242,7 @@ class XlaCompiledCpuFunction { const char** arg_names_ = nullptr; const char** result_names_ = nullptr; const xla::ProgramShape* program_shape_ = nullptr; + const xla::HloProfilePrinter* hlo_profile_printer_ = nullptr; }; } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc index 1dd454ea8d..f727f20464 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function.cc @@ -90,21 +90,6 @@ xla::StatusOr ComputeResultIndex( return result_slice.index(); } -// Adapt ComputeFunctionType, which includes a final profile_counters arg, to -// RawFunction, which doesn't include that final arg. -// -// TODO(toddw): Change RawFunction and AOT to also pass the final -// profile_counters arg, and remove this adapter. -XlaCompiledCpuFunction::RawFunction RawFunctionAdapter( - xla::cpu::CpuExecutable::ComputeFunctionType compute_function) { - return [compute_function](void* result, - const xla::ExecutableRunOptions* run_options, - const void** args, void** temps) { - return compute_function(result, run_options, args, temps, - /*profile_counters=*/nullptr); - }; -} - // Collect names from `entries`, where T is one of tf2xla::{Feed,Fetch}. We hold // the actual strings in nonempty_names, and hold arrays of pointers in // name_ptrs, terminated by a nullptr entry. @@ -177,7 +162,7 @@ XlaJitCompiledCpuFunction::Compile( const xla::cpu::CpuExecutable* cpu_executable = static_cast(executable->executable()); XlaCompiledCpuFunction::RawFunction raw_function = - RawFunctionAdapter(cpu_executable->compute_function()); + cpu_executable->compute_function(); const xla::BufferAssignment& buffer_assignment = cpu_executable->buffer_assignment(); @@ -211,6 +196,14 @@ XlaJitCompiledCpuFunction::Compile( jit->static_data_.arg_names = jit->arg_names_.data(); jit->static_data_.result_names = jit->result_names_.data(); jit->static_data_.program_shape = jit->program_shape_.get(); + + if (cpu_executable->hlo_profiling_enabled()) { + jit->static_data_.hlo_profile_printer = + &cpu_executable->hlo_profile_printer(); + jit->static_data_.profile_counters_size = + cpu_executable->hlo_profile_printer().profile_counters_size(); + } + return std::move(jit_unique_ptr); } diff --git a/tensorflow/compiler/xla/service/cpu/ir_function.cc b/tensorflow/compiler/xla/service/cpu/ir_function.cc index fa88627156..701bce2cbf 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_function.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_function.cc @@ -99,7 +99,7 @@ void IrFunction::Initialize(const string& function_name, // // /---------------------------------------------\ // prof counters -> | counter 0 | counter 1 | ..... | counter N-1 | - // (elided for aot) \---------------------------------------------/ + // \---------------------------------------------/ // Even though the type of params and temps is void** in the host's view, in // LLVM IR this is represented by i8*, similarly to void*. It's up to the code diff --git a/tensorflow/compiler/xla/service/hlo_execution_profile.cc b/tensorflow/compiler/xla/service/hlo_execution_profile.cc index ba75e2ef1b..0809fe780d 100644 --- a/tensorflow/compiler/xla/service/hlo_execution_profile.cc +++ b/tensorflow/compiler/xla/service/hlo_execution_profile.cc @@ -109,7 +109,8 @@ std::unique_ptr CreateHloProfilePrinter( }; return MakeUnique( - computation_infos, hlo_profile_index_map.computation_count(), deleter); + computation_infos, hlo_profile_index_map.computation_count(), + /*profile_counters_size=*/max_profile_index, deleter); } HloExecutionProfile::HloExecutionProfile( diff --git a/tensorflow/compiler/xla/service/hlo_profile_printer.h b/tensorflow/compiler/xla/service/hlo_profile_printer.h index 316753a82a..2f056490ae 100644 --- a/tensorflow/compiler/xla/service/hlo_profile_printer.h +++ b/tensorflow/compiler/xla/service/hlo_profile_printer.h @@ -65,9 +65,11 @@ class HloProfilePrinter { HloProfilePrinter( HloComputationInfo* computation_infos, int64 computation_infos_size, + int64 profile_counters_size, std::function deleter = nullptr) : computation_infos_(computation_infos), computation_infos_size_(computation_infos_size), + profile_counters_size_(profile_counters_size), deleter_(std::move(deleter)) {} HloProfilePrinter(HloProfilePrinter&& other) { @@ -79,10 +81,13 @@ class HloProfilePrinter { HloProfilePrinter(const HloProfilePrinter&) = delete; HloProfilePrinter& operator=(const HloProfilePrinter&) = delete; - // Convert the profile counter sequence `counters` to a human readable string + // Converts the profile counter sequence `counters` to a human readable string // representation. string ToString(const int64* counters, double clock_rate_ghz) const; + // Returns the size of the profile buffer expected by this printer. + int64 profile_counters_size() const { return profile_counters_size_; } + ~HloProfilePrinter(); private: @@ -90,6 +95,7 @@ class HloProfilePrinter { // is manifested as the deleter_ function. HloComputationInfo* computation_infos_ = nullptr; int64 computation_infos_size_ = 0; + int64 profile_counters_size_ = 0; std::function deleter_; }; } // namespace xla -- GitLab From af36437e3937e6e532579e9c42d7f45353b88990 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 12:29:06 -0800 Subject: [PATCH 0155/1924] Add rules to replace nodes corresponding to operations with the neutral and absorbing elements for addition and multiplication with Identity. Get rid of a gratuitous copy of the entire graph in the main optimizer loop. PiperOrigin-RevId: 177491247 --- tensorflow/core/grappler/optimizers/BUILD | 1 + .../grappler/optimizers/constant_folding.cc | 274 ++++++++++++++++-- .../grappler/optimizers/constant_folding.h | 9 +- .../optimizers/constant_folding_test.cc | 96 ++++++ 4 files changed, 359 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 24e6f8847a..7b4ed10e7e 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -96,6 +96,7 @@ cc_library( ":graph_optimizer", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", "//tensorflow/core/grappler:grappler_item", "//tensorflow/core/grappler:op_types", diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index b5172a4833..cf913d6f48 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -30,13 +30,16 @@ limitations under the License. #include "tensorflow/core/grappler/costs/graph_properties.h" #include "tensorflow/core/grappler/grappler_item.h" #include "tensorflow/core/grappler/op_types.h" +#include "tensorflow/core/grappler/utils.h" #include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/tensor_coding.h" #include "tensorflow/core/public/version.h" #include "tensorflow/core/util/bcast.h" +#include "tensorflow/core/util/saved_tensor_slice_util.h" namespace tensorflow { namespace grappler { @@ -95,7 +98,38 @@ class DeviceSimple : public DeviceBase { std::unique_ptr eigen_device_; }; +template +bool AllValuesAre(const TensorProto& tensor, const T& value) { + // TensorProto represents the content of the tensor in either _val or + // tensor_content. + typename checkpoint::SaveTypeTraits::RepeatedField* tensor_values = + checkpoint::MutableTensorProtoData(const_cast(&tensor)); + if (!tensor_values->empty()) { + for (const T& tensor_value : *tensor_values) { + if (tensor_value != value) { + return false; + } + } + return true; + } + const auto tensor_content_size = tensor.tensor_content().size(); + if (tensor_content_size > 0) { + CHECK_EQ(0, tensor_content_size % sizeof(T)); + std::vector raw_values(tensor_content_size / sizeof(T)); + port::CopyToArray(tensor.tensor_content(), + reinterpret_cast(raw_values.data())); + for (int i = 0; i < tensor_content_size / sizeof(T); ++i) { + if (raw_values[i] != value) { + return false; + } + } + return true; + } + return false; +} + } // namespace + ConstantFolding::ConstantFolding(RewriterConfig::Toggle opt_level, DeviceBase* cpu_device) : opt_level_(opt_level), cpu_device_(cpu_device) { @@ -202,9 +236,9 @@ Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { // We may add some nodes to the graph to encode control dependencies: there is // no need to process these, so only iterate over the nodes of the input // graph. - const int node_count = graph_.node_size(); + const int node_count = graph_->node_size(); for (int i = 0; i < node_count; ++i) { - NodeDef& node = *graph_.mutable_node(i); + NodeDef& node = *graph_->mutable_node(i); const string op = node.op(); if (op != "Shape" && op != "Size" && op != "Rank" && op != "ShapeN") { continue; @@ -248,7 +282,7 @@ Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { // cases where the shape/rank/size would have been run in // the original graph. Additional inputs are extra control string ctrl_dep = - AddControlDependency(node.input(0), &graph_, node_map_.get()); + AddControlDependency(node.input(0), graph_, node_map_.get()); node.set_input(0, ctrl_dep); node_map_->AddOutput(NodeName(ctrl_dep), node.name()); } else { @@ -263,7 +297,7 @@ Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { AddPrefixToNodeName(strings::StrCat(node.name(), "-", j), kConstantFoldingConst); if (node_map_->GetNode(const_name) == nullptr) { - NodeDef* added_node = graph_.add_node(); + NodeDef* added_node = graph_->add_node(); added_node->set_name(const_name); added_node->set_op("Const"); added_node->set_device(node.device()); @@ -274,7 +308,7 @@ Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { // We add a control dependency to the original ShapeN node, // so that the node will only be run if all inputs of the // original ShapeN node are run. - string ctrl_dep = AddControlDependency(node.name(), &graph_, + string ctrl_dep = AddControlDependency(node.name(), graph_, node_map_.get()); *added_node->add_input() = ctrl_dep; node_map_->AddOutput(NodeName(ctrl_dep), added_node->name()); @@ -293,6 +327,25 @@ Status ConstantFolding::MaterializeShapes(const GraphProperties& properties) { } namespace { +bool ShapesEqual(const TensorShapeProto& shape1, + const TensorShapeProto& shape2) { + if (shape1.unknown_rank() || shape2.unknown_rank()) { + return false; + } + if (shape1.dim_size() != shape2.dim_size()) { + return false; + } + for (int i = 0; i < shape1.dim_size(); ++i) { + if (shape1.dim(i).size() != shape2.dim(i).size()) { + return false; + } + if (shape1.dim(i).size() == -1 || shape2.dim(i).size() == -1) { + return false; + } + } + return true; +} + bool ExtractShape(const NodeDef& shape_node, const GraphProperties& properties, BCast::Vec* shape, int64* min_id) { if (shape_node.op() == "Shape") { @@ -383,13 +436,13 @@ Status ConstantFolding::MaterializeBroadcastGradientArgs( strings::StrCat(node.name(), "-", j), kConstantFoldingConst); out[j] = node_map_->GetNode(const_name); if (out[j] == nullptr) { - out[j] = graph_.add_node(); + out[j] = graph_->add_node(); Tensor value(type, TensorShape({0})); *out[j] = CreateNodeDef(const_name, TensorValue(&value)); out[j]->set_device(node.device()); node_map_->AddNode(const_name, out[j]); string ctrl_dep = - AddControlDependency(node.name(), &graph_, node_map_.get()); + AddControlDependency(node.name(), graph_, node_map_.get()); *out[j]->add_input() = ctrl_dep; node_map_->AddOutput(NodeName(ctrl_dep), const_name); } @@ -470,7 +523,7 @@ Status ConstantFolding::MaterializeReductionIndices( if (node_map_->GetNode(const_name)) { return Status::OK(); } - NodeDef* reduction_indices = graph_.add_node(); + NodeDef* reduction_indices = graph_->add_node(); Tensor value(dtype, TensorShape({rank})); for (int i = 0; i < rank; ++i) { if (dtype == DT_INT32) { @@ -482,7 +535,7 @@ Status ConstantFolding::MaterializeReductionIndices( *reduction_indices = CreateNodeDef(const_name, TensorValue(&value)); reduction_indices->set_device(node->device()); string ctrl_dep = - AddControlDependency(node->input(1), &graph_, node_map_.get()); + AddControlDependency(node->input(1), graph_, node_map_.get()); *reduction_indices->add_input() = ctrl_dep; node_map_->AddNode(const_name, reduction_indices); node_map_->AddOutput(NodeName(ctrl_dep), const_name); @@ -496,9 +549,9 @@ Status ConstantFolding::MaterializeReductionIndices( Status ConstantFolding::MaterializeConstants( const GraphProperties& properties) { - const int node_count = graph_.node_size(); + const int node_count = graph_->node_size(); for (int i = 0; i < node_count; ++i) { - NodeDef& node = *graph_.mutable_node(i); + NodeDef& node = *graph_->mutable_node(i); const string& op = node.op(); if (op == "BroadcastGradientArgs") { TF_RETURN_IF_ERROR(MaterializeBroadcastGradientArgs(node, properties)); @@ -602,6 +655,32 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const { return true; } +namespace { + +#define SET_TENSOR_VAL_CASE(DTYPE, TYPE) \ + case DTYPE: \ + t->add_##TYPE##_val(static_cast(value)); \ + break; + +Status CreateConstantTensorAttrValue(DataType type, double value, + const TensorShapeProto& shape, + AttrValue* attr_tensor) { + TensorProto* t = attr_tensor->mutable_tensor(); + *t->mutable_tensor_shape() = shape; + switch (type) { + SET_TENSOR_VAL_CASE(DT_FLOAT, float); + SET_TENSOR_VAL_CASE(DT_DOUBLE, double); + SET_TENSOR_VAL_CASE(DT_INT64, int64); + SET_TENSOR_VAL_CASE(DT_INT32, int); + default: + return errors::InvalidArgument("Unsupported type: ", type); + } + return Status::OK(); +} + +#undef SET_TENSOR_CAL_CASE +} // namespace + // static NodeDef ConstantFolding::CreateNodeDef(const string& name, const TensorValue& tensor) { @@ -945,8 +1024,8 @@ Status ConstantFolding::FoldNode(NodeDef* node, GraphDef* output_graph) { Status ConstantFolding::FoldGraph(GraphDef* output) { std::unordered_set processed_nodes; std::deque queue; - for (int i = 0; i < graph_.node_size(); i++) { - auto node = graph_.mutable_node(i); + for (int i = 0; i < graph_->node_size(); i++) { + auto node = graph_->mutable_node(i); if (IsFoldable(*node)) { queue.push_back(node); } @@ -985,7 +1064,7 @@ Status ConstantFolding::FoldGraph(GraphDef* output) { output->mutable_node()->DeleteSubrange(last + 1, output->node_size() - last - 1); - for (const auto& node : graph_.node()) { + for (const auto& node : graph_->node()) { // If no fetch nodes is provided, we conservatively // keep all nodes in the original graph in case users need to fetch // their values. @@ -1080,6 +1159,104 @@ bool ConstantFolding::IsSimplifiableReshape( return shape.IsCompatibleWith(new_dims); } +#define IS_VALUE_CASE(DTYPE, VALUE) \ + case DTYPE: \ + return AllValuesAre::Type>( \ + node.attr().at("value").tensor(), EnumToDataType::Type(VALUE)) + +#define IS_ONES_CASE(TYPE) IS_VALUE_CASE(TYPE, 1) +#define IS_ZEROS_CASE(TYPE) IS_VALUE_CASE(TYPE, 0) + +bool ConstantFolding::IsOnes(const NodeDef& node) const { + if (feed_nodes_.find(node.name()) != feed_nodes_.end()) { + return false; + } + if (node.op() == "OnesLike") { + return true; + } + if (node.op() != "Const") { + return false; + } + const auto dtype = node.attr().at("dtype").type(); + switch (dtype) { + // IS_ONES_CASE(DT_HALF); + IS_ONES_CASE(DT_FLOAT); + IS_ONES_CASE(DT_DOUBLE); + IS_ONES_CASE(DT_UINT8); + IS_ONES_CASE(DT_INT8); + IS_ONES_CASE(DT_UINT16); + IS_ONES_CASE(DT_INT16); + IS_ONES_CASE(DT_INT32); + IS_ONES_CASE(DT_INT64); + IS_ONES_CASE(DT_COMPLEX64); + IS_ONES_CASE(DT_COMPLEX128); + default: + LOG(ERROR) << "Unexpected type " << DataTypeString(dtype); + return false; + } + return false; +} + +bool ConstantFolding::IsZeros(const NodeDef& node) const { + if (feed_nodes_.find(node.name()) != feed_nodes_.end()) { + return false; + } + if (node.op() == "ZerosLike") { + return true; + } + if (!IsConstant(node)) { + return false; + } + const auto dtype = node.attr().at("dtype").type(); + switch (dtype) { + // IS_ZEROS_CASE(DT_HALF); + IS_ZEROS_CASE(DT_FLOAT); + IS_ZEROS_CASE(DT_DOUBLE); + IS_ZEROS_CASE(DT_UINT8); + IS_ZEROS_CASE(DT_INT8); + IS_ZEROS_CASE(DT_UINT16); + IS_ZEROS_CASE(DT_INT16); + IS_ZEROS_CASE(DT_INT32); + IS_ZEROS_CASE(DT_INT64); + IS_ZEROS_CASE(DT_COMPLEX64); + IS_ZEROS_CASE(DT_COMPLEX128); + default: + LOG(ERROR) << "Unexpected type " << DataTypeString(dtype); + return false; + } + return false; +} + +void ConstantFolding::ReplaceAddOrMulWithIdentity(int input_to_forward, + NodeDef* node) { + node->set_op("Identity"); + // Propagate the designated input through the identity. + node->mutable_input()->SwapElements(0, input_to_forward); + // Add all other inputs as control dependencies. + for (int i = 1; i < node->input_size(); ++i) { + node->set_input(i, AsControlDependency(node->input(i))); + } + graph_modified_ = true; +} + +Status ConstantFolding::ReplaceAddOrMulWithConstant( + double value, const TensorShapeProto& shape, NodeDef* node) { + AttrValue tensor_attr; + TF_RETURN_IF_ERROR(CreateConstantTensorAttrValue(node->attr().at("T").type(), + value, shape, &tensor_attr)); + node->mutable_attr()->insert({"value", tensor_attr}); + node->set_op("Const"); + // Convert all inputs to control dependencies. + for (int i = 0; i < node->input_size(); ++i) { + if (IsControlInput(node->input(i))) { + break; + } + node->set_input(i, AsControlDependency(node->input(i))); + } + graph_modified_ = true; + return Status::OK(); +} + Status ConstantFolding::SimplifyGraph(GraphDef* output, const GraphProperties& properties, bool use_shape_info) { @@ -1125,6 +1302,63 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output, *node.add_input() = input; } } + + // Simplify multiplication by ones or zeros, and addition of zeros. + bool is_mul = IsMul(node); + bool is_add = IsAdd(node); + if (opt_level_ == RewriterConfig::AGGRESSIVE && use_shape_info && + (is_mul || is_add) && properties.HasInputProperties(node.name()) && + properties.HasOutputProperties(node.name())) { + const NodeDef* x = node_map_->GetNode(node.input(0)); + const NodeDef* y = node_map_->GetNode(node.input(1)); + if (x == nullptr || y == nullptr) { + return errors::InvalidArgument("Invalid inputs to node: ", + node.DebugString()); + } + const TensorShapeProto& output_shape = + properties.GetOutputProperties(node.name())[0].shape(); + const TensorShapeProto& x_shape = + properties.GetInputProperties(node.name())[0].shape(); + + // Simplify multiplication by or addition of zeros. + const bool x_is_zero = IsZeros(*x); + const bool x_matches_output_shape = ShapesEqual(output_shape, x_shape); + if (x_is_zero && x_matches_output_shape) { + // 0 * y = 0 or 0 + y = y. + ReplaceAddOrMulWithIdentity(is_mul ? 0 : 1, &node); + continue; + } + const TensorShapeProto& y_shape = + properties.GetInputProperties(node.name())[1].shape(); + const bool y_is_zero = IsZeros(*y); + const bool y_matches_output_shape = ShapesEqual(output_shape, y_shape); + if (y_is_zero && y_matches_output_shape) { + // x * 0 = 0 or x + 0 = x. + ReplaceAddOrMulWithIdentity(is_mul ? 1 : 0, &node); + continue; + } + + if (is_mul) { + // Simplify multiplication by zeros where the output shape does not + // match the shape of the zero input. + if (x_is_zero || y_is_zero) { + TF_RETURN_IF_ERROR( + ReplaceAddOrMulWithConstant(0, output_shape, &node)); + continue; + } + + // Simplify multiplication by ones. + if (IsOnes(*x) && y_matches_output_shape) { + // 1 * y = y. + ReplaceAddOrMulWithIdentity(1, &node); + continue; + } else if (IsOnes(*y) && x_matches_output_shape) { + // x * 1 = x. + ReplaceAddOrMulWithIdentity(0, &node); + continue; + } + } + } } return Status::OK(); } @@ -1132,7 +1366,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output, Status ConstantFolding::RunOptimizationPass(Cluster* cluster, const GrapplerItem& item, GraphDef* output) { - node_map_.reset(new NodeMap(&graph_)); + node_map_.reset(new NodeMap(graph_)); nodes_whitelist_.clear(); // Fold fetch nodes iff it has a single fanout. Note that if a fetch node // has a single fanout, it would be rewritten as a constant with the same @@ -1189,13 +1423,13 @@ Status ConstantFolding::Optimize(Cluster* cluster, const GrapplerItem& item, *output = item.graph; int64 node_count; do { - graph_.Swap(output); - item_to_optimize.graph = graph_; + graph_modified_ = false; + item_to_optimize.graph.Swap(output); + graph_ = &item_to_optimize.graph; *output = GraphDef(); - node_count = graph_.node_size(); + node_count = graph_->node_size(); TF_RETURN_IF_ERROR(RunOptimizationPass(cluster, item_to_optimize, output)); - } while (output->node_size() != node_count); - + } while (graph_modified_ || output->node_size() != node_count); *output->mutable_library() = item.graph.library(); *output->mutable_versions() = item.graph.versions(); diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h index 8af5b5fbe6..3bb9926338 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.h +++ b/tensorflow/core/grappler/optimizers/constant_folding.h @@ -72,6 +72,12 @@ class ConstantFolding : public GraphOptimizer { Status FoldNode(NodeDef* node, GraphDef* output_graph); + bool IsOnes(const NodeDef& node) const; + bool IsZeros(const NodeDef& node) const; + void ReplaceAddOrMulWithIdentity(int input_to_forward, NodeDef* node); + Status ReplaceAddOrMulWithConstant(double value, + const TensorShapeProto& shape, + NodeDef* node); Status FoldGraph(GraphDef* output); bool IsSimplifiableReduction(const NodeDef& node) const; @@ -89,12 +95,13 @@ class ConstantFolding : public GraphOptimizer { std::unique_ptr owned_device_; std::unique_ptr resource_mgr_; - GraphDef graph_; + GraphDef* graph_; std::unique_ptr node_map_; std::unordered_set nodes_to_preserve_; std::unordered_set nodes_whitelist_; std::unordered_set feed_nodes_; bool has_fetch_; + bool graph_modified_; }; } // end namespace grappler diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index b2d9b02c68..c72ed96520 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -77,6 +77,102 @@ TEST_F(ConstantFoldingTest, SimpleFolding) { test::ExpectTensorEqual(tensors_expected[0], tensors[0]); } +TEST_F(ConstantFoldingTest, NeutralElement) { + for (bool use_const : {true, false}) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT, + ops::Placeholder::Shape(TensorShape({1, 2}))); + Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT, + ops::Placeholder::Shape(TensorShape({1, 2}))); + Output zeros = + !use_const ? ops::ZerosLike(s.WithOpName("zeros"), x) + : ops::Const(s.WithOpName("zeros"), {0.0f, 0.0f}, {1, 2}); + Output zeros_broadcast = + ops::Const(s.WithOpName("zeros_broadcast"), {0.0f}, {1, 1}); + Output ones = !use_const + ? ops::OnesLike(s.WithOpName("ones"), x) + : ops::Const(s.WithOpName("ones"), {1.0f, 1.0f}, {1, 2}); + Output mul1 = ops::Mul(s.WithOpName("mul1"), x, zeros); + Output mul2 = ops::Mul(s.WithOpName("mul2"), zeros, y); + Output mul3 = ops::Mul(s.WithOpName("mul3"), x, ones); + Output mul4 = ops::Mul(s.WithOpName("mul4"), ones, y); + Output mul5 = ops::Mul(s.WithOpName("mul1"), x, zeros_broadcast); + Output mul6 = ops::Mul(s.WithOpName("mul2"), zeros_broadcast, y); + Output add1 = ops::Add(s.WithOpName("add1"), x, zeros); + Output add2 = ops::Add(s.WithOpName("add2"), zeros, y); + Output addn = ops::AddN(s, {mul1, mul2, mul3, mul4, add1, add2}); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ConstantFolding optimizer(RewriterConfig::AGGRESSIVE, + nullptr /* cpu_device */); + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(14, output.node_size()); + for (int i = 0; i < output.node_size(); ++i) { + const NodeDef& node = output.node(i); + const string& name = node.name(); + if (name == "mul1") { + if (use_const) { + EXPECT_EQ("Const", node.op()); + EXPECT_EQ("^x", node.input(0)); + } else { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("zeros", node.input(0)); + EXPECT_EQ("^x", node.input(1)); + } + } else if (name == "mul2") { + if (use_const) { + EXPECT_EQ("Const", node.op()); + EXPECT_EQ("^y", node.input(0)); + } else { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("zeros", node.input(0)); + EXPECT_EQ("^y", node.input(1)); + } + } else if (name == "mul3") { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("x", node.input(0)); + EXPECT_EQ("^ones", node.input(1)); + } else if (name == "mul4") { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("y", node.input(0)); + EXPECT_EQ("^ones", node.input(1)); + } else if (name == "mul5") { + EXPECT_EQ("Const", node.op()); + EXPECT_EQ("^x", node.input(0)); + EXPECT_EQ("^ones", node.input(1)); + TensorProto t = node.attr().at("value").tensor(); + EXPECT_EQ(1, t.float_val_size()); + EXPECT_EQ(0, t.float_val(0)); + EXPECT_EQ(2, t.tensor_shape().dim_size()); + EXPECT_EQ(1, t.tensor_shape().dim(0).size()); + EXPECT_EQ(2, t.tensor_shape().dim(1).size()); + } else if (name == "mul6") { + EXPECT_EQ("Const", node.op()); + EXPECT_EQ("^y", node.input(0)); + EXPECT_EQ("^ones", node.input(1)); + TensorProto t = node.attr().at("value").tensor(); + EXPECT_EQ(1, t.float_val_size()); + EXPECT_EQ(0, t.float_val(0)); + EXPECT_EQ(2, t.tensor_shape().dim_size()); + EXPECT_EQ(1, t.tensor_shape().dim(0).size()); + EXPECT_EQ(2, t.tensor_shape().dim(1).size()); + } else if (name == "add1") { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("x", node.input(0)); + EXPECT_EQ("^zeros", node.input(1)); + } else if (name == "add2") { + EXPECT_EQ("Identity", node.op()); + EXPECT_EQ("y", node.input(0)); + EXPECT_EQ("^zeros", node.input(1)); + } + } + } +} + TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) { // Build a simple graph with a few trivially prunable ops. tensorflow::Scope s = tensorflow::Scope::NewRootScope(); -- GitLab From 5e54d87f94271ce671ddc874cca8d34c83c180cc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 13:14:09 -0800 Subject: [PATCH 0156/1924] Add an option to override maximum number of elements in the quantile accumulator. PiperOrigin-RevId: 177497240 --- tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py index 7e8e15e7d8..294e04002a 100644 --- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py +++ b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py @@ -45,6 +45,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject): init_stamp_token, epsilon, num_quantiles, + max_elements=None, name=None, container=None): """Creates a QuantileAccumulator object. @@ -53,6 +54,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject): init_stamp_token: The initial value for the stamp token. epsilon: Error bound on the quantile computation. num_quantiles: Number of quantiles to produce from the final summary. + max_elements: Maximum number of elements added to the accumulator. name: the name to save the accumulator under. container: An optional `string`. Defaults to `""` """ @@ -67,6 +69,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject): self._quantile_accumulator_handle, init_stamp_token, epsilon=epsilon, + max_elements=max_elements, num_quantiles=num_quantiles) is_initialized_op = gen_quantile_ops.quantile_accumulator_is_initialized( self._quantile_accumulator_handle) -- GitLab From 62b70f5566768e0fd57013e3042a402830b2c4f0 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 30 Nov 2017 13:29:12 -0800 Subject: [PATCH 0157/1924] Support binary operations with a scalar and a 4d tensor as input; refactor connectivity check code. PiperOrigin-RevId: 177499365 --- .../grappler/optimizers/layout_optimizer.cc | 87 ++++++++---- .../optimizers/layout_optimizer_test.cc | 128 ++++++++++++++++++ 2 files changed, 189 insertions(+), 26 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index ef4b015295..97c8e6f907 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -761,24 +761,52 @@ class AgnosticNodeProcessor : public NodeProcessor { bool IsNodeAfterNCHWToNHWC() const { std::set ops_format_agnostic = GetOpsFormatAgnostic(); - auto node = node_map_->GetNode(node_->name()); - while (node->input_size() > 0) { - int data_input_pos = 0; - if (IsConcatV1(*node) || IsSplit(*node)) { - data_input_pos = 1; - } - node = node_map_->GetNode(node->input(data_input_pos)); - if (IsNodeNCHWToNHWC(node->name())) { + std::deque queue; + auto first_node_pos = DataInputPos(*node_); + for (const auto& pos : first_node_pos) { + auto input_node = node_map_->GetNode(node_->input(pos)); + queue.push_back(input_node); + } + // The code will exit this while loop in one iteration in most cases, as the + // graph is already topologically sorted. + while (!queue.empty()) { + NodeDef* current_node = queue.front(); + queue.pop_front(); + if (IsNodeNCHWToNHWC(current_node->name())) { return true; } - bool connected = - ops_format_agnostic.find(node->op()) != ops_format_agnostic.end(); - if (!connected) { - return false; + // We only continue searching if the path is connected through + // format-agnostic nodes. + if (ops_format_agnostic.find(current_node->op()) != + ops_format_agnostic.end()) { + auto current_node_pos = DataInputPos(*current_node); + for (const auto& pos : current_node_pos) { + auto input_node = node_map_->GetNode(current_node->input(pos)); + queue.push_back(input_node); + } } } return false; } + + private: + std::vector DataInputPos(const NodeDef& node) const { + std::vector pos; + if (IsSplit(node)) { + return {1}; + } + if (IsConcatV1(node)) { + return {1}; + } + if (IsAdd(node) || IsMul(node) || IsRealDiv(node) || + IsSquaredDifference(node) || IsSub(node)) { + return {0, 1}; + } + if (node.input_size() > 0 && !IsControlInput(node.input(0))) { + return {0}; + } + return {}; + } }; class AddNProcessor : public AgnosticNodeProcessor { @@ -801,42 +829,49 @@ class BinaryOpProcessor : public AgnosticNodeProcessor { public: explicit BinaryOpProcessor(const OptimizeContext& opt_cxt) : AgnosticNodeProcessor(opt_cxt) { - is_4d_with_vector_ = Is4DOperateWithVector(); + is_4d_with_vector_ = IsNDOperateWithMD(4, 1); } protected: bool ShouldProcess() const override { + // TODO(yaozhang): Support IsNDOperateWithMD(1, 4): first input is a vector + // and the second input is a 4D tensor; and update CustomizedProcessing() + // accordingly. return !MustPreserve() && IsDimsFour(*node_) && HasOutputs() && IsNodeAfterNCHWToNHWC() && - (Is4DOperateWithND(4) || Is4DOperateWithScalar() || - Is4DOperateWithVector()) && + (IsNDOperateWithMD(4, 0) || IsNDOperateWithMD(4, 1) || + IsNDOperateWithMD(4, 4) || IsNDOperateWithMD(0, 4)) && IsOnGPU(); } std::vector GetInputPos() const override { - std::vector input_pos = {0}; - if (Is4DOperateWithND(4)) { + std::vector input_pos; + auto input0 = node_map_->GetNode(node_->input(0)); + auto input1 = node_map_->GetNode(node_->input(1)); + if (IsDimsFour(*input0)) { + input_pos.push_back(0); + } + if (IsDimsFour(*input1)) { input_pos.push_back(1); } return input_pos; } - bool Is4DOperateWithND(int n) const { + bool IsDimsFour(const NodeDef& node) const { + return NodeProcessor::IsDimsFour(node) || IsNodeNCHWToNHWC(node.name()); + } + + bool IsNDOperateWithMD(int n, int m) const { auto input0 = node_map_->GetNode(node_->input(0)); auto input1 = node_map_->GetNode(node_->input(1)); if (input0 && input1) { - return (IsDimsFour(*input0) || IsNodeNCHWToNHWC(input0->name())) && - ((n == 4) - ? (IsDimsFour(*input1) || IsNodeNCHWToNHWC(input1->name())) - : IsDimsN(*input1, n)); + bool input0_is_n = (n == 4) ? IsDimsFour(*input0) : IsDimsN(*input0, n); + bool input1_is_m = (m == 4) ? IsDimsFour(*input1) : IsDimsN(*input1, m); + return input0_is_n && input1_is_m; } return false; } - bool Is4DOperateWithScalar() const { return Is4DOperateWithND(0); } - - bool Is4DOperateWithVector() const { return Is4DOperateWithND(1); } - NodeDef* AddNodeShapeConst(const string& name, int num_channels) { NodeDef* node = graph_->add_node(); node_map_->AddNode(name, node); diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index e8f7b8ac3c..363b4c3fd8 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -298,6 +298,39 @@ TEST_F(LayoutOptimizerTest, Connectivity) { EXPECT_EQ(node_i2_output->input(0), "i1"); } +TEST_F(LayoutOptimizerTest, ConnectivityBinaryOpWithInputScalarAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto i1 = ops::Identity(s.WithOpName("i1"), conv); + auto i2 = ops::Identity(s.WithOpName("i2"), i1); + auto scalar_sub = ops::Const(s.WithOpName("scalar_sub"), 3.0f, {}); + auto sub = ops::Sub(s.WithOpName("sub"), scalar_sub, i2); + auto i3 = ops::Identity(s.WithOpName("i3"), sub); + auto i4 = ops::Identity(s.WithOpName("i4"), i3); + auto i5 = ops::Identity(s.WithOpName("i5"), i4); + auto scalar_mul = ops::Const(s.WithOpName("scalar_mul"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), scalar_mul, i5); + auto i6 = ops::Identity(s.WithOpName("i6"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + // Make the graph not in topological order to test the handling of multi-hop + // connectivity (here we say two nodes are connected if all nodes in the + // middle are layout agnostic). If the graph is already in topological order, + // the problem is easier, where layout optimizer only needs to check + // single-hop connectivity. + NodeMap node_map_original(&item.graph); + auto node_i1 = node_map_original.GetNode("i1"); + auto node_mul = node_map_original.GetNode("mul"); + node_mul->Swap(node_i1); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map_output(&output); + auto mul_node = node_map_output.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "scalar_mul"); + EXPECT_EQ(mul_node->input(1), "i5"); +} + TEST_F(LayoutOptimizerTest, PreserveFetch) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); auto conv = SimpleConv2D(&s, 3, 2, "VALID"); @@ -571,6 +604,101 @@ TEST_F(LayoutOptimizerTest, Sum) { */ } +TEST_F(LayoutOptimizerTest, MulScalarAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), scalar, conv); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "scalar"); + EXPECT_EQ(mul_node->input(1), "Conv2D"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAndScalar) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), conv, scalar); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "scalar"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto i = ops::Identity(s.WithOpName("i"), conv); + auto mul = ops::Mul(s.WithOpName("mul"), conv, i); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "i"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAndVector) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); + auto mul = ops::Mul(s.WithOpName("mul"), conv, vector); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "LayoutOptimizerReshapeNHWCToNCHW-mul-vector"); + auto mul_const = node_map.GetNode("LayoutOptimizerReshapeConst-mul-vector"); + Tensor tensor; + EXPECT_TRUE( + tensor.FromProto(mul_const->mutable_attr()->at({"value"}).tensor())); + Tensor tensor_expected(DT_INT32, {4}); + test::FillValues(&tensor_expected, {1, 2, 1, 1}); + test::ExpectTensorEqual(tensor_expected, tensor); +} + +TEST_F(LayoutOptimizerTest, MulVectorAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); + auto mul = ops::Mul(s.WithOpName("mul"), vector, conv); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + // TODO(yaozhang): Support vector as the first input and 4d tensor as the + // second input for BinaryOpProcessor. + EXPECT_EQ(mul_node->input(0), "vector"); + EXPECT_EQ(mul_node->input(1), + "LayoutOptimizerTransposeNCHWToNHWC-Conv2D-mul-1"); +} + } // namespace } // namespace grappler } // namespace tensorflow -- GitLab From 15b06e060af59a1e30f4a9079679718aaa68dbc7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 13:48:37 -0800 Subject: [PATCH 0158/1924] Add R1 slice tests. PiperOrigin-RevId: 177502259 --- tensorflow/compiler/xla/tests/slice_test.cc | 90 +++++++++++++++------ 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc index c21124750a..4db566f784 100644 --- a/tensorflow/compiler/xla/tests/slice_test.cc +++ b/tensorflow/compiler/xla/tests/slice_test.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -211,6 +212,13 @@ class SliceR1Test : public ClientLibraryTestBase, } }; +string SliceR1TestDataToString(const ::testing::TestParamInfo& data) { + const R1Spec& spec = data.param; + return ::tensorflow::strings::Printf("%lld_%lld_%lld_%lld", spec.input_dim0, + spec.slice_start, spec.slice_limit, + spec.slice_stride); +} + XLA_TEST_P(SliceR1Test, DoIt_F32) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_F64) { Run(GetParam()); } @@ -223,30 +231,66 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run(GetParam()); } XLA_TEST_P(SliceR1Test, DoIt_S64) { Run(GetParam()); } -INSTANTIATE_TEST_CASE_P( // - SliceR1TestInstantiation, // - SliceR1Test, // - ::testing::Values( // - R1Spec{10, 0, 0, 1}, // - R1Spec{10, 7, 7, 1}, // - R1Spec{10, 2, 4, 1}, // - R1Spec{10, 2, 4, 2}, // - R1Spec{10, 0, 10, 1}, // - R1Spec{1024, 1024 - 4, 1024, 1}, // - R1Spec{4096, 7, 7 + 1024, 1}, // - R1Spec{10, 0, 10, 2}, // - R1Spec{10, 0, 10, 3}, // - R1Spec{10, 0, 10, 4}, // - R1Spec{10, 0, 10, 5}, // - R1Spec{10, 0, 10, 10}, // - R1Spec{500, 200, 400, 7}, // - R1Spec{4096, 1, 4095, 3}, // - R1Spec{2047, 1024 - 24, 1024 + 160, 31}, // - R1Spec{2047, 1, 2046, 3 * 128}, // - R1Spec{4096, 1024 + 3, 4095, 500}, // - R1Spec{8192, 0, 8192, 1024 * 3 + 400} // - ) // +// Tests for R1 slice ops. +// The format for each testcase is {input size, start, limit, stride}. +// clang-format off +INSTANTIATE_TEST_CASE_P( + SliceR1TestInstantiation, + SliceR1Test, + ::testing::Values( + R1Spec{10, 0, 0, 1}, + R1Spec{10, 7, 7, 1}, + R1Spec{10, 0, 5, 1}, + R1Spec{10, 3, 5, 1}, + R1Spec{10, 0, 10, 1}, + R1Spec{1024, 0, 5, 1}, + R1Spec{1024, 3, 5, 1}, + R1Spec{1024 + 17, 0, 5, 1}, + R1Spec{1024 + 17, 3, 5, 1}, + R1Spec{1024 + 17, 1024, 1024 + 6, 1}, + R1Spec{1024 + 17, 1024 + 1, 1024 + 6, 1}, + R1Spec{1024, 1024 - 4, 1024, 1}, + R1Spec{4 * 1024, 7, 7 + 1024, 1}, + R1Spec{4 * 1024, 0, 4 * 1024, 1}, + R1Spec{4 * 1024, 1, 4 * 1024 - 1, 1}, + R1Spec{4 * 1024, 1024, 3 * 1024, 1}, + R1Spec{4 * 1024, 1024 + 1, 3 * 1024 - 1, 1}, + R1Spec{16 * 1024, 0, 5, 1}, + R1Spec{16 * 1024, 3, 5, 1}, + R1Spec{16 * 1024 + 17, 0, 5, 1}, + R1Spec{16 * 1024 + 17, 3, 5, 1}, + R1Spec{16 * 1024 + 17, 16 * 1024, 16 * 1024 + 6, 1}, + R1Spec{16 * 1024 + 17, 16 * 1024 + 1, 16 * 1024 + 6, 1}, + R1Spec{16 * 1024, 4 * 1024 - 17, 8 * 1024 - 18, 1}, + R1Spec{64 * 1024, 0, 64 * 1024, 1}, + R1Spec{64 * 1024, 1, 64 * 1024 - 1, 1}, + R1Spec{64 * 1024, 1024, 63 * 1024, 1}, + R1Spec{64 * 1024, 1024 + 1, 63 * 1024 - 1, 1}, + R1Spec{64 * 1024, 32 * 1024, 33 * 1024, 1}, + R1Spec{64 * 1024, 32 * 1024 + 1, 33 * 1024 - 1, 1}, + R1Spec{64 * 1024, 32 * 1024 - 17, 36 * 1024 - 18, 1}, +// TODO(b/69425338): This uses too much memory on GPU. +#ifndef XLA_TEST_BACKEND_GPU + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1}, + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1}, + R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1}, +#endif + R1Spec{10, 2, 4, 2}, + R1Spec{10, 0, 10, 2}, + R1Spec{10, 0, 10, 3}, + R1Spec{10, 0, 10, 4}, + R1Spec{10, 0, 10, 5}, + R1Spec{10, 0, 10, 10}, + R1Spec{500, 200, 400, 7}, + R1Spec{4096, 1, 4095, 3}, + R1Spec{2047, 1024 - 24, 1024 + 160, 31}, + R1Spec{2047, 1, 2046, 3 * 128}, + R1Spec{4096, 1024 + 3, 4095, 500}, + R1Spec{8192, 0, 8192, 1024 * 3 + 400} + ), + SliceR1TestDataToString ); +// clang-format on struct R2Spec { int64 input_dim0; -- GitLab From 39cac0519176d1244b0e29d6c28691189ea755ec Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 30 Nov 2017 13:50:13 -0800 Subject: [PATCH 0159/1924] [TF:XLA] Allow bfloat16 types in more places. PiperOrigin-RevId: 177502497 --- .../compiler/tf2xla/kernels/matmul_op.cc | 4 +- tensorflow/compiler/tf2xla/lib/util.cc | 3 ++ tensorflow/compiler/tf2xla/xla_helpers.cc | 7 ++- tensorflow/compiler/xla/literal_util.cc | 6 ++- tensorflow/core/framework/bfloat16_test.cc | 12 ++++++ tensorflow/core/framework/numeric_types.h | 43 ++++++++++++++++--- 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc index fcef497e58..a62d233526 100644 --- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc @@ -23,8 +23,8 @@ limitations under the License. namespace tensorflow { namespace { -constexpr std::array kMatmulTypes = { - {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}}; +constexpr std::array kMatmulTypes = { + {DT_HALF, DT_BFLOAT16, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}}; class MatMulOp : public XlaOpKernel { public: diff --git a/tensorflow/compiler/tf2xla/lib/util.cc b/tensorflow/compiler/tf2xla/lib/util.cc index 7ffe0aa6df..943248aedb 100644 --- a/tensorflow/compiler/tf2xla/lib/util.cc +++ b/tensorflow/compiler/tf2xla/lib/util.cc @@ -40,6 +40,9 @@ xla::ComputationDataHandle FloatLiteral(xla::ComputationBuilder* builder, case xla::F16: return builder->ConstantR0(static_cast(value)); break; + case xla::BF16: + return builder->ConstantR0(static_cast(value)); + break; case xla::F32: return builder->ConstantR0(static_cast(value)); break; diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc index 9c3e15d2fa..ec9e535b70 100644 --- a/tensorflow/compiler/tf2xla/xla_helpers.cc +++ b/tensorflow/compiler/tf2xla/xla_helpers.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// This file defines helper routines for Tla JIT compilation. +// This file defines helper routines for XLA compilation. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/lib/util.h" @@ -121,6 +121,8 @@ xla::ComputationDataHandle XlaHelpers::One(xla::ComputationBuilder* b, xla::ComputationDataHandle XlaHelpers::Epsilon(xla::ComputationBuilder* b, DataType data_type) { switch (data_type) { + case DT_BFLOAT16: + return b->ConstantR0(bfloat16::epsilon()); case DT_FLOAT: return b->ConstantR0(std::numeric_limits::epsilon()); case DT_DOUBLE: @@ -169,6 +171,9 @@ xla::ComputationDataHandle XlaHelpers::IntegerLiteral( case xla::S16: case xla::U16: LOG(FATAL) << "u16/s16 literals not yet implemented"; + case xla::BF16: + literal = *xla::Literal::CreateR0(static_cast(value)); + break; case xla::F16: literal = *xla::Literal::CreateR0(static_cast(value)); diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index 93d3cd425f..250df5f4d5 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -252,6 +252,10 @@ Status Literal::Copy(const Literal& src_literal, return *Literal::CreateR0(1); case S64: return *Literal::CreateR0(1); + case F16: + return *Literal::CreateR0(static_cast(1.0f)); + case BF16: + return *Literal::CreateR0(static_cast(1.0f)); case F32: return *Literal::CreateR0(1); case F64: @@ -263,8 +267,6 @@ Status Literal::Copy(const Literal& src_literal, case S16: case U16: LOG(FATAL) << "u16/s16 literals not yet implemented"; - case F16: - return *Literal::CreateR0(static_cast(1.0f)); case TUPLE: LOG(FATAL) << "tuple element type cannot take on value of 1"; case OPAQUE: diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc index 6e45338751..17e6209f8e 100644 --- a/tensorflow/core/framework/bfloat16_test.cc +++ b/tensorflow/core/framework/bfloat16_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/framework/bfloat16.h" +#include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/lib/core/casts.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" @@ -104,6 +105,17 @@ TEST(Bfloat16Test, Conversion) { } } +TEST(Bfloat16Test, Epsilon) { + EXPECT_LT(1.0f, static_cast(bfloat16::epsilon() + bfloat16(1.0f))); + EXPECT_EQ(1.0f, static_cast((bfloat16::epsilon() / bfloat16(2.0f)) + + bfloat16(1.0f))); +} + +TEST(Bfloat16Test, Negate) { + EXPECT_EQ(-3.0f, static_cast(-bfloat16(3.0f))); + EXPECT_EQ(4.5f, static_cast(-bfloat16(-4.5f))); +} + static void BM_FloatToBFloat16(int iters) { testing::StopTiming(); static const int N = 32 << 20; diff --git a/tensorflow/core/framework/numeric_types.h b/tensorflow/core/framework/numeric_types.h index 2b080e13fd..29cac26244 100644 --- a/tensorflow/core/framework/numeric_types.h +++ b/tensorflow/core/framework/numeric_types.h @@ -121,15 +121,48 @@ struct bfloat16 { return static_cast(float(*this)); } + static bfloat16 epsilon() { + bfloat16 x; + x.value = 0x3c00; // 0x1.0p-7 + return x; + } + uint16_t value; }; -inline bool operator==(const bfloat16 a, const bfloat16 b) { - return a.value == b.value; +inline bfloat16 operator+(bfloat16 a, bfloat16 b) { + return bfloat16(static_cast(a) + static_cast(b)); } - -inline bool operator!=(const bfloat16 a, const bfloat16 b) { - return a.value != b.value; +inline bfloat16 operator-(bfloat16 a, bfloat16 b) { + return bfloat16(static_cast(a) - static_cast(b)); +} +inline bfloat16 operator*(bfloat16 a, bfloat16 b) { + return bfloat16(static_cast(a) * static_cast(b)); +} +inline bfloat16 operator/(bfloat16 a, bfloat16 b) { + return bfloat16(static_cast(a) / static_cast(b)); +} +inline bfloat16 operator-(bfloat16 a) { + a.value ^= 0x8000; + return a; +} +inline bool operator<(bfloat16 a, bfloat16 b) { + return static_cast(a) < static_cast(b); +} +inline bool operator<=(bfloat16 a, bfloat16 b) { + return static_cast(a) <= static_cast(b); +} +inline bool operator==(bfloat16 a, bfloat16 b) { + return static_cast(a) == static_cast(b); +} +inline bool operator!=(bfloat16 a, bfloat16 b) { + return static_cast(a) != static_cast(b); +} +inline bool operator>(bfloat16 a, bfloat16 b) { + return static_cast(a) > static_cast(b); +} +inline bool operator>=(bfloat16 a, bfloat16 b) { + return static_cast(a) >= static_cast(b); } } // end namespace tensorflow -- GitLab From bc2b4b0679dc6e4ad6dc543d475d759f3ad6cadf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 13:57:33 -0800 Subject: [PATCH 0160/1924] Enable tests that pass now with the new copy insertion. PiperOrigin-RevId: 177503567 --- tensorflow/compiler/xla/tests/while_test.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc index 49f673f5f0..f3f10517e3 100644 --- a/tensorflow/compiler/xla/tests/while_test.cc +++ b/tensorflow/compiler/xla/tests/while_test.cc @@ -357,8 +357,7 @@ TEST_F(WhileTest, WhileWithVectorResultIntoTuple) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0.0001)); } -// TODO(b/63003356): 11-06-2017: fails on all back-ends with incorrect result. -TEST_F(WhileTest, DISABLED_WhileWithPermutationAndTupleResult) { +TEST_F(WhileTest, WhileWithPermutationAndTupleResult) { std::vector shape_elements = { ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3})}; @@ -411,8 +410,7 @@ TEST_F(WhileTest, DISABLED_WhileWithPermutationAndTupleResult) { ComputeAndCompareTuple(&builder, *expected, {}, ErrorSpec(0.0001)); } -// TODO(b/63003356): 11-06-2017: fails on all back-ends with incorrect result. -TEST_F(WhileTest, DISABLED_WhileWithPermutationAndVectorResult) { +TEST_F(WhileTest, WhileWithPermutationAndVectorResult) { std::vector shape_elements = { ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3}), ShapeUtil::MakeShape(F32, {3})}; -- GitLab From 99525c7e1e21b3548eafdc7ae606ac1df2bf06fe Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 30 Nov 2017 14:11:17 -0800 Subject: [PATCH 0161/1924] Automated g4 rollback of changelist 177499365 PiperOrigin-RevId: 177505909 --- .../grappler/optimizers/layout_optimizer.cc | 87 ++++-------- .../optimizers/layout_optimizer_test.cc | 128 ------------------ 2 files changed, 26 insertions(+), 189 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index 97c8e6f907..ef4b015295 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -761,52 +761,24 @@ class AgnosticNodeProcessor : public NodeProcessor { bool IsNodeAfterNCHWToNHWC() const { std::set ops_format_agnostic = GetOpsFormatAgnostic(); - std::deque queue; - auto first_node_pos = DataInputPos(*node_); - for (const auto& pos : first_node_pos) { - auto input_node = node_map_->GetNode(node_->input(pos)); - queue.push_back(input_node); - } - // The code will exit this while loop in one iteration in most cases, as the - // graph is already topologically sorted. - while (!queue.empty()) { - NodeDef* current_node = queue.front(); - queue.pop_front(); - if (IsNodeNCHWToNHWC(current_node->name())) { + auto node = node_map_->GetNode(node_->name()); + while (node->input_size() > 0) { + int data_input_pos = 0; + if (IsConcatV1(*node) || IsSplit(*node)) { + data_input_pos = 1; + } + node = node_map_->GetNode(node->input(data_input_pos)); + if (IsNodeNCHWToNHWC(node->name())) { return true; } - // We only continue searching if the path is connected through - // format-agnostic nodes. - if (ops_format_agnostic.find(current_node->op()) != - ops_format_agnostic.end()) { - auto current_node_pos = DataInputPos(*current_node); - for (const auto& pos : current_node_pos) { - auto input_node = node_map_->GetNode(current_node->input(pos)); - queue.push_back(input_node); - } + bool connected = + ops_format_agnostic.find(node->op()) != ops_format_agnostic.end(); + if (!connected) { + return false; } } return false; } - - private: - std::vector DataInputPos(const NodeDef& node) const { - std::vector pos; - if (IsSplit(node)) { - return {1}; - } - if (IsConcatV1(node)) { - return {1}; - } - if (IsAdd(node) || IsMul(node) || IsRealDiv(node) || - IsSquaredDifference(node) || IsSub(node)) { - return {0, 1}; - } - if (node.input_size() > 0 && !IsControlInput(node.input(0))) { - return {0}; - } - return {}; - } }; class AddNProcessor : public AgnosticNodeProcessor { @@ -829,49 +801,42 @@ class BinaryOpProcessor : public AgnosticNodeProcessor { public: explicit BinaryOpProcessor(const OptimizeContext& opt_cxt) : AgnosticNodeProcessor(opt_cxt) { - is_4d_with_vector_ = IsNDOperateWithMD(4, 1); + is_4d_with_vector_ = Is4DOperateWithVector(); } protected: bool ShouldProcess() const override { - // TODO(yaozhang): Support IsNDOperateWithMD(1, 4): first input is a vector - // and the second input is a 4D tensor; and update CustomizedProcessing() - // accordingly. return !MustPreserve() && IsDimsFour(*node_) && HasOutputs() && IsNodeAfterNCHWToNHWC() && - (IsNDOperateWithMD(4, 0) || IsNDOperateWithMD(4, 1) || - IsNDOperateWithMD(4, 4) || IsNDOperateWithMD(0, 4)) && + (Is4DOperateWithND(4) || Is4DOperateWithScalar() || + Is4DOperateWithVector()) && IsOnGPU(); } std::vector GetInputPos() const override { - std::vector input_pos; - auto input0 = node_map_->GetNode(node_->input(0)); - auto input1 = node_map_->GetNode(node_->input(1)); - if (IsDimsFour(*input0)) { - input_pos.push_back(0); - } - if (IsDimsFour(*input1)) { + std::vector input_pos = {0}; + if (Is4DOperateWithND(4)) { input_pos.push_back(1); } return input_pos; } - bool IsDimsFour(const NodeDef& node) const { - return NodeProcessor::IsDimsFour(node) || IsNodeNCHWToNHWC(node.name()); - } - - bool IsNDOperateWithMD(int n, int m) const { + bool Is4DOperateWithND(int n) const { auto input0 = node_map_->GetNode(node_->input(0)); auto input1 = node_map_->GetNode(node_->input(1)); if (input0 && input1) { - bool input0_is_n = (n == 4) ? IsDimsFour(*input0) : IsDimsN(*input0, n); - bool input1_is_m = (m == 4) ? IsDimsFour(*input1) : IsDimsN(*input1, m); - return input0_is_n && input1_is_m; + return (IsDimsFour(*input0) || IsNodeNCHWToNHWC(input0->name())) && + ((n == 4) + ? (IsDimsFour(*input1) || IsNodeNCHWToNHWC(input1->name())) + : IsDimsN(*input1, n)); } return false; } + bool Is4DOperateWithScalar() const { return Is4DOperateWithND(0); } + + bool Is4DOperateWithVector() const { return Is4DOperateWithND(1); } + NodeDef* AddNodeShapeConst(const string& name, int num_channels) { NodeDef* node = graph_->add_node(); node_map_->AddNode(name, node); diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index 363b4c3fd8..e8f7b8ac3c 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -298,39 +298,6 @@ TEST_F(LayoutOptimizerTest, Connectivity) { EXPECT_EQ(node_i2_output->input(0), "i1"); } -TEST_F(LayoutOptimizerTest, ConnectivityBinaryOpWithInputScalarAnd4D) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto i1 = ops::Identity(s.WithOpName("i1"), conv); - auto i2 = ops::Identity(s.WithOpName("i2"), i1); - auto scalar_sub = ops::Const(s.WithOpName("scalar_sub"), 3.0f, {}); - auto sub = ops::Sub(s.WithOpName("sub"), scalar_sub, i2); - auto i3 = ops::Identity(s.WithOpName("i3"), sub); - auto i4 = ops::Identity(s.WithOpName("i4"), i3); - auto i5 = ops::Identity(s.WithOpName("i5"), i4); - auto scalar_mul = ops::Const(s.WithOpName("scalar_mul"), 3.0f, {}); - auto mul = ops::Mul(s.WithOpName("mul"), scalar_mul, i5); - auto i6 = ops::Identity(s.WithOpName("i6"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - // Make the graph not in topological order to test the handling of multi-hop - // connectivity (here we say two nodes are connected if all nodes in the - // middle are layout agnostic). If the graph is already in topological order, - // the problem is easier, where layout optimizer only needs to check - // single-hop connectivity. - NodeMap node_map_original(&item.graph); - auto node_i1 = node_map_original.GetNode("i1"); - auto node_mul = node_map_original.GetNode("mul"); - node_mul->Swap(node_i1); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map_output(&output); - auto mul_node = node_map_output.GetNode("mul"); - EXPECT_EQ(mul_node->input(0), "scalar_mul"); - EXPECT_EQ(mul_node->input(1), "i5"); -} - TEST_F(LayoutOptimizerTest, PreserveFetch) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); auto conv = SimpleConv2D(&s, 3, 2, "VALID"); @@ -604,101 +571,6 @@ TEST_F(LayoutOptimizerTest, Sum) { */ } -TEST_F(LayoutOptimizerTest, MulScalarAnd4D) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); - auto mul = ops::Mul(s.WithOpName("mul"), scalar, conv); - auto o = ops::Identity(s.WithOpName("o"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map(&output); - auto mul_node = node_map.GetNode("mul"); - EXPECT_EQ(mul_node->input(0), "scalar"); - EXPECT_EQ(mul_node->input(1), "Conv2D"); -} - -TEST_F(LayoutOptimizerTest, Mul4DAndScalar) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); - auto mul = ops::Mul(s.WithOpName("mul"), conv, scalar); - auto o = ops::Identity(s.WithOpName("o"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map(&output); - auto mul_node = node_map.GetNode("mul"); - EXPECT_EQ(mul_node->input(0), "Conv2D"); - EXPECT_EQ(mul_node->input(1), "scalar"); -} - -TEST_F(LayoutOptimizerTest, Mul4DAnd4D) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto i = ops::Identity(s.WithOpName("i"), conv); - auto mul = ops::Mul(s.WithOpName("mul"), conv, i); - auto o = ops::Identity(s.WithOpName("o"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map(&output); - auto mul_node = node_map.GetNode("mul"); - EXPECT_EQ(mul_node->input(0), "Conv2D"); - EXPECT_EQ(mul_node->input(1), "i"); -} - -TEST_F(LayoutOptimizerTest, Mul4DAndVector) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); - auto mul = ops::Mul(s.WithOpName("mul"), conv, vector); - auto o = ops::Identity(s.WithOpName("o"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map(&output); - auto mul_node = node_map.GetNode("mul"); - EXPECT_EQ(mul_node->input(0), "Conv2D"); - EXPECT_EQ(mul_node->input(1), "LayoutOptimizerReshapeNHWCToNCHW-mul-vector"); - auto mul_const = node_map.GetNode("LayoutOptimizerReshapeConst-mul-vector"); - Tensor tensor; - EXPECT_TRUE( - tensor.FromProto(mul_const->mutable_attr()->at({"value"}).tensor())); - Tensor tensor_expected(DT_INT32, {4}); - test::FillValues(&tensor_expected, {1, 2, 1, 1}); - test::ExpectTensorEqual(tensor_expected, tensor); -} - -TEST_F(LayoutOptimizerTest, MulVectorAnd4D) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - auto conv = SimpleConv2D(&s, 3, 2, "VALID"); - auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); - auto mul = ops::Mul(s.WithOpName("mul"), vector, conv); - auto o = ops::Identity(s.WithOpName("o"), mul); - GrapplerItem item; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - LayoutOptimizer optimizer; - GraphDef output; - Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); - NodeMap node_map(&output); - auto mul_node = node_map.GetNode("mul"); - // TODO(yaozhang): Support vector as the first input and 4d tensor as the - // second input for BinaryOpProcessor. - EXPECT_EQ(mul_node->input(0), "vector"); - EXPECT_EQ(mul_node->input(1), - "LayoutOptimizerTransposeNCHWToNHWC-Conv2D-mul-1"); -} - } // namespace } // namespace grappler } // namespace tensorflow -- GitLab From 3a011f904112fe8c61017248d343798569b174f0 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Thu, 30 Nov 2017 14:12:54 -0800 Subject: [PATCH 0162/1924] Disable state_saving_rnn_estimator_test in asan mode. PiperOrigin-RevId: 177506166 --- tensorflow/contrib/learn/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index 94920db574..26bbcab307 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -461,6 +461,7 @@ py_test( size = "medium", srcs = ["python/learn/estimators/state_saving_rnn_estimator_test.py"], srcs_version = "PY2AND3", + tags = ["noasan"], deps = [ ":learn", "//tensorflow/contrib/layers:layers_py", -- GitLab From 6ebb6d6465ddf2380430de7aa287676e9440df7e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 14:14:29 -0800 Subject: [PATCH 0163/1924] TF server should not crash when -v=1 is enabled. These WriteTextProto() calls are purely for diagnostics (and are usually called within IF_VLOG_IS_ON(1) guards), but if they fail to write to a file, they'll take down the entire calling process. Which makes debugging difficult, and seems rather astonishing. PiperOrigin-RevId: 177506379 --- tensorflow/compiler/tf2xla/dump_graph.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/tf2xla/dump_graph.cc b/tensorflow/compiler/tf2xla/dump_graph.cc index ddd912b873..03603ee9ba 100644 --- a/tensorflow/compiler/tf2xla/dump_graph.cc +++ b/tensorflow/compiler/tf2xla/dump_graph.cc @@ -63,7 +63,12 @@ string MakeUniquePath(string name) { string DumpGraphDefToFile(const string& name, GraphDef const& graph_def) { string path = MakeUniquePath(name); - TF_CHECK_OK(WriteTextProto(Env::Default(), path, graph_def)); + Status status = WriteTextProto(Env::Default(), path, graph_def); + if (!status.ok()) { + VLOG(1) << "Failed to dump GraphDef to file: " << path << " : " << status; + path.clear(); + path = "(unavailable)"; + } return path; } @@ -79,7 +84,13 @@ string DumpGraphToFile(const string& name, Graph const& graph, string DumpFunctionDefToFile(const string& name, FunctionDef const& fdef) { string path = MakeUniquePath(name); - TF_CHECK_OK(WriteTextProto(Env::Default(), path, fdef)); + Status status = WriteTextProto(Env::Default(), path, fdef); + if (!status.ok()) { + VLOG(1) << "Failed to dump FunctionDef to file: " << path << " : " + << status; + path.clear(); + path = "(unavailable)"; + } return path; } -- GitLab From 6bfc73a0b3c6810725a5eb0020470457cc5cc23e Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 30 Nov 2017 14:26:58 -0800 Subject: [PATCH 0164/1924] Extract out a MathUtil::GCD helper This fixes a TODO. PiperOrigin-RevId: 177508258 --- .../compiler/xla/service/cpu/ir_emitter.cc | 22 ++++---------- tensorflow/core/lib/core/arena.cc | 18 ++---------- tensorflow/core/lib/math/math_util.h | 17 +++++++++++ tensorflow/core/lib/math/math_util_test.cc | 29 +++++++++++++++++++ 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index f242e0acb8..bb75d3f49e 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -24,6 +24,7 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/math/math_util.h" #include "tensorflow/core/platform/logging.h" // IWYU pragma: no_include "llvm/IR/Intrinsics.gen.inc" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -1651,19 +1652,6 @@ void IrEmitter::EmitShardedVectorStore( } } -namespace { -// TODO(sanjoy): This is duplicated in tensorflow/core/lib/core/arena.cc. -// Extract out a common implementation to tensorflow/core/lib/math/math_util.h -uint32 GCD(uint32 x, uint32 y) { - while (y != 0) { - uint32 r = x % y; - x = y; - y = r; - } - return x; -} -} // namespace - StatusOr IrEmitter::EmitVectorizedReduce( HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, tensorflow::gtl::ArraySlice dimensions, HloComputation* function, @@ -1686,9 +1674,9 @@ StatusOr IrEmitter::EmitVectorizedReduce( std::find(dimensions.begin(), dimensions.end(), arg->shape().layout().minor_to_major(0)) != dimensions.end(); - unsigned element_alignment = - GCD(ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type()), - MinimumAlignmentForPrimitiveType(reduce->shape().element_type())); + unsigned element_alignment = tensorflow::MathUtil::GCD( + ShapeUtil::ByteSizeOfPrimitiveType(reduce->shape().element_type()), + MinimumAlignmentForPrimitiveType(reduce->shape().element_type())); if (is_reduction_over_minor_dimension) { // TODO(sanjoy): Implement vectorized reduction over the minor dimension. @@ -2463,7 +2451,7 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source, const llvm_ir::IrArray& source_array) { unsigned primitive_type_size = ShapeUtil::ByteSizeOfPrimitiveType(primitive_type); - unsigned element_alignment = GCD( + unsigned element_alignment = tensorflow::MathUtil::GCD( primitive_type_size, MinimumAlignmentForPrimitiveType(primitive_type)); llvm::Type* primitive_ptr_type = llvm::PointerType::getUnqual( llvm_ir::PrimitiveTypeToIrType(primitive_type, module_)); diff --git a/tensorflow/core/lib/core/arena.cc b/tensorflow/core/lib/core/arena.cc index 2a04f7bd39..55e481d0e6 100644 --- a/tensorflow/core/lib/core/arena.cc +++ b/tensorflow/core/lib/core/arena.cc @@ -28,6 +28,7 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/math/math_util.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mem.h" @@ -113,24 +114,11 @@ void Arena::MakeNewBlock(const uint32 alignment) { CHECK(SatisfyAlignment(alignment)); } -// The following simple numeric routines also exist in util/math/mathutil.h -// but we don't want to depend on that library. - -// Euclid's algorithm for Greatest Common Denominator. -static uint32 GCD(uint32 x, uint32 y) { - while (y != 0) { - uint32 r = x % y; - x = y; - y = r; - } - return x; -} - static uint32 LeastCommonMultiple(uint32 a, uint32 b) { if (a > b) { - return (a / GCD(a, b)) * b; + return (a / MathUtil::GCD(a, b)) * b; } else if (a < b) { - return (b / GCD(b, a)) * a; + return (b / MathUtil::GCD(b, a)) * a; } else { return a; } diff --git a/tensorflow/core/lib/math/math_util.h b/tensorflow/core/lib/math/math_util.h index 6f279865e7..9e71598622 100644 --- a/tensorflow/core/lib/math/math_util.h +++ b/tensorflow/core/lib/math/math_util.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_LIB_MATH_MATH_UTIL_H_ #define TENSORFLOW_LIB_MATH_MATH_UTIL_H_ +#include + #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" @@ -59,6 +61,9 @@ class MathUtil { template static IntegralType CeilOrFloorOfRatio(IntegralType numerator, IntegralType denominator); + + template + static IntegralType GCD(IntegralType x, IntegralType y); }; // ---- CeilOrFloorOfRatio ---- @@ -107,6 +112,18 @@ IntegralType MathUtil::CeilOrFloorOfRatio(IntegralType numerator, } } +template +IntegralType MathUtil::GCD(IntegralType a, IntegralType b) { + static_assert(std::is_unsigned::value, + "signed GCD not supported!"); + while (b != 0) { + IntegralType r = a % b; + a = b; + b = r; + } + return a; +} + } // namespace tensorflow #endif // TENSORFLOW_LIB_MATH_MATH_UTIL_H_ diff --git a/tensorflow/core/lib/math/math_util_test.cc b/tensorflow/core/lib/math/math_util_test.cc index eaf8c31a43..a96e5467c3 100644 --- a/tensorflow/core/lib/math/math_util_test.cc +++ b/tensorflow/core/lib/math/math_util_test.cc @@ -195,4 +195,33 @@ TEST(MathUtil, CeilOfRatio) { #endif } +struct GCDTestCase { + unsigned int x; + unsigned int y; + unsigned int gcd; +}; + +TEST(MathUtil, GCD) { + std::vector testcases({ + {10, 20, 10}, // + {27, 8, 1}, // + {4, 3, 1}, // + {6, 8, 2}, // + {5, 0, 5}, // + {5, 5, 5}, // + {0, 0, 0} // + }); + + for (const auto& tc : testcases) { + EXPECT_EQ(tc.gcd, MathUtil::GCD(tc.x, tc.y)); + EXPECT_EQ(tc.gcd, MathUtil::GCD(tc.y, tc.x)); + EXPECT_EQ(tc.gcd, MathUtil::GCD(tc.x, tc.y)); + EXPECT_EQ(tc.gcd, MathUtil::GCD(tc.y, tc.x)); + } + + const uint64 biggish_prime = 1666666667; + EXPECT_EQ(biggish_prime, + MathUtil::GCD(biggish_prime * 3, biggish_prime * 4)); +} + } // namespace tensorflow -- GitLab From ce4200eae990d7f5efdfb727939d38bf48001ba2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 15:20:30 -0800 Subject: [PATCH 0165/1924] Fix profiler to track some missed persistent bytes. PiperOrigin-RevId: 177516249 --- tensorflow/core/profiler/g3doc/options.md | 11 +- .../core/profiler/internal/tfprof_node.cc | 28 +++++ .../core/profiler/internal/tfprof_node.h | 10 +- .../profiler/internal/tfprof_show_test.cc | 37 +++--- .../profiler/internal/tfprof_stats_test.cc | 105 ++++++++++-------- tensorflow/core/profiler/tfprof_log.proto | 5 +- .../python/profiler/model_analyzer_test.py | 40 ++++++- 7 files changed, 157 insertions(+), 79 deletions(-) diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md index 4c73e372e3..dd12f76d6f 100644 --- a/tensorflow/core/profiler/g3doc/options.md +++ b/tensorflow/core/profiler/g3doc/options.md @@ -60,11 +60,14 @@ Currently, profiler only tracks the allocation of memory. As a result, the accumulated memory request is uaually larger than the peak memory of the overall model. -bytes: The memory allocations requested by the operation. -peak_bytes: The peak requested memory (not de-allocated) by the operation. -residual_bytes: The memory requested by the operation and not de-allocated +It's recommended to generate timeline to see the allocator memory usage over +time. + +`bytes`: The memory allocations requested by the operation. +`peak_bytes`: The peak requested memory (not de-allocated) by the operation. +`residual_bytes`: The memory requested by the operation and not de-allocated when Compute finishes. -output_bytes: The memory output by the operation. It's not necessarily requested +`output_bytes`: The memory output by the operation. It's not necessarily requested by the current operation. For example, it can be a tensor forwarded from input to output, with in-place mutation. diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc index 671b65d708..5cd1050bcc 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.cc +++ b/tensorflow/core/profiler/internal/tfprof_node.cc @@ -139,6 +139,25 @@ void ExecStep::AddMemoryStats(const string& dev, exec_.accelerator_persistent_bytes() + step_stat.memory_stats().device_persistent_memory_size()); } + + // TODO(xpan): Make this more accurate: + // High level: Memory tracking is suspicous and requires large scale + // clean up. + // Investigte the memory usage difference between CPU/GPU with OpViewTest. + // + // 1. OpKernelConstruction::allocate_xxx is not traced. Below, we only + // discuss OpKernelContext-related allocations. + // 2. allocate_output calls allocate_tensor, which is properly tracked in + // 'NodeExecStats.memory'. + // 3. allocate_temp is only tracked through record_xxx_temp. It appears + // in 'NodeExecStats.memory_stats'. + // 4. allocate_persistent calls allocate_tensor, which is properly tracked + // in 'NodeExecStats.memory'. However, there is no way to count it as + // persistent now. + // 5. record_xxx_persistent is called when allocate_persistent + // is not used and hence tracks some complementary bytes. It appears in + // 'NodeExecStats.memory_stats'. It's suspicious. But we should + // use it now since it covers constant op. int64 residual_bytes = 0; int64 requested_bytes = 0; int64 peak_bytes = 0; @@ -147,6 +166,15 @@ void ExecStep::AddMemoryStats(const string& dev, requested_bytes += mem.total_bytes(); peak_bytes += mem.peak_bytes(); } + residual_bytes += + exec_.host_persistent_bytes() + exec_.accelerator_persistent_bytes(); + requested_bytes += exec_.host_persistent_bytes() + + exec_.accelerator_persistent_bytes() + + exec_.host_temp_bytes() + exec_.accelerator_temp_bytes(); + peak_bytes += exec_.host_persistent_bytes() + + exec_.accelerator_persistent_bytes() + exec_.host_temp_bytes() + + exec_.accelerator_temp_bytes(); + exec_.set_requested_bytes(requested_bytes); exec_.set_residual_bytes(residual_bytes); exec_.set_peak_bytes(peak_bytes); diff --git a/tensorflow/core/profiler/internal/tfprof_node.h b/tensorflow/core/profiler/internal/tfprof_node.h index e2d0563a07..77c14cb792 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.h +++ b/tensorflow/core/profiler/internal/tfprof_node.h @@ -593,17 +593,11 @@ class TFGraphNode { int64 accelerator_persistent_bytes() const { int64 persistent_bytes = 0; for (const auto& exec : execs_) { - persistent_bytes += exec.second.accelerator_persistent_bytes(); + persistent_bytes = std::max(persistent_bytes, + exec.second.accelerator_persistent_bytes()); } return persistent_bytes; } - int64 host_persistent_bytes(int64 step) const { - auto exec = execs_.find(step); - if (exec == execs_.end()) { - return 0; - } - return exec->second.host_persistent_bytes(); - } const std::map>& output_memory( int64 step) const { auto exec = execs_.find(step); diff --git a/tensorflow/core/profiler/internal/tfprof_show_test.cc b/tensorflow/core/profiler/internal/tfprof_show_test.cc index 1f19f8c322..98773ae19e 100644 --- a/tensorflow/core/profiler/internal/tfprof_show_test.cc +++ b/tensorflow/core/profiler/internal/tfprof_show_test.cc @@ -105,12 +105,13 @@ TEST_F(TFProfShowTest, DumpScopeMode) { "node name | # parameters | # float_ops | requested bytes | peak bytes | " "residual bytes | output bytes | total execution time | accelerator " "execution time | cpu execution time\n_TFProfRoot (--/451 params, --/0 " - "flops, --/0B, --/0B, --/0B, --/2.56KB, --/13us, --/0us, --/13us)\n DW " - "(3x3x3x6, 162/162 params, 0/0 flops, 0B/0B, 0B/0B, 0B/0B, " - "1.28KB/1.28KB, 2us/2us, 0us/0us, 2us/2us)\n DW2 (2x2x6x12, 288/288 " - "params, 0/0 flops, 0B/0B, 0B/0B, 0B/0B, 1.28KB/1.28KB, 11us/11us, " - "0us/0us, 11us/11us)\n ScalarW (1, 1/1 params, 0/0 flops, 0B/0B, 0B/0B, " - "0B/0B, 0B/0B, 0us/0us, 0us/0us, 0us/0us)\n", + "flops, --/2.56KB, --/2.56KB, --/2.56KB, --/2.56KB, --/13us, --/0us, " + "--/13us)\n DW (3x3x3x6, 162/162 params, 0/0 flops, 1.28KB/1.28KB, " + "1.28KB/1.28KB, 1.28KB/1.28KB, 1.28KB/1.28KB, 2us/2us, 0us/0us, " + "2us/2us)\n DW2 (2x2x6x12, 288/288 params, 0/0 flops, 1.28KB/1.28KB, " + "1.28KB/1.28KB, 1.28KB/1.28KB, 1.28KB/1.28KB, 11us/11us, 0us/0us, " + "11us/11us)\n ScalarW (1, 1/1 params, 0/0 flops, 0B/0B, 0B/0B, 0B/0B, " + "0B/0B, 0us/0us, 0us/0us, 0us/0us)\n", dump_str); EXPECT_EQ(dump_str, TestToFromProto("scope", opts)); @@ -178,22 +179,22 @@ TEST_F(TFProfShowTest, DumpOpMode) { EXPECT_EQ( "nodename|requestedbytes|totalexecutiontime|acceleratorexecutiontime|" "cpuexecutiontime|#parameters|#float_ops|opoccurrence(run|defined)|" - "inputshapes\nVariableV20B(0.00%,0.00%),13us(100.00%,0.26%),0us(100.00%," - "0.00%),13us(100.00%,0.29%),451params(100.00%,100.00%),0float_ops(100.00%" - ",0.00%),2|3\n\ninput_type:\t(run*2|defined*3)\texec_time:13us\n\nAdd0B(" - "0.00%,0.00%),0us(99.74%,0.00%),0us(100.00%,0.00%),0us(99.71%,0.00%)," - "0params(0.00%,0.00%),0float_ops(100.00%,0.00%),0|3\n\ninput_type:0:1," - "\t1:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:2x2x6x12,\t1:1\t(" - "run*0|defined*1)\texec_time:0us\ninput_type:0:3x3x3x6,\t1:1\t(run*0|" - "defined*1)\texec_time:0us\n\nAssign0B(0.00%,0.00%),0us(99.74%,0.00%)," - "0us(100.00%,0.00%),0us(99.71%,0.00%),0params(0.00%,0.00%),0float_ops(" - "100.00%,0.00%),0|3\n\ninput_type:0:1,\t1:1\t(run*0|defined*1)\texec_" + "inputshapes\nVariableV22.56KB(100.00%,8.40%),13us(100.00%,0.26%),0us(" + "100.00%,0.00%),13us(100.00%,0.29%),451params(100.00%,100.00%),0float_" + "ops(100.00%,0.00%),2|3\n\ninput_type:\t(run*2|defined*3)\texec_time:" + "13us\n\nAdd0B(0.00%,0.00%),0us(99.74%,0.00%),0us(100.00%,0.00%),0us(99." + "71%,0.00%),0params(0.00%,0.00%),0float_ops(100.00%,0.00%),0|3\n\ninput_" + "type:0:1,\t1:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:2x2x6x12," + "\t1:1\t(run*0|defined*1)\texec_time:0us\ninput_type:0:3x3x3x6,\t1:1\t(" + "run*0|defined*1)\texec_time:0us\n\nAssign0B(0.00%,0.00%),0us(99.74%,0." + "00%),0us(100.00%,0.00%),0us(99.71%,0.00%),0params(0.00%,0.00%),0float_" + "ops(100.00%,0.00%),0|3\n\ninput_type:0:1,\t1:1\t(run*0|defined*1)\texec_" "time:0us\ninput_type:0:2x2x6x12,\t1:2x2x6x12\t(run*0|defined*1)\texec_" "time:0us\ninput_type:0:3x3x3x6,\t1:3x3x3x6\t(run*0|defined*1)\texec_" "time:0us\n\nConst0B(0.00%,0.00%),2us(99.74%,0.04%),0us(100.00%,0.00%)," "2us(99.71%,0.04%),0params(0.00%,0.00%),0float_ops(100.00%,0.00%),1|" - "10\n\ninput_type:\t(run*1|defined*10)\texec_time:2us\n\nConv2D14.59KB(" - "100.00%,100.00%),4.89ms(99.70%,98.87%),404us(100.00%,100.00%),4.49ms(99." + "10\n\ninput_type:\t(run*1|defined*10)\texec_time:2us\n\nConv2D27.90KB(" + "91.60%,91.60%),4.89ms(99.70%,98.87%),404us(100.00%,100.00%),4.49ms(99." "67%,98.77%),0params(0.00%,0.00%),10.44kfloat_ops(100.00%,100.00%),2|" "2\n\ninput_type:0:2x3x3x6,\t1:2x2x6x12\t(run*1|defined*1)\texec_time:" "597us\ninput_type:0:2x6x6x3,\t1:3x3x3x6\t(run*1|defined*1)\texec_time:4." diff --git a/tensorflow/core/profiler/internal/tfprof_stats_test.cc b/tensorflow/core/profiler/internal/tfprof_stats_test.cc index 2f2101d76b..b86a83cb1b 100644 --- a/tensorflow/core/profiler/internal/tfprof_stats_test.cc +++ b/tensorflow/core/profiler/internal/tfprof_stats_test.cc @@ -89,21 +89,27 @@ TEST_F(TFProfStatsTest, CustomOpType) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( - "name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_parameters: " - "451\nchildren {\n name: \"DW\"\n exec_micros: 2\n parameters: 162\n " - "total_exec_micros: 2\n total_parameters: 162\n devices: " + "name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_requested_bytes: " + "2560\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n exec_micros: " + "2\n requested_bytes: 1280\n parameters: 162\n total_exec_micros: 2\n " + " total_requested_bytes: 1280\n total_parameters: 162\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 2\n " "total_cpu_exec_micros: 2\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n output_bytes: 1280\n total_output_bytes: " - "1280\n}\nchildren {\n name: \"DW2\"\n exec_micros: 11\n parameters: " - "288\n total_exec_micros: 11\n total_parameters: 288\n devices: " + "total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n " + " output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: " + "1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"DW2\"\n " + "exec_micros: 11\n requested_bytes: 1280\n parameters: 288\n " + "total_exec_micros: 11\n total_requested_bytes: 1280\n " + "total_parameters: 288\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 11\n " "total_cpu_exec_micros: 11\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n output_bytes: 1280\n total_output_bytes: " - "1280\n}\nchildren {\n name: \"ScalarW\"\n parameters: 1\n " - "total_parameters: 1\n total_definition_count: " + "total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n " + " output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: " + "1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"ScalarW\"\n " + "parameters: 1\n total_parameters: 1\n total_definition_count: " "1\n}\ntotal_cpu_exec_micros: 13\ntotal_run_count: " - "2\ntotal_definition_count: 3\ntotal_output_bytes: 2560\n", + "2\ntotal_definition_count: 3\ntotal_peak_bytes: " + "2560\ntotal_residual_bytes: 2560\ntotal_output_bytes: 2560\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); @@ -119,21 +125,27 @@ TEST_F(TFProfStatsTest, CheckPointOpType) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( - "name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_parameters: " - "451\nchildren {\n name: \"DW\"\n exec_micros: 2\n parameters: 162\n " - "total_exec_micros: 2\n total_parameters: 162\n devices: " + "name: \"_TFProfRoot\"\ntotal_exec_micros: 13\ntotal_requested_bytes: " + "2560\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n exec_micros: " + "2\n requested_bytes: 1280\n parameters: 162\n total_exec_micros: 2\n " + " total_requested_bytes: 1280\n total_parameters: 162\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 2\n " "total_cpu_exec_micros: 2\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n output_bytes: 1280\n total_output_bytes: " - "1280\n}\nchildren {\n name: \"DW2\"\n exec_micros: 11\n parameters: " - "288\n total_exec_micros: 11\n total_parameters: 288\n devices: " + "total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n " + " output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: " + "1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"DW2\"\n " + "exec_micros: 11\n requested_bytes: 1280\n parameters: 288\n " + "total_exec_micros: 11\n total_requested_bytes: 1280\n " + "total_parameters: 288\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n cpu_exec_micros: 11\n " "total_cpu_exec_micros: 11\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n output_bytes: 1280\n total_output_bytes: " - "1280\n}\nchildren {\n name: \"ScalarW\"\n parameters: 1\n " - "total_parameters: 1\n total_definition_count: " + "total_definition_count: 1\n peak_bytes: 1280\n residual_bytes: 1280\n " + " output_bytes: 1280\n total_peak_bytes: 1280\n total_residual_bytes: " + "1280\n total_output_bytes: 1280\n}\nchildren {\n name: \"ScalarW\"\n " + "parameters: 1\n total_parameters: 1\n total_definition_count: " "1\n}\ntotal_cpu_exec_micros: 13\ntotal_run_count: " - "2\ntotal_definition_count: 3\ntotal_output_bytes: 2560\n", + "2\ntotal_definition_count: 3\ntotal_peak_bytes: " + "2560\ntotal_residual_bytes: 2560\ntotal_output_bytes: 2560\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); @@ -150,7 +162,7 @@ TEST_F(TFProfStatsTest, TestGraph) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( "name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: " - "14592\ntotal_parameters: 451\nchildren {\n name: " + "30464\ntotal_parameters: 451\nchildren {\n name: " "\"DW/Initializer/random_normal/mul\"\n children {\n name: " "\"DW/Initializer/random_normal/RandomStandardNormal\"\n children {\n " " name: \"DW/Initializer/random_normal/shape\"\n " @@ -166,7 +178,7 @@ TEST_F(TFProfStatsTest, TestGraph) { "4\n}\ntotal_float_ops: 10440\ntotal_accelerator_exec_micros: " "404\ntotal_cpu_exec_micros: 4541\ntotal_run_count: " "6\ntotal_definition_count: 32\ntotal_peak_bytes: " - "9984\ntotal_residual_bytes: 1280\ntotal_output_bytes: 4864\n", + "25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); @@ -181,9 +193,9 @@ TEST_F(TFProfStatsTest, TestFloatOps) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( "name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: " - "14592\ntotal_parameters: 451\nchildren {\n name: \"Conv2D\"\n " - "exec_micros: 4292\n requested_bytes: 9472\n total_exec_micros: 4292\n " - " total_requested_bytes: 9472\n devices: " + "30464\ntotal_parameters: 451\nchildren {\n name: \"Conv2D\"\n " + "exec_micros: 4292\n requested_bytes: 18176\n total_exec_micros: " + "4292\n total_requested_bytes: 18176\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 5832\n " "total_float_ops: 5832\n input_shapes {\n key: 0\n value {\n " "dim {\n size: 2\n }\n dim {\n size: 6\n " @@ -194,11 +206,11 @@ TEST_F(TFProfStatsTest, TestFloatOps) { "6\n }\n }\n }\n accelerator_exec_micros: 226\n " "cpu_exec_micros: 4066\n total_accelerator_exec_micros: 226\n " "total_cpu_exec_micros: 4066\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n peak_bytes: 5888\n residual_bytes: 768\n " - "output_bytes: 768\n total_peak_bytes: 5888\n total_residual_bytes: " + "total_definition_count: 1\n peak_bytes: 14592\n residual_bytes: 768\n " + " output_bytes: 768\n total_peak_bytes: 14592\n total_residual_bytes: " "768\n total_output_bytes: 768\n}\nchildren {\n name: \"Conv2D_1\"\n " - "exec_micros: 597\n requested_bytes: 5120\n total_exec_micros: 597\n " - "total_requested_bytes: 5120\n devices: " + "exec_micros: 597\n requested_bytes: 9728\n total_exec_micros: 597\n " + "total_requested_bytes: 9728\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 4608\n " "total_float_ops: 4608\n input_shapes {\n key: 0\n value {\n " "dim {\n size: 2\n }\n dim {\n size: 3\n " @@ -209,12 +221,12 @@ TEST_F(TFProfStatsTest, TestFloatOps) { "12\n }\n }\n }\n accelerator_exec_micros: 178\n " "cpu_exec_micros: 419\n total_accelerator_exec_micros: 178\n " "total_cpu_exec_micros: 419\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n peak_bytes: 4096\n residual_bytes: 512\n " - "output_bytes: 512\n total_peak_bytes: 4096\n total_residual_bytes: " + "total_definition_count: 1\n peak_bytes: 8704\n residual_bytes: 512\n " + "output_bytes: 512\n total_peak_bytes: 8704\n total_residual_bytes: " "512\n total_output_bytes: 512\n}\ntotal_float_ops: " "10440\ntotal_accelerator_exec_micros: 404\ntotal_cpu_exec_micros: " "4541\ntotal_run_count: 6\ntotal_definition_count: 35\ntotal_peak_bytes: " - "9984\ntotal_residual_bytes: 1280\ntotal_output_bytes: 4864\n", + "25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); @@ -231,9 +243,9 @@ TEST_F(TFProfStatsTest, TestAccountShownNameOnly) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( "name: \"_TFProfRoot\"\ntotal_exec_micros: 597\ntotal_requested_bytes: " - "5120\nchildren {\n name: \"Conv2D_1\"\n exec_micros: 597\n " - "requested_bytes: 5120\n total_exec_micros: 597\n " - "total_requested_bytes: 5120\n devices: " + "9728\nchildren {\n name: \"Conv2D_1\"\n exec_micros: 597\n " + "requested_bytes: 9728\n total_exec_micros: 597\n " + "total_requested_bytes: 9728\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n float_ops: 4608\n " "total_float_ops: 4608\n input_shapes {\n key: 0\n value {\n " "dim {\n size: 2\n }\n dim {\n size: 3\n " @@ -244,12 +256,12 @@ TEST_F(TFProfStatsTest, TestAccountShownNameOnly) { "12\n }\n }\n }\n accelerator_exec_micros: 178\n " "cpu_exec_micros: 419\n total_accelerator_exec_micros: 178\n " "total_cpu_exec_micros: 419\n run_count: 1\n total_run_count: 1\n " - "total_definition_count: 1\n peak_bytes: 4096\n residual_bytes: 512\n " - "output_bytes: 512\n total_peak_bytes: 4096\n total_residual_bytes: " + "total_definition_count: 1\n peak_bytes: 8704\n residual_bytes: 512\n " + "output_bytes: 512\n total_peak_bytes: 8704\n total_residual_bytes: " "512\n total_output_bytes: 512\n}\ntotal_float_ops: " "4608\ntotal_accelerator_exec_micros: 178\ntotal_cpu_exec_micros: " "419\ntotal_run_count: 1\ntotal_definition_count: 2\ntotal_peak_bytes: " - "4096\ntotal_residual_bytes: 512\ntotal_output_bytes: 512\n", + "8704\ntotal_residual_bytes: 512\ntotal_output_bytes: 512\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); @@ -265,8 +277,9 @@ TEST_F(TFProfStatsTest, TestShowTensorValue) { GraphNodeProto expected; CHECK(protobuf::TextFormat::ParseFromString( "name: \"_TFProfRoot\"\ntotal_exec_micros: 4945\ntotal_requested_bytes: " - "14592\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n " - "exec_micros: 2\n parameters: 162\n total_exec_micros: 2\n " + "30464\ntotal_parameters: 451\nchildren {\n name: \"DW\"\n " + "exec_micros: 2\n requested_bytes: 1280\n parameters: 162\n " + "total_exec_micros: 2\n total_requested_bytes: 1280\n " "total_parameters: 162\n devices: " "\"/job:localhost/replica:0/task:0/gpu:0\"\n tensor_value {\n dtype: " "DT_FLOAT\n value_double: -0.000534315\n value_double: " @@ -351,11 +364,13 @@ TEST_F(TFProfStatsTest, TestShowTensorValue) { "value_double: 0.000374641\n value_double: -0.00149603\n " "value_double: -0.000317367\n value_double: -0.000417829\n }\n " "cpu_exec_micros: 2\n total_cpu_exec_micros: 2\n run_count: 1\n " - "total_run_count: 1\n total_definition_count: 10\n output_bytes: " - "1280\n total_output_bytes: 1280\n}\ntotal_float_ops: " - "10440\ntotal_accelerator_exec_micros: 404\ntotal_cpu_exec_micros: " - "4541\ntotal_run_count: 6\ntotal_definition_count: 35\ntotal_peak_bytes: " - "9984\ntotal_residual_bytes: 1280\ntotal_output_bytes: 4864\n", + "total_run_count: 1\n total_definition_count: 10\n peak_bytes: 1280\n " + "residual_bytes: 1280\n output_bytes: 1280\n total_peak_bytes: 1280\n " + "total_residual_bytes: 1280\n total_output_bytes: " + "1280\n}\ntotal_float_ops: 10440\ntotal_accelerator_exec_micros: " + "404\ntotal_cpu_exec_micros: 4541\ntotal_run_count: " + "6\ntotal_definition_count: 35\ntotal_peak_bytes: " + "25856\ntotal_residual_bytes: 3840\ntotal_output_bytes: 4864\n", &expected)); EXPECT_EQ(expected.DebugString(), root.DebugString()); } diff --git a/tensorflow/core/profiler/tfprof_log.proto b/tensorflow/core/profiler/tfprof_log.proto index f92301133a..b49bdf64ac 100644 --- a/tensorflow/core/profiler/tfprof_log.proto +++ b/tensorflow/core/profiler/tfprof_log.proto @@ -124,9 +124,10 @@ message ExecProfile { int64 residual_bytes = 9; // Total bytes output by the op (not necessarily requested by the op). int64 output_bytes = 10; - // Total temporary bytes allocated and released by the op. + // NOTE: Please don't depend on the following 4 fields yet. Due to + // TensorFlow internal tracing issues, the numbers can be quite wrong. + // TODO(xpan): Fix the TensorFlow internal tracing. int64 host_temp_bytes = 11; - // Total persistent bytes (e.g. variable) allocated by the op. int64 host_persistent_bytes = 12; int64 accelerator_temp_bytes = 13; int64 accelerator_persistent_bytes = 14; diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index 698f8906d4..c39d0fa5b1 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -23,12 +23,15 @@ import os import random import re +import numpy as np + from tensorflow.core.profiler import profile_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.client import session from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile @@ -346,8 +349,8 @@ class PrintModelAnalysisTest(test.TestCase): with gfile.Open(outfile, 'r') as f: # pylint: disable=line-too-long self.assertEqual( - 'nodename|requestedbytes|peakbytes|residualbytes|outputbytes|totalexecutiontime|acceleratorexecutiontime|cpuexecutiontime|#parameters|opoccurrence(run|defined)|inputshapes\nConst0B(0', - f.read().replace('\t', '').replace(' ', '')[0:180]) + 'nodename|requestedbytes|peakbytes|residualbytes|outputbytes|totalexecutiontime|acceleratorexecutiontime|cpuexecutiontime|#parameters|opoccurrence(run|defined)|inputshapes', + f.read().replace('\t', '').replace(' ', '')[0:170]) # pylint: enable=line-too-long total_children = 0 @@ -694,6 +697,39 @@ class PrintModelAnalysisTest(test.TestCase): exception_str) self.assertTrue(mat is None) + def testTrackPersistentBytes(self): + ops.reset_default_graph() + a = array_ops.constant(np.ones((100, 100))) + b = array_ops.constant(np.ones((100, 100))) + c = a * b + + with session.Session() as sess: + run_options = config_pb2.RunOptions( + trace_level=config_pb2.RunOptions.FULL_TRACE) + run_metadata = config_pb2.RunMetadata() + sess.run(c, options=run_options, run_metadata=run_metadata) + + options = option_builder.ProfileOptionBuilder.time_and_memory() + options['min_bytes'] = 0 + options['select'] = ('bytes', 'peak_bytes', 'output_bytes', + 'residual_bytes') + ret = model_analyzer.profile( + sess.graph, run_meta=run_metadata, cmd='scope', options=options) + + run_metadata = config_pb2.RunMetadata() + sess.run(c, options=run_options, run_metadata=run_metadata) + ret2 = model_analyzer.profile( + sess.graph, run_meta=run_metadata, cmd='scope', options=options) + + n = lib.SearchTFProfNode(ret, 'mul') + n2 = lib.SearchTFProfNode(ret2, 'mul') + self.assertGreater(n.peak_bytes, 0) + self.assertGreater(n.output_bytes, 0) + self.assertGreater(n.residual_bytes, 0) + self.assertEqual(n.peak_bytes, n2.peak_bytes) + self.assertEqual(n.output_bytes, n2.output_bytes) + self.assertEqual(n.residual_bytes, n2.residual_bytes) + if __name__ == '__main__': test.main() -- GitLab From f69915dc152c5516e6bc88b93515cbb02a1fbfc5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 15:33:39 -0800 Subject: [PATCH 0166/1924] [XLA] Sanitize hlo names to match regexp "[a-zA-Z_][a-zA-Z0-9_.-]*". PiperOrigin-RevId: 177518046 --- tensorflow/compiler/xla/service/hlo_module.cc | 7 ++-- .../compiler/xla/service/name_uniquer.cc | 32 +++++++++++++++++++ .../compiler/xla/service/name_uniquer.h | 13 ++++++-- .../compiler/xla/service/name_uniquer_test.cc | 26 ++++++++++++--- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index faaf73ea1c..6fe2134466 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -35,14 +35,15 @@ namespace xla { HloModule::HloModule(const string& name, const VersionedComputationHandle& entry_computation_handle, const HloModuleConfig& config) - : name_(name), + : name_(NameUniquer::GetSanitizedName(name)), config_(config), has_entry_computation_handle_(true), entry_computation_handle_(entry_computation_handle) {} -HloModule::HloModule(const string& name) : name_(name) {} +HloModule::HloModule(const string& name) + : name_(NameUniquer::GetSanitizedName(name)) {} HloModule::HloModule(const string& name, const HloModuleConfig& config) - : name_(name), config_(config) {} + : name_(NameUniquer::GetSanitizedName(name)), config_(config) {} HloComputation* HloModule::AddComputationInternal( std::unique_ptr computation, bool is_entry, diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc index a0d08c288d..7d8c05fffa 100644 --- a/tensorflow/compiler/xla/service/name_uniquer.cc +++ b/tensorflow/compiler/xla/service/name_uniquer.cc @@ -17,12 +17,44 @@ limitations under the License. #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" namespace xla { +namespace { + +bool IsAllowed(char character) { + auto c = static_cast(character); + return (isalnum(c) != 0) || c == '_' || c == '.' || c == '-'; +} + +} // namespace + +NameUniquer::NameUniquer(const string& separator) { + CHECK(std::all_of(separator.begin(), separator.end(), IsAllowed)) + << "separator should comprises allowed characters only"; + separator_ = separator; +} + +/*static*/ string NameUniquer::GetSanitizedName(const string& name) { + string result = name; + CHECK(!result.empty()) << "name should not be empty"; + char c = static_cast(result[0]); + if (!isalpha(c) && c != '_') { + result[0] = '_'; + } + for (int i = 1; i < result.length(); i++) { + if (!IsAllowed(result[i])) { + result[i] = '_'; + } + } + return result; +} + string NameUniquer::GetUniqueName(tensorflow::StringPiece prefix) { string root = prefix.empty() ? "name" : prefix.ToString(); + root = GetSanitizedName(root); // Strip away numeric suffix (if any). Only recognize separator if it is in // the middle of the name. diff --git a/tensorflow/compiler/xla/service/name_uniquer.h b/tensorflow/compiler/xla/service/name_uniquer.h index ed379b5225..4139c2700b 100644 --- a/tensorflow/compiler/xla/service/name_uniquer.h +++ b/tensorflow/compiler/xla/service/name_uniquer.h @@ -28,14 +28,21 @@ namespace xla { // Simple stateful class that helps generate "unique" names. To use it, simply // call GetUniqueName as many times as needed. The names returned by // GetUniqueName are guaranteed to be distinct for this instance of the class. +// Note that the names will be sanitized to match regexp +// "[a-zA-Z_][a-zA-Z0-9_.-]*". class NameUniquer { public: - explicit NameUniquer(const string& separator = "__") - : separator_(separator) {} + // The separator must contain allowed characters only: "[a-zA-Z0-9_.-]". + explicit NameUniquer(const string& separator = "__"); - // Get a unique name in a string, with an optional prefix for convenience. + // Get a sanitized unique name in a string, with an optional prefix for + // convenience. string GetUniqueName(tensorflow::StringPiece prefix = ""); + // Sanitizes and returns the name. Unallowed characters will be replaced with + // '_'. The result will match the regexp "[a-zA-Z_][a-zA-Z0-9_.-]*". + static string GetSanitizedName(const string& name); + private: // The string to use to separate the prefix of the name from the uniquing // integer value. diff --git a/tensorflow/compiler/xla/service/name_uniquer_test.cc b/tensorflow/compiler/xla/service/name_uniquer_test.cc index 9f0747a6e2..4258cf1687 100644 --- a/tensorflow/compiler/xla/service/name_uniquer_test.cc +++ b/tensorflow/compiler/xla/service/name_uniquer_test.cc @@ -60,12 +60,30 @@ TEST_F(NameUniquerTest, NumericSuffixes) { EXPECT_EQ("bar", uniquer.GetUniqueName("bar.-1000")); EXPECT_EQ("bar.1", uniquer.GetUniqueName("bar.-2000")); EXPECT_EQ("bar.2", uniquer.GetUniqueName("bar.1")); +} + +TEST_F(NameUniquerTest, Sanitize) { + NameUniquer uniquer("_"); + + EXPECT_EQ("foo", uniquer.GetUniqueName("foo")); + EXPECT_EQ("foo_1", uniquer.GetUniqueName("foo")); + EXPECT_EQ("foo.54", uniquer.GetUniqueName("foo.54")); + EXPECT_EQ("foo_54", uniquer.GetUniqueName("foo_54")); + EXPECT_EQ("foo_54.1", uniquer.GetUniqueName("foo_54.1")); + EXPECT_EQ("foo_55", uniquer.GetUniqueName("foo")); + + // Invalid characters will be replaced with '_'. + EXPECT_EQ("bar", uniquer.GetUniqueName("bar<-1000")); + EXPECT_EQ("bar_1", uniquer.GetUniqueName("bar<-2000")); + EXPECT_EQ("bar_2", uniquer.GetUniqueName("bar_1")); // Separator is only recognized in the middle of the prefix. - EXPECT_EQ(".10", uniquer.GetUniqueName(".10")); - EXPECT_EQ(".10.1", uniquer.GetUniqueName(".10")); - EXPECT_EQ("foobar.", uniquer.GetUniqueName("foobar.")); - EXPECT_EQ("foobar..1", uniquer.GetUniqueName("foobar.")); + EXPECT_EQ("_10", uniquer.GetUniqueName( + ".10")); // the leading '.' is replaced with '_'. + EXPECT_EQ("_10_1", uniquer.GetUniqueName(".10")); + EXPECT_EQ("_10_2", uniquer.GetUniqueName("_10")); + EXPECT_EQ("foobar_", uniquer.GetUniqueName("foobar_")); + EXPECT_EQ("foobar__1", uniquer.GetUniqueName("foobar_")); } } // namespace -- GitLab From 0472116d163eeb77d51cabdc5fc67be917048870 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 15:34:36 -0800 Subject: [PATCH 0167/1924] [TF:XLA] Make tf_cnn_benchmarks run on CPU with XLA. Adds _cpu_jit to tf_cnn_benchmarks_xla BUILD rule and fixes an issue in XLA bridge triggered by XLA CPU compilation of whole graphs. In particular, modifies mark_for_compilation_pass.cc to skip _Retval nodes when looking for compilation candidates in the top level function. _Retval nodes are introduced in the input subgraph as a replacement for fetches. Including _Retval nodes into XLA clusters confuses encapsulate subgraph pass that expects a graph with no pre-existing _Retval nodes. PiperOrigin-RevId: 177518178 --- .../compiler/jit/mark_for_compilation_pass.cc | 7 +++++ .../jit/mark_for_compilation_pass_test.cc | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index 74c9791f5e..aceedeb823 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -210,6 +210,13 @@ Status FindCompilationCandidates( !IsCompilableWhile(*node, jit_device_type, 0, lib_runtime)) { continue; } + // _Retval nodes in a top-level function represent fetches. + // Do not compile them. + if (node->type_string() == "_Retval") { + VLOG(2) << "Compilation rejected node: return value " << node->name() + << ": " << node->type_string(); + continue; + } candidates->insert(node); } return Status::OK(); diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index b3d258aea1..454f0aeae9 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -525,5 +525,32 @@ TEST(XlaCompilationTest, IllegalCycle_UsefulErrorMessage) { "+-- c\n")); } +TEST(XlaCompilationTest, Retval) { + std::unique_ptr graph(new Graph(OpRegistry::Global())); + GraphDef graphdef; + { + GraphDefBuilder builder(GraphDefBuilder::kFailImmediately); + Node* a = ops::SourceOp("Const", builder.opts() + .WithName("A") + .WithAttr("dtype", DT_FLOAT) + .WithAttr("value", Tensor())); + Node* b = ops::UnaryOp("Relu", a, builder.opts().WithName("B")); + ops::UnaryOp("_Retval", b, + builder.opts() + .WithName("R") + .WithAttr("T", DT_FLOAT) + .WithAttr("index", 0)); + + TF_EXPECT_OK(builder.ToGraph(graph.get())); + } + + TF_ASSERT_OK(MarkForCompilation(&graph)); + auto clusters = GetClusters(*graph); + + EXPECT_EQ(2, clusters.size()); + EXPECT_TRUE(clusters.find("R") == clusters.cend()); + EXPECT_EQ(clusters["A"], clusters["B"]); +} + } // namespace } // namespace tensorflow -- GitLab From 186caed810c0e9a9ee9a3f1e0f8bea50764ce5df Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 15:48:06 -0800 Subject: [PATCH 0168/1924] Add int64 support to XLA Shape op. PiperOrigin-RevId: 177519992 --- .../compiler/tf2xla/kernels/shape_op.cc | 76 +++++++++++-------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/shape_op.cc b/tensorflow/compiler/tf2xla/kernels/shape_op.cc index 24a99f253d..06838d1625 100644 --- a/tensorflow/compiler/tf2xla/kernels/shape_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/shape_op.cc @@ -25,58 +25,72 @@ limitations under the License. namespace tensorflow { namespace { +// Converts a TensorShape to a constant Tensor. +// +// The input TensorShape input_shape is used to populate the elements of +// shape_constant, which is modified in place. +Status TensorShapeToConstant(const TensorShape& input_shape, + Tensor* shape_constant) { + const int dims = input_shape.dims(); + if (shape_constant->dtype() == DT_INT32) { + auto vec = shape_constant->vec(); + for (int i = 0; i < dims; ++i) { + int64 dim_size = input_shape.dim_size(i); + if (!FastBoundsCheck(dim_size, std::numeric_limits::max())) { + return errors::InvalidArgument( + "Shape with out_type=int32 does not support tensors > int32max", + " but dim ", i, " is ", dim_size); + } + vec(i) = static_cast(dim_size); + } + } else { + auto vec = shape_constant->vec(); + for (int i = 0; i < dims; ++i) { + int64 dim_size = input_shape.dim_size(i); + vec(i) = dim_size; + } + } + return Status::OK(); +} + class ShapeOp : public XlaOpKernel { public: - explicit ShapeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + explicit ShapeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("out_type", &out_dtype_)); + } void Compile(XlaOpKernelContext* ctx) override { const TensorShape input_shape = ctx->InputShape(0); - const int rank = input_shape.dims(); - Tensor shape_constant(DT_INT32, TensorShape({rank})); - auto vec = shape_constant.vec(); - // TODO(dga): support int64. b/28119922. - for (int i = 0; i < rank; ++i) { - int64 dim_size = input_shape.dim_size(i); - OP_REQUIRES( - ctx, FastBoundsCheck(dim_size, std::numeric_limits::max()), - errors::InvalidArgument("Shape does not support tensors > int32max", - " but dim ", i, " is ", dim_size)); - vec(i) = static_cast(dim_size); - } - + Tensor shape_constant(out_dtype_, TensorShape({input_shape.dims()})); + OP_REQUIRES_OK(ctx, TensorShapeToConstant(input_shape, &shape_constant)); ctx->SetConstantOutput(0, shape_constant); } + + private: + DataType out_dtype_; }; REGISTER_XLA_OP(Name("Shape"), ShapeOp); class ShapeNOp : public XlaOpKernel { public: - explicit ShapeNOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + explicit ShapeNOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("out_type", &out_dtype_)); + } void Compile(XlaOpKernelContext* ctx) override { for (int i = 0; i < ctx->num_inputs(); ++i) { - const TensorShape shape = ctx->InputShape(i); - const int dims = shape.dims(); - Tensor shape_constant(DT_INT32, TensorShape({dims})); - auto vec = shape_constant.vec(); - - // TODO(dga): support int64. b/28119922. - for (int j = 0; j < dims; ++j) { - int64 dim_size = shape.dim_size(j); - OP_REQUIRES( - ctx, FastBoundsCheck(dim_size, std::numeric_limits::max()), - errors::InvalidArgument("Shape does not support tensors > int32max", - " but shape ", i, " dim ", j, " is ", - dim_size)); - vec(j) = static_cast(dim_size); - } - + const TensorShape input_shape = ctx->InputShape(i); + Tensor shape_constant(out_dtype_, TensorShape({input_shape.dims()})); + OP_REQUIRES_OK(ctx, TensorShapeToConstant(input_shape, &shape_constant)); ctx->SetConstantOutput(i, shape_constant); } } bool IsExpensive() override { return false; } + + private: + DataType out_dtype_; }; REGISTER_XLA_OP(Name("ShapeN"), ShapeNOp); -- GitLab From 0438ac79bdb503ed267bec2146e7136ac8e99ff9 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Thu, 30 Nov 2017 16:07:24 -0800 Subject: [PATCH 0169/1924] [TF:XLA] Use output spatial dimensions instead of a transpose for conv backwards filter PiperOrigin-RevId: 177522710 --- .../compiler/tf2xla/kernels/conv_ops.cc | 31 ++++------- .../xla/service/gpu/convolution_folding.cc | 55 +++++-------------- .../service/gpu/convolution_folding_test.cc | 42 +++++--------- .../compiler/xla/service/gpu/pad_insertion.cc | 16 +----- 4 files changed, 41 insertions(+), 103 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc index 61f4d1993a..aaddbe811c 100644 --- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc @@ -540,9 +540,7 @@ class ConvBackpropFilterOp : public XlaOpKernel { // Swap n_dim and c_dim in the activations. dnums.set_input_batch_dimension(c_dim); - dnums.set_output_batch_dimension(c_dim); dnums.set_input_feature_dimension(n_dim); - dnums.set_output_feature_dimension(n_dim); // The gradients become the RHS of the convolution. // The gradients have shape [batch, out_rows, out_cols, ..., out_depth] @@ -554,11 +552,17 @@ class ConvBackpropFilterOp : public XlaOpKernel { std::vector rhs_dilation(num_spatial_dims_); std::vector ones(num_spatial_dims_, 1); + // Tensorflow filter shape is [ H, W, ..., inC, outC ]. + for (int i = 0; i < num_spatial_dims_; ++i) { + dnums.add_output_spatial_dimensions(i); + } + dnums.set_output_batch_dimension(num_spatial_dims_); + dnums.set_output_feature_dimension(num_spatial_dims_ + 1); + for (int i = 0; i < num_spatial_dims_; ++i) { int64 dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i); dnums.add_input_spatial_dimensions(dim); dnums.add_kernel_spatial_dimensions(dim); - dnums.add_output_spatial_dimensions(dim); // We will also need to pad the input with zeros such that after the // convolution, we get the right size for the filter. @@ -615,26 +619,11 @@ class ConvBackpropFilterOp : public XlaOpKernel { /*window_strides=*/ones, padding, /*lhs_dilation=*/ones, rhs_dilation, dnums); - // The layout of filter_backprop will match the layout of - // padded_activations - // and so will have layout: [out_feature, h, w, ..., in_feature] - // Tensorflow filter shape is [ H, W, ..., inC, outC ], so we transpose the - // output. - std::vector transpose_dims; - transpose_dims.reserve(num_dims()); - for (int i = 0; i < num_spatial_dims_; ++i) { - transpose_dims.push_back(dnums.output_spatial_dimensions(i)); - } - transpose_dims.push_back(c_dim); - transpose_dims.push_back(n_dim); - xla::ComputationDataHandle filter_backprop_reshaped = - b->Transpose(filter_backprop, transpose_dims); - if (depthwise_) { - filter_backprop_reshaped = ContractFilterForDepthwiseBackprop( - ctx, filter_shape, ctx->input_type(0), filter_backprop_reshaped, b); + filter_backprop = ContractFilterForDepthwiseBackprop( + ctx, filter_shape, ctx->input_type(0), filter_backprop, b); } - ctx->SetOutput(0, filter_backprop_reshaped); + ctx->SetOutput(0, filter_backprop); } protected: diff --git a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc index 828ae675d7..f198c4c08e 100644 --- a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc +++ b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc @@ -55,19 +55,7 @@ MatchBackwardFilter(HloInstruction* conv) { // v v // Convolution // conv - // | - // v - // Transpose (optional if identity transposition) CHECK_EQ(HloOpcode::kConvolution, conv->opcode()); - // If the forward convolution is followed by a transpose, we can fuse the - // transpose into the backward convolution as well. - HloInstruction* transpose = nullptr; - if (conv->user_count() == 1) { - HloInstruction* single_user = *conv->users().begin(); - if (single_user->opcode() == HloOpcode::kTranspose) { - transpose = single_user; - } - } // Step 2: match paddings and dimension numbers of the forward convolution. const ConvolutionDimensionNumbers& conv_dnums = @@ -75,6 +63,9 @@ MatchBackwardFilter(HloInstruction* conv) { auto input_batch_dim = conv_dnums.input_batch_dimension(); auto input_feature_dim = conv_dnums.input_feature_dimension(); auto input_spatial_dims = conv_dnums.input_spatial_dimensions(); + auto kernel_input_feature_dim = conv_dnums.kernel_input_feature_dimension(); + auto kernel_output_feature_dim = conv_dnums.kernel_output_feature_dimension(); + auto kernel_spatial_dims = conv_dnums.kernel_spatial_dimensions(); auto output_batch_dim = conv_dnums.output_batch_dimension(); auto output_feature_dim = conv_dnums.output_feature_dimension(); auto output_spatial_dims = conv_dnums.output_spatial_dimensions(); @@ -98,7 +89,8 @@ MatchBackwardFilter(HloInstruction* conv) { } // Padding high will be checked in Step 3. } - if (transpose == nullptr && !window_util::HasWindowDilation(conv->window())) { + if (input_batch_dim == output_batch_dim && + !window_util::HasWindowDilation(conv->window())) { VLOG(1) << conv->ToString() << " is a regular forward convolution. No need " "to fold it to a backward filter convolution."; @@ -169,53 +161,32 @@ MatchBackwardFilter(HloInstruction* conv) { } } - // To make future HLO passes easier, we canonicalize the fused expression by - // adding an identity transposition if it's omitted in the pattern. - if (transpose == nullptr) { - // Create an identity transposition with the same rank as the forward - // convolution. - HloComputation* parent_computation = conv->parent(); - std::vector transpose_dimensions(ShapeUtil::Rank(conv->shape())); - std::iota(transpose_dimensions.begin(), transpose_dimensions.end(), 0); - transpose = - parent_computation->AddInstruction(HloInstruction::CreateTranspose( - conv->shape(), conv, transpose_dimensions)); - TF_CHECK_OK(conv->ReplaceAllUsesWith(transpose)); - } - // Restore the dimension numbers of the backward convolution from the forward // convolution. The two activation dimensions are reversed (batch and // feature). ConvolutionDimensionNumbers backward_conv_dnums; backward_conv_dnums.set_input_batch_dimension(input_feature_dim); backward_conv_dnums.set_input_feature_dimension(input_batch_dim); - backward_conv_dnums.set_output_batch_dimension(output_feature_dim); - backward_conv_dnums.set_output_feature_dimension(output_batch_dim); for (int i = 0; i < input_spatial_dims.size(); ++i) { backward_conv_dnums.add_input_spatial_dimensions(input_spatial_dims[i]); } - for (int i = 0; i < output_spatial_dims.size(); ++i) { - backward_conv_dnums.add_output_spatial_dimensions(output_spatial_dims[i]); + backward_conv_dnums.set_output_batch_dimension(kernel_input_feature_dim); + backward_conv_dnums.set_output_feature_dimension(kernel_output_feature_dim); + for (int i = 0; i < kernel_spatial_dims.size(); ++i) { + backward_conv_dnums.add_output_spatial_dimensions(kernel_spatial_dims[i]); } // The dimension numbering of the output of the forward convolution (before // transposition) is the same as that of the activations (according to the // semantics of kConvolution). The batch dimension of the activations should // be treated as the input feature dimension, and the feature dimension should // be treated as the output feature. - // - // The output of the forward convolution needs to be transposed to fit into - // the dimension numbering of the weight gradients. This transposition maps - // dimension i to PositionInContainer(transpose->dimensions(), i). - backward_conv_dnums.set_kernel_input_feature_dimension( - PositionInContainer(transpose->dimensions(), output_batch_dim)); - backward_conv_dnums.set_kernel_output_feature_dimension( - PositionInContainer(transpose->dimensions(), output_feature_dim)); + backward_conv_dnums.set_kernel_input_feature_dimension(output_batch_dim); + backward_conv_dnums.set_kernel_output_feature_dimension(output_feature_dim); for (int i = 0; i < output_spatial_dims.size(); ++i) { - backward_conv_dnums.add_kernel_spatial_dimensions( - PositionInContainer(transpose->dimensions(), output_spatial_dims[i])); + backward_conv_dnums.add_kernel_spatial_dimensions(output_spatial_dims[i]); } - return std::make_tuple(true, std::vector({transpose, conv}), + return std::make_tuple(true, std::vector({conv}), backward_conv_window, backward_conv_dnums); } diff --git a/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc b/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc index 112c496e1f..34e6bdb117 100644 --- a/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc +++ b/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc @@ -46,18 +46,18 @@ class ConvolutionFoldingTest : public HloTestBase { // // TODO(jingyue): Add more tests on NCHW input order which TF also supports. tf_default_dnums_for_backward_filter_.set_input_batch_dimension(3); - tf_default_dnums_for_backward_filter_.set_output_batch_dimension(3); tf_default_dnums_for_backward_filter_.set_input_feature_dimension(0); - tf_default_dnums_for_backward_filter_.set_output_feature_dimension(0); tf_default_dnums_for_backward_filter_.add_input_spatial_dimensions(1); - tf_default_dnums_for_backward_filter_.add_output_spatial_dimensions(1); tf_default_dnums_for_backward_filter_.add_input_spatial_dimensions(2); - tf_default_dnums_for_backward_filter_.add_output_spatial_dimensions(2); tf_default_dnums_for_backward_filter_.set_kernel_input_feature_dimension(0); tf_default_dnums_for_backward_filter_.set_kernel_output_feature_dimension( 3); tf_default_dnums_for_backward_filter_.add_kernel_spatial_dimensions(1); tf_default_dnums_for_backward_filter_.add_kernel_spatial_dimensions(2); + tf_default_dnums_for_backward_filter_.add_output_spatial_dimensions(0); + tf_default_dnums_for_backward_filter_.add_output_spatial_dimensions(1); + tf_default_dnums_for_backward_filter_.set_output_batch_dimension(2); + tf_default_dnums_for_backward_filter_.set_output_feature_dimension(3); tf_default_dnums_for_backward_input_.set_input_batch_dimension(0); tf_default_dnums_for_backward_input_.set_output_batch_dimension(0); @@ -86,7 +86,7 @@ class ConvolutionFoldingTest : public HloTestBase { ConvolutionDimensionNumbers tf_default_dnums_for_backward_input_; }; -TEST_F(ConvolutionFoldingTest, BackwardFilterConvolveWithoutTranspose) { +TEST_F(ConvolutionFoldingTest, BackwardFilterConvolve) { HloComputation::Builder builder(TestName()); HloInstruction* activations = builder.AddInstruction(HloInstruction::CreateParameter( @@ -136,7 +136,7 @@ TEST_F(ConvolutionFoldingTest, auto module = CreateNewModule(); module->AddEntryComputation(builder.Build()); - EXPECT_FALSE(FoldConvolution(module.get())); + EXPECT_TRUE(FoldConvolution(module.get())); } // Extracted from block35 training. @@ -155,13 +155,9 @@ TEST_F(ConvolutionFoldingTest, BackwardFilterConvolveWithPaddedActivations) { conv_window.mutable_dimensions(i)->set_padding_low(1); conv_window.mutable_dimensions(i)->set_padding_high(1); } - HloInstruction* convolution = - builder.AddInstruction(HloInstruction::CreateConvolve( - ShapeUtil::MakeShape(F32, {32, 3, 3, 32}), activations, gradients, - conv_window, tf_default_dnums_for_backward_filter_)); - - builder.AddInstruction(HloInstruction::CreateTranspose( - ShapeUtil::MakeShape(F32, {3, 3, 32, 32}), convolution, {1, 2, 3, 0})); + builder.AddInstruction(HloInstruction::CreateConvolve( + ShapeUtil::MakeShape(F32, {32, 3, 3, 32}), activations, gradients, + conv_window, tf_default_dnums_for_backward_filter_)); auto module = CreateNewModule(); HloComputation* entry_computation = @@ -189,13 +185,9 @@ TEST_F(ConvolutionFoldingTest, BackwardFilterConvolveWithPaddedGradients) { conv_window.mutable_dimensions(i)->set_padding_high(-1); conv_window.mutable_dimensions(i)->set_window_dilation(2); } - HloInstruction* convolution = - builder.AddInstruction(HloInstruction::CreateConvolve( - ShapeUtil::MakeShape(F32, {320, 3, 3, 192}), activations, gradients, - conv_window, tf_default_dnums_for_backward_filter_)); - - builder.AddInstruction(HloInstruction::CreateTranspose( - ShapeUtil::MakeShape(F32, {3, 3, 192, 320}), convolution, {1, 2, 3, 0})); + builder.AddInstruction(HloInstruction::CreateConvolve( + ShapeUtil::MakeShape(F32, {320, 3, 3, 192}), activations, gradients, + conv_window, tf_default_dnums_for_backward_filter_)); auto module = CreateNewModule(); HloComputation* entry_computation = @@ -222,13 +214,9 @@ TEST_F(ConvolutionFoldingTest, BackwardFilterConvolveWithUnevenPadding) { // Uneven padding: padding_low=0, padding_high=1 conv_window.mutable_dimensions(i)->set_padding_high(1); } - HloInstruction* convolution = - builder.AddInstruction(HloInstruction::CreateConvolve( - ShapeUtil::MakeShape(F32, {32, 2, 2, 32}), activations, gradients, - conv_window, tf_default_dnums_for_backward_filter_)); - - builder.AddInstruction(HloInstruction::CreateTranspose( - ShapeUtil::MakeShape(F32, {2, 2, 32, 32}), convolution, {1, 2, 3, 0})); + builder.AddInstruction(HloInstruction::CreateConvolve( + ShapeUtil::MakeShape(F32, {32, 2, 2, 32}), activations, gradients, + conv_window, tf_default_dnums_for_backward_filter_)); auto module = CreateNewModule(); HloComputation* entry_computation = diff --git a/tensorflow/compiler/xla/service/gpu/pad_insertion.cc b/tensorflow/compiler/xla/service/gpu/pad_insertion.cc index 11290eda4f..c29fee0879 100644 --- a/tensorflow/compiler/xla/service/gpu/pad_insertion.cc +++ b/tensorflow/compiler/xla/service/gpu/pad_insertion.cc @@ -202,8 +202,7 @@ bool PadInsertion::CanonicalizeBackwardFilterConvolution( // ABCD0 = Pad(ABCD, padding_high=1) // BackwardFilterConv(ABCD0, xyz, padding_low=pading_high=1) // We choose the lesser of padding_low and padding_high as the new padding. - HloInstruction* transpose = backward_conv->fused_expression_root(); - HloInstruction* forward_conv = transpose->mutable_operand(0); + HloInstruction* forward_conv = backward_conv->fused_expression_root(); HloInstruction* input = backward_conv->mutable_operand(0); Window new_forward_conv_window = forward_conv->window(); Window new_backward_conv_window = backward_conv->window(); @@ -269,19 +268,10 @@ bool PadInsertion::CanonicalizeBackwardFilterConvolution( .ConsumeValueOrDie(), padded_input, output, new_forward_conv_window, forward_conv_dnums)); - HloInstruction* new_transpose = - computation->AddInstruction(HloInstruction::CreateTranspose( - ShapeInference::InferTransposeShape(new_forward_conv->shape(), - transpose->dimensions()) - .ConsumeValueOrDie(), - new_forward_conv, transpose->dimensions())); - - // Fuse the new forward convolution and the new transpose to the new backward - // convolution. + // Fuse the new forward convolution to the new backward convolution. HloInstruction* new_backward_conv = computation->CreateFusionInstructionForBackwardConvolution( - {new_transpose, new_forward_conv}, - HloInstruction::FusionKind::kConvBackwardFilter, + {new_forward_conv}, HloInstruction::FusionKind::kConvBackwardFilter, new_backward_conv_window, backward_conv_dnums); VLOG(1) << "Canonicalizing backward filter conv"; -- GitLab From b2db981a6731e978453862a73dab892bc674db68 Mon Sep 17 00:00:00 2001 From: Sourabh Bajaj Date: Thu, 30 Nov 2017 16:37:11 -0800 Subject: [PATCH 0170/1924] Merge changes from github. PiperOrigin-RevId: 177526301 --- .gitignore | 5 + tensorflow/compiler/tf2xla/xla_op_kernel.cc | 4 +- tensorflow/compiler/xla/BUILD | 1 + tensorflow/compiler/xla/ptr_util.h | 47 +- .../xla/service/buffer_assignment_test.cc | 20 +- .../xla/service/buffer_liveness_test.cc | 34 +- .../compiler/xla/service/cpu/cpu_compiler.cc | 25 +- .../xla/service/cpu/simple_orc_jit.cc | 130 ++--- tensorflow/compiler/xla/xla_data.proto | 2 +- .../boosted_trees/lib/utils/batch_features.h | 2 +- .../contrib/boosted_trees/lib/utils/example.h | 10 +- .../lib/utils/sparse_column_iterable.cc | 4 + tensorflow/contrib/cmake/external/re2.cmake | 1 + tensorflow/contrib/cmake/tf_shared_lib.cmake | 10 +- tensorflow/contrib/cmake/tf_tests.cmake | 6 +- tensorflow/contrib/crf/python/ops/crf.py | 2 +- .../contrib/data/python/kernel_tests/BUILD | 3 + .../python/ops/poisson_lognormal.py | 2 +- .../contrib/eager/python/metrics_impl.py | 2 +- .../contrib/factorization/python/ops/wals.py | 2 +- tensorflow/contrib/ffmpeg/BUILD | 47 ++ tensorflow/contrib/ffmpeg/__init__.py | 3 +- tensorflow/contrib/ffmpeg/decode_audio_op.cc | 25 +- tensorflow/contrib/ffmpeg/decode_video_op.cc | 118 ++++ .../contrib/ffmpeg/decode_video_op_test.py | 69 +++ .../contrib/ffmpeg/default/ffmpeg_lib.cc | 247 ++++++--- .../ffmpeg/default/ffmpeg_lib_utility_test.cc | 3 +- tensorflow/contrib/ffmpeg/ffmpeg_lib.h | 25 +- tensorflow/contrib/ffmpeg/ffmpeg_ops.py | 17 + tensorflow/contrib/ffmpeg/testdata/small.mp4 | Bin 0 -> 383631 bytes .../contrib/ffmpeg/testdata/small_100.bmp | Bin 0 -> 537654 bytes .../framework/python/framework/graph_util.py | 28 +- .../python/framework/graph_util_test.py | 14 + tensorflow/contrib/gan/python/train.py | 2 +- .../contrib/layers/python/layers/layers.py | 5 +- .../layers/python/layers/layers_test.py | 22 +- tensorflow/contrib/lite/BUILD | 3 + tensorflow/contrib/lite/Makefile | 147 +++++ tensorflow/contrib/lite/README.md | 66 +-- .../contrib/lite/build_ios_universal_lib.sh | 31 ++ .../contrib/lite/download_dependencies.sh | 99 ++++ .../lite/examples/ios/camera/.gitignore | 2 + .../ios/camera/CameraExampleAppDelegate.h | 21 + .../ios/camera/CameraExampleAppDelegate.m | 44 ++ .../ios/camera/CameraExampleViewController.h | 48 ++ .../ios/camera/CameraExampleViewController.mm | 506 ++++++++++++++++++ .../lite/examples/ios/camera/Info.plist | 44 ++ .../camera/MainStoryboard_iPhone.storyboard | 46 ++ .../contrib/lite/examples/ios/camera/Podfile | 5 + .../lite/examples/ios/camera/data/.gitignore | 0 .../contrib/lite/examples/ios/camera/main.mm | 28 + .../project.pbxproj | 419 +++++++++++++++ .../lite/examples/ios/simple/AppDelegate.h | 21 + .../lite/examples/ios/simple/AppDelegate.mm | 47 ++ .../contrib/lite/examples/ios/simple/Podfile | 5 + .../examples/ios/simple/RunModel-Info.plist | 47 ++ .../ios/simple/RunModelViewController.h | 24 + .../ios/simple/RunModelViewController.mm | 221 ++++++++ .../ios/simple/RunModelViewController.xib | 46 ++ .../examples/ios/simple/data/grace_hopper.jpg | Bin 0 -> 73746 bytes .../lite/examples/ios/simple/ios_image_load.h | 23 + .../examples/ios/simple/ios_image_load.mm | 80 +++ .../contrib/lite/examples/ios/simple/main.mm | 22 + .../simple/simple.xcodeproj/project.pbxproj | 359 +++++++++++++ tensorflow/contrib/lite/g3doc/apis.md | 2 +- .../lite/g3doc/tf_ops_compatibility.md | 2 +- tensorflow/contrib/lite/ios_makefile.inc | 31 ++ .../contrib/lite/java/demo/app/build.gradle | 4 +- .../lite/models/testdata/g3doc/README.md | 29 +- .../contrib/lite/nnapi/NeuralNetworksShim.h | 6 +- .../lite/schema/upgrade_schema_test.py | 2 +- tensorflow/contrib/lite/testing/BUILD | 1 + .../contrib/lite/testing/parse_testdata.cc | 2 +- tensorflow/contrib/lite/testing/test_runner.h | 2 +- tensorflow/contrib/lite/toco/model.h | 2 +- .../contrib/lite/tools/benchmark_model.cc | 95 ++++ .../contrib/lite/tools/mutable_op_resolver.h | 10 + tensorflow/contrib/mpi/BUILD | 1 + .../contrib/nn/python/ops/cross_entropy.py | 2 +- .../contrib/nn/python/ops/sampling_ops.py | 2 +- tensorflow/contrib/rnn/python/ops/rnn_cell.py | 1 - tensorflow/contrib/slim/README.md | 9 +- .../contrib/slim/python/slim/evaluation.py | 4 +- tensorflow/contrib/summary/BUILD | 14 - .../contrib/summary/summary_ops_graph_test.py | 5 +- .../contrib/summary/summary_ops_test.py | 7 +- .../contrib/summary/summary_test_util.py | 39 +- .../models/decisions_to_data_then_nn_test.py | 6 +- tensorflow/core/BUILD | 6 + tensorflow/core/graph/graph.h | 1 - .../optimizers/arithmetic_optimizer.cc | 2 +- tensorflow/core/kernels/BUILD | 9 +- .../core/kernels/batch_matmul_op_complex.cc | 2 + .../core/kernels/batch_matmul_op_real.cc | 2 + tensorflow/core/kernels/cwise_op_asinh.cc | 2 +- tensorflow/core/kernels/decode_bmp_op.cc | 15 +- .../core/kernels/dynamic_partition_op_test.cc | 58 ++ .../core/kernels/mkl_batch_matmul_op.cc | 238 ++++++++ .../core/kernels/prefetch_dataset_op.cc | 2 + tensorflow/core/kernels/summary_interface.cc | 4 +- tensorflow/core/lib/io/path.cc | 66 ++- tensorflow/core/lib/io/path.h | 3 + tensorflow/core/ops/math_ops.cc | 24 +- tensorflow/core/ops/math_ops_test.cc | 11 + .../platform/default/build_config_root.bzl | 22 +- tensorflow/core/util/ptr_util.h | 80 +++ tensorflow/docs_src/extend/adding_an_op.md | 28 +- tensorflow/docs_src/get_started/input_fn.md | 4 +- .../docs_src/install/install_windows.md | 10 +- tensorflow/docs_src/mobile/ios_build.md | 2 +- tensorflow/docs_src/mobile/mobile_intro.md | 2 +- tensorflow/docs_src/mobile/optimizing.md | 2 +- .../performance/xla/operation_semantics.md | 2 +- .../docs_src/tutorials/image_recognition.md | 2 +- .../reading_data/convert_to_records.py | 24 +- .../examples/speech_commands/input_data.py | 3 +- tensorflow/examples/speech_commands/train.py | 3 +- tensorflow/examples/udacity/1_notmnist.ipynb | 6 +- .../java/org/tensorflow/OperationBuilder.java | 35 +- .../src/main/native/operation_builder_jni.cc | 36 ++ .../src/main/native/operation_builder_jni.h | 8 + .../org/tensorflow/OperationBuilderTest.java | 37 ++ tensorflow/python/BUILD | 2 + tensorflow/python/data/util/nest.py | 10 + tensorflow/python/data/util/nest_test.py | 9 + tensorflow/python/estimator/export/export.py | 2 +- tensorflow/python/estimator/training_test.py | 4 +- tensorflow/python/keras/BUILD | 1 + .../python/keras/_impl/keras/backend.py | 2 +- .../python/keras/_impl/keras/callbacks.py | 2 +- .../keras/_impl/keras/callbacks_test.py | 61 ++- .../keras/_impl/keras/engine/training_test.py | 6 + .../keras/_impl/keras/estimator_test.py | 5 + .../python/keras/_impl/keras/models_test.py | 21 +- .../_impl/keras/utils/data_utils_test.py | 7 + .../keras/_impl/keras/utils/io_utils.py | 11 +- .../keras/_impl/keras/utils/vis_utils.py | 2 +- tensorflow/python/kernel_tests/BUILD | 16 +- .../python/kernel_tests/decode_bmp_op_test.py | 75 +++ .../kernel_tests/prefetch_dataset_op_test.py | 59 ++ tensorflow/python/layers/convolutional.py | 6 +- tensorflow/python/ops/math_ops_test.py | 2 +- tensorflow/python/ops/variable_scope.py | 2 +- tensorflow/python/ops/variables.py | 4 +- .../python/profiler/model_analyzer_test.py | 2 +- tensorflow/python/training/saver_test.py | 24 +- .../training/sync_replicas_optimizer.py | 2 +- tensorflow/python/util/nest.py | 15 +- tensorflow/python/util/nest_test.py | 13 +- ...orflow.keras.callbacks.-tensor-board.pbtxt | 2 +- tensorflow/tools/benchmark/benchmark_model.cc | 2 +- .../tools/ci_build/builds/test_user_ops.sh | 28 +- .../tools/ci_build/ci_parameterized_build.sh | 4 +- .../ci_build/windows/cpu/cmake/run_build.bat | 2 +- .../windows/cpu/pip/build_tf_windows.sh | 4 +- .../ci_build/windows/gpu/cmake/run_build.bat | 2 +- .../windows/gpu/pip/build_tf_windows.sh | 4 +- .../dist_test/python/census_widendeep.py | 3 +- tensorflow/tools/docker/Dockerfile.devel | 1 - .../tools/docker/Dockerfile.devel-cpu-mkl | 85 +++ tensorflow/tools/docker/Dockerfile.devel-gpu | 2 - .../docker/Dockerfile.devel-gpu-cuda9-cudnn7 | 2 - .../docker/notebooks/2_getting_started.ipynb | 12 +- .../notebooks/3_mnist_from_scratch.ipynb | 2 + .../tools/pip_package/pip_smoke_test.py | 3 - tensorflow/tools/pip_package/setup.py | 27 +- tensorflow/workspace.bzl | 2 +- third_party/flatbuffers/flatbuffers.BUILD | 7 +- third_party/mkl/build_defs.bzl | 2 +- third_party/nccl.BUILD | 2 +- third_party/py/python_configure.bzl | 91 ++-- 171 files changed, 4446 insertions(+), 578 deletions(-) create mode 100644 tensorflow/contrib/ffmpeg/decode_video_op.cc create mode 100644 tensorflow/contrib/ffmpeg/decode_video_op_test.py create mode 100644 tensorflow/contrib/ffmpeg/testdata/small.mp4 create mode 100644 tensorflow/contrib/ffmpeg/testdata/small_100.bmp create mode 100644 tensorflow/contrib/lite/Makefile create mode 100755 tensorflow/contrib/lite/build_ios_universal_lib.sh create mode 100755 tensorflow/contrib/lite/download_dependencies.sh create mode 100644 tensorflow/contrib/lite/examples/ios/camera/.gitignore create mode 100644 tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.h create mode 100644 tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.m create mode 100644 tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.h create mode 100644 tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.mm create mode 100644 tensorflow/contrib/lite/examples/ios/camera/Info.plist create mode 100644 tensorflow/contrib/lite/examples/ios/camera/MainStoryboard_iPhone.storyboard create mode 100644 tensorflow/contrib/lite/examples/ios/camera/Podfile create mode 100644 tensorflow/contrib/lite/examples/ios/camera/data/.gitignore create mode 100644 tensorflow/contrib/lite/examples/ios/camera/main.mm create mode 100644 tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj create mode 100644 tensorflow/contrib/lite/examples/ios/simple/AppDelegate.h create mode 100644 tensorflow/contrib/lite/examples/ios/simple/AppDelegate.mm create mode 100644 tensorflow/contrib/lite/examples/ios/simple/Podfile create mode 100644 tensorflow/contrib/lite/examples/ios/simple/RunModel-Info.plist create mode 100644 tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.h create mode 100644 tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.mm create mode 100644 tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.xib create mode 100644 tensorflow/contrib/lite/examples/ios/simple/data/grace_hopper.jpg create mode 100644 tensorflow/contrib/lite/examples/ios/simple/ios_image_load.h create mode 100644 tensorflow/contrib/lite/examples/ios/simple/ios_image_load.mm create mode 100644 tensorflow/contrib/lite/examples/ios/simple/main.mm create mode 100644 tensorflow/contrib/lite/examples/ios/simple/simple.xcodeproj/project.pbxproj create mode 100644 tensorflow/contrib/lite/ios_makefile.inc create mode 100644 tensorflow/contrib/lite/tools/benchmark_model.cc create mode 100644 tensorflow/core/kernels/mkl_batch_matmul_op.cc create mode 100644 tensorflow/core/util/ptr_util.h create mode 100644 tensorflow/python/kernel_tests/prefetch_dataset_op_test.py create mode 100644 tensorflow/tools/docker/Dockerfile.devel-cpu-mkl diff --git a/.gitignore b/.gitignore index 9ae0d9c96f..d11a504bdc 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ Pods Podfile.lock *.pbxproj *.xcworkspacedata +/tensorflow/contrib/lite/downloads/** +/tensorflow/contrib/lite/gen/** +/tensorflow/contrib/lite/examples/ios/simple/data/*.txt +/tensorflow/contrib/lite/examples/ios/simple/data/*.tflite +xcuserdata/** \ No newline at end of file diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index f16472cac8..79d501b511 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -346,9 +346,9 @@ void XlaOpKernelContext::SetConstantOutput(int index, const Tensor& constant) { } void XlaOpKernelContext::SetInvalidOutput(int index) { - const TensorShape shape; Tensor* output = nullptr; - OP_REQUIRES_OK(context_, context_->allocate_output(index, shape, &output)); + OP_REQUIRES_OK(context_, + context_->allocate_output(index, TensorShape({}), &output)); XlaExpression* expression = CastExpressionFromUninitializedTensor(output); xla::ComputationDataHandle handle; handle.set_handle(0); diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 515b572b0e..d3f292207f 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -175,6 +175,7 @@ cc_library( ":types", ":xla_data_proto", "//tensorflow/core:lib", + "//tensorflow/core:ptr_util", ], ) diff --git a/tensorflow/compiler/xla/ptr_util.h b/tensorflow/compiler/xla/ptr_util.h index fa67030313..c58c19db2c 100644 --- a/tensorflow/compiler/xla/ptr_util.h +++ b/tensorflow/compiler/xla/ptr_util.h @@ -16,7 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_PTR_UTIL_H_ #define TENSORFLOW_COMPILER_XLA_PTR_UTIL_H_ -// Utility functions for pointers. +// As this was moved to tensorflow/core/util, provide indirections here to +// maintain current functionality of the library. #include @@ -24,55 +25,27 @@ limitations under the License. #include #include -namespace xla { - -namespace internal { - -// Trait to select overloads and return types for MakeUnique. -template -struct MakeUniqueResult { - using scalar = std::unique_ptr; -}; -template -struct MakeUniqueResult { - using array = std::unique_ptr; -}; -template -struct MakeUniqueResult { - using invalid = void; -}; +#include "tensorflow/core/util/ptr_util.h" -} // namespace internal +namespace xla { -// Transfers ownership of a raw pointer to a std::unique_ptr of deduced type. -// Example: -// X* NewX(int, int); -// auto x = WrapUnique(NewX(1, 2)); // 'x' is std::unique_ptr. -// -// WrapUnique is useful for capturing the output of a raw pointer factory. -// However, prefer 'MakeUnique(args...) over 'WrapUnique(new T(args...))'. -// auto x = WrapUnique(new X(1, 2)); // works, but nonideal. -// auto x = MakeUnique(1, 2); // safer, standard, avoids raw 'new'. -// -// Note: Cannot wrap pointers to array of unknown bound (i.e. U(*)[]). template std::unique_ptr WrapUnique(T* ptr) { - static_assert(!std::is_array::value || std::extent::value != 0, - "types T[0] or T[] are unsupported"); - return std::unique_ptr(ptr); + return tensorflow::WrapUnique(ptr); } template -typename internal::MakeUniqueResult::scalar MakeUnique(Args&&... args) { - return std::unique_ptr(new T(std::forward(args)...)); +typename tensorflow::helper::MakeUniqueResult::scalar MakeUnique( + Args&&... args) { + return tensorflow::MakeUnique(std::forward(args)...); } // Overload for array of unknown bound. // The allocation of arrays needs to use the array form of new, // and cannot take element constructor arguments. template -typename internal::MakeUniqueResult::array MakeUnique(size_t n) { - return std::unique_ptr(new typename std::remove_extent::type[n]()); +typename tensorflow::helper::MakeUniqueResult::array MakeUnique(size_t n) { + return tensorflow::MakeUnique(n); } } // namespace xla diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index 75c71dfeb1..09681b34e7 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -85,7 +85,7 @@ class BufferAssignmentTest : public HloTestBase { std::unique_ptr RunBufferAssignment(HloModule* module, int64 alignment = 1) { return BufferAssigner::Run( - module, MakeUnique(module), + module, xla::MakeUnique(module), backend().compiler()->BufferSizeBytesFunction(), [alignment](LogicalBuffer::Color) { return alignment; }) .ConsumeValueOrDie(); @@ -94,7 +94,7 @@ class BufferAssignmentTest : public HloTestBase { std::unique_ptr RunColoredBufferAssignment( HloModule* module, BufferLiveness::Colorer colorer, int64 alignment = 1) { return BufferAssigner::Run( - module, MakeUnique(module), + module, xla::MakeUnique(module), backend().compiler()->BufferSizeBytesFunction(), [alignment](LogicalBuffer::Color) { return alignment; }, false, std::move(colorer)) @@ -1451,7 +1451,7 @@ class WhileBufferAssignmentTest : public HloTestBase { auto sequence = CreateMemoryMinimizingSequence(*module, ByteSizeOf).ConsumeValueOrDie(); return BufferAssigner::Run( - module, MakeUnique(module, sequence), + module, xla::MakeUnique(module, sequence), ByteSizeOf, [alignment](LogicalBuffer::Color) { return alignment; }) .ConsumeValueOrDie(); @@ -1472,7 +1472,7 @@ static void RunCopyInsertion(HloModule* module) { } TEST_F(WhileBufferAssignmentTest, TwoForwardWhileLoops) { - auto module = MakeUnique(TestName()); + auto module = xla::MakeUnique(TestName()); auto builder = HloComputation::Builder("entry"); auto input0 = builder.AddInstruction( @@ -1529,7 +1529,7 @@ TEST_F(WhileBufferAssignmentTest, TwoForwardWhileLoops) { } TEST_F(WhileBufferAssignmentTest, OneForwardBackwardWhileLoopSet) { - auto module = MakeUnique(TestName()); + auto module = xla::MakeUnique(TestName()); auto builder = HloComputation::Builder("entry"); auto input0 = builder.AddInstruction( @@ -1574,7 +1574,7 @@ TEST_F(WhileBufferAssignmentTest, OneForwardBackwardWhileLoopSet) { } TEST_F(BufferAssignmentTest, TwoCalls) { - auto module = MakeUnique(TestName()); + auto module = xla::MakeUnique(TestName()); Shape r0f32 = ShapeUtil::MakeShape(xla::F32, {}); HloComputation* sub_computation; { @@ -1639,7 +1639,7 @@ static bool IsPostOrderTraversal( } TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) { - auto module = MakeUnique(TestName()); + auto module = xla::MakeUnique(TestName()); auto builder = HloComputation::Builder(TestName()); auto zero = builder.AddInstruction( @@ -1710,15 +1710,15 @@ TEST_F(WhileBufferAssignmentTest, WhileLoopsInterferingResultRange) { auto assignment = BufferAssigner::Run( module.get(), - MakeUnique(module.get(), sequence), ByteSizeOf, - [](LogicalBuffer::Color) { return 1; }) + xla::MakeUnique(module.get(), sequence), + ByteSizeOf, [](LogicalBuffer::Color) { return 1; }) .ConsumeValueOrDie(); EXPECT_TRUE(BuffersDistinct({while0}, {while1}, *assignment)); } TEST_F(WhileBufferAssignmentTest, WhilesDontShareEntryParamIfLiveOut) { - auto module = MakeUnique(TestName()); + auto module = xla::MakeUnique(TestName()); auto builder = HloComputation::Builder("entry"); auto input0 = builder.AddInstruction( diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc index 56600b5838..13825fe05b 100644 --- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc +++ b/tensorflow/compiler/xla/service/buffer_liveness_test.cc @@ -120,7 +120,7 @@ TEST_F(BufferLivenessTest, ElementwiseChain) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, negate)); @@ -167,10 +167,10 @@ TEST_F(BufferLivenessTest, MultipleEntryParameters_Sequential) { SequentialHloOrdering::HloModuleSequence sequence; sequence.insert({entry, {param0, negate, param1, exp, add}}); - auto liveness = BufferLiveness::Run( - module.get(), - MakeUnique(module.get(), sequence)) - .ConsumeValueOrDie(); + auto liveness = + BufferLiveness::Run(module.get(), xla::MakeUnique( + module.get(), sequence)) + .ConsumeValueOrDie(); // Entry parameters interfere as if they are defined simultaneously at // the very beginning. @@ -216,7 +216,7 @@ TEST_F(BufferLivenessTest, NonElementwiseOperand) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, exp)); @@ -250,7 +250,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffers) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); EXPECT_TRUE(InstructionsMayInterfere(*liveness, param, negate)); @@ -294,7 +294,7 @@ TEST_F(BufferLivenessTest, OverlappedBuffersSequentialOrder) { std::vector order = {param, negate, exp, add}; module_sequence.emplace(computation, order); auto liveness = - BufferLiveness::Run(module.get(), MakeUnique( + BufferLiveness::Run(module.get(), xla::MakeUnique( module.get(), module_sequence)) .ConsumeValueOrDie(); @@ -334,7 +334,7 @@ TEST_F(BufferLivenessTest, TupleLiveOut) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // All buffers should be live out except the param @@ -370,7 +370,7 @@ TEST_F(BufferLivenessTest, EmbeddedComputation) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // Buffers in different computations should always interfere. @@ -409,7 +409,7 @@ TEST_F(BufferLivenessTest, TupleConstantLiveOut) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // Only the element buffers of the tuple constant which are pointed to by @@ -474,7 +474,7 @@ TEST_F(BufferLivenessTest, IndependentTupleElements) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // We compare tuple element pairs that are input/output to the computation: @@ -536,7 +536,7 @@ TEST_F(BufferLivenessTest, DependentTupleElements) { auto liveness = BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // We compare tuple element pairs that are input/output to the computation: @@ -624,8 +624,8 @@ class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest { // Run BufferLiveness on 'module'. auto liveness = - BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + BufferLiveness::Run( + module.get(), xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // Return whether or not buffers interference is detected between // 'tuple_param0' and 'tuple_root' at shape index '{1}'. @@ -736,8 +736,8 @@ class DynamicUpdateSliceLivenessTest : public BufferLivenessTest { module->AddEmbeddedComputation(builder.Build()); // Run BufferLiveness on 'module'. auto liveness = - BufferLiveness::Run(module.get(), - MakeUnique(module.get())) + BufferLiveness::Run( + module.get(), xla::MakeUnique(module.get())) .ConsumeValueOrDie(); // Return whether or not buffers interference is detected between // 'tuple_param0' and 'tuple_root' at shape index '{1}'. diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 99dae793ab..988f632748 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -469,11 +469,11 @@ StatusOr> CpuCompiler::RunBackend( &pre_optimization_ir_hook, &post_optimization_ir_hook)); // Compile must be thread-safe so create a new LLVM context for the module. - auto llvm_context = MakeUnique(); + auto llvm_context = xla::MakeUnique(); auto llvm_module = - MakeUnique("__compute_module", *llvm_context); + xla::MakeUnique("__compute_module", *llvm_context); - auto jit = MakeUnique( + auto jit = xla::MakeUnique( CompilerTargetOptions(module->config()), CodeGenOptLevel(module->config()), options::OptimizeForSizeRequested(module->config()), @@ -528,9 +528,9 @@ StatusOr> CpuCompiler::RunBackend( // uses data dependencies for determining order. TF_ASSIGN_OR_RETURN( std::unique_ptr assignment, - BufferAssigner::Run(module.get(), - MakeUnique(module.get()), - BufferSizeBytesFunction(), memory_alignment)); + BufferAssigner::Run( + module.get(), xla::MakeUnique(module.get()), + BufferSizeBytesFunction(), memory_alignment)); // BufferAssignment::ToString() includes a header, so no need for us to // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); @@ -557,7 +557,7 @@ StatusOr> CpuCompiler::RunBackend( const void* data = instruction->literal().InternalData(); int64 size = CpuExecutable::ShapeSizeBytes(instruction->shape()); auto iter = aligned_constants.emplace( - instruction, MakeUnique(size)); + instruction, xla::MakeUnique(size)); CHECK_EQ(iter.second, true); unsigned char* aligned_data = iter.first->second.get(); memcpy(aligned_data, data, size); @@ -642,10 +642,10 @@ StatusOr> CpuCompiler::RunBackend( // temporary buffers are required to run the computation. TF_ASSIGN_OR_RETURN( std::unique_ptr assignment, - BufferAssigner::Run( - module.get(), - MakeUnique(module.get(), module_sequence), - BufferSizeBytesFunction(), memory_alignment)); + BufferAssigner::Run(module.get(), + xla::MakeUnique( + module.get(), module_sequence), + BufferSizeBytesFunction(), memory_alignment)); // BufferAssignment::ToString() includes a header, so no need for us to // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); @@ -824,7 +824,8 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, TF_ASSIGN_OR_RETURN( std::unique_ptr assignment, BufferAssigner::Run( - module, MakeUnique(module, module_sequence), + module, + xla::MakeUnique(module, module_sequence), BufferSizeBytesFunction(), memory_alignment)); // BufferAssignment::ToString() includes a header, so no need for us to // print one ourselves. diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc index db6c201876..cda2783307 100644 --- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc +++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc @@ -213,71 +213,75 @@ bool RegisterKnownJITSymbols() { #undef REGISTER_CPU_RUNTIME_SYMBOL -#define REGISTER_LIBM_SYMBOL(name) \ - do { \ - /* Register both the F32 and F64 variants of the libm symbol. */ \ - registry->Register(#name "f", reinterpret_cast(name##f)); \ - registry->Register(#name, reinterpret_cast(name)); \ +// Register both the f32 (float) and f64 (double) versions of a libm symbol. +// Unfortunately the double versions are overloaded on some systems, e.g. +// Mac so we need an explicit cast. This requires passing the function signature +// for that case. +#define REGISTER_LIBM_SYMBOL(name, double_sig) \ + do { \ + registry->Register(#name "f", reinterpret_cast(name##f)); \ + registry->Register( \ + #name, reinterpret_cast(static_cast(name))); \ } while (false) - REGISTER_LIBM_SYMBOL(acos); - REGISTER_LIBM_SYMBOL(acosh); - REGISTER_LIBM_SYMBOL(asin); - REGISTER_LIBM_SYMBOL(asinh); - REGISTER_LIBM_SYMBOL(atan); - REGISTER_LIBM_SYMBOL(atan2); - REGISTER_LIBM_SYMBOL(atanh); - REGISTER_LIBM_SYMBOL(cbrt); - REGISTER_LIBM_SYMBOL(ceil); - REGISTER_LIBM_SYMBOL(copysign); - REGISTER_LIBM_SYMBOL(cos); - REGISTER_LIBM_SYMBOL(cosh); - REGISTER_LIBM_SYMBOL(erf); - REGISTER_LIBM_SYMBOL(erfc); - REGISTER_LIBM_SYMBOL(exp); - REGISTER_LIBM_SYMBOL(exp2); - REGISTER_LIBM_SYMBOL(expm1); - REGISTER_LIBM_SYMBOL(fabs); - REGISTER_LIBM_SYMBOL(fdim); - REGISTER_LIBM_SYMBOL(floor); - REGISTER_LIBM_SYMBOL(fma); - REGISTER_LIBM_SYMBOL(fmax); - REGISTER_LIBM_SYMBOL(fmin); - REGISTER_LIBM_SYMBOL(fmod); - REGISTER_LIBM_SYMBOL(frexp); - REGISTER_LIBM_SYMBOL(hypot); - REGISTER_LIBM_SYMBOL(ilogb); - REGISTER_LIBM_SYMBOL(ldexp); - REGISTER_LIBM_SYMBOL(lgamma); - REGISTER_LIBM_SYMBOL(llrint); - REGISTER_LIBM_SYMBOL(llround); - REGISTER_LIBM_SYMBOL(log); - REGISTER_LIBM_SYMBOL(log10); - REGISTER_LIBM_SYMBOL(log1p); - REGISTER_LIBM_SYMBOL(log2); - REGISTER_LIBM_SYMBOL(logb); - REGISTER_LIBM_SYMBOL(lrint); - REGISTER_LIBM_SYMBOL(lround); - REGISTER_LIBM_SYMBOL(modf); - REGISTER_LIBM_SYMBOL(nan); - REGISTER_LIBM_SYMBOL(nearbyint); - REGISTER_LIBM_SYMBOL(nextafter); - REGISTER_LIBM_SYMBOL(nexttoward); - REGISTER_LIBM_SYMBOL(pow); - REGISTER_LIBM_SYMBOL(remainder); - REGISTER_LIBM_SYMBOL(remquo); - REGISTER_LIBM_SYMBOL(rint); - REGISTER_LIBM_SYMBOL(round); - REGISTER_LIBM_SYMBOL(scalbln); - REGISTER_LIBM_SYMBOL(scalbn); - REGISTER_LIBM_SYMBOL(sin); - REGISTER_LIBM_SYMBOL(sincos); - REGISTER_LIBM_SYMBOL(sinh); - REGISTER_LIBM_SYMBOL(sqrt); - REGISTER_LIBM_SYMBOL(tan); - REGISTER_LIBM_SYMBOL(tanh); - REGISTER_LIBM_SYMBOL(tgamma); - REGISTER_LIBM_SYMBOL(trunc); + REGISTER_LIBM_SYMBOL(acos, double (*)(double)); + REGISTER_LIBM_SYMBOL(acosh, double (*)(double)); + REGISTER_LIBM_SYMBOL(asin, double (*)(double)); + REGISTER_LIBM_SYMBOL(asinh, double (*)(double)); + REGISTER_LIBM_SYMBOL(atan, double (*)(double)); + REGISTER_LIBM_SYMBOL(atan2, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(atanh, double (*)(double)); + REGISTER_LIBM_SYMBOL(cbrt, double (*)(double)); + REGISTER_LIBM_SYMBOL(ceil, double (*)(double)); + REGISTER_LIBM_SYMBOL(copysign, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(cos, double (*)(double)); + REGISTER_LIBM_SYMBOL(cosh, double (*)(double)); + REGISTER_LIBM_SYMBOL(erf, double (*)(double)); + REGISTER_LIBM_SYMBOL(erfc, double (*)(double)); + REGISTER_LIBM_SYMBOL(exp, double (*)(double)); + REGISTER_LIBM_SYMBOL(exp2, double (*)(double)); + REGISTER_LIBM_SYMBOL(expm1, double (*)(double)); + REGISTER_LIBM_SYMBOL(fabs, double (*)(double)); + REGISTER_LIBM_SYMBOL(fdim, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(floor, double (*)(double)); + REGISTER_LIBM_SYMBOL(fma, double (*)(double, double, double)); + REGISTER_LIBM_SYMBOL(fmax, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(fmin, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(fmod, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(frexp, double (*)(double, int*)); + REGISTER_LIBM_SYMBOL(hypot, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(ilogb, int (*)(double)); + REGISTER_LIBM_SYMBOL(ldexp, double (*)(double, int)); + REGISTER_LIBM_SYMBOL(lgamma, double (*)(double)); + REGISTER_LIBM_SYMBOL(llrint, long long (*)(double)); + REGISTER_LIBM_SYMBOL(llround, long long (*)(double)); + REGISTER_LIBM_SYMBOL(log, double (*)(double)); + REGISTER_LIBM_SYMBOL(log10, double (*)(double)); + REGISTER_LIBM_SYMBOL(log1p, double (*)(double)); + REGISTER_LIBM_SYMBOL(log2, double (*)(double)); + REGISTER_LIBM_SYMBOL(logb, double (*)(double)); + REGISTER_LIBM_SYMBOL(lrint, long (*)(double)); + REGISTER_LIBM_SYMBOL(lround, long (*)(double)); + REGISTER_LIBM_SYMBOL(modf, double (*)(double, double*)); + REGISTER_LIBM_SYMBOL(nan, double (*)(const char*)); + REGISTER_LIBM_SYMBOL(nearbyint, double (*)(double)); + REGISTER_LIBM_SYMBOL(nextafter, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(nexttoward, double (*)(double, long double)); + REGISTER_LIBM_SYMBOL(pow, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(remainder, double (*)(double, double)); + REGISTER_LIBM_SYMBOL(remquo, double (*)(double, double, int*)); + REGISTER_LIBM_SYMBOL(rint, double (*)(double)); + REGISTER_LIBM_SYMBOL(round, double (*)(double)); + REGISTER_LIBM_SYMBOL(scalbln, double (*)(double, long)); + REGISTER_LIBM_SYMBOL(scalbn, double (*)(double, int)); + REGISTER_LIBM_SYMBOL(sin, double (*)(double)); + REGISTER_LIBM_SYMBOL(sincos, void (*)(double, double*, double*)); + REGISTER_LIBM_SYMBOL(sinh, double (*)(double)); + REGISTER_LIBM_SYMBOL(sqrt, double (*)(double)); + REGISTER_LIBM_SYMBOL(tan, double (*)(double)); + REGISTER_LIBM_SYMBOL(tanh, double (*)(double)); + REGISTER_LIBM_SYMBOL(tgamma, double (*)(double)); + REGISTER_LIBM_SYMBOL(trunc, double (*)(double)); #undef REGISTER_LIBM_SYMBOL diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 7efdf8552e..6800c3d7fa 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -450,7 +450,7 @@ message ConvolutionDimensionNumbers { message ConvolveRequest { ComputationDataHandle lhs = 2; ComputationDataHandle rhs = 3; // This is the filter/kernel. - Window window = 4; // Describes the filter/kenel. + Window window = 4; // Describes the filter/kernel. ConvolutionDimensionNumbers dimension_numbers = 5; } diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h index 7a550d6f73..badc629a11 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h @@ -56,7 +56,7 @@ class BatchFeatures { *num_sparse_int_features = sparse_int_feature_columns_.size(); if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 && *num_sparse_int_features == 0) { - return errors::FailedPrecondition("Not intialized yet."); + return errors::FailedPrecondition("Not initialized yet."); } return Status::OK(); } diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h index e388cf332c..54f60e1dee 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/example.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/example.h @@ -63,7 +63,7 @@ class SparseFloatFeatureColumn { public: void Reserve(const int32 size) { if (!single_dimensional_) { - mutlidimensional_values.Reserve(size); + multidimensional_values.Reserve(size); } } @@ -76,7 +76,7 @@ class SparseFloatFeatureColumn { DCHECK_EQ(0, feature_idx); single_value_ = value; } else { - mutlidimensional_values.Add(feature_idx, value); + multidimensional_values.Add(feature_idx, value); } initialized_ = true; } @@ -84,7 +84,7 @@ class SparseFloatFeatureColumn { void Clear() { single_dimensional_ = false; initialized_ = false; - mutlidimensional_values.Clear(); + multidimensional_values.Clear(); } OptionalValue operator[](int feature_idx) const { @@ -94,7 +94,7 @@ class SparseFloatFeatureColumn { if (single_dimensional_) { return OptionalValue(single_value_); } else { - return mutlidimensional_values[feature_idx]; + return multidimensional_values[feature_idx]; } } @@ -102,7 +102,7 @@ class SparseFloatFeatureColumn { bool single_dimensional_; bool initialized_; T single_value_; - SparseMultidimensionalValues mutlidimensional_values; + SparseMultidimensionalValues multidimensional_values; }; // Holds data for one example and enables lookup by feature column. diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc index bc0a93db8c..ccee9530b6 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.cc @@ -96,6 +96,10 @@ class IndicesRowIterator return (row_idx_ != other.row_idx_); } + bool operator<(const IndicesRowIterator& other) const { + return (row_idx_ < other.row_idx_); + } + bool operator==(const IndicesRowIterator& other) const { QCHECK_EQ(iter_, other.iter_); return (row_idx_ == other.row_idx_); diff --git a/tensorflow/contrib/cmake/external/re2.cmake b/tensorflow/contrib/cmake/external/re2.cmake index b56f4b0898..d10f5959f7 100644 --- a/tensorflow/contrib/cmake/external/re2.cmake +++ b/tensorflow/contrib/cmake/external/re2.cmake @@ -45,4 +45,5 @@ ExternalProject_Add(re2 endif() -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_INSTALL_PREFIX:STRING=${re2_INSTALL} + -DRE2_BUILD_TESTING:BOOL=OFF ) diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake index dcedabb333..571d2b0dec 100644 --- a/tensorflow/contrib/cmake/tf_shared_lib.cmake +++ b/tensorflow/contrib/cmake/tf_shared_lib.cmake @@ -95,10 +95,18 @@ if(WIN32) add_dependencies(tensorflow tensorflow_static) endif(WIN32) -install(TARGETS tensorflow +target_include_directories(tensorflow PUBLIC + $ + $) + +install(TARGETS tensorflow EXPORT tensorflow_export RUNTIME DESTINATION bin LIBRARY DESTINATION lib ARCHIVE DESTINATION lib) + +install(EXPORT tensorflow_export + FILE TensorflowConfig.cmake + DESTINATION lib/cmake) # install necessary headers # tensorflow headers diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 5d6ba9ca8d..2e3ee2c96b 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -153,7 +153,7 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/contrib/data/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/factorization/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/image/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/integration_test.py" + "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/python/kernel_tests/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/python/kernel_tests/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/stateless/python/kernel_tests/*_test.py" @@ -171,7 +171,6 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/contrib/graph_editor/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/bayesflow/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/framework/*_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/keras/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/distributions/*_test.py" "${tensorflow_source_dir}/tensorflow/contrib/learn/*_test.py" ) @@ -225,6 +224,9 @@ if (tensorflow_BUILD_PYTHON_TESTS) # Numerical issues, calculations off. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py" "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" + "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py" + "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/backend_test.py" + "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py" # Float division by zero "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py" # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces. diff --git a/tensorflow/contrib/crf/python/ops/crf.py b/tensorflow/contrib/crf/python/ops/crf.py index ec395e41d0..7f5ae937b2 100644 --- a/tensorflow/contrib/crf/python/ops/crf.py +++ b/tensorflow/contrib/crf/python/ops/crf.py @@ -420,7 +420,7 @@ class CrfDecodeBackwardRnnCell(rnn_cell.RNNCell): """Initialize the CrfDecodeBackwardRnnCell. Args: - num_tags: The number of tags. + num_tags: An integer. The number of tags. """ self._num_tags = num_tags diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index 43431ca2c5..ffb5655c3e 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -161,6 +161,7 @@ py_test( size = "small", srcs = ["flat_map_dataset_op_test.py"], srcs_version = "PY2AND3", + tags = ["no_pip"], deps = [ ":dataset_serialization_test", "//tensorflow/contrib/data/python/ops:dataset_ops", @@ -278,6 +279,7 @@ py_test( size = "medium", srcs = ["map_dataset_op_test.py"], srcs_version = "PY2AND3", + tags = ["no_pip"], deps = [ ":dataset_serialization_test", "//tensorflow/contrib/data/python/ops:dataset_ops", @@ -348,6 +350,7 @@ py_test( size = "medium", srcs = ["reader_dataset_ops_test.py"], srcs_version = "PY2AND3", + tags = ["no_pip"], deps = [ ":dataset_serialization_test", "//tensorflow/contrib/data/python/ops:readers", diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 96dff85665..2701c36fb5 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -293,7 +293,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): # where, # # Z|v ~ interpolate_affine[v](distribution) - # V ~ mixture_distrubution + # V ~ mixture_distribution # # thus, # diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index aa359b7a0d..2f8016ede3 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -73,7 +73,7 @@ class Metric(object): * `result()`: Computes and returns a final value for the metric from the variables in `self`. - Decendants may override `aggregate()`, but usually won't need to. It + Descendants may override `aggregate()`, but usually won't need to. It adds in the state from a list of metrics of the same type as `self`. (Default is to sum all the variables.) Note that users should not call `aggregate()`, it is for use by TensorFlow infrastructure. diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py index 2bde3e0dd7..4fe22ea26e 100644 --- a/tensorflow/contrib/factorization/python/ops/wals.py +++ b/tensorflow/contrib/factorization/python/ops/wals.py @@ -183,7 +183,7 @@ def _wals_factorization_model_function(features, labels, mode, params): # TRAIN mode: if mode == model_fn.ModeKeys.TRAIN: - # Training consists of the folowing ops (controlled using a SweepHook). + # Training consists of the following ops (controlled using a SweepHook). # Before a row sweep: # row_update_prep_gramian_op # initialize_row_update_op diff --git a/tensorflow/contrib/ffmpeg/BUILD b/tensorflow/contrib/ffmpeg/BUILD index 7a5a4cb8c9..eccce99071 100644 --- a/tensorflow/contrib/ffmpeg/BUILD +++ b/tensorflow/contrib/ffmpeg/BUILD @@ -47,10 +47,25 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "decode_video_op_cc", + srcs = ["decode_video_op.cc"], + copts = tf_copts(), + linkstatic = 1, + visibility = ["//visibility:private"], + deps = [ + "//tensorflow/contrib/ffmpeg/default:ffmpeg_lib", + "//tensorflow/core:framework_headers_lib", + "//third_party/eigen3", + ], + alwayslink = 1, +) + tf_custom_op_library( name = "ffmpeg.so", deps = [ ":decode_audio_op_cc", + ":decode_video_op_cc", ":encode_audio_op_cc", ], ) @@ -59,6 +74,7 @@ cc_library( name = "ffmpeg_op_lib", deps = [ ":decode_audio_op_cc", + ":decode_video_op_cc", ":encode_audio_op_cc", ], ) @@ -81,6 +97,15 @@ tf_gen_op_wrapper_py( ], ) +tf_gen_op_wrapper_py( + name = "decode_video_op_py", + require_shape_functions = True, + visibility = ["//visibility:private"], + deps = [ + ":decode_video_op_cc", + ], +) + tf_py_test( name = "decode_audio_op_test", srcs = ["decode_audio_op_test.py"], @@ -115,6 +140,27 @@ tf_py_test( tags = ["manual"], ) +tf_py_test( + name = "decode_video_op_test", + size = "small", + srcs = ["decode_video_op_test.py"], + additional_deps = [ + ":ffmpeg_ops_py", + "@six_archive//:six", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:platform", + "//tensorflow/python:image_ops", + ], + data = [ + ":test_data", + ], + tags = [ + "manual", + "notap", + ], +) + py_library( name = "ffmpeg_ops_py", srcs = [ @@ -126,6 +172,7 @@ py_library( visibility = ["//visibility:public"], deps = [ ":decode_audio_op_py", + ":decode_video_op_py", ":encode_audio_op_py", "//tensorflow/contrib/util:util_py", "//tensorflow/python:framework_for_generated_wrappers", diff --git a/tensorflow/contrib/ffmpeg/__init__.py b/tensorflow/contrib/ffmpeg/__init__.py index 2bcb7284e1..484ffee3e7 100644 --- a/tensorflow/contrib/ffmpeg/__init__.py +++ b/tensorflow/contrib/ffmpeg/__init__.py @@ -26,9 +26,10 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.ffmpeg.ffmpeg_ops import decode_audio +from tensorflow.contrib.ffmpeg.ffmpeg_ops import decode_video from tensorflow.contrib.ffmpeg.ffmpeg_ops import encode_audio from tensorflow.python.util.all_util import remove_undocumented -_allowed_symbols = ['decode_audio', 'encode_audio'] +_allowed_symbols = ['decode_audio', 'encode_audio', 'decode_video'] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/ffmpeg/decode_audio_op.cc b/tensorflow/contrib/ffmpeg/decode_audio_op.cc index 4b1c8a337e..92fad70b1f 100644 --- a/tensorflow/contrib/ffmpeg/decode_audio_op.cc +++ b/tensorflow/contrib/ffmpeg/decode_audio_op.cc @@ -37,29 +37,6 @@ namespace { // https://www.ffmpeg.org/ffmpeg-formats.html const char* kValidFileFormats[] = {"mp3", "mp4", "ogg", "wav"}; -// Writes binary data to a file. -Status WriteFile(const string& filename, tensorflow::StringPiece contents) { - Env& env = *Env::Default(); - std::unique_ptr file; - TF_RETURN_IF_ERROR(env.NewWritableFile(filename, &file)); - TF_RETURN_IF_ERROR(file->Append(contents)); - TF_RETURN_IF_ERROR(file->Close()); - return Status::OK(); -} - -// Cleans up a file on destruction. -class FileDeleter { - public: - explicit FileDeleter(const string& filename) : filename_(filename) {} - ~FileDeleter() { - Env& env = *Env::Default(); - env.DeleteFile(filename_).IgnoreError(); - } - - private: - const string filename_; -}; - /* * Decoding implementation, shared across V1 and V2 ops. Creates a new * output in the context. @@ -69,7 +46,7 @@ void Decode(OpKernelContext* context, const string& file_format, const int32 samples_per_second, const int32 channel_count) { // Write the input data to a temp file. - const string temp_filename = GetTempFilename(file_format); + const string temp_filename = io::GetTempFilename(file_format); OP_REQUIRES_OK(context, WriteFile(temp_filename, file_contents)); FileDeleter deleter(temp_filename); diff --git a/tensorflow/contrib/ffmpeg/decode_video_op.cc b/tensorflow/contrib/ffmpeg/decode_video_op.cc new file mode 100644 index 0000000000..d44032968d --- /dev/null +++ b/tensorflow/contrib/ffmpeg/decode_video_op.cc @@ -0,0 +1,118 @@ +// Copyright 2016 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +#include + +#include +#include + +#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace ffmpeg { + +class DecodeVideoOp : public OpKernel { + public: + explicit DecodeVideoOp(OpKernelConstruction* context) : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + OP_REQUIRES( + context, context->num_inputs() == 1, + errors::InvalidArgument("DecodeVideo requires exactly 1 input.")); + const Tensor& contents_tensor = context->input(0); + + OP_REQUIRES(context, TensorShapeUtils::IsScalar(contents_tensor.shape()), + errors::InvalidArgument( + "contents must be a rank-0 tensor but got shape ", + contents_tensor.shape().DebugString())); + const tensorflow::StringPiece contents = contents_tensor.scalar()(); + + // Write the input data to a temp file. + string extension; + const string temp_filename = io::GetTempFilename(extension); + OP_REQUIRES_OK(context, WriteFile(temp_filename, contents)); + FileDeleter deleter(temp_filename); + + uint32 width = 0; + uint32 height = 0; + uint32 frames = 0; + + // Run FFmpeg on the data and verify results. + std::vector output_data; + const Status result = ffmpeg::ReadVideoFile(temp_filename, &output_data, + &width, &height, &frames); + if (result.code() == error::Code::NOT_FOUND) { + OP_REQUIRES( + context, result.ok(), + errors::Unavailable("FFmpeg must be installed to run this op. FFmpeg " + "can be found at http://www.ffmpeg.org.")); + } else if (result.code() == error::UNKNOWN) { + LOG(ERROR) << "Ffmpeg failed with error '" << result.error_message() + << "'. Returning empty tensor."; + Tensor* output = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape({0, 0}), &output)); + return; + } else { + OP_REQUIRES_OK(context, result); + } + OP_REQUIRES(context, !output_data.empty(), + errors::Unknown("No output created by FFmpeg.")); + OP_REQUIRES( + context, output_data.size() == (frames * height * width * 3), + errors::Unknown("Output created by FFmpeg [", output_data.size(), + "] does not match description [", frames, ", ", height, + ", ", width, ", 3]")); + Tensor* output = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output( + 0, TensorShape({frames, height, width, 3}), &output)); + auto output_flat = output->flat(); + std::copy_n(output_data.begin(), output_data.size(), &output_flat(0)); + } +}; + +REGISTER_KERNEL_BUILDER(Name("DecodeVideo").Device(DEVICE_CPU), DecodeVideoOp); + +REGISTER_OP("DecodeVideo") + .Input("contents: string") + .Output("output: uint8") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->UnknownShapeOfRank(4)); + return Status::OK(); + }) + .Doc(R"doc( +Processes the contents of an audio file into a tensor using FFmpeg to decode +the file. + +One row of the tensor is created for each channel in the audio file. Each +channel contains audio samples starting at the beginning of the audio and +having `1/samples_per_second` time between them. If the `channel_count` is +different from the contents of the file, channels will be merged or created. + +contents: The binary audio file contents, as a string or rank-0 string + tensor. +)doc"); + +} // namespace ffmpeg +} // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/decode_video_op_test.py b/tensorflow/contrib/ffmpeg/decode_video_op_test.py new file mode 100644 index 0000000000..b43b6b8919 --- /dev/null +++ b/tensorflow/contrib/ffmpeg/decode_video_op_test.py @@ -0,0 +1,69 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================= +"""Tests for third_party.tensorflow.contrib.ffmpeg.decode_video_op.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os.path + +import six # pylint: disable=unused-import + +from tensorflow.contrib import ffmpeg +from tensorflow.python.ops import image_ops +from tensorflow.python.platform import resource_loader +from tensorflow.python.platform import test + + +class DecodeVideoOpTest(test.TestCase): + + def _loadFileAndTest(self, filename, width, height, frames, bmp_filename, + index): + """Loads an video file and validates the output tensor. + + Args: + filename: The filename of the input file. + width: The width of the video. + height: The height of the video. + frames: The frames of the video. + bmp_filename: The filename for the bmp file. + index: Index location inside the video. + """ + with self.test_session(): + path = os.path.join(resource_loader.get_data_files_path(), 'testdata', + filename) + with open(path, 'rb') as f: + contents = f.read() + + bmp_path = os.path.join(resource_loader.get_data_files_path(), 'testdata', + bmp_filename) + with open(bmp_path, 'rb') as f: + bmp_contents = f.read() + + image_op = image_ops.decode_bmp(bmp_contents) + image = image_op.eval() + self.assertEqual(image.shape, (height, width, 3)) + video_op = ffmpeg.decode_video(contents) + video = video_op.eval() + self.assertEqual(video.shape, (frames, height, width, 3)) + self.assertAllEqual(video[index, :, :, :], image) + + def testMp4(self): + self._loadFileAndTest('small.mp4', 560, 320, 166, 'small_100.bmp', 99) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc index 545a4386d0..1245f515fe 100644 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc @@ -16,6 +16,7 @@ #include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" @@ -38,28 +40,45 @@ namespace { const char kFfmpegExecutable[] = "ffmpeg"; const int32 kDefaultProbeSize = 5000000; // 5MB -std::vector FfmpegCommandLine(const string& input_filename, - const string& output_filename, - const string& input_format_id, - int32 samples_per_second, - int32 channel_count) { - return { - "-nostats", // No additional progress display. - "-nostdin", // No interactive commands accepted. - "-f", input_format_id, // eg: "mp3" - "-probesize", StrCat(kDefaultProbeSize), - "-i", input_filename, - "-loglevel", "info", // Enable verbose logging to support debugging. - "-map_metadata", "-1", // Copy global metadata from input to output. - "-vn", // No video recording. - "-ac:a:0", StrCat(channel_count), - "-ar:a:0", StrCat(samples_per_second), - // Output set (in several ways) to signed 16-bit little-endian ints. - "-codec:a:0", "pcm_s16le", "-sample_fmt", "s16", "-f", "s16le", - "-sn", // No subtitle recording. - "-y", // Overwrite output file. - StrCat(output_filename) - }; +std::vector FfmpegAudioCommandLine(const string& input_filename, + const string& output_filename, + const string& input_format_id, + int32 samples_per_second, + int32 channel_count) { + return {"-nostats", // No additional progress display. + "-nostdin", // No interactive commands accepted. + "-f", input_format_id, // eg: "mp3" + "-probesize", StrCat(kDefaultProbeSize), "-i", input_filename, + "-loglevel", "info", // Enable verbose logging to support debugging. + "-map_metadata", "-1", // Copy global metadata from input to output. + "-vn", // No video recording. + "-ac:a:0", StrCat(channel_count), "-ar:a:0", + StrCat(samples_per_second), + // Output set (in several ways) to signed 16-bit little-endian ints. + "-codec:a:0", "pcm_s16le", "-sample_fmt", "s16", "-f", "s16le", + "-sn", // No subtitle recording. + "-y", // Overwrite output file. + StrCat(output_filename)}; +} + +std::vector FfmpegVideoCommandLine(const string& input_filename, + const string& output_filename) { + return {"-nostats", // No additional progress display. + "-nostdin", // No interactive commands accepted. + "-i", + input_filename, + "-f", + "image2pipe", + "-probesize", + StrCat(kDefaultProbeSize), + "-loglevel", + "info", // Enable verbose logging to support debugging. + "-vcodec", + "rawvideo", + "-pix_fmt", + "rgb24", + "-y", // Overwrite output file. + StrCat(output_filename)}; } // Is a named binary installed and executable by the current process? @@ -106,7 +125,7 @@ bool IsBinaryInstalled(const string& binary_name) { ::execvp(kFfmpegExecutable, args_chars.data()); // exec only returns on error. const int error = errno; - LOG(ERROR) << "FFmpeg could not be executed: " << error; + LOG(ERROR) << "FFmpeg could not be executed: " << strerror(error); ::_exit(error); } @@ -198,52 +217,101 @@ string BuildWavFile(int32 samples_per_second, int32 channel_count, return data; } -// Returns a unique number every time it is called. -int64 UniqueId() { - static mutex mu(LINKER_INITIALIZED); - static int64 id = 0; - mutex_lock l(mu); - return ++id; -} - -} // namespace - -string GetTempFilename(const string& extension) { - for (const char* dir : std::vector( - {getenv("TEST_TMPDIR"), getenv("TMPDIR"), getenv("TMP"), "/tmp"})) { - if (!dir || !dir[0]) { +Status ReadInfoFile(const string& filename, uint32* width, uint32* height, + uint32* frames) { + string data; + TF_QCHECK_OK(ReadFileToString(Env::Default(), filename, &data)) + << "Could not read FFmpeg file: " << filename; + bool in_output = false; + bool in_mapping = false; + uint32 frames_value = 0; + uint32 height_value = 0; + uint32 width_value = 0; + for (const string& line : str_util::Split(data, '\n')) { + // Output starts with the first line of `Output #..`. + // Further processing output region starts next line so we could continue + // the loop. + if (!in_output && line.find("Output #") == 0) { + in_output = true; + in_mapping = false; continue; } - struct stat statbuf; - if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) { - // UniqueId is added here because mkstemps is not as thread safe as it - // looks. https://github.com/tensorflow/tensorflow/issues/5804 shows - // the problem. - string tmp_filepath = io::JoinPath( - dir, - StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX.", extension)); - int fd = mkstemps(&tmp_filepath[0], extension.length() + 1); - if (fd < 0) { - LOG(FATAL) << "Failed to create temp file."; - } else { - close(fd); - return tmp_filepath; + // Stream mapping starts with the first line of `Stream mapping`, it also + // signals the end of Output section. + // Further processing of stream mapping region starts next line so we could + // continue the loop. + if (!in_mapping && line.find("Stream mapping:") == 0) { + in_output = false; + in_mapping = true; + continue; + } + if (in_output) { + // We only look for the first stream in output `Stream #0`. + // Once processed we will not further process output section. + if (line.find(" Stream #") == 0) { + size_t p = line.find(", rgb24, ", 24); + if (p != std::string::npos) { + string rgb24 = line.substr(p + 9, line.find(" ", p + 9)); + rgb24 = rgb24.substr(0, rgb24.find(",")); + string rgb24_width = rgb24.substr(0, rgb24.find("x")); + string rgb24_height = rgb24.substr(rgb24_width.length() + 1); + if (strings::safe_strtou32(rgb24_width, &width_value) && + strings::safe_strtou32(rgb24_height, &height_value)) { + in_output = false; + } + } + } + continue; + } + if (in_mapping) { + // We only look for the first stream mapping to have the number of the + // frames. + // Once processed we will not further process stream mapping section. + if (line.find("frame= ") == 0) { + string number = line.substr(8, line.find(" ", 8)); + number = number.substr(0, number.find(" ")); + if (strings::safe_strtou32(number, &frames_value)) { + in_mapping = false; + } } + continue; } } - LOG(FATAL) << "No temp directory found."; + if (frames_value == 0 || height_value == 0 || width_value == 0) { + return errors::Unknown("Not enough video info returned by FFmpeg [", + frames_value, ", ", height_value, ", ", width_value, + ", 3]"); + } + *width = width_value; + *height = height_value; + *frames = frames_value; + return Status::OK(); } -Status ReadAudioFile(const string& filename, - const string& audio_format_id, - int32 samples_per_second, - int32 channel_count, +} // namespace + +FileDeleter::~FileDeleter() { + Env& env = *Env::Default(); + env.DeleteFile(filename_).IgnoreError(); +} + +Status WriteFile(const string& filename, StringPiece contents) { + Env& env = *Env::Default(); + std::unique_ptr file; + TF_RETURN_IF_ERROR(env.NewWritableFile(filename, &file)); + TF_RETURN_IF_ERROR(file->Append(contents)); + TF_RETURN_IF_ERROR(file->Close()); + return Status::OK(); +} + +Status ReadAudioFile(const string& filename, const string& audio_format_id, + int32 samples_per_second, int32 channel_count, std::vector* output_samples) { // Create an argument list. - string output_filename = GetTempFilename("raw"); + string output_filename = io::GetTempFilename("raw"); const std::vector args = - FfmpegCommandLine(filename, output_filename, audio_format_id, - samples_per_second, channel_count); + FfmpegAudioCommandLine(filename, output_filename, audio_format_id, + samples_per_second, channel_count); // Unfortunately, it's impossible to differentiate an exec failure due to the // binary being missing and an error from the binary's execution. Therefore, @@ -256,7 +324,8 @@ Status ReadAudioFile(const string& filename, // Execute ffmpeg and report errors. pid_t child_pid = ::fork(); if (child_pid < 0) { - return Status(error::Code::UNKNOWN, StrCat("fork failed: ", errno)); + return Status(error::Code::UNKNOWN, + StrCat("fork failed: ", strerror(errno))); } if (child_pid == 0) { ExecuteFfmpeg(args); @@ -285,5 +354,63 @@ Status CreateAudioFile(const string& audio_format_id, int32 bits_per_second, return Status::OK(); } +Status ReadVideoFile(const string& filename, std::vector* output_data, + uint32* width, uint32* height, uint32* frames) { + if (!IsBinaryInstalled(kFfmpegExecutable)) { + return Status(error::Code::NOT_FOUND, StrCat("FFmpeg could not be found.")); + } + + string output_filename = io::GetTempFilename("raw"); + string stderr_filename = io::GetTempFilename("err"); + + // Create an argument list. + const std::vector args = + FfmpegVideoCommandLine(filename, output_filename); + + // Execute ffmpeg and report errors. + pid_t child_pid = ::fork(); + if (child_pid < 0) { + return Status(error::Code::UNKNOWN, + StrCat("fork failed: ", strerror(errno))); + } + if (child_pid == 0) { + const int fd = + open(stderr_filename.c_str(), O_RDWR | O_CREAT | O_APPEND, 0600); + if (fd < 0) { + const int error = errno; + LOG(ERROR) << "FFmpeg stderr file could not be created: " + << strerror(error); + ::_exit(error); + } + close(STDERR_FILENO); + dup2(fd, STDERR_FILENO); + ExecuteFfmpeg(args); + } else { + int status_code; + if (::waitpid(child_pid, &status_code, 0) < 0) { + return Status(error::Code::UNKNOWN, + StrCat("waitpid failed: ", strerror(errno))); + } + if (status_code) { + return Status(error::Code::UNKNOWN, + StrCat("FFmpeg execution failed: ", status_code)); + } + + TF_QCHECK_OK(ReadInfoFile(stderr_filename, width, height, frames)) + << "Could not read FFmpeg stderr file: " << stderr_filename; + + string raw_data; + TF_QCHECK_OK(ReadFileToString(Env::Default(), output_filename, &raw_data)) + << "Could not read FFmpeg output file: " << output_filename; + output_data->resize(raw_data.size()); + std::copy_n(raw_data.data(), raw_data.size(), output_data->begin()); + + TF_QCHECK_OK(Env::Default()->DeleteFile(output_filename)) + << output_filename; + TF_QCHECK_OK(Env::Default()->DeleteFile(stderr_filename)) + << stderr_filename; + return Status::OK(); + } +} } // namespace ffmpeg } // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc index 7176f3b550..d6c885a324 100644 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc @@ -21,6 +21,7 @@ #include #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/test.h" @@ -49,7 +50,7 @@ TEST(FfmpegLibTest, TestTempDirectoryThreading) { pool.Schedule([&mu, &temp_filenames, environment]() { std::array buffer; for (int32 j = 0; j < kStringsPerItem; ++j) { - buffer[j] = GetTempFilename("mp3"); + buffer[j] = io::GetTempFilename("mp3"); TF_QCHECK_OK(environment->DeleteFile(buffer[j])); } mutex_lock l(mu); diff --git a/tensorflow/contrib/ffmpeg/ffmpeg_lib.h b/tensorflow/contrib/ffmpeg/ffmpeg_lib.h index f64007c81d..c5ea1432bf 100644 --- a/tensorflow/contrib/ffmpeg/ffmpeg_lib.h +++ b/tensorflow/contrib/ffmpeg/ffmpeg_lib.h @@ -24,16 +24,24 @@ namespace tensorflow { namespace ffmpeg { -// Gets a temp filename in an appropriate location. -string GetTempFilename(const string& extension); +// Cleans up a file on destruction. +class FileDeleter { + public: + explicit FileDeleter(const string& filename) : filename_(filename) {} + ~FileDeleter(); + + private: + const string filename_; +}; + +// Writes binary data to a file. +Status WriteFile(const string& filename, tensorflow::StringPiece contents); // Reads an audio file using ffmpeg and converts it into an array of samples in // [-1.0, 1.0]. If there are multiple channels in the audio then each frame will // contain a separate sample for each channel. Frames are ordered by time. -Status ReadAudioFile(const string& filename, - const string& audio_format_id, - int32 samples_per_second, - int32 channel_count, +Status ReadAudioFile(const string& filename, const string& audio_format_id, + int32 samples_per_second, int32 channel_count, std::vector* output_samples); // Creates an audio file using ffmpeg in a specific format. The samples are in @@ -45,6 +53,11 @@ Status CreateAudioFile(const string& audio_format_id, int32 bits_per_second, int32 samples_per_second, int32 channel_count, const std::vector& samples, string* output_data); +// Reads an video file using ffmpeg adn converts it into a RGB24 in uint8 +// [frames, height, width, 3]. The w, h, and frames are obtained from ffmpeg. +Status ReadVideoFile(const string& filename, std::vector* output_data, + uint32* width, uint32* height, uint32* frames); + } // namespace ffmpeg } // namespace tensorflow diff --git a/tensorflow/contrib/ffmpeg/ffmpeg_ops.py b/tensorflow/contrib/ffmpeg/ffmpeg_ops.py index 18b0b8b812..5bb011f41c 100644 --- a/tensorflow/contrib/ffmpeg/ffmpeg_ops.py +++ b/tensorflow/contrib/ffmpeg/ffmpeg_ops.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.ffmpeg.ops import gen_decode_audio_op_py +from tensorflow.contrib.ffmpeg.ops import gen_decode_video_op_py from tensorflow.contrib.ffmpeg.ops import gen_encode_audio_op_py from tensorflow.contrib.util import loader from tensorflow.python.framework import ops @@ -89,3 +90,19 @@ def encode_audio(audio, file_format=None, samples_per_second=None): ops.NotDifferentiable('EncodeAudio') + + +def decode_video(contents): + """Create an op that decodes the contents of a video file. + + Args: + contents: The binary contents of the video file to decode. This is a + scalar. + + Returns: + A rank-4 `Tensor` that has `[frames, height, width, 3]` RGB as output. + """ + return gen_decode_video_op_py.decode_video(contents) + + +ops.NotDifferentiable('DecodeVideo') diff --git a/tensorflow/contrib/ffmpeg/testdata/small.mp4 b/tensorflow/contrib/ffmpeg/testdata/small.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fc478842f51e7519866f474a02ad605235bc6a6 GIT binary patch literal 383631 zcmZQzV30{GsVvAXFfn3aU|;~znZ^0JiDk)#3=9k{X+^22An^gitfyb(rX-dyFfehl zvHt(hddKzU-e)&GwolW!p>R|A-TMk7GZO_}h2;FAR0VTO1ziOXJtH#{ecu3AcU=<& z$1rDwWCdLX=lp`ooYb@u1tS9kV_hQy1CZ*Bl9B=|ef{$Ca=o(5l+^s3#5}$H zqI7+bv3eOLxj72D3i$;knfZCeRtm|9Nr}m}1`0)~X|_fRDXB?0`N`R~hE@hv1`3IJ zi8+(ATquv#nw>Iz{)_+Kp{7? zA~hu*WFAO;d{JUvdaA9VnL=_#QGRY>d~T|(p+ZSfYEDjOv8{oEMTJF5a*3^hLULiQ zt${*HYGO)NeqO4rk)e*EAxuwUL41B%T5)QLt*((mNk&m>VoI^CnL=I>SVwYZZej_@ z)ZC=Rv^0=W`FX`9MTwbtsVVW9c_l@Owgw7GX+?>-sURE63yM-x3TzD&vQsNF^Ga-u z4Peap+{`>%V*`can_vp{)74@{>T){F>z-{XfmOWYhe@Z)o=;FFzE9>PLL;dGfu>5}gi}i`d z!T)!%o9{K}MYb&NF0$tOG|w>ddFNF7|NpvH9bWQ4!(#e}YYav!?zC(-fAk^Mb(zuqQ5f%m1f8rkAz7?{8yZjr8$wQYq-T%N)oZk-D$Jx{u8* z;biNFH}%Z2iJ~*SwZ24M-oH=X_oDSi>8=k6Ut;|K_)U4@X{J=|vipe0v4qV|>3;Xm zw@bzfHoSN-i6`Khxp2V#E72>~2WNkNRC88rn|;3B%84JI;6fQ{+bzqyBmI11hW+Vg%T;v+An9>}B}3Rzl?f-`-92YfL@!M1=M-H!yzI zOrFrtrN_uHf%8H4S>tt%hgb9O{g5CP$dRU%!t(SzV~dZuSW2(y)Ni>z87`d>*&O)H z{=kut*)tt16`p0}{E=S0@u7`0TNes@i|fFK2L1;n9fj z+?Dcs=U%puU#?sB32@yL5IHtY;oGGJ4_0(!dglsV5xuR#>Sh)2mtV-xlf#~Y?b?R? z13cM@rxRSCfB)zvU;Ff5*#AYRW^P)4`_xp1{92<9Hk11?&;IS4ze7gi(FdIsSpwoh zhZ!BCB5xc}=zrOe&^ae{$>~%BiD`aJ31FOH+Gr#N4;`skTSM~EV$A)8~Rt8}wlFly)=X&(8WRD{Q=gdP4ee>j=-Z^wI zPVUsqh_9T=>YWor1(u07>i#|U_1LZ5M-)3(GN?y9+_ufpg@4QG-R2Lro|7tVUOQ#s zvYPYdWzvn)((YGkI(r+ukjp*nq*E8PZuzGL6&DN`G>$nkFqSW#Q*ho@>yh!3PwBer z8ZY=Z3FMxsP5SvGQdc@_MeO2m;qsGht0(TfIy+cAv@UGV*=m#L|Noz*`nksZJ#iudy7uRnUtF^7@b%7{6Bij+8P0Tb zdwpi>&yQtuUH{upQeLs!Z`I}3r_Ogi4D;Kj8gcjO%|E+)1EVk23vIW`ZvQ1)AX94c zt?q+ixYI+hBQuhgs&U z%&fQRu3e@_7pbT!FD*L!UH?&OVb)^}*HtHH)>-_qx-Y3?Xms13H8X~NX6AGwF&*Ce zW@DZ&^+qdi{oK$Konkv*YM1D>Z`ZtbhU^N+kA1Z=c+K}U;fDhsHoajfVqex7)>rhW z<>^eL*%seTLl}cO&A#3;GI%rN*g1dJyP3zIT@A{b6aJL%zx|KTIbH7sJA>Ay?J{`4 zEW_Er#Ivl%=YR>%s8qq>7X(_p=QUC8+?7eifNrE=gswd8<4YV8aOGZ($O{&DDU|Bw6fUke{dmD>94#~h7y6?4L+O1ZQ?O2`^; zPM7Ws^uM;Ml)H8Exm6dnG{5-io2*)X_4%jyWt~S0{nsUx9@llT5OuXU|Nlc=dz}55 zzwIsw3cVtx_kOS|tnK1TGqt^KbU^FB$mxG~rY0|&v|V5Br>pmm&l`VpaBul@bg%ib z^4DzN=Kq+IJ^yE5Vd>1ND+@&e?=x^5*_zmvE~0L1pu=GC@POxOo;_TgUAt@h-6L{} zW5q8ssPzdbPkU?-{lD&Q?F{`*9qUy3*Kg=jnmExz_TUjF&F+_19*Fk#ZdonHAiGY~ zp)!I&=wr}ytJF6U%c`30PB^{Hjp6KV#f9~I5+i3!s;)4yxYb-OqkeeKiaCX@o0TTj z2>&uKd--SgbkV7)D$YmVeR{M}OFPND=>0dR@*l!`Ppo10XILV8clwb>|CK%$-RnNG zW~Ej4*GDs^#(S-*zdYHaf4|qptejZ;>H3f7Z;m<2Vk2bR;qh++myVh}!<}hP3-qcV z@P1psY{3&}S!}XtJHr9D@RJheCJP!EZZL6a2?(rKGRa^_Gwhpjzwyli$CbP{xRWgy z%gr}mX`R||ae+N&4|8DiBp2@(Mum-ee^}a2*plBH9Pg$?@vj#=61`#TGUGpDeQxz1J73h+2AVxvAbx!1g!{22Mu$Z|?|UaSL4tjm zlV8Mo+sq@0ihmylq)W^y3stbPkea2joYm$oZAlg36#9tqdk9G zW`BYB8Pz6*tHSG87D#zNm}75T+EI7U$3c+gw=F~2{zuOno2NTeJji>T+2vr#Bf!U~ zz|PV8m`|W*(^bz$fd}O_C`)o3tZ9sKUiRR`!50lvm;SIg@FAlAq!9ZXk=@J*Ukb#v z^7b(NpOUhMf%DJIhcmoaf2sc9G2iA2+kx%5+ZZ-{uaMwtc(9t$eb4dMeMSvit2Z(# z?4380{j%Mnl7%ii3olx%>1V2WvZU+Czo{`xx{geGH#NrZjoXTL@#~#O%UxC$PvtmJ zQlc`i^RLjh&d|!jLan_O%WC4+#`qtqJU>~^{^jaJZ0m16V18}f$nj-$iM3xEg8*ko zfzdmLRu9R86Z-pDm%aG!y<^9u35v%~f7ri#3qw)zhMsWG$H|V<*X`f0I$tWNuP*2P zhN8J0+qE3pL|YZU-V0w}Cd8lWQ;>Z4|APq(0!#@81`OX+gcaKEtZWso5o{DGkKE+u za=?W>>dEOw{^OS)%zn0f|L%LMggw;{ty=k_p0VZ%`<_MYXpOx^l}=#hIB|g5(q@%h=#!jR$EvZ*p;);#m*?fY)I`t}K@hIbueIl!R6#m>TdK%!blTh6)0C-0>^ z17p;rme`ke=E>~}oxOWb@Z_(Y*v0s0fk@q>9<7GZ?`K_@UuD+EX>rVB^x3gf^2NFa zV`gEKFY}Cp(l@*=lL%Va{pI4s;+Z9UcLih?-m(*5;N|CG;!CWm(qfQED99+-Y5UfC zN|wlkCnpyOB`C&fFwavsc4e*>Gq1w*t1j6Q4l|u<8vJi1c;2^XxWAhD#Z+6zz3Y_g zjg6=O2o>G3*qiZaCEJs__{AJf2@C?lOe@OXEqKx7G;P5(r}J(FL6tift)8ZmA9pWG zclQ<7kaW3CUX1JNN^;W`7xnKtE7+WPN&n*nv-9%NmJf`t*!_trfA;@-t6bqkjV0!j zkLB}3_@A?nUipmiLYmzYhSjF;7C&m|Jg6lT>GyOV=auz~@*NbW`R!kp)hxyz#_OC_ z`R}q#hGXs49~&a}O_}-s@{2=LWuoUFh}gK`+UpWMW>>>`_8bg?G8qnnPV$n6^u8R} zd@#;XlP~^x#LX=mo(U~6WWCMevGd;Vh-2>-nZ{q`c$6s5y0%X09W%ouKBZTxhZ#;6 zx6RRYG_-AC(o`smzv%LA@rxZ+0r%eoBs0|)|2}eWeyg>%#dM!3k0iNgF)<`ekZ86o zxbl3(as!9ihJ_4FJ1@(ir^A1W zXk{K~U{T`c5EGr_7%arTW7Z9gNfQ_{tX?>9uFXye;oB4|x=lkwd`@j)YI*7ti@0;|6bzv1~UW^sv(O^Z~f&ztd!CA#;}%(}1Hvp+C76mm`vxV~YQ z_{JJ$ku^4-cfa9ncRGKkTJdn;2BXu{S8=d5TyxTiZrJHiHb=^>iAB51tRb&|Uslth z^RYW-WZub;iM*#J+4;@&+K;uLQv3w@Z#MPU=tl;3-Z|8ZhFVk?>mR%_3b>qf@%w{MZ(jiyBZE8s8pSO=fMB$fM;X_ z*Tvh#Z=yfjeBdsM$WoH}*0Am?laR(;`@gRW{~SC2&D)zvdh^u2&4DvY)lYnDIkb6; z026}%2XA8o!^&f63}LDtmNJ~OZfm-5h|96y*QF+pzm0D9q&vCK>`6U(=jySQ;><1& z)$BVI_N?50(0y{^qVGzb`ATnpd|JBkv8f<`T|rpwOkumP`?C~gq%!|MQn`sexcYP? z-_1O+9Rd2TDz{Jf^}gAXrGHbXKXT?cB75*J8#O#sdrvF03{R6CMZ{ zEm`0s(^}=r@Yq3zF(he~m+18^6)cJG4+s7{_Le>9^0J$fza>o`$8BOOV~*0fYq>~9 zJ9m>`(;m6*2v!4L9mk88oZlz@WHeXqZ=0ez$K#RW`8{8|H|oh>t1(Z$a9O_N{^Qx- z&xB6dR;l^K)QnZ8y21Hwc~8&k|Nk#4Rg|^FPh5IUseb>L`%xSW4(T5p4Sd&M9;4ooK@a<~j+> zBZV)Rn_~9x#BnV?!aC)6&b9wJn***mBuL9lv|Yhjz}Uvlz&MSQ`PaL{2Ud9NFv`D( z+O5N$T)^D7W(HdVb0X(jzVP!o9|Ac49#|Uq@9NReQdU8;B4!WiXm_{G``8>)k0p4N z-+TJ(h33H()!BZ&57gZ2_&Fm}HwN&mZ2JEBux6cv;JU+wn+RRw!%9M z|INde?p?6X^xp*s7M2B#Jk9e986;P~O^$d^Zm@1&d*@hVS~=rI^%kEEN^4Kt%#AZVEpzR7fAz*T2B(cI3>-(f zy#yZ}u*piSF=+hz;h2%Y^v!d`6-=3~BryK#w7F4xg^_0#uY~@EW%2(`o?I`*_C?{`lXWzl&syU2{tQzs}8@rT&>|oAX`9U;AGl5xB-%^Fh-!K|Esr4GmQb z8TO_+<|WS#FgVC`CUk#Lu=Te$+9GiF|C8r>dQIA~At7hI7qU3DSe9cevciAjGDlsSUiyPye4_e1Mr_8))XZ*lu zl`m82^gBB|R#$plkG_&qCRO@B=f3GJm##Tay?YO(x?T9SJjuPwZMl(3nMjs_$Hv>D z$*cbOnyS@*ws&OYjxN|4{CQd@vnKPcx36ZqF}l6(>iGYc|JACM+3RlC@8>?VUa&B4 zio-|dmWGvwPgKua#l|R7xj?bg_gw?)zZ9K^`OCK54d0~r|0%-}xwSGY7?>Z1?F?*Q z+bU_|dUb&l1JeS5gNh8UQPP}pw;HtEHar$d&TtT1IQOC9We4W16VBb{_qVaqo^8Bv zajRH?&4h@VGdOe(9C+uN#ITBq(Ll`pVVA+hM=?uJ8{4|RzPT;2yY}sKJ%^pw9RGA> zg-*_VdHG0F$|0$oUrTZv{WmY`Y-G79%~eY7m1tZjs9!dDd)}5S&1?Ssos;Ime6`f@QTP9U z+iPa!Zw<}gpe$?tk^5f2g);HVyEkSU`EP%i{_l)rUD?zv>vEk#4u-4opvv!6BX?BNQ{U%DV-`S&$@#DCA~ zt^alP_L>sI{VWa!77gRWHiS9t!P$SSRQkw5xxkjP<##oFI5?)Z87mn2Imh4AoWGVWVArFcDQZD?Pq(^$EeleRYRWqid%XUVi7Zd< zuj#)hDi(CU_-4mte}ki6Va1#0XAd(lGO97iNw6;M`XK4VyZOk6Jq&717dLY#J6T=` zF0eA2o{}ucAhwG?amAIVOl=zT-`6(YJ!gN!dJDs{uD8;xS`NDwOkRCz+m4cyzYez! z$5|}0zVgE9L7H@gv`yuI%`;U&TLf5qZ4{*Lek%*#cQc&vZ5(qNqkqMp*1bXe4S!^I z*i^g`uwFPxd68LIe&d-d#h2E-ce!*!((Ki`M|+%@oz`|RaCDf&~{F|7tXQLELz)YuKW#Iy@fiJqWxdJz^i#N{RcIs-y%ZE&o z$>$B`ck>E;II*m>;mP$0_9v<>*JNFO^?dSY6B%av&SZr~-lr@K4&^MHY7Rx7m6Dil zb2+x|kgA)edY{K*21W+v2{wlUUQS48WVVex)?Kae(>hHqp)hiW#{MVo=dpe}IFGf* zTFpUa&nNjyN42V!X|}C%UjCH%wpFN4RkB*srgDMn*ncDU9qgrrBDe1^=~{WeMQFl9 z27xfSyu#B>3u+c#w9VdNmcQNTF7N7}>@KYW?VA_=dUb>4qt z&qIMHAMej#HVB!)({FsfS&YwS<(JdejNA7XerRBqacBq(3jE!8AV*H*ds1)Fhmh^St)h}rZq)O`g<`P9n$`Ttj~b(eopC|CTW_}c1F zrE~N1|NmcjDKU6{`i?)k`=XXUa!Qu<$nb^Lewe1_ShG^;$Sd`=l}o#BKFnIQJG1IavZR+%{PWv_yQ(!D zd3cv|>fTx3v|o1ry&JCH7w0Lj6Y1b!Vlr!AEK)Hq|3`TNlT4MksrXh2rL(V@7^E09 zHXLFJxXd6_^)F4QBi-}CF~-HhqQ_=_)>*lV$M!(S{PR5g#pbLE;_d%RcoYO0<2l%GmfJ2+`p$6ga-ghq?5m?I zF9a1x7;U{P+hoUM%`-`VQ`Wx|HL-cOGutL!Q%>6RHAAZ?GFD@UuSbV*xw77U`CUoz ze5=e_PVURznYg7?Fur+1k%pT8KgsL_=KqZx;=O*)mi1cQf1J+nP3U&PhDEyj6&^=? zTDtr4L6&s~5_RS}aR(oq?$N+p{Z8c1Ua{aQ3Ee(imi03GI6|M;c?TP8*T4VnO{K&v z{|{mM%dTa=vR=(G*ZkkZC0zoK5||p}dE5CH zf7)?FW7Qmq#g~@{(GtMRyDUbH|DR_W0N!Af7<_ji5iRZvkb>KTem1Ct=yZ} z&X>42uI_XO!;MY1qJDL~P7bljx!-w2Sf#w{sDf3WYVUj7)3SlCP79xlYFezf=Q7nf z_JWPAgpGY8pW5xe??e?(JLT(IIanIxmHp`s@jQ7|{`j^&IFEaU^en!KKdYvOzVA`84*1s^s>pmsF5}BV zBW<=|qiY<37alY)K2!=&eRRn}izDoYNnLo8@MF=NYfo}YU9&zXdj9a1bMLqpya}A= zxcK1SPkVXppIK^|bT`d$V_3if1_ibS3XvaOthIX*-`LN*b92pw|IZm%nGY|}Ss z8SCMEJ>;*tDZ97q>)9>dYU&SoQXafhKD7Pan#X6gWRutz9AJ}vSR`{GIbsjna|a&A ztu{Q4WlEikzyIeKU4H8Di?6)twU-X)JUS=ZwnFY9^W=Z~kFYe@T6($J1u#U%`=4UZIDg1i|Y5w%Uv|$9wqU-jJ`dcadRf~BUu(k zKF>x5#tV$itWQ=&&tq~k>~G|MCO)yD#3N|xi31-j4Uez!VF|BS70{TFe4_8<-9&-y z><3zx8>GuOy|uVez}Vnk@Sy0IWqOF@jfZz7nRmot|3CdTH0pKK`Y*dWZ$EL33=?$c*V_Mo`Trut_mkJXZru4H zt~ucICkHui1zF{8kuMD8Ia?pky~HDX_kM5~``IKJN#_YRjVmYeO=gig%~PUow&}1e z=NZ%M{E3b$8kLT2<0;kpyC(2^=c63GU%S+q3x59Avku?ja3z_WrAX{vMgGirJxqVt zO)XOX*WQh=J(gDfEk1jC^F7BCeY)=z{p`4-3#ztU4t)9Y)s5x{3Jly92E06L5@n{^ zN`;sHy}U+TeS_wdg2nq8j$S_c(_YTP_J)PV%AW#kd%w!WCQad9YjpRB<5Pt?gRl4c znGT!ehbJ`nsKq`FFRizpt(Elu7L48UFP0TRUe2Rm#swaLxP7{NKXx;f}1= zUS+0hZqH=Ra94Wnd{*L(!ee$jx&EE^t&gui#K6pWQbvM7Ktp1?$hX85m$YwXEQoBA zKRdx#>s{LYjk~A1*1z)qoD(ek+00q-hE_W7^SF*Al=?BnI8V2+tF_xK+O1}A+!|L~&S^TlkTks+O)19|&njD`rnMu7V^C8##C;Ls#UeA_)IqOLLukS2-^VVPGVDa6t z|E~kXcW%GXOoogfC0@-ZKDgSId=O+3*J9+#7)@1{rQ~KJE=OB z@|=`62WIRyF*g@i`ZL#nkta#BH{#stWKjkU#@G)U+!4C#C%nA4h~chSX1PGhL7)0& zk^a9C7WVrOFflcKF=k(=^UN`bPKiQt!zrw`Spj4+jHInY##y)d`d(Eemc{R;<@Ov4wl8Q|ydi{j5_ec5*PyQAprtb@r2G z{ZbHO^ly`b*lWe*zm`q;+xcJINuuzi^U{oMdpLF+^fQ^d>1j@^+S<-3F*ZLdVw%@2 zmiSbmm5}j}(Q0CiXR`kSC$*M0eKG|M%jX{C$-aEmc+c*>t;@`k&J}EPTks|Q#=H}} z5qXY}&B9i^bQ4!dWZ-jT7cfxTQ6(@#;?x}xFXkDF_nDF=U8oUayTZESzw86U#j1t6 z`@eEJx@(x6F|JeKVH7_rz~|^WwZ6$&*HVAyZ`=OO2cH&HZ<_n=_+g%j32a>LEB0}- zrL&)wt_s+&(n3Q$Vlq=_x6Z*61>weaT5`*|qR$)|m3<^pLQVbko3Cg~9WoODh{pn%a$RL>f+4p&|?T@#Rbld!IRV8fu82+aoEO@VdP#>B$Qo29{=p&ITTdo5|Nyl^esf zCSCaRr!w!|7JrqW&F5yY>r}_FziiOd((amcGTf8rq1p+XD#afXhP-V3+RvFZ^esYk zVqDA?d7E)8;Fz|)^u+hVu0@kGvmL#sHqg2E%)xY4Ao;NuU0sT$tWz?YA>#%X_os1}>i;j} zZs*_sy|DU`)u$)C_nCAeuRLy@*RYn|WdmOoH%HwZ!z&uU7{AYNX|U(xo>3FidB42Q zbAx`b)9--l1J8Ja+q9M&u4du&R;cM+yL#Vsb7sy51+QlJ{jFbR_ueNO<#M*xp3aQyeWpF2R^2;xjd+xB7foH?Ug+n)*Y?sYwz6r z+htLpU{;$xn{`kQ>&?vjLcbS(p5iy-kkD?SPx9YxrzZNEv}~6iE&bFQgB_{6Nzz^a*ED}PZ?lrw?-=Mjf{_snM0bg#PlqLjn_ z`%8oBeLCE7e-6G1=X6neV6}eb`U}FPZ@!f99la_(dERc*C)-*xE-p5lCz8KGe_9*U zt0Mv1qZr~|+`6|uM|`5^kKYgSgHD}4VQA31vgqH#J1eZi^Q4wmXl3(CTt2GJ;ke+^ zjknfvzrNq@W@2RP&WpaR|FHhxvD3!fpXzgu@&!qp< z`CCV94cATlwWVhd8(YHpe}>-jJKp}fy{26z`MhCqeqnt9v&4%)uLHKsWpkyCmu{Bv zo@l(&MYe%aU=qXZJPG@>!wVdja-6);oAqzH!;FR`-zL?7J-dGGII_b{hquf@;oY8( z*PqS%TmQ@QzOa@DyENCH?Fsr*-Pn7l?_+LYWN6ROq*u3%-=1wqB?~~AbFmff^~o6y@k3Ayp^vk-*|PnPW>CZv|xTf*){`5 zEq4oxZ55^oUcq0&s%n-CsZ{UZ($f&)_V>MFRRaTqivoiU2g6@B#ch!i$G%RLkTIU~ znX@wpp!9d`=1hqd1V&BTHRh8t|`+$}7Yi&!Is*%Bm< z$e&{f{>iOb$*HE)k>f1!-)YT`@F@p`c8P?y)ZV+l^__dtWMR)w2kf6{v3!5>+l%$I zL!6m!2PCoKX?2Ss*Iq#~##~*N$ z``e+OSuY%ZbJrAU_P4S}8EubWlF#M0A?en+xe8h3x%WPl#m~7g`*8l+7DkOSHpw4m z+SzW-&%{hT#6&KBh`n%o$)#(bjM*=2kWJ56DX>DCxz3UE$9$dkEe!KC8!xnp|6@3- zsJY%HO0>P?>&)k@c|Vk%|6BCn%9@4Vb_}QE)9rc}h=)$ANY1}8>)!{}2#J}>3#Mc! zovk?T*Z%Cl)7f`;L^;2hd~X!p7i%y(x>hY*B127k)|#FRkyc%6CMBNGeH$JAO+Rww zrYBnuyzMeR`)he%XnXV>(+TPJs_AdfnKrQg@D`nxHCy^n-^YNZmK(m>-rOMeskLGE z*ZWVrQJ{>sGhNa98k!d0IRg8R{ojK7vJ9#)T{9Qou+`jsr96A?Ye^x(&h_geaPTp;52e- zU}#vuz`?xB`uD!Hr4MFU_Xabl&I$=qn&idsY+9n@-Ii%qCpq{&FbFW8FOEq!J#*tz z%>kV&RrUvF?zlI7>w)_m47?42F4n7l2l45fGCc@d{@~QI_POm3tw5cenD9%FHd?el z?|kZ6lrV2<*A(_*rk3gckIHtbXX#y#X4rRnAKS$*7bI=k7!ww%w8(9^)A-U!v4d%S ztZS&YqRI*V0>hk&2mK0rpS-`eU!`44^M!wJOvCT)j_2}PCX0HXB_tboAMrb=qG7Rv zhn<6uNlTn9aCU3b@=c7#)wtv(UH+x>9smEQqGYw|?^oOl)%LWkPe0ps_+Rp~s#iPM z9)4NC{BerXasdODHF05^WZpBn-~7bpaj->)uhU90?+L&8g=_KF->cS%MwoO&zmjfp z$eP1__{~OB|DJ~-Mt0F+^*~5Z|9?q){b2e}b2naP?7BafZ ze?wZ)bHnN%t2DFBn;h$;u8VU0d8yTonB zcPYulDTS99XG=@Hhze!@5&R=bF!I^^&xaWUEfu6(V<&bB+Az+&aLBAcbGLK<-v8w; zLb_k4sR-|!m!6)D{W@u$EWYL}tzL6PWIs{|k0}-=(UX*%AKoOLeIAF2Di?2V8miKjVE^Yh6G-KK1i~E01n7q#8_N&`Ex3||ZsjX}_{A|z= zBiP8<$tE)=v6SON@`Rwa&o|cTOV`vKy8ryFt7xdE#ezxhFD_-6n|!R^e5T3SCE$Jb zMvvsM=0`$%S?`qX@A<3CuXvM(K|8hQ2AA@l_>i`L8<=@G4n^2H{gpCs{=2zh#*{<- z3A-Ya{MCAP_5In#^=E$ezMdoRET*12k>~Vp*6IuGT4|P_9&_%q*}nKQmzL(0_m4Cg z8aRLNR?~kyXU&2oCQUYRzO`0I6V?PXYp54p{`pP8gHe3vq@xBBc_w!B55rzH!H@6HZ!i};%k8eZDu-1;;IW$h+Q)K=4d+kJzmd}{z#zcN?9{`s z;1Q!{o{+$Mei^^*KOa7k<=gVkoLSw6(_`WB7jO2mPh{M{5OYQHM?(GIi4&JPr>yz+ zP&IA&yO@6Q$jk-%|EstBpKrZw>RvsreB1x&K0AcV*%$B7+%SpLiF1~?f#;-8(om#!-%L<7H4=7Ao zcDPJoAFt6d$%?kV0`Z#PCK{_FZ@&G((w%f_7DHnF?l98>@`hz@W(6Mk|FauYBz_IXQ8kBSG9MX!d>@c ze%@U8CFx9K)7&rlrEecJGH9?0a4|T@IT-ZoG4bWyU;Xvr!l^Nd2G>>uBzLjBVA)$B zv802AZNaXmapEgf^Jlg{_n32XsnX3pi-uU;=9PhLYR@;`-yOR~=b+jCcMS8kU%%e& z?!n`DA>Ad1p|Rl%vp~ZC6F;AI_jrD0Y+!H@VbDHsfWbpZ>+?*82!Wt~j%^J3KRYf5 zEL{A{Y~y#G0%IN~#?J!xbB#nxjj9wU{K+?1Asf_{BE={3;<~*73+lczqhULjw*V=#H{b9WNVb})2^j*c%J~Hn}4Dk+W&bD^&Vv=EHFl>$% zEUGDbxPD9K-1YbRrYkTAuqVf>vIuNB=E-ZW_Ea{3({a^IV`gUN6;?l7%?x6fxrP4V zvhO-^%d_lP%k(>6k2rWXOf_Kr6Xmlz=?v>@u_sTtl;&r?Q0rjrDfumCx_81a!wr%C zhn^VlSQ*_t`PM^Pi&uYz9gAb*Q5MC7bsnWspWB&DjoS%O|-+|1PHtYsrsYwZJ0#OVu zth)D-12e)!`);L4ePc`fcr{63{b8OntaaCIB(g1}TFlw@9*FLazaaj%VCmhR)AC%F zYdAP9WSD6F#Pfs8R-L+tjokYjxu<+Ot+#H6$c)g|)%8m>*8j9;e5sIi+{ba%wfbaijPC0y}uHgLR11_q+_?91-IfHeZ#cR3OEw}O; z85kMwY-nIfUGe1t=Yb+684kT4H!@814Jr&1?q3z#rM)mHk>HBWBShu){l%O0t~WjVjEtF-ff9^Th}`M z^ct6K3GOTZYQ`AqK8+QhlX&6RGGXU+Q@)B^<PCf`wM4mf+7H9lM@{e9jMkXGI-T`BsgJ0&kQM_c!ei7Bm>sV$0qL+OZ+%JWj)uS zMXMf${w*-=>)yyWYr%?2Lq`7gleJre4rlN@dg#1{-SGOgB?pclfB22L^!2KB@*mc+ z>!h(39{(!(@6h3jre%DQ`tz9t8oeZTBpoSKlFM;c>kIe^?Nwe+T!f)sd&g9 z42hTiP*Ae{`hyjRKjjLBEij7_*xW34@8TP$NKLOND^|{SQ?RkZwyy1hrIUS6=d5GLF0Qegc*{^DUE#-r>8CbKti748 zcv1Gk)mZDBLgH8QKNzt-TE=>HzsJ96P1<*_e0t&EXP0R8qGXf1!FQjDIagMOeJ~Z$ z`mtT&%7-ahyuXqSo^Xf?BI;5G7-Kf(pPuZ z>`;tY59@Ka1oM5oi+->deu&flT-WxdcdDH0^@WwgSzDlT^QKT&7WJ!A?9GvZeGp&PqgXVrC+lXuZnK3f6wFbzA1Wd zN_D8(^A>JKSH?yW_a3GR77Ewi@I|HEY`kzOwBeMUkM4ylTax)JYOXoib6bZsH0=8s zBJ#Cr_kMNlPj)ZrMSgye|E;5$_2|VCUJZuVc3(X1F-T;e|0_FtZJYSFsj=49@s}Eo zR+t>%;b3Ucx^nrEiLENj8u15;^*1(1KC}xv$UdLNP&3}oF7eY@jeku4Qr^8^yr0wH ze#0K;*J^?6$181`Hy_yc-rnliY-x@9?)bH-d@>wNavuyBtQe#PN<<76>dUM!bA9lu zaSK;~{N+EQ%NzT{?}#s8%u$=bF=tbQ=7x)W>;F3IM`=wJv-@*m^TlP1yI!Rjoqf-9 zsHaIFzeqT6ZQ1l?MfXfSwQ~E|zRvb2`h6^S;`wvarnZJY{ro^Kcfg#2TR)p{5M68Ha%TcGin$>QPTJGEzhODD|y{7B8a*5Vy=&=a2B6)3MX%T$ zglm%G>T@RfTc)1i`C|4zGRjujFjC^d92f8FzCO+?!e2-VialDXeJYUAf;IR{_A~_x z0mC_^cAMhjzqppZFGyrB;*jlGHq9t0I#p5KR=#Ni`v02r* zvEllyC-+=*dRF$V6ZF1Uzh&wibCt>LayJG?p-%l%zh=ptQ#_^$2|~5U zc3%(ke*LJ&?4p*?B4%qjafSC{nrZIoD`b*m*)B~lU(k2D_M|rR&xpC^8R`o!Px~t! zuwG`vu`31L4Ts)*`0-*hYyQ7ZF_v^bgM5}-T;FOsXE2^&w%MC+vP?TJLix|Mbxu=) z7`pd4Y?^xJ!=HP_7Pnf}ed5Lbe)v$pl_Ng!D*tTxmCR)hTV6em_w7uh%CTKAv^x*$}+;n^1Z6VS~v!2YVl$l3CUGdzP__e*eugbGyV7 zRIaIeCvU!)XM3xk$AEDy55s{Eu19!|uqI3n788q2VeL{BKDcZFtHz1+z^D$U)m=<& zewQj2-3k?XvSoAU<-ghO5fU?Pp7ThR7ORx3JY_1`Cigkk^4ZjKNreZi|3A=OmvF^E zhD|M@(EGs*cOB*p6*XZxMU8(0=V>jpP<;N{l~bqLvm!R>e7?`~?#%g(2Ur!K8-K9i zDLZB}x4ZdFRL=by)dHp+OM6N?jz737^oVn2Wr|i>qX5gBju#u=Maq8v!uwNDIfa); zZ%gq5r=2oY7uKwlkT>UL+{>O`AOAcpXKgTj@F(I<_fC#r zgUhHLx90ShC9Nj!t2hF$BwV=ct7$n zzHQ0gx#nMK?AKRqhZz{QE)0EE?)XxPaq1y^`{TY3JEs5Ms=B4YN?xgBZGg4Vt;WVa zA<@Kugo9S>X0m(TXEmmTCtv^i{B=a1h47sZ7D@|?Px<$h81j{K{1HEvzH#^WA70n( z?L_1XEKi)dZXmlOI(^1IHV5<6&!UF@|Lk{NahGJf>LZwOz)$jLz0wJpf*A}ik1YB< zZ&&|o{%uO_O^#&;cC_nRC~M!46OH;G&T;MVuEcV~vn<~jr|2*Lk)&|cbwb)5&WT|z zrMIuW{Q5O8Ytme~>)8Q`U)Bj0DjZ%cSUuC^-E*;r!s}M6ia6}gthm(tDlfAxl5N6M z{r_iIUE8-_SRP6&M&eei&Gm{%>RFa_ebQ@6Jl5IeQ@5< zeDC@CMD1Arpz__%e}7;TU@(v{5v)mkx?!RpqwUs!FFXv3xmI$$?&9gcET6N)sKL{v z-h< z+Xb9KexvlUy0C*&@BfeyWH2G;*8CLj6PhEv9NIx{YQFxb#0|hDc}#g!ws#8a!)w%A?S*n(N`ieZdAY z=C7M}+&y>g^M1M82Nqme<)|RL{d93s-q|^&jsZ&H_X1MFS zpG4b^J2hV}WLz?C+kdL;X4#>xD`t02hUPTfe`**g%I%;UwM4#%Q$fO^kw=fMxt2L^ zMb5!FJ3hFEP7^9-ciUu8!N1tqa8}>84$V84-193p-bnE3z0mRH$3dCUDp}QbQ|T`s zzs@eNa8WZzaC4N+pO^jbTL0AaX$}kKBz@LP)4s6$dVK8lJZ8b?MLRSS@0~oIbn$BI z9I<;j&l}qXJa&4S=LhW#>a(1{`d>VD$_dMp2Q=e+gUffU(ONNe*4N#fMK+PU*MH1& zVmu(MYhbhF&?vd{G@+4Kq0rvF7X*r2gb}KVJ*pnM-^D22_THDqu3XL^(%*``88sjJ3 z%_>NI?Vl^AVHe%i2+$1INJq3ypA=SeLL3XDB$EyAoqjS3P%A7u7P zi&)6y%Za-FGk1&mSjpbUe5>Wju8G^78zcA|t7foieLes8xz26bjSf4$KQPYR7_d+K z=js2P!P{;(hyFQKq`$t4_wcj>2^oza>L)U8bPsP0fGys+za5$(}lO{t$}dZhR7`_}6LQ#$Re-A~U*9K~bS7_E!Y+TyL|ftd0|$-@I6Cb< zusnW8QtVg7yZ@vnepz$jKX1EgkA2g=)Xm3adfQz-_wkrWvhOW#TwnT{`9>Q9qg=5= zn0EUIgQ*+7?)Y(jDPKpP!OUWR>J9dG9u13;ZHsFripXe@fj&xw0hwZ7I|p-5<~N(mg^pK zo3eOYj{MKIacoHl4b6G><-ep8Y*_;{H%#S+tSR%9t29bigS47UC2LnnoHg5 zR&v1Qs)H6285kPYKWMwwy!np;ll3VMmQ=f91^vkO%?mHH=rrF@uvz;5(bre|mR3xQ zK4dZ1{=Z+i$gvf3#kW3;;rsOe-+YeLV|)KEn>VX;`M-;^{wcSe{~Z zWk}iKR=&8pt>}r}`7*I9%=TQnzeg@#cmI4jW84l^^&di3C**_W=YEq2@z7Cn2>R$C z?OuDWApGK#&wGzgTr$@&?5orPo<6~VhQ;OkR_Z347rAj^gFM??38y~=3?(n#a#SZ( zUbyBaSZ^+}-ajx^<3*;T!$LkkQ-0o=B4c-AX3`l) z)=a(SUD{{rIbuKXJmJ%j+A+Umwt__wqrvR0dFaOd;V{ui3E z{Qbr97V$CC)wlUd`%&y|G^vTG83_D9thvhx}GK43c_ z%l6`N_cv+Vwa*f>Sryuv&Q1C9vg47S$eAb6Yl^tOeXz_uY@zTrb*8V-fs!*SDI(d+ ze>d10RBp=g=44ts zMev-F=keEyuhlmDo#*>eVd>y9rAapU_=Wom4lFv%#Cmhl;w4undFe5&Tapvdu=q!q z;F)3u1AbA%t?BH(3O;(0SF?gzp9rkDwOm5}ke>7&wSpIO+jJ%shWtGzRO!d+&@i!u zp<$koj(LaFmpzZN6GU4+Jjgq_L12o@x>im-14e0$TcR_#pRJH}SnxhS?aL(1BXQ@$ z{OSx%w|Cp^S`qldl~a^qeP!4F7vfya4skQQU&NQ(X5zeJ;J{GAEW^^1srfvl_k+oS z(|b4yb#@q>FDqhOY@j3I$#Crg6Ic9AVVmX4dVbnm`)1*NadF@hotufAvxU~wi+VST zJ2~e5%r#zoKj(*oz@Z-k4Gayd8yFgvDc`u`Wx%7nYsRMK60>*TuVB{TRDI~T^Mk`@ zNzHc7b5-wuU6*U>P;7fTf7(ti?Z%}|=JPJ;Jmr&#V|8;lAN%P$gOP~j%O)|0$qbAP z4H|wHrVPP*6Q>#Q?8x^^I`Z_umjD)Xo=*aLa-EK6r#8q(Gk@R+_;RDJ#Je#%Emh!2 zvR_^-yrri2fa#Ef4pU+)@f7hJLBek0CVC&S|X>`op3y9-UH84q81nKN6%sOHayNf|un4m(becCFjC zsNjF%>;KN1`!pKoYk#n4z16hf{>MtmC5Pls)o;?^KXUHlH={fJ7y9M}GCwZ)zh>R^ zUn^EH_sM@gd9kT1$$Zg)jLI^JX6dzCP79x7JW$Ma$NTfQ>v?s8+|?gMx?|OD-e@g; z&ug)&>YP>YwTCg`YRr`fHa>67IDXv+X+S0DhcJ60}S8G3bgDyU}6(j!)S7v zV_~WHf4c|$M~w_LUTxtoaQA*+$8>^Ah|OYsAA967745l2lfy+5FY}1je!Wn5RGQJk z{lT3BvW4?!woGhX;r&5hs??yi;lYG1J>4IQP7i-f@BL7|N!Ol5@WAaOXNv3=DR$xN$U&6lFVyZmr4{S-k$3116Tzz+P z`03L9KQAuWw^8oFwMA||UpH$oh;nBcep*vf&HPME)@yd${6iC*npo-_3xDxulznP4 z+_G)qQqRd(WQ!8zHZks5mSC{VX~(`R@{TGS)||+fdcM6`XcW|{I+i&^dr6o%VmyJv`(~0uupRAl1Toen9B4)SwZRS>KDRW7E;KcN5$iHxvF4Iw7k&StyM6LNcC$GExHhIwB|>` zM_IMkzukZLF}iRY+-MILF)({OPjsv9We$fq2N@V2FcfvJGvBJy_&4qNgKOPV>}m&9 z-a8&r&{<~RnsHl{`M}>0wcXsFy>IVV^xaR?PJQcD#&<)#$BZfGjHB7VV+@m+HF#}R zMN3%{4fbS}NLsKkFex%9Bsk^zoDk-G_hiA{&_xeUSVS#c(tZ9RugA2*avCh^zYflw zD01}fy`SgmnrHXuuaa)RuAXZ+%YY$#%H`O`XCmdAXCoF|Rg(C9_~8l%?gn-)4hD`D z50hpIr{7w4ov&4AiH!B3|LrP5F>2Z^`&rH;Gp>DaUTmLq?C7!$8S|Pei`?#~WEcf^ zr1Z^kob~ZW^}Zdp{w#85|LsU;WPG$|_NM*+bDx<{{{JuI{;&I&@3VWozozQY8EUan zu;r_FfjWPiOXM=E!Yc*Ow@2)+oWJz{j$c*&&K_5;+Fy7up*7F+=ik<)hXS(!K%tS1duO1Mn;7ysxCkYQBaxUqkV%bbqa6jsfwkmu7v*$ zP2`uHVtwz{((?yrNI7l@kL=AkR4Cud#U7(_?#~@-?gp2G3S0Icx*YN-&PgD}MNss* z!VXD;f+IT^r>!}W@R#Ru!i}u8zR9AGe|2Tu2%XjXT3hewT2qD-way7DJ{sZAT5=^e zq;5=%(fH21@aWU}iHyfTHa4w)QD5u!bd}&4mnSnCngR_IN~H|A9!N`FW@0yHwt7CB z>EUF)4J%HYJSbPuKC&b3>rA+e>Qov)jV%n?0xUYm5_=5CqMf1a=G>A z!?)$nGc>JOrBm;HE@GPN58bbi^WWdKj@oh1p0)m78dJm1a@qg?kMceEvU>T-rzt#D zJ|{L*gg^aXE9JLu$Eux=s%js-QfvLEEzZY>G|&_uKoc@?S0781?^Ou>3F8s@HMX zH(UsR?V`#zu0nx)QV^J1#5M_2dGRK3o#pLzo}pMJCdk#^wYhuNRI?^{25 z-Xy zwu!br|H8t5yIu)1zbJXJV1}(=$7|;!nmikyS#8TVUe4^W!I5!F=lYr&KBb@bUs z?sY9pb~0&Q!a~JDj%;#Q)~B$POptD1xVNXZG3VHu{YS%Hr&){cE#0zbnbA}gz& z9*m9Kc`nb1IrUpR)D##S-@Uy5 zxBgphKcDXL)rL2I>)YzYY0GNowoJdDztQDGe%P<;SMUE?wlMzMzFo&PH|`1AGsFG$ zdf~5GD^$-M>wUgsU#n}a_`a?Q;Vg!S&cBb17oF1+_VjgVOO5jX)rP0D7i<44?5;^K zes#4o?&B53k1>0EZ{)X4Ie-7+oL{yPOGUS~*7k)@w|dpCYxV5&>KSWIG;7MQtzKJO z+7-XHa^L!|Qw!!CU{JiH@!R9BW!=3MBJZvU%yxb9;wDGNt?XA`?HX$i{JP#5X1Dcg za*Wc}t8uNFh1z;YweN=?uP=?be*Itms+{o0*{i2ttj=3>wvWeEf9(<{u~~I3*DtOP zZ?z2H$g#3jIfZ%u>UZIZuKP4ESZ+x6Pb{yJjQ7gCa@Ojf=DU4qHmBpGeqFP!Oqvq* zu}*E*@<5j#TYY}TEj<19?8<|&*=PSGF9|tTe$>t|S!Ckvc8$9(A6CxJ{I}3A<}t7J z+sN2CWy;s$ZrtK~3F7KX~eW-FH-WO#G^^A@8_x`u~7djgzGY z+fQs0%i)Z^t^ccx<9Cs_{*xcUeW`)&lTw1+CDcPq^gPmDG5r(Zd(t6X@-=72L{qn= zEw3hj{9|;SRsQO|kg|!hUq?LsGB^9o`bRZh`>dY^{u95k^ksG2QC;!p|2}JfkGkJI z-K_mjQRc^lgP#se-T&o&_jw7^pY^v7m1Ny|Y`UhVl*?sHrl9^UgLOK;U-s$Uc68nJ zLg@IDn5J2r*KcQTJ$!u2HKjb;Gog#cW@v4Ax+J_qMV~8@ZP|qX{eihDyuZ2QBp0Ng zIa)dQ%Mst`yDzUFbI-of_w3IK?>QV_&RXyt692{Oyr>}1Pkn{<-LunMe*ORaE6iq1 z@A;O58Gp}-zQ$P#l6Uu@7rgf_er^C>7o#K zlk5HKudkLR)Ek=m;EVsp;Awa?bfoBDcvxa<4My7E#@#mLTY+D9YY^d>b4 z9B8=ul|O@BLj6;lGPnAbr)sH9CSIFgnhDr!-aO&LdZ|XMyvCgFHPTs$%>T?=w*O|C zudS)-n3n!ug8%We#HeKPHM6#*EjV$J%k%iV_$IrYCx@4C+DOoNrQNwJ z%JYJ|)DB;K);5Vv@Q$mFJ+~d-%e^n`g(OeKN5w6e`dUDh?fZnkCBh#WuWj92yHjD2 z>{J0^t<&mN<^Ni>IS-y>mF4CFCeO@vOy@nPSpLtx=gH#3U01eB+MQ~ZU-FSPdX8dR>GRhc=UkO-UnlFp zm$Yz7$J3OvK@YrNPEEMG>u;^<@3JYGx*LutPCqi$m^M!{t?1L>;-$ z79amJ)v9W?uWaV^VwV9u?+4j-D^WY6@ zt_NR~$(5$TC&ucsC>-LVgU zI9V@LG-~bYkl*@$lPl!@9+Y4A_7Ia^$m<6y6qtgPI2(>n^nK`8y0Ls_$fgH8?e>R6 zKUye1t^LLSw@$W0%5lYiX^(~mqo_pb3Nh`MQRn9i^qQZ$`0ZyvbLTP(=l|>ZK8kB9 z)_#w!{*%XM{5a}w%#=127NaNU<7e}oIV`tu<){DtEj#~CFsn7$6#Jkd$0|nPp0?<* z+c#7`9{M|5OG&@xg!e&tHTJ|sf-_a(WSZaH(|=>wvwEY=j5NddRrl17{(L3!H(qwS z>dEMz8@-*UUAr>ZC~4QK^^d=AwwOL4;lTM*2DKNS{@-KG;+rtPgmq6*S&mkITYUAT z_`X$7;|~4Tsn%cgcj*lc)wDBVoV!hpZTsfmIMUH@IaJ2yMrKDQi~6tAf-6^wU(LJC zGAZrHKhExL^WIIZP*rG|Ao$m-?b*%xq)+Qbq80CR{)k?wFD%zhg3{t-5<@ z>C$;0Yg=lwcHb$~3YP6@IKBG;|Fp{bdk5WSB`Z%2KRW$?z!cS(>3()spX~3xAwB89 z@uZR^3J!Unv^ifg@#?RCxIggy`Du@zuRp)}hp_Fg`h}<7FQ0e%|Np3cTce^w9o^^M z{gZxBaowMzMV#f=4VRh)t1H_DH5clIgq>XU|K~bw+j`EJ-&^A*>F0e|C>6SSJ{Q;V zv-c(nNo>EFUdSRLcCn1}q}{}qJ9&=p*8jfrie63TWDZvuzuq||_3Nk4iJi?_qwLOk z!^ZLJtk*LOSZeQxW-kje`g`AD!G#~6IU+nhS!d?#+uu~&UbFc=$DU7B$q%CqS|&be z+fdK6Q+@jVFMhpIotO6IuUH?yo+Y?YZJx|@#ig&cS?}I4T+v|80fck3VjoQ=v9b=Hm0eAKb!N9l6$P zFDk#jO5pSBMSHpa?GK+A{$VL!h*olb%b9l`>y{TP`Tw$(kN3N> z?5fx7M?V;D*zOQ*@Z|g>57)p(#~`F9JWI1){nM;^x;ETpulV1>nn}<0Pj4@8W&S9jSY@L2{>S6~ zr@E6|98UlH#*zHy$4V{7T9%p}Nuu$~Z`V&h8UFp5VNuNwC1JkboA+N?nELn5gwE9| z%(tg=|2?&T@A>}|rrKGpN{&&ln*D0tf+_1IdRX);Sr#$B&Tk5eJM(7o_jP_z`DZuqot+w5*w1|i&rJ`?Tu{N_3P{F;;*mD*UA3ds-AsU`pcThR~EJ=t~n&WGClIu zPuu^C^QNb--;y}#Xo60i+QY5cFM4P59**kz?4=g3X=M58y3pDX-O11YKRcd!c#+1_ zy;U2^e?3+8dQ&u`>0s5>`&sveWBgxFS@K=Ciu=<2@_@_Tw||$j1i9qJ-@LVy)#Z#e)3~*_PK#buI{j6tJmmD}eM{GOgoUo2akOrqYSQ1N-7ejG z!b?xT*!cVB)fwLFuTJ+__c!Tp)!v9h-o@G9U)Rn4x@zC`*^ffI_JoIVD&JqY!PueV zQ&s7cMuk5g|KEQtUUKW|zPyj8w3|!vf8D)jt>^iQ^Xl)2-QoMRc1{ni{i3w@VeIFt zi|jM}VxRn2e@-!P_TT;CaeF_x)NlVA^JV+%ki{HJ6Z4;Z`pM_Wks4uG#f|zr{3Ta{lfs zyey^pgFmrq^V@j-l^hY_8}3!Sx>xh}<)5NQlYWOR+FP}1b(r17*e{!9Yu~?~bmQ0S zecvmZ3)@pov)+D~*HkvQW2JS?(k&IQ&Yn$v7F3n`BW|kJtHo&>4Q8LH^Oygy-QjS- zv|D>`OqqP|czxban@XckwZEUQHgOKxf9v4-1+K^M*A;%Xs@q%OeE*2I->!v`|F6YO zP}zN+XU`O6t@zm58Jn)}{!~-{g)!VM=qszc3At(^X5%gcZ3zi_XyxVgE+aXI^! z*Cl@s@{}1*Y&~}J`Rk}^mYTY-u&4YHW->G0ZOx62__g5X^sGOBSBI~y+w}66^6ss5 zv-e+L|97wbgWhi^6ABM3yfM3E?PUw+@X3w0qn3QX`tP8^?2N9ztu_1P~I<>t*Vkx~Df>VJKMm-FAo0BK3X z-u?GqY&7w>OZd-Cu9AB`a6vchH0zP;UUDtUGKduPK%+qTSSU7~9x z`?E~AZ^FF7l@-+?Qtz~Ch4oMG{JegFTiMpaPlDUOaa0x^xo$q=_pYypzt&fMa8oXu zfBoyBJqppG8f)J#p8h{{YL?K}rSrvKoj+gxa{fiLFX9^U@;=_MZ%3&*6m)0-67Ug&Sb!=19wk67|KIzNi7i)L|8JQ=q7gZmQwW8tkHt6yp@T2@(`-){K0y!+{`(1(-G?%qG` z*5vD7!#L*KKkChS(4c*|Dt6V?gFIi#pHK6@^tb$V{{Mr5SLS<2n{4?V_E)%IGRM`t z?)$5Me{p|RP^+_R&Hmq;i)|}ZbhV1s$NXRG`Jiuh7;o?E&B;Yu_pjge`v29=@E89! z-rC=?L^s|CHtPU#Spzw{@|kZ>2wVP1M5ASgw#u>Ie|KXgT%eEfg%`Tzf}>(=VVt_@#Xw>56-#lNe+T)^cHT!Sy?H%j3)|BV%n?7x8)ZZW#<7I)7#_rxFhM8ekS2!)tP3XGBq^QS# zIB)_(#*OLgCpz-hI$7K-nPzq2R=Ifki-vPE3bI0*Snm6UtbN&--#)k5yY6km$=BO2 zls$;Km{O6|D|jJuxo1a5*fJi~m;5@tj%U9|Vz8)Ny(YTr+pQ!m}xeztne?xULfeyb3sUx~gYyv(`_Iz5hDm{QcG936tlSmKEtA?by-^P> zCUQH8UcLJLenQLQh7aNV?+@=gC~vSLbyl%8|7|1jo4tKGD@y%%FS}jZwg2nkuU#>U ze+5@xUH)ueYTobFYk%K<&Hh^a)zqA)fj#w~n{uDdy>#$?qMhb3p2}~H)g>Gyiw}g@ zaeOP>BcFmM?&R$wyyZ2w)7=LPwZ~f}Jt*6!$ zXWbXRbv^5PV)v#j$*XqT+7kcYc-gz4BXil}nN#Po_HW(f3nKYs~^0*E*iwPFQ_bgzwdPU zzpi8Ng*A36mhZ1!^z^2ug@fVa()h{G|6H5&ZR^dqyK{ek&H8_BcjVRA(`!$@mOA;! z|JCy+r+!Ab{CxF&@2tz|8X^#6Qy?0fw;vU6tl7j^!_W!tZAKDuqm|3%*F_kXOo)+1r^(&|%X<`2J| zgo2H`wk-Mm|8j-i{A0YkUvAiacwS8Svi%<-@0^vAG+&?g;XwHtp+K2m%wM{nO)-eS zb?a!6)Akn|YxTDNI(RSs`jcn>zTqA z6HD%AY+E0;KI}u^Udh#0@9&vk7xd}>Y`4aF@-Y*=fB%2bZua(Y(EiwaZ$DN4<@m?(Qt$fC)$x_JQhFyUj{mq_o7uWCcS#BMSxwCL2wKDH`V7yi6yY4FY;pBwL!44K{7=zOQFg zTxmM>n3rz&?mN?3UKIVmWOu*$i(S&kPw#Ke{`{|h^`6wt)2CR!xWBn>kDcvQ_RX)a zep=UDulatSwMzPYD|ipmMr~Cb{y!SVi_$vO+W&FEuVpBu&{VNl? zoV;re#q7_$@=tBy|9`e2rCPzdf0~)t_w%ng^8XKmTtAz_;+hqj+mAlhS#acOD68gg zlPiKp|6A_n7-bDhCE&s}Y7cmL0Jn>1m$MecKU z!};MKC$KZBh)mHjkrre+;jm$Xj)*nup@j;aA;B?E*6isFa#?YDWn__%6@O#qgDTk7>C?BKS#7iR-bofLZe?^6iii=l=l#ga zvvJ8ggQpv&XbSnYTvvQ|K(pw8`)saTK2NgW-tAs+MRn`Hix+ft7gqnPFO{42&|<6A z>$=dlhdH;2Gt9rf#M#WQx_Z-%qA!+lj0)91E;DMj97}8caID5|Vd|vcQi`^vqJ_89 zt>@elxG{}ut%R55pDSyRCf%R6e>U^C(zKHt%?*rdB0U~%T&zuv4I3}yh-me)94uJj z;w30??dIA0Mxm}x*Q*4iZ0%h5&;HTBYPY)j)@S>Fyk7p}hK>5eSMy?;FNRhbZEBbQ zzHZ*RE3f*JCfApG=AZpOFZ|Sg=`;4+$16`6^mtyV^ZBx`)It51MR=6P`Gd3GRH?}S zSXy`B?!3hHj~ag`*J}1I=Zk(?ckkDeSH*7yLkhyV-OUUBa`|NKy7m0Uw4%Gah(0({lJ!)y6Z}>ADpw*MPlWoow}Khy|!1@ z-S61`vTwlyMir3>XH>X_m<~29i0}}TVr^ z|NeSx>5shTXV>4o+5g}6{>~%$5y}8`n>f=zJhP`n^_BL z9=VpfC0NOFcSp{hz5J<-|DEuSg|+YRvAw?A{&0tW`r$e+wj`~y`nC-ruDfGC{ho8Q z?Cz~yYql1ws(E{Vl>%d^L2ALS1eul2pH?bw_X>0DY7JZ)`tZJj1*4kCgf~}&IS(Cl zSP`Hj&ehto&|yMLzTJ}Y%F1AcEmw;VEe(C=tsQ#c(cg0uuUWmm{C{mlcJ}}9+Q;sx zbFwZkZv9huEcCFwlY#$3yY7wojNr! z>33L~*XB1?*N()^kBiu*S^ITr*krZ{NOIK><6?DcN_3c@Bf-tebg*H=1RWD@PNs#4 z8Ub30nyp%)DSPg1<6NZ`8x*Q{RqKcRqr2wfvNihxe|&#;Y2~ZpRE78l+j`0?8griu zT={9e|4HroZ8c3I#q#?Mc4i(f_#*Xii{9iEskE|38xyOqJk#9x?R?`w(K&WU=B{77 zX;D$)){E?px6~daIp)ADO(Ufo?;)NL~e+Fy_&{5^&IHYW_Z@+HQ)ZN#sR!!fzH08>( zxRa%&>JGc=`Jc!i{B!?b@9t8&@6$K`h`(mF{`>hj>(I3|GP{d?MNXfy@V@Zk)wEMp zz6SrfSNj&TW>+b{+CKGV{-u9em*ttJKa-FCY5uJ%>r3lfk3)a{U)!g_%fq}RB&R(- z-rx^coMr?|Ki5C2&V(P|A~}Q-lbzm&r}K!OxPGTNfA;64>QlHn=QvH?>=-j=<{bvh zsT`fZR!*8<{NUph-^Xw38>dfN#rC`;Y%)XuS5%*sJru%yKAT zU|`+rIE#s)i?t+v{e>C(>aYGY+PVMzj#*lNtKR07g!!M}|MmUVz59OuO?omh`_VPM zsDC{_=6!Ve^Y_d5W8d#*wEbPP*HOLi;qmMCF8??UmuT~4M?L7C);O)?S^VqhuQe)L zf4#bHGDX>3_0`7s)n$7>+SlH13cA3#c%#?VPyzE}@lV!Yx2m3H6Z5@h!?|r2t+zIR zxBXcgfA#K3*FWkXZG~^`2wQVpcb{5(m{pX_zQ55=*2nD*oAmqg7LM#qg&FRgeNvOZ{O)l@_7U;F=i{as(S)$iYRy{o4--TS&eD|%1Rqt5qV zwL)5Mya~Vl`tF+WY|*OS65H3u{jI+!Y4iUtXWYWm_FC6B)z;5n@#cAM{Ygi+b@|=L z>*d#ndzJi+-)j47s`cAyk@q6&+BNHUM^@?mto!J-`uhH`u-D%dQ?3YAyl>Tf)fIBI zswPs$>{kE6hkviRNOWpnx?&bXZ@wq0do@E-l%HFy)cDrVmzS{ck_3P$eP?_ITDv?EUX?$p{7M!{111bd!-s+Tngi~qOj&yn&^|2X=J z+=_ypKU2EC|N8!@zj1rBw;HUzdgI-@o``e5Bg^BqYF%fQxZvQ%dtinDYnga;+xLy@ zH|Xds;FN}Bo(l`(r$5`S?RTxesH)zz zS+VD2*w-gF{io}BTv^UpZZmD`-KwYYPSfI@wf@+d&)d4D?Y;bEbddglD9uGefh`OXUnIV-rK)EIVHiXzG~yos?{5uk1;WI8wl?Do_#g@ z)VK9cQ+94$xkr6bP1^pg*OPlrMfe=sKIPSisVv7n9Jo8bPTI~{SAR~TOG>`^hcnC0 z=g68$+s%xR>o$MX;TUN@S-fqV;^|qZPs?9lTedYkZfE(e{Ts9OuW9X{EwaVGwtT(Q z{Q8V+(FYq^Pjha~RH}=)TB#_`{qXTKJC2`+7&iHaCBF-?W@*X0&1i8WFEeq0>W|Ai z+pISW2+B ztckNS*squWLhyvbvH*v4>kVI*FW9MIwrgSKj^Dx28&-NPt8W*$^p$5VEtDJt#`l0vMN^!qU z=>M!=`}duVNl1xX@A7T`;+RQK{=Q!oTDAJ>`e$|~Hd50!+oWjPwmw*}MLB3z;YI(i zM}99Ze|p;gsdeCt_*ZU!JvRM~f4BYJzLQV?ah{Ocdb_mXoL^O_5VHIof)t=6Y+E@FpFZud@(%YEnS9iyq{Q3XabL+Uj@pF@7MK9da zt=q1$#c%r5*gJE!)~?pRrnObEZcVs&-{~9rQ7a4MwAW4T?+P!dkJ}pb$2aPlVfZua zsn+lIW@dNC?EWJ-b?#r6Z+{(mB4=|sX0q}9XAyp7*SO(WR_46dpWXJm+`N&VsrAdm z^{Dcb=DEMt=KtDnwU8A1{JTGtpKO`AJ~VNc$lBWMJ@f5< zJQcmJwKw+H{#S3pV!rIX%N6}~YM_bnt{r>-?fl3c9_M+4^K6>v7oR=xuXW0&yZqbw z>uKDY^6;>Aq2Jb-U;4Ms^FjCSr@u~h|No->{%O>NvY7v`{{HdWb=m$?-P&LKE0+F^ zU;WieJ8W-Xv@qB2@V5#3qmJsHzqV`P>5BPr>%(Ww<}Cg8Gj84bpL=G0O|5zwuBvtY z)w!BL6OCCRgZ#|9G zcPUw~zWVCyRmaknmoEC|KmU>Mo#$Tjx&55|!+I1}c)tF=|Bp!Qw(Y9#f2gb~liqJ= zmc9RcXiu;FzqLO$U0G?qcv0>DMWyjYzrMeI{_54rxc6VX{_?y(XIkgi)&BX)qscBy z>L0Z1xy}A$qwdy+H=fn7CaCdzzA|&F(PC+Zf1NMCCd`itjKBJPeec5HrPKd?xXL)^ z5|_}FA1^ug|4S-dxMbftyLGX`xAxnuU)}Y=dBf9yyR&EA;D7Q~Bx%p{c%w~EC)PZR zH=U!v|LN(SX`D7wuJ%3kzrOyO*8Ze-QD46=%)UBpZQS1J|30{ve_FlXQZz*?vH5?D zV@%7}|4F`nd!EFbZfgybd~Whr_pFKd8)G-Kz5frJslNJqO<+yvYrS8qLo5HrOxfrE z`rlPEfBR2Uum4h>zefC_ke7m{a>mof#IN;pUTwU{?)~zk+TE9H9Nu5ez7qf6wd(8X zPhmT2|Hqdt_rLOG-mdkweij~!YWAmZQN3^fYx9&=ZE@%C<{6BcHAY@(h1f6du9Ln(A9TO>^;43+PvTEYdI39 z?7x3~^`u`ad#}#A>nJXn_n_cz%yiYo-z;7D!d8b~lP`X~weg3_ z#cQ$uII7l%uD;%+bMNon{L{hz_RRg@p0&1mZT7ya@BT+-O%lBQ`b+EAzY%}qyd$0_ z8CUJ!p>&$}+b{DF1_l;p$63q_r&#M38vc*jI8XnY_S4 zM*I$Gug>~?@^}2deb%nYYoq5zDL7p>*?ypX^5y>_;h!pNIHHulttnR%y}mm1>gs)3 zJHuCZ#n$bO+uNb4{`%^r@8z%C->ekfa!odU#hPX4zIHv6~#>&pe_J_N51E00^^-#PjE?uGaDuO;rg+O>6R zY~<^Y&1TX6UN5a)?f*UK%KG5dDbtqYs~s=PgY{gu$~zrMeo{&&sp-A}`JuTqSA zTdMDUZO6RzyYw&rh+Z5r|EAohMZ1%o-AuP|$E^up6TddLcD4Ab?@@chqXU)qulgTh zAGg}PJxKa%@#i~>cmF>q`f}SdfBXN}c2_1G_`9lf^PMygiO$d8Z(7`qV^L)(oh39q z>Wlh|t*_&f%q3ahpp4^1@xNt$X>X?Gl-+Mq zTAKC0>-5IOt!JwLCamWD@v?7@&4H^m`?yb}YUVqy?-7xk^&vVWZbfB?%0G40($|-& zwDm9U+FP~fX}p7{T4DYAy1l=)Ka9S0?`#0iq^wiv>Ty*ICRydR|Bc@iwttp=|Jyka z&Hr&{FS~hjvsGQs^Vj>I&T46XeRR52`0Cm-Gwm;KTJQR5_t#h7JTu?#+MKrhOW2>= zYpV{PTIN;$t(X0YaLE2}d6ud7R$r8O_g8T4y3iLfNqe3}E?F=2_x@^*myutq9=;CS zeKKKEU)J5p@2{IZJ|x8c!OivZs;}?QzrL0KPio!1If)iVLW|;8uMgusD7x*n*j>-_ zzWeIW-<~X8|8;t{cy@eS@sg>^f9_i@vHt#I{pq`poJ3!*zpnLlRr%u3pWAY+{rqbe z?S1<8|B|S`(t0oE*X`VGzWR0O;j>Y*ri%Xln|f=8Y{|N*`=;r2hJ5l~ulF}5BQ*bq z;=A{SM;`ADUl+Q5x8cYCs|^?!m^V4jVs3cFT6rwtUw*RO+p9nYW(Bh$raQ!>{kJufMuK?6{s`gxbF~ z;p=PMgT80q*RA@pZ*SPzzUS<(UVpI;iC6!BkmKX@YnRML%GQ+ab>8{)*4L;#(|_%) zy1wdEbZmLunuXq7(T~!0Z+*43HzvG(-PiK0qg8vWj@~(_e=Yu`%Rk|LXPI^PfAFl< z`Mdhxy4X)Av-5t1UY-6>{(sb8pIepHVY8>kfAQHpx&Gqk6Ytb^Oz}Hf7_Po1^mY5| z*!ycjUwwV8_3=`V;D?1K)>XIS4;wCA{`!5doxOSJ>O)59({0bs53k!0u5VZr^)+fg z$9MbHfA0QU8?x;_OU%xz(U^pj3U$Se4Hx|;{CxUj#v}3b+|2t~ToqZ2d7eMLk(IN5(@U=ZAEHmI z$ptEz?>7|9`PF21?CXt?CtH{Bi0U+UT` zci~T?6YtKF>-YT8DQ_fd?UHxf(=D#)ZYf5i@e;sLW|NcVXe%IF@Hu}q#gnn*g zI>iP zElAK&k!EE%=x7k2W6H&Hs8O-QH`#c-WT9)U_sYc2(3(HfAKVXm__$7W!v5}c8!~;b=jTN(_?cZ||DbTKX8hCtR;#kw zx-3@A4O9O;Kjh8XSB0-LbS7s=T@9c5R=C;Y+T6^XgSN28I!j2Zf8IuxcE#7?o{{ucov6=sP)y{d$ESjXXY>Pzc zH&vmoq|~oRX1s4O&i@?K!1->Hq`Kwy44aiRj>$|7b~!O&<2%b*d8vHU|EHeqe*5;* z)ys$G+C?Qjv$9vccE+N5`MYI_bELY|TfGC7A|&^$-fMp~=5}}arse>#c%Q3x4w%L& zUEcMoW}4%<@GVi^9$sIAt_YkIzOp$zPkQdkjmw3UTUR;F(47{4FmPg9qfcI5vj4Kw zXIWcL_OgaPebU{T@#b}UobHM9oSnT>{3`Ny6ckzM6t03!_hpu9axo;l7vfJ>> zY#HrYvv|5UUwo?()&4bkMWfuq3m<1aE^m0isB~q*nkmwpPE866PI#ztak?lfboeL< zb4Q-_7oA@pQ!KV>XLw{$YsjNs3pu^3|K?>cn4DFk^{>9~+@1~c#($a87MB0N{b#ZJ zfBq@|)Bm6UU0-=ze94Ph;(X7ZAHH~@*gn!IYR&ekSIr-;bC^Bt?|%1+eP-VT&s6QP zV|V#-@5vty-|qkQn~uLWcUsl7S<*Cm-KGXzo#&G4^1}B$m75=b)%4l*YbElfZ+O4% zd64w(lit~)-wup=dPG~GyG=gnYbzJj0z%C zG}O2_XEX+E4Db>WW_3CkFu_Gdn3Kij*^Xz$)mz_|YOPx?yYy(N)YMWF*;(_y%n1*u z+4OGK*LPR;Zt_3ub9Ygb?fxUPs^c~HHMcF>reA9uY5aM=(?9!HzrX%JIoo~xzGU9F zwoKpo7j(R1JYIi=BX7!nOp(ii`BturIaMFM2|BQ;cILEsZkDHAY#1IeDv3<+5mDx4I_EecMng=BrA0BI zW9F)pGiHi(ww}Fpb6v_g&C*!D_n~I*@4ddO{8y?k=0)zD?qAEK*2mWV5%=4^`n}ES zeQArjcB-ge$`_g~@@P|ed7-oO$LUsQG8X@1xNg#}{(wX3rS0|$70b<5X>WdR`pVU! z{O|qWvrj*p`tse+D`#Z)AK|-vZ|+*p<$>!tkIt$3;rDkdhvFKm=U@L_e_wx#2UQdiqn!W+r4z37rY8hir>Gu zHg-qvy000pw(K}-zb`((f>A}JM?;j0x2Z94!2%r#Zb_y?P8y(*a+X7m0ux+(*G3-i zUVXqTR#$78YgUkz{M=u&AOG~7{g?CVU;UW3&%eiiwBKFKzpUnxY|P=`eVtXFb2*m1 z)u=l@Sz`Ku)FpMlHU@Z~KKnB5+RCez=AG-ltzx$*Ji@y+SU9`7Cx@^2XwiiWa(nkR zhA-GtrFXbE{AcC2ViW7AzsE)Ur+xf%ezjJQdOLSj{`G3cEQYYH2X)y}w!HG6AZhED z?610S!Tqb3Io^Lyyt|Mi_hod9cVF+sBfr+XiCjEK>ch!zM-`G!m=$SE`ChkW`?o&_ z9)KpYa#VylnGQJ|i0}{-Vr^+mSQw!p#?{K?=+NPjCUPrj`u)(Z4O2_iRJCGOO+WBN zO8NPm$%+4&f7Th7Pk&WA>B}_l9n){`+o^oy;P!1swQBQq1J}Cew`@&LNnSVkxBr!V z)!#e4o|f*;o+Is2&(?Ub-3bxw=#_0Jb-J@4nH?@PG(-)Elfy>>QZ-G_jIxUSthQT+OGC4jxUal{dsCPi}bzyuN>+=&Fx=* z&GOg&?BMHyuXVx$)@S)Ai{87oV{ed>R9dQz!ud8ug##aR+viCs&bXQ}!+;~4v;Lqe zdqV1&{al~RW-ZhTeabgiWO4oppNqM_u0OK5e7M6Yx=jN8!nSA6|-t>pEdFki{^@dv3mR#}wuIX%;3d&u`K>ys5%AuE_9xlg4NLQ>#N3Y+vWKb^6ISo6FYg_ins;$X$P5fYICT z6rlwcDz*0cd$`p3efF%Z@12$SB16b>rRK+L2kt8Ami+(wk%NC`*SritCZ70@;j_Zn z?9^3KvyziOUocc?SRb3H{IKcN&qvMMm*l_xzG~YpmZ0RGGfU3}gdgfz_3pilS;_OK zdv#W7oxa4kaIvb_*Ql#^KZpMD`L`kaO`^pG1!KFl@8!*;YO>e)_g@j3cEOc{J;tC` zntNTho|~U+ZIdwjM2(UPd!0^WCWiH?nV0YF)h(~y5HG(#<7dJ9Z_ocPo+$apQNR0% zQ~%H9{p$nf9W_0#|KRYkRFRM*>-FuyJ03)Rjk>dIXUL|g*ITD16^Z}X3(eB5xE6l- zhR4k6SJw_|79LJ!YABarjhDZ%h%e^j!RDCcH%lLu+^jl#oK0_U%pbwiFDkrVG|k?g zzn!z!!8J+B|B$R}o9 z{O{vGi)^gyb65SZci7$W)NtZ7lU0?kKTkV;tT%i0r|jH{{O~I`DsKM!&$3%uNl};C z>BoMqxP{EQaskSpZ`Xa@^=JR}?k7jw&j0M6VdQ4U%`r3mmn_q)8>SntR%$yM|9&C( z@IyesrC;BL73!q~^q$vzu9~+hPW*fL|GOMjUZ+q0(9jMza%Re`zR%Ymy-92oqc-y*Su@K&1Cudvx2mS4LvL>+y>*6q+-?Zr|VhgW?&fYqqW{(lcNGUtE**^R)**d z{TmVRievrs?~i(y8eI0>6;!=%N8iIHpSwS8ub;eMG1I^El6UR@@b}Rx)|+>Tsh{1v z>iyT(`>*d0pCtF>O3NX$Sk2mcMRULX*Pq#0zslIZzOwGu_DS(4_Z(t>^e3Aq_x+j{ zy{O9ieXEZoF8L~Q^sxy`&8NEcug>oKF8@9Hp`Bdlv>!k0Qr}&@bY=b9(pTTN{FU8! z*Frzy(%O=cJui-|t~&XZjJUq*+x{i5idWpew*Ehh8PCN0 ztM^Z>jb6R~-rtDNDnE-q{G0t?`mrdpOC{Hz&X<3+VZZyk$W`I{BX&n-sef4a{{Hp( zTY{JB|95$HcF*2gCGW4@XS}}4$FE>uV9s}(#nPz7T3##E`t-k(ym>|M_ghDQZLHgS z=ezytFzL0sRAc^MU-fg!;duFuNZYE|@ZI~vwXdJr_4SqPuT~xRwc)EhU%&oZe{Fx% z-$c;|?OB$o`&rB~J&N`3d+M)~Uy~FaU+q0LHfm?x+N&Sy*R8&~RPbh<_19NhSFV}o zle|03_twsj_R?kR?;p{wnZIU#Z0z;bld5)QO+C4``s@0uyQ9`cf6dyOFT470%-@*5 zQF~+lR(~!2T68q4z0Q5>`m5Jpi@z4F+NgWFXMOzo^O36S!{gW0l+Rq>{cgK?zoLG) zY5a|}+*h4b()Y{`U3K)=(Z{RKE9R@M3jbTxs{OWDer@Q{3YFt`qeS+zioX4S(u}Ed z{^6P{Ve(6=!*4zObu4_fP`&uAt&g|9`dXfK_0`w-Jp1zR7Ds!uua8x3@h$tNQ-Q`lw~KRa@rm z{dH6IU)7ljY4TrROf0__U;k$R_xp3+)&BbR{avT{>-S#X!P_r=z4bl&ivRkbwpIRN z^0svs1ILYTeK7hgVYa6}QQo`X`E#8~U01Ao>NiKO#v%#c$k&s)Uti-czrOn4)?e43DldBV z^|bo)Crv`jc1!Pzc1_;ueZRC$|I5~@CcXF5U(8yss(7HO_U#v+t?$F@x^=Jq750m} z_kaKTy8lyJ_Fg@G3Q2FkOFC-1YKs=f4oiFNhq*YV!^S5B6E ztz8%TD&=kT*HyAl!>_u2{TsK(^yx8qH>=#t{i|n%v;HaAe7sPo>2yrWIqxm+f5*Lz zytSukd(g_s`(Itt+q&+3=ih9JeY^igtIz*)Z{>d$JC{s@AF4e!{=Vg2Vjo$rx^DOH z@b~+E>Bhf|TX($NEB^oLzgegMAJPteb@Z3;n^>ygtZ_Ss(Fz3Sh)CySmgs*gBcRJn`I==zKG_ttYvQVq76c=2L^f~AX& zs-MX`_eUwQ{Rdx)abMS363(q#^wR3t%c!4MSH)l79kJx=%NmiKuFhQP>xCb@+ozT7 z(~kNh?8dZwS%k^|sS&dS%i|B4nH22oczWLW>-+Aw^|3V*!^3v1k1r1o@!J0S>a&}R znP;5`1&^dd; z0n?Q|QrCqa&M0!<_2=t@u15Y-AM2EM?EHL}+2n<7)5@)@`1KZi`_#s!_o(dG)ivcW z_r!;UBx(!nZ#g+*^V{_oAN&yvJpIbAx#7Tzs*~^k{|Hay>27!A^vPf=`su{v$QZZY zQvBoVIrFdoPnfLK7A-%?`{l3IQ~dANOv#wjZpXE8`SVYanf%S)Pq<9`n|tf)r~4dw zvF5L@XWhBAU%ZS%*DZ;Q_4wIYoB9Qu9(L{darke`!yjB0j1v@O*S5TBJH+S0%VWC# zYw?yc^Lxnw{eP!RrA&M4Dq6L(Zf#uE&xcjh=e^&#W8N#h_xrwm$%(3L#U+XtDxnuv<_ot8f-v0IE!@jj~ zd*gn&PTBs$;qsQwGA8GCbHPu->%;F>*{1$~Q>faWZkKiM#r{8c|E;mBTHt$lyY@fr z|HC$ z$`)c+ugbEf_WShf{;n;j|83JgHd)+k-nHEG^nBO8gXM+S+|NDxFT8taXQtloKa*$7NqD~h z`tEmcWAE2~zh4_0rS#uZE@^Y<>#Ogt?+<(9cJ6)MlP_-i2L+R@|7TvU`V+aCYxVnm z7jNX4+6vvNHoD3#e^Bg_N$d|5!*5e}>wOQuwavf(yI0)X&&mgY#6k=&wHVoVig|wcodIN)$VreIw7_Bl?u*ll@WOZ*s~=c)#Cw z_0^mG-xqzmeJDj~(&pWN>qOSOZ|D3ZC@}qG|Cag*t%mpO_x8Sg^>DTnui1w`U#IOa zy8Oq{m*2{T^AYpcBi>cz@An^6XwSZ08vg3}Dq9cjpQ$g4-k*H`_0W6u{eOSyd9}vI zIxsLWD>=?$Z3JzJ(Kk0!K2p&aBk=qGKbCvH|L_0*zp3trW%%EsU%%ttM*RB!b@j7% z{xR#XzA5iM$eJDFe>|(~>ZPaKC!gOPeBucA${89Uw_?_XbEx7#*c zd*5}guQR%yF8a59Qt7{^N3V8o?f$y@YxT|+E$?edf3rpQuYNQwdVR#Q{nyrCi+DG8 z`qfp>cCHl*nLq7Op7&qDQ~R%L+7?Z|7QdSHb=dsa@>|b6vyS?|)LvD!HSg)xS6{Qf zR&A}cjsM#HHUHXvJ~y{k{?$F5(r)~J{HpAN^#2CQ?@RLh8@*kuWK!Hl&$_Cwcc!fi zU)!~H)t9L6;Y;fuwA{HLzb3TO-u}tgx|i?Q#ftwr^?q*lH1FMi#dUwL6na!2Rvx$M zjq0&Odl>Ux9}0VU`L6cc-Cf&eFYep)dwpQRq|ViouIk-iy&^B|pMU+Y{a62+o7h)} z*1hw`P3mSNm&=~7@2)OetGhV$hKly$n?KV(wJ^SkRBzd|scOU3FV`O% zyTsRZ&5>_P+Wm92?=R*3EL$%fi`~-wTL1JU9<^uxh5aUMLU0~?T*kt?skDcPo4Sg{knx{OK7DT@5og zhnK%A&)F&3UK(ufBTGw2q^jPa&^Dx_`Duj^=;f7w?Pa)&E!A_@Pnc zXSC*9xxJE4_TSjOwrcA=>$Irnt0T+fgakDlM5_J&PPx5c`|G-`%fI~dpZ_VFW6!gT zoA$>|KAYv_arMc+Ka)iJzCP&b)w{p=ijx2Rm|52!wE2H6w_oe|U`B7s*}I+Zzm)yW z@($i!zuI24JZ`=Ej@ueLc4$WI3E$va{Omet`LdP>XfdzDh5#K2Zc(Pgjs_tbpq7(4Xi2aqSxBt)9YW&##ut zUiq=DxBdMu@5TE)#KP-dfBSkvY+vuEqs!kH#ILP!ez5Q(zhGAH#rsAj;r!(<%-N~~ z-e3B)U+$R-!>2{sr?YTAewUY0T2Gt*r{qMe0{aaO@P9}?m%!Y|sOFQ0vs?Fq&{TewXJ#P_2rNAceg!$e(p_re1UkV(z>r^*Y@7$(_I(&YmZ#azE+>vPoEgn z3*Pw?Tvt9fICo)aPr;wEuUCI-@xS@9^VOR6KRQ1R?cT~Iz6|*M)bsDJtriR4M(F)& zR&EH*T))+zWnnzu92dzi<+5zkeb)y+pK--4a(!w+`?NOpTP6yT48a|ByvEb_?c4t( ze%VLq#$NT!Jx#5D-3PrD9rlz#fUc4Hlrm${rEU zPAAZ+`we1ToK1}Z3qv&2xLP_C4LV}3ZC)R4wBz1Gwbd#swYt1#W^X+5?f;!D|H_L0 zYyX-bb)WC|{ML)Txpi|}uTR_feZRx`+HfC>&)iZ6&HkwArrp2LyL`!(1O6J)Y=MRc z=Z3qSFWzK+CXJ2LY|B$mOS79zPcBC8m>=w*d;LY)%D}r-j{9HTxLl(eFZ%1-(^MNh zw)q>*`t9elI?ug#;-8#-E_uwIvlEzm%R=I6xL(}3b6sxAbonf`V2J{8>EF8ytloR| zO|$CcTV_;q^{DghX>IPIvzO12VN?>Cq9G>4+SHt=5XlAFR?wKRVuF`ACy$e(pz-|t zm}32Do4HR<>Uy-&%VSqu^Y+v@&eQ*G-M{3||9|aX{8x*BWT))4ne)$V`YpG6KkurO zOKt=geqFP(!}Wgeq>qj(BA@lot$nw=RA^eB4x$HWemRDFAYYs$MF@BZD=RWlMa zc((ZYkqVc~v8Rg;&f#28Fv%>uKJxsFnJ#-TyxSd?RjB&>gx%Dg`yJ2U&;Iu7_-7|= zm0#;C#XrB@Hbpx>eZkIapZN}k3BQWB*uBatOlaz5?R^V=*Xno9O6f`h3X?K0|;tC^Lf|j&icRupB{;s+Irx)ux zGk2D~+Eve;`cUO{yx69Cm8YAR&i;`mRsFKtTUQopcK#9JFIza-;(LzEeaoD0Pnmws zeja?|gy@1=vU z{$88E`F6Lrt86~s?CI+kw(qX{{8mR?>(;{M>$ZzdR2KYOAJ;N_yBlNrvni+bKYUP{ zu0%O`W^!HadpvlEXm3cvB|_ix)3PfvEp-nnkq zvFDl6ttYP4-P2a(>4zN>z$ha!MWYzB zUm)o~M1~lbL6c&@gciM@*W&yto@qud+g>d6DlE=B^wb0EZSp4yez!gRF8{4=@q-)3 zJ$~}7ypvsXD_cM0PImowuG#hS51Tf>Zrb?z(j~9pzl?g`e5mP@@WpBR3_TB2&kMPqwr|&*B?`+3|+{1jE(l=H2 zl>YLwEZF(-+idsEy$`CFZx%S&Vn02#QuTbu^IpjX3ntbj-n!K9?)N=jFOPS!|!l_;S;}l>XCr{t*Zq$3%@$@RVUK6{#Di1%69$L z-IHD~?P|@d5BnSW>-E>vTUV<#^!=?~ANp_C-&Fmzo>^ayeAT?Y^<}Mx{`r_0_h*EE z{=B+w*W2J5OV_SjdirYA-l}8znyV*YoEL8~^;_4}HPdUgMPD7ge`$Mc)c4ogUz_%p za+K_iTNiqDb?oDvT3=tX>vQTxEcCwhFkIp3#$V60ukP+T`fO{LfB2-UCvCGoxZgPJ z;pzOh^w#O!U%TRCy|Z3@t-q$VwO)Guwf*6{*RvGK*a&Fxo_l{~`)j_xwPAb9Zz*5@ z&T)I)?srP>MJIFQ&sz6aY{|agyyY(2l5e|A{;$4f_xkWZOFWLq*Dikk=Sj75as0o% z;q`HQ%T?Ewzgn-lx_34IqK9!OwuNV`(fc!hfAppJ!$xh9Zi}a1Xi;Q%f5`COH;%_A zUM_KbGd->|{72^{E$h0UReyc;HTkRUuczv(?QNr%8Q$Al z_5W+>NBi}$Cnv`(Wq;9Qcj{LB_1%VgVb4Fu{jC4)6#sc*&AqNQ#;#Qh*A%q>FMiB- zFh?lA>TSf^s&#v_zZ%&7KC1sZer@d8w5RK9o4ntKKU`NnZPGv6E7Rw#40%=mDk1jY z8u5ASSI2*zSm&M8v0wexyLEo-NRyy;gIl<`@;=U8_9#O75U$pZ6vC~#oM0)q$YL}OLqecI%E%B}T z8k^>^CVWx(i$$xyYG~~Lx5NL%1dsi0hZz`{%N=L2HLe0peyx}v|M5=z|L>pbe?9%R z`SJeM(cjj@ua;jlSO20_eAJJvFZBgBRee7y-n>y?VSQKl|7S0!-fgltefrJnAG7up z-P;@Se(Rcf_kX;fUKamv-8D_yxV>?IBmPd=`srug{Rf9R*)>-c&o^ZDcK>qUQy zI_^fr$3$1m`_1wzYmVqkQKj7eJGI9DwN80IkNfL!sp@;}zqPlnR(&n5+iUu7eI3Ww zZI53af6!KuA;L9%wS3#B4S~DkWp@Alw`SF!-s-zqfw7l#&hT|Xd}p0au5F7w8?+O&xbk)QYn}P`|F?ds78JU-+%Mt8NA0Ko_T=|1 z^8PvRZsL!+Kmpa?g3EsIKU6-$c;mJA>sjVW9$5b)W1Y{YAQp)aAB*IFC@Onarv8r% zyIi%sDC+C4FW2-v->3BXzyIJ~6uj<9ywmgV_vW36tluPb)ad9o*CttG(-`lV#C;I6OhWsCXNn$&)yWW4k<`&)k^#2-@itMt|gf8(}f2)m_ z|9bzieBbSVHSt+yQ~yT%`TECr#ns6>eQWDx{k__MeSh4$zcGK~-d4Ril3%;%VfdYS zFGY?v{x9L(o?X9a=P%Xx*_+h<2>!SyQ5&=_n)4y{J$OlJ@@xt_5S=V zInEjXB~1IsM9WKIT`Q|Rx&QvJ->EiX+TKL^FMeE9oh_wL-k>tc5%{Q8=>@3gVt zljGkld;hCG3l1wt7h(3{qnDq{Y6yz7WcY1J3C%J zf4hC__6lqCyJvl^`ucjOcx;5)8|5=aQY$uJ-oN4M^5ln~FBx%Isb=)ZPS_>&_h$XC zt&?B>|B{)1|Lft+nWt_4$p{_$6MU=wkEvwlE9a1t(eLim2~)U=i3zZSc`?xbCP z%B75d9R8c?-U@}*?wzY-)#KOt*SPmx+x@rn*TGxHr?lUHiLcvzhk=3F&2bhx<0;nKg}KtN zWWY5>*O2?mmEi4n5cb|8A+0K*6 zS^KYUJaTVu_SQr35sPe#{wDpc-oLc?YnF)r^ZieEt>+7^tvNMq_Ad47>sSBNjd^rs zn^4fBzx!W(fA#n7zm1+HhTeVg!f~6!SBI|&j?a4cerer5)tDE%_JsY8*k!ghHaz^- z-l&~1Ti&Vf*#A>+-)pYwA6qMHg*4?qRq4jBeSP)y)$@m>S#^!pXB?5zomBkt@~>0> zcI@j4Po5I?W9?tvvNe-lcOK6dR?$rRdZun&{QA&EGeZ33KXYua{j&Ye7vG|A>Hogr zcRw0MUAKb?H^d-_S<`>QY3Uw?awLo~7O*RuB; zyq6S3Zcfbpe8qgJ_wRG-UhiE0b^TQ(*I)nk2)x_!Xx$$5$NJlZeCxLUnE8Iwj<;9l z+XgPm`1JMF|F$)E7fr0Jj&k;2|IIJvwq~btO5v;N-1mybMAg@Cv-tX7QShYn`<=RR zt2=M6e&-$ad-}zs-8`@44|I4xHI>uZ{`>yRwe``f_q$XDeSNXMZq4rECZ&9=CnFyFPesJoRGL{}>V3HBawskE=R< zV!Ougui@*XCjb69_1TxGKR@0Jh4=2&`y+Nw=ip$Ovp?|=r*Q1IHM6h2KH+=u@2BUlPJap7{#x?a z(eQhJBTZ75A3c|`^HCf5 zH@-~zwfn;}1A|FdSGh~rf5};XeShdJt&6YTRw*CkR9$|5a`|7M(APKX@9P$)U#yOP zazZ;}-g?u;7IEuqm3M8~8+vH#tFKe5u14+Jo3KB8efj?K;@GI|8cH+z-v2VK^Q*DE z!N9(JL)U)KbEzh9!6_1`j-&3t|DV%Mn6H+~21 zTNEbW@Aj`}`poa^H!hqh)SPzoX=3~1wN_hvc89;);JVN3Z~WKf@1@z{uf<>e-Z8a$k@esBuj!=*({*;ZJ55`+H(?|9ooQ?8HvD&O55IG*Q}oOF z=%A=C{yNjPuV%4%B4%OFGLwD&&R-92Z48eV+V7k7IsVcq?}$~``LC{8y>EnUsKZ?%%vmaH9RLSOlE_2xqnlKfA zw@3Zdv13`;=SqLg>HdGPm1*{+d3wiB88{nB`X0XY>vxw?beQQIfn%3y)~3vRoVZJt zv2JS}r*Yi7->d(Mi@&-Q^<`SJ*Rj9idcWiT9+&!GX(#`|d+%%Qoq4~$9{F4S_5Hz2 zGk5XCuC-6T>;Anu;q&EN-=?43<@Y!HVWf4~$>0Bfz4CUedw((Z{_~yD*}s>5(|5^x z6PT-V`F-7=K%^^@PGHGUw!s<*}*?TFdrgMK=U$^`JYtSH#LT4f1MMs`NMzh zU)9$6+p8Y2eKoN%e#_>xe9iInuQMxeRQ%dmZoF^uwzKaq?SoK)!K9= zg;)K{=BM)VxyoB_U#`8jT-JO4_Mk^qRYqUdJ~vsnZh!h|KJGSE?!!x_ywTDNI968A zxov$@$eg85H}3qP-tKfHef^t*2S9@?IwG8{ObZhgW)y(B>5WN>5*n)9oJ>v*6MDRE z-8{bc$ldI+Jrh>XZdH19mwm?DANeoL^8?M}*ME&aRpD~UU9SAktt!pe;XI!MPrKEa zmmj;d?EOD03%Ij^$bPV0?+C=xlcDs_wtD;zRj63)uC#I_Aa7{AJ_G+Nve^?gJOcOR% zT{Wi{70ZhLjE%kT{-tMiZiq+inP2^@{ZdVq)`@K2 zY0`C9mcMDVQ$KxR!#}YD&Y5exW&QGdUR_#T^ZZN3>_4}~&DWH>y<29%CpXbc+Rx|e z3ZsDjXug}Sr604FrUgb;9Zy zjvE4cRJd7L7CKC1(T}aZUA0ha)hpN7kQq}a?U*Whpu+R~g1!Cx7ylfO(qH@Uzv8K! zw!3$kcADmhpS$$9Zf~T1C->ke3<#4xZz_%O5|xFDLj$Tkrn*S*73p`*Ezyo0T1D`#!?| zOjtzOjAed0nF{Z`JeP{6`5cjabaI6ZqlUBT;ZW7sJQ#w z&b8&oBxaoq3JN(nWpbBWd;90}>tD#9&DwwRe@Dr(t?#-5>p5rC?9{F(y*i=nw9%cV zs{{RWbAzsLneNc~DM6R>TJA>Pm&v7t<w)|J-<}5kU6nPlS8OuZ0lm9t*W6{_yQ%Bn)Kd#AwKIQ7UQOPf=H#7wBE|Kl zR{!iL((jUG8&4~(UC#OV^Jks&9l;B&4!(K+<)f+Vt#@kg8!uT+UHa`sCr{yDfyNIX z^4LKh)Dm%KWjX>r6vwHRC-qriP zm!o!n+Q02j{7Fl_#PfCWZenjI$IrBtTFPbnWB&i1M$LH6EjLqblDGWY-ctIGQSRBg zyGQ%(JiW~z{$O{zKzDHFMsm%D7#d*Ue{X@!RHo=LS1yd9jw55N}I!!hr}4F)7Zb zg)RmGI-(q{PL6^S(LPh7KX294SDET5krdlL@#p;s+vm&WuiIZTPx{Hv_&fXF?xaJswJ+3!nj;$la!h?WZ>4lIAeKWx&QTQjAPZ+ph2?`dIsk4>$-NX&{##xDZEv~i%B+ft$O-1O+-erY;UT0+xA_X&Hul?-)^;Wze$(l|2gZ`c%?R^2C9GSDD}?1 z>wSNH$=ua{_N#lEX_x!)Ae|F?Q==|`@M z|334i@1OHp?y&gN{TKSCJ-J(@etq|T*VUh9m;cgkzW?On&W{x;T55CWhTc6{bhk=9 zcG0Uni^M&vbXK1aVR_NcAHOy0Yty=TmKJ{3)ql&nwZ3-k4bop*x4FFQXw9tMUn}#! zgsgwiz2tgXP~OwsUt(_m(hr;Zy*xaw$xx-{czxcTboMNvXiYlGU) zrADPm*cleT`oCy@?dtHgfi~BFzA<<5os{d#WI6r$-=sIf$CMQ0dcU5oPw1{H@Xk7% z(DysCyo`IrIR)EH$&2B)_VV5Q8vbkFy;m2N*4uMAw`JIWzZ)C*`lqkd^s4veKh@5% z{87Cq(sU+u|KA@69H)G$x-#R=jrp&yXM7Fo6}|r|Abiu__p|q1-KGBZrH$b1c{lX+ zPQAN$6SNSU^VhOfOM|bzx^%y^s7fug7Svn!aGb@_q{Uj)@%qBFN$;Mls*l=xb$873 z+W1eUrLRR}&ioQQS^9c<>8poT8_M?HnYZq^?e5jC$*UY?f2qXk#wpDDx+ZA<$&>3p zPp$d1$+Pm!Z^PHz?>F2_S?bJi=T92bL9u_*y)Q4UQ`!E?>Zoi>^vdfoEDlHJ>MhEP zk6Q1t{%XS+&D{|z!nEJ5SsiXSJ$_}@*PmRsz8#wKyudQ1fx6%Ln zM-yw&@0E7@Utib!8yVxXZ>xIuO5-1iaZ{==JsW*43ZNGb7Zu%(FPVh=2Nn_VZ7Fo!WDEX8l3l z$Y&hOv+}J)-c|jRSz6n4X+FQ*!q@Bn|5Dk_5@utY^pf-A|65z?XP3Oc9zV@lC{g|I z**2G^@3-{iCrsaWUH{tZE3M}1cJKTj*YhD!oAK821r}~c-5;*#(f+@)w6TAF_K^tA z#yJi-TfW7={QN5Pw)c}=U%UPmyv_1(-wIw&nYDA(yzI!jB*n8^pX!-ZUY{{()0KL@ zFK$gb*QMDx=ePrz2YyZB!dJ;O}5Qw`0Q_TK73N|ogK%A=Y|e9W(&^0 z+J1HQHT^@?`^)1LCVjs>wQJwehet2f6dV5Ry7!;O_TKJ40xuVruhuk~UO#VTUoUgZ zi~3I&_4WVP&vxE_a=K}uUH3}Q*ZaQKFS3eW{kLFyd05}m3Kc`Q;QUL~CqGqaXq~hQ zS<~^ga!soflO?B~ z_OD;Gr}f{TBju-GyubCe?{E3t*q^U@O=kb!y8YRBO3O}l8tyQYJvcvsQdD)poTJK)|UcK)f z=q!VmHi8`d&&ukcs%;;ycH_O&Ty-?Ih=2BtE{S)5E>ti^Zkwd{X= z`J=se{J&i%U;h99fAtr|?+4ds&8+(Wzw`am$=6n2UsL|_ZPKdLb=S^az5Y7%^_uYT zm{sR1ZL{_NpLY2kuD(XLNvTKtJYR)e=jVSOuk}o9|7Coxwf~y1Z>^BF=*jvui)&L= zPrqJ$ZN<8GtdTqBUA&Cv@dN*;-b&aLdeyM+bANsXw!>_OL#lMb*YL|X5b^WzJ`|_{v&(|+- z7X7!G#Z!*%a$eJyfty`m==d%tUQpS^O3c}Ms9h^5t^D{Hpg{mx&n_x0K8f9us3fBmIex4&ZTt)sf@ z!&fDLID0YH&&lNaPur~byT7>AwQl^8*IwoPa$2|CldVnhZhk*@>1~tKAs6{=>wl>4efPcjYx?t5f(rlt+_@6^I_U1}pRQt0l=JV`dD*YO z7_t1-dKH_}_}5oo9e=O-?#Wl9x*f0X-oO9q@5TDR6IHf9+^fERy71Bu?x#D{wGU5S z;>h?hvi!-eeXq^t{nFVVwvsz;cig9xudDW7l=F}Jo8=$3f`NhQu;VPwCeR8g!>u*3 zXJ7qaeX=;VD123kFX#TackAlo>$Zl6uf6^?#rNO3)!`aJ_t$%da=f1QKJ3l<-Jd7c z+5X#`wQl`ZBmdgfU$v*cj$a?X`tDlK*B8ZqzDn%gf23sXMf2>rp=*Mx_RQO|de;{B z4CjCSnz>nOC1I{nJM~wGublksZ^WOwf|LI2_Wif^*HhuFqlLk@?yNqdU-4dK`^}mg ze_E1~W{OPj*dH0b`o??FPe+UX+;Z+cL|i^M~Ss-tT{IrSq_fU+GxzNA8i+U%YBU$f43 ze3)rj`S9*^72_MLx%~a+KNmZ{Ve$HH@;_Yy@@CprZ+-FCdu4fC*Rx$GJRWX}E)W0d zA39_I{!P|#|0DJb_?aYky7<@an|@F*>)-4n{vCRs*6(|DG+)*+Tkh@mj%}wNi+#Si z(L(zD$$5p%fA3E_Wb*gN_wZ9YL)Ywa-x)|JzZUNLBqn!l-|Oq)ucyBb`eiKoI_SyuMOUV6oqYV?TG92_ z^_OOUm0hw^f7kwtQlQ}uI45-lESRAo%Ei*6Qn7l^p6AXg%VMXr`h-mqZixQM ze$;eb^F#jlk9EpntD1AKey}k78yx;$eACxknal3StWBA(tE^eS{jt>I2m32!*$;o~ z_0B)pbg1r2q2tPZ$YdV%^muo!J6wq*|%qtPE=3+oG!cK?X}Fi+TU#dsI68$ zdbs{n{5S6F$IGQcwtd*J_v+f}%zeLfTs}Vdzk~1oiCrf5XZ|lUn4k5};BE7>e@hM7 z1ztt)ZaTvsw8ybEcdK2-L$6&E{|lenC38F3MfpuTLjhWKYO(jZ1uYLwmfin#c=EsO--mKORxMnq zxXj?%r~64x5vLu!mn9b1xH#W(XJk(Lsu^*J6&Vzt2ASR@EWMG2ryG(b!tvn;GrQ3TCSux!zJ&vzOb6x8@*7^kg0}ct5+^u z`bRo%(qH|@_r>r3PQSXhw#p(*_0E&B9ozy(SLW|qe!F+>jk~YdT;@)Hs<->O315WB zwtw#V)}Pw0-Yu#9aR2RhnV_9hl3h60NA_1LR35T=Bk+7(PU!wN(QQ+H`9GiSCcVeG zKk;jLtj;N~$;X%Pc)s^X{O_=-<^Gy4*kCJ6`MAc4dZW z)rsj5bw?N6D@|Fx;nC&mRa2d!XS3~*e&^jVT2ffcA?v2P~MPQ7SOy z(Br+uyJt+boVqkdYpKtal`Fkx$*=R{&v~*Z<>&i4zj<+)7qW$JtDlOS`nq33-}dhO zVB6oDcIRbF28Uf7?;^?LiyEj^lmJ?`LQ4 zUq5B*WmogZzZBjv|+vC>fF*9HF zvT!WZjB?qsS9O1KBiGNIe97_lr4M^K11>C^{@N;khGS{vOJ&Wap||}zgmxDOwI${( zmGBTQnB9{&m*vod=6yXc%O}i9pECF2QjSTU(c6~E_em6-tDJvaYiZ%cwUKMe6ShT7 zPN~{3^TyK78A0c|^Y(vlVr6adIWU3!^uF5C)8=V4OsSqpca|;U^WjX;%!yp4t-f;O zT7f;{HXG)>T7Tf(w{H*2?qzMwJ+tJ2Oiy8A-;%d-;>?czp^MiyzSveYS^v&{Q?1$4 zwk$Y$?%#qs)~lDxzG1P6q=vPS6pNYHeywSg=4x zm1E|6KjYYwXYLiA>kXW+(q*lZ1%Ln3+mAn;UEWzW=hY+GqXixBGHc5p&y(`%Q}xxq z`@Qbk=0}> zrwa|@nDtC9?hecKFf?1ZCB(r{K7N&6N{2Rgl*r;T-`(%r%$JoYCvH+(r)sUKvOPwI zQAMOfBbAG_rLke-L>^EMe5lYF8Ln-)HrP0{-U*KXc>t$FtE>ds1pFE9Uh_e$@@V}GXb zAC9~5VD}Y|EwZ1#C+g?#|FbB-$wlQ+*|a^||F70axoaHxd;Zla!A;M+mm8dGFrED` z`P!qX#mk=T<7b!8N~@Yv`Ltr+svH4%l?G>~#JJo{A$B>&_l}*e-0H5H8p^j$xu0k? zT-=-DopQCHfPsNY%W)PLQx|LP^@Tdzm-ftCAGg(XefawL)l*+jxh?!zTYmL?!B~CY zS1M zcgF3HEb$M2{c6|l9UtSjdiVazn)hRMb6#z6ZKUupze7*ncTZ=6J zrdhw+8~=61+@Fs!c7M(Onk;(y#m4X^Vcp#mLwxW2+bWv9vS#umQ`7p@+8xt({GNQB zd)N1~d4IE%tDn#3{ChELZ`IzS^(^Mna}HRocbWcX>Ak&0e{KdU7loF-`oCtsOXbdY zl_x)m{5=x$Z?C=S-=MWFRYy7Y^#0d7w|r*m*UUdpV~yTVJ{%cwrIP(`tnJl_uYX-# zzhPd6=~}+OMZ4eajobfxpHH~@;@4k1gTJni4ZXhlev_Y7bICp5obBrCmMjhL{JQ7f z2H%oj*ERQi{ctKGAnMJus{K*lquzh*TG0M=MZaz8?)a@u@j5jM_x<*4d%H>R;pC~V zuYddt{qvuhC*aPb_%G@9ag(oWYW?~v`uf_slc9J2aq9i`O<(_T%D4UVRz7TbWB=-G z{QZncf4;v+{kK+3c+;2sO~Doi|0VP9x7#bu{Ut;1obJ-Q_V=F%&Jx>o@m_mwdHHtz zJyY+`kDs)jH~ztdizn}E?fkPT{+Dw6=6B!UtKKoL>$@12tXm(WeYI=Xm$J9Qp*xi3 z^Cf?Oaclb3^PNsvN0YW@ulsm!{q_CfY&-79?AcnWz`(#P5J&8ukW6$ z7rbb{+Vgenzm4&$cdz~%wKwK_xP9=|SIUH6|z4vQ%qc*(JNJU&pU^ z`*qy+$%an<*F5V&Pd?jv`s<6?Z==7~U0-CG{<2D~GwxOR?y&H+p+-l49S!jXZ8Z3@ zrmo{_dHIpkZFN)TTzc}a>r&PB(7Sz4gzNT(*Kw{1{Br)JOU3!C^Vf)a*X{Oy^>VM^ z@~KRL?VWwynzc#m!~Ryi+bgu_?{xh?LPZk&?pb?d>vB$?7O!7q@&2c2WNp&y zS=r$opX4m~tbca&`W*ip@HA=9l$Tmkn=VN1P5poE&rY74`Y@HRT|ahzt+dUmXEAxX z!0*qV{a@VI>m1XceyAkp-kekQPVpbr7ERZ${pN9U^MA90y)$l136?H3ak`}2Dq);?S_!DUCG&HSf7(rb2~{-u^h)kd(^^SWG*~W|LKOZ;sozUzIka}r&~1myZj$p?v&npYJ+`YyI^&CvdHtO|Z(kpBQK`+WIl4ge#!n8D89%wd{t}v*m;ELQMH+SMzkl`oLBIO%&jRz;JH_vNeNbLbYs$>fYQx>U8T5UuE3R$aSHJ%fc>JKlr$n?W|Del;-wU7q+Wy3} zD@xy0Lv{aA@r#GOzdhf4p&zoF-cJI^Qe}qN;?}~~2 zxAxaj_4WQQ+{?pSwjT}sw)gI>@1?J=zZRL-mEam%W&HlC)G7uBCMCyN+)Srfix#h+ zy?ow3{r{7{sQwQ7zyHdC|NH;{{`LE(b-4Bar+>57@4I?>*+%a>P1Bdfy`Q-+HUC%s z%$S!`r@y`?6!rf5>uZ{Mzuzwpn-G3!ZN&Q-f3xbrOOaSLbW>s?!Q z_54Bc@Yi2$^%sRkX_s-Vue<(Y(!A7Fi`M14AKLR-{G37W!cXzfl}_cZ{8V-Q^}VR4 z5zevtPIgtBR|eMKlrJrRu`zt|iU$$paeH6wJ+)UzyW?wWCFilm8rSpOXJ737_517l z>r1{~na8nl;_auPv#2K2b=_YryLdxfuXy?Em3vd|*ZaOIdG|MAb=h_G_1?viAKm_1 z|G%z((K~!``Rj}3>g$B=7yr^<^7YjhkEOf%_q$hJtv+P;$Y%Cs!PgR#?>0`ZJ^i5n z^u5I%S=V2@|8@1W>h%ee?_O`xbde78eI0buvQ2aH)yq})R&1@Fza@Ol)TXA`!|_-D z?Jes#TtBIh(d+E3gOeZaUwyM}Q;hrCx$KXc6-;Y)mXnZ&ARMu#QKPJFUnti?aE#se))6Ev^ABl&R^Yo?a06NzpiH; zxwo}$>#e<2yP}uop00ZxUh4X7ecXEemGcE(Pk%lAb){`;+16jz+g68e)%TA5w)NN1 zQ!CxGW52BnJ^5PX_L?Q1t_H|f?R>YcTxHei=Wnx?zu(9IKm6tObsxDk7G=HsR~N;-_mCKbJ1|tgZf<^!#t!-uOdx z|CHWld3T6xeRcimy6#tBpX{=Yndg^wf1Ui=uuY$m_Uu3T`s!;<-J(U6|L?E$em(tB z8JoT3{=aWOotSg~Yr@|6t=(Vq{>QAo{*|kCb!E}7FV}?jO}&x(N3cA%(e~qqMnk#1 zKVO~Ae}DCd;y=&d`@`NwDW8wmIOw2uTWC}1p+fJN>qS+2Z+!`$)8)QbUwzRg|FUI< ze*{DGwLV|@u>WNDYVXolN8H!P-d`U+E9Q6S)_3pU%WwY^zE`#`nrYtjS4XxBmVWxR zf7M&vNiSpeN#A>Qp!vl<`Tsn(dm67#f6l_#)5Q5V>d%zQm0vZzU&l{B>lE{C*YmJx zH_J<32mR#LKbi4v?L~!Y*Yq#OU7uzj_S9nDm;G*2=5hGlH2$+aZttzHzu!IkIz4G; z%}bMbo*Q3dGX8IuQg{=4`|bXRo%VCPq>gR)|89@i6T^j1moMMBcXD~S>f)o}LD$#B z-aYxCE~DV^;@n4?Y!r&|ENAjP5u4< zLb1;GSJyGhg-p>f;^J&+Y*-NSqLqucX|Y0Q>^txKa$3H<$&r;=Q^O<0#e=!m7~bh5ucr#dB{<_uu_z+sZWmMOleq+dThGRN~t-dBY`Z{yXa!Shl81 zUBA04V}-o@w>!IDJ@@$3zijcR@7XRo{z1`G|4)C`w=3-`=UT2l(dzXF)+NnvZ`R+P z7W|hrdONgzckiflJznn~9yxz=+x0WyE;_fkYHPW=P9C~e z`+M!39`$$gs_o5p6>jtq(wB&ed0tS!C^hq+bV1AMj4tmBs>1opbXXPBX0R(rJ-Kc* zTfD;F#YUaA$@|Kt1lQw{&qFq^*t&jtWP$K`^R)H+Tg|??*VS4Swbf1i?s?)@a@*cl zrBf&8Jy_fF^!0?sW{H?I9r5|u?31;mvQ`zSWC=WKf4m~yCrW1Xa=DX1e6QY1>NKD1 z`}KO&AJId_drQMJ?j}kkdkZX<&$xV0RI==&Rx!^uh9&Q{c)xfTrRHj1WIwAQ#dTN zeBXj z3EZ9+-*7*hk)_OYbG_dL#?YIJ=MJW=t?>PK#?3%@W7~qV7@z-s=NEOe2nyR3-e;b9 zbmr2TyKh2*E}Ohn3eINsm%6suNsUwSzTrs&`?>9<2c=AM z^43ZAtX-xkYPso>*_#`EQ#W*o?(pOfm)ZGLsz6J*Y)4$JL&Ji*e7XF|PKOm9Flt=s z$PxnenF2N>fYyjG9SWG}q9w@L;+7<=mAAZZ&)J7Y@0QP;)O~Z3%C5P5|0I9fTI$y8 zf7xDM82oeV#r{6SUai)Z{mT8S8iZ-OJtf-ym&CiF(wbOGh_~MH`45td{#FcTxH5ckQnN zOIp)IzO&{P3HV**=2rgBcz{t&WQvLi=wPZw0}mY)Zb4=yMU4(8C9W1H$AB3w$Hc3y zS6BU-r>(Z>x%pGkpq^O0pkI7-vnRLrF1a%=`Y*d{{=#{^NB``Xn)III(-MzM>S-@; zt*JUI_TCgrWD7(gja97cZIk?D&G0g;7^zo6DK< z^j7kGHV=robYi){Bgu6(t{T70c_&SMyYO$hwrD_T^~$t}vOMnh;Va7A7|z~dXH*jb zg>D;5B4`DU2{$K`qrrp#6-7Z7Cr5)W>DT*q&fk82_uB3~JCCu7?3wg^S^NC&+Uv`e zC#!f z&GM%YbiMxcJLyqkiZkX=PI|j??+)9QlLB@Iy>^vc7QqftQr95-5czyWW zYkz*%nKLemvY+4Rs-*w_>3*#6tANRd!#Azoh3syPiS^Db@pZm8`J}H*e-@1pOD zh%o&y-JdBk;WX>jze_<~X2utevv`<6YYuYn-rD-<+s^vB_2KHPWlxr0`tdvNZ;<|K zQ5Ku1&0q7bZ~Ff?YJb*|`B#7K{q?kN`=2{EZhfu#T3YqqdAICZ@3#Nz>$a~Ey!w0h zv7+U#!(U&0t$+2T{?{V^7&Y15Wt&z<-EUg8r}h7@eZOZt3ArnL{>O{?;kDXUnsF=o zq$hcA&DxpgAGR*_(E9m1^*n#SjoAMxOzU@;*8hmr*Vnzi9-4hQ_LoZh>d@~Md*e&{ zkA&K<3)X)1{OhSL`=d|4da`e??cIMnW7q9I{eOqw>Zc#eWwcjMz8Sk_U+4YR(_Lb| zXUF+$7k%<%&2-g&U7zn~2h~=cTDLWB&i?w{vSz0569sBB1*;CKZ{jtn^%u^*{=N46 z`?;011rzVb?TlY77<%k?uZ-Ou*DaDv3>PM6d6&mY_3}51M`upWzkBEM>DKIkx8HAF zpRd09zQ;!IS5Llet=sfhSnFdH=S2RtiS>4WHXA0d{+sky{(r`F%dj0^SL%PQ{t}aa z_1Tw9|5r!0|N46O_lx6^_U_YD->ts-^!nDKc(PP=yd#k=yJ&o(Szb1B#Nc{gvPd;B+)O7gIw5|8;vrgJ_z32_t7VaJT zD$)D3cxh49)yePwe@UOmp{c;Y!1&B@7EiMlYv~K`xn47DUQgMsdi7-XMX|lNwpMMu zrCXjPzE)^cP5qaxr%f+B-CK2K-kR0ntN%uS&E8tHwt7*RPVt>-ua;)IX@3v>^|kWU zed~*-GN%7hW_^8seZc$stCRPKuZ~^q@=<dDwuK%~DZsX}iVtUut6up-H{WW_=*N)kdm#V%-ef_fb z))v>3udBYqdcT+&wJx-N-J-uq`-4=wZi_#^ZmBr$>+A6Bm9<&=SD(z=b8SgNlEgn> z_1*i_GTi*5{#M0D++P#8Z)17f&hRzyt53eZZ~ALVWXk@?y{SL9$6uZPcYTW8)W$Xxjg}t->jHwZ0yk_xkGli&5TjvoA`9$8{x^bq3w7QY|;1(<^QX% zyqL5;LiOvXz13e|?=SscbZhh1?5(f9Cao9Q`s&Bhzmd0&s;@6wy7AQPy5(M9-*?Ww z7%}bLf0tLM-&P-zzrJK^{q-d)i*~>NdMaw`N2BTY?{BTYzUm2RF;MlQ(05l4eO(iD z^w*W?s7Oq@8FYkG zVOh7{vrm&14u-$x5w<^G^XqP0et7s=-`5-JIxkggZQt+u?&9vwfYzh4xg{5jnG`TARS z`=Z}}aOG9B__y?X^}oG$f9<)(bDadK)i#_51y;g4+dmfhPaoUtPz*!1&y87H@MFX!O&vZezUB z_lmyk?ys-DzSNCKU!~Pnx2UrI?w`*eMPIMX%MUpl6S_KlP1V=rty7EsZLIMLUmLRH z>HeTMwc4k%{(i6VFL$|mea*T@dab%ud%2G0Ut9HLYUG_i_g7zE>+;()$fIfJU)gup zR$p7KTC_0!_4M+ypX=7Eua;jE`&awA;IAvL$9_GHTPMH1ZbSL=f1Z)sj(_#*vYxg& zvTWD)s7b57zPcKeSw-ze&AYt;*~6)@}M*{k8gP_12ep)$gj;tCG*WuZsG50pjdVMXl^g(-6)Zf=C z;_K_y@YeoYulp}s_3qjFO-py??a@Cwg-^c2INnL`RQq$K<+^t7cOS`ref4B+T=w6j z>Gx)GM%D^(>UBQ-uYY~DTh*)jsQDKoPX7Go^z_sH+SQ?}L)W`h9?_{&U4Qbq)<28p zGEU3ltEIKy<2(}U`LFLRU9VbP_#mrav3r-)g%lsTskurJSw>Q5uuAOnSxl-D=g^NY4R>{bB)_;y)6SkL0y}fC= zxaha^MYoI)2AhSpQ9>(E>^}J5iut}}Kj(>W6gu9xl!$-Z+m za6!Y3Nf%S2zjPQbu$&+@Vco9IDd_Rz2Ex0|)STerVmzH8}evmf2Pta)lR3sR($N)8n;DqNYM zBErcDxOj_}vJom#+mA4z;cSvdQA9)ni)&zXHwA9+h+ap|EIl)y#K9}v-)nUG3%AD zkM~Ii?Ej!CS2APq#~sE3b57j12-&o)ALmnvJO?VL^-zXsWVBQNqJZM5ra; z)YX(fzeB^Mv>v@SHJUy>JZRl6neG*@KW?{E_utJ{FZFoWyQzqUE&H1}( z=e2}~hj*+sWiRw|J^ias^HubtV11?+YjeK)HSCeEOb?$Jqpa(p`FlZptGT+Q$2x{N zixgLSmohgn>WEBn5#{7`df2ccz(<3Nx9OpiK!}nNXG_yUgU&m}*M8pbzrJf#L?N3Yo)R?i}3r&foiY>Gyf}Kj(1laOIu1YW1^OL393#{$AF>=Xj~~)Nzq2 z>s;Fl_LF)y&XfLCqIUk?x#+nbcazPsSS&O%qhfQnS)4rF&t&oGoMPtgri910{G$)Q zSrIUAGhax}6rE@6JHCBVnsqbPW%8sM>?Lbi0xc(NmZUmmn=m&pYKTnHP~vK7I@qva zhK86_E8D|>86H|9934%G0xNv9ggNF~?)mod=kKejp{rgVbBzi)STu3LyL;yA_~wa! z_CNPCZl(Gq>xr{Z^=$H(;xr9Z5j@@1a?qrYxvGG1F=y?y!r{p*FrH(tk1 zy>UAD+SlaP-FnxOUv{xfW_qEqe9`BSy}uT=X{s~KzIi^JcFCP^K>3$}08 z&x|`^p)d2@+TaOq%kxaPKTJbDk_{UZHWOh zTGRv$r)t&BE#0H^YIX13C%aa9?akBM#b@VMXZo^GZOi|gg^w?Ndp$X$%|9l4{$HDX z^97xIkKcU$N8uG~RQ>bJ#r7fMEDP_hbBIZEKjC|Q>WRv2vDc?G&y^^e9w8huT{iC0 z%x?ZmoIky`?hZJi+RQ0``ETdR^(S9G%Z&=!nRBy#e#x2wuUi=_LylaTxAyHr)@gGV zZPXJky1@=wW}qd)$;z}a;lP9(5usL=g$*kLS_C;+og4!?e3V2vrW|@2TKe-(}v6E zUx6tbxZgB)hrVd$KXkh$hC!g(vwB1P;=L`ee|G;h>eicH7dGSK6$XAdqwZWt`M!g8Ee-TZ?5Uc-oqiJdBy485g7&s z#&3?Z_?Wv`OEY}SV?u+I>^s+ae_eO=k!{@judBYUzxqE+dhOL#t3`XG)RujHclGL} zqra|p#b5m$?oze!qv)$w`@6!|hv{F}=zbR;HT`P)bxzxuh4**iz58o|@~ppE zD)Zz1zdCSl@7>s6TVlWe+IVNT$&KE?t*>6+`kSTn`ufs&dsE{!i|n5L_1%-5wtn+o zUq7O4AHMPQ*Q)QQe|=fB^W9%R&b@xCzE01&zuNQl)I0Nb`o6lK_4xIw@BOC#^=Ek> zc|KjiL8R*I;jgd0u6(sWGJNexy;k9vz5llUT)Xve!T#{+*Oxp}`MQ4EI`6;!q2Jd1 zO?qeb`pU}cHKAF5%**eDuL~`Gx;Q<%{MF@1%jJTv?|xhJWA2{(?5S0I;@9lAt$n^W z{+0LUtYuF=2W|Fn6eg0b|7*r*8 z_`&vGN#_l{rnx)AC#_k@HP7GF+diXby=&jJC!ZpeyDtCC&Hc0erqJ`RD{GS`cTB&y zVAA{R`&n8QyG?w)sxJzE5v}^cDK~7{rdKi77wxUuzx3MLsP$n-d>{05b)T}n7W7MJ z_vEYnSAT!qA6d5d&i1&i;bEPuQ7VCV!nD_2nDEvV{_j;Q=3SWgH15~?|NDRc-2MOm z|G(M$|Nl+;|2F^E{=fg%UyR>cer$Ew*0|2=XK!71tz8@YX#cETcH3vyZ`snke`(jY zsjoB3>qFN@tlJyCHhcZnbr)mR1@*71R;k6WyezWm=8DyK|CeaL4PFZxcKyG)_4Ot3 z|1nO-s$PH1J1BRxVAjgLaa!-M1(oczzpyBM|JBvk@2?MEUvu~5>)@&_({`4BzPdi_ z`pH_|T5av6<=NM)|Evss^0n&!)vK=#{<{9Uy?(v-_w2jtR$brSz4i6){nxyh_T{C% zUh=p4(G19rtYSx@-HxR!@8XN^fsfe%#OX*H>Sic(3zn(W$-91iIH<+Z{F8 zHvZ}UxW7fyuN)2k^~yi&-Madyb;?`c>F>Y#>Sy)pdB4}YzRdIIyt%5Xm|x%RrK^6| zr_+VgqA%R=dwo5q>Xi2D>v~uByT13*y3nJ2@oC+zX`lYq3s0|GYx{4n?$f{j_7?4r zSa-^2rujnKTgvm-hOaS=t$J1Ol9~5cWP71`+)nQ;-S@p$nohd5?2E^%uU&s5`gYwF zZTea@%|Gnv@=RUN=j+?2JN|oh_5Nh_C*R&*<8Z&aCj4~qeeabvEgoMs$P$EsGxf#CWu z&{pr)yY^k*AGSX3ZS|LUJMCS&xA-URTetK5*3{S^DvPc~+P(kubXmLp?)7m;YgdQ- zSmat5y+2I5bA52DzF^JOS5KpM{Ms6|_1E|88Gozyy}r6n{blv-^>Oc?c5nR}b>+J0 zYR|h|uS#EkEyz>en!WpL^4FyLu-Vy<)?Ysws>YFh_3Zb$t+$S<{#)n$>T2oaFKbo@ zZ9iWVU!FbxYu0YTyWiHmEt>S|_us78m4CChZiw4zzP4^%xYysitNCYVF5A~}VBgY> zK@+;azSK>4=e<8_((m=*Kc-gNT$z98-g@1&@1AX)GQDnVtkO~G(l98;{dN56 ztEY=AHy!?^C;QJSDlJ*O^tskT{#l#;mp^}1$~&R;Sj~k0udWvPt#P{>Tj{x}ynelR z>8tN8Mpe6vt_8>J+j_V9b70jK->d(+ufD$i`f~h>b#{&-li#V|>wU=Y+P(YJzp6vb-{x|NeG^bv?W?KE%BJV%bgzs$E{g(7<$$qulG50!7 z))sVpdg<%@OS`;ot8O;yRrS@@??1Rcsn$9hazW+#q7vnxZI;t7-Xp zcfDuoa9`)>^~iqt+N)2$_=Ydq|2{Ky-cOa?JJ;{5jd;Ja_TA*~llsC}W$rClAGI^| z)Yq=HNuVTiZGUXo>%CPw1^r9^uAUq8Ysp8=bV`eVpW+iM`cqccZ_)dRf1}FFasfbj1FMWrE-1cKE-T82Vi$ zO1D;L+1WLQlit1hAG4n&#*{18D!gt(_-r{Ac8_U0>o(SH+8?t@F#K=!^yh!8{Bptt zi(_M4ddt>clwZGY_Se;4-y zUAwnt=B=?PnZ0Sp-?IwsH|zG#zRw(4QrrfJuLsy2$hIOqT3eNmveaQTa673a3zdMc&m z^6C5C*!UM~)fd^n*s-W!{zup+PB>(#E)$iv0+O)RlZIoZ=`)ljtC#|%by6S)2 zueddq@7l}DZNKddc$aphyyBha>WdQl!{3)*{kK)MwCGBg%R0@tr2j|i*CvZ!y7lE= z)gPCHr(aV`HpUBVoa=SDYWLUIUr+t5itg+VU+8`Q+uGodCjRSxHOl2j?9yKntX{vi zy!`I3-MhcO+WPL=(*0MvMX#?e1Kq#+?&_1>&wtFn+O_rLFSSL**|S!?^Yb)HD&0N( z>Vb8;kJnB5nQ}GyYu2uPPrk&ydiP{&_SSdTISb}i<%gG-o1QOy9pr!bzgqnNmYP{- zlJZ?%?k!qVzV%e>$~?VS?_Qq^wL6vl$ZpxY-_t{1?6%FhGv!l!p~$D0dz(5VEqAhV z{XPCZylRonc9+lp-|d;jkrS7??_!iy#Qd zbB+0~>wR(FR@2{yj{NvP=~HF>qIuJ=YwQ1CY;Ln}XWe=g1_s7ejv_QuV$ghp} zy3zMH`s=&dU-O^t+JE)es#ojw{g3)|_1yno@soe;iVt7E`)|_z@H$SD()0(D!y>C! z|J$!MU-tLbch|H;udl9KGi%n{h*uMees6t$oqPA!gmvCw*5NOvzTSFh!?N{XS61$N zwl!n-+xUI&q}Bxg+Zz7s>aG1c4om-Lsb78d`nuMC)%dS9<#RuQ7V`*yid|dc{yL8% zWoy;`(uzLieOKE)#_Zczx9?r_de>v_G4+c~Z~az3|GI3W@2is!Chc8XwQ5~<)cUZ$ z9-Qm`#(&OvrF1>^qwPtT`zt4{-@W5}#FDS;uW27m*?P+E{ngXEzE1tMdDo$P*H_h- zt^yr9^*j8lOGUp^>YeMql!gC3(ylX(3|+nP+?y7gt9$&{)~%^q`!`Yc*SF`-c7LtE z+WAiV+S09+deN%)!++n3T)(5~|9Sz<%J4$+F|jZ?ms zq;`GTmoV$DuX%Sif4|pi{^ISeFWWYhi@thZ(EKmE`-_KK{J)md_fPM4%joexx#<6g z-l{9}x-Yp3)~%1*nfhV*&%aeillN|Y2^tPQ;y6o?#f!D5=+D-Qj;V1QcfVV=V{7-{ z8rA#XUx#Kr^;;j~zyJEr`oH4)Bld2cGC!`f(l*NbsQ$vR{clqzKVLuT>gliR9@!R6 zdM2`|YN!5c>&cTv_Z00fy1st(HO`$WJ6UzYS9{m5p1USq{nGBI8}F>Vb8l_k)+BS) zYu)Ty1jRA>4$4Q{af`l;X;}A`uL__ z+h50i|Fz}bn(#HdCtnMF9lvJvwe`E2MDwp~jek}3w`xQG*3-)t-gznb`udBRo-ek( z+Vw5;)vUjfF>`AiX+p;9W{yF{;ES6Q^ZuEjLIsOsoF|GmH0yLK60Uo}hgi_elzIn(xv?|%30 ztCP_rtzGdKwojJt{91B7ZoTBI&oTFUgC-yUr+z*@=X;{+Yx5aSJ!}0Wmj30>=XmyY z{ne#AZT%)yef@P*@OAuZ@9b&XyVup0uJ?Vues$)*qE*)E{lC^G*$ROx#;uOC1Y5E| zxo-OFt7m_G_uH8A=l??43;nM8_4l*>|G%|;`>pS_>uWXNUyJ|#ZR^+Tr@A%UJof(D z{C?lFuhY-0+x>G_YT>fjxDF)mE;kmoe+(cJDX6|LbVn+W1>vW9F^z zdVO{M_5E?{!xnP?+S?Hxyx#R|OxIQUJJa^w`X==LZ@T0!h4NEs^S|bI{9Kj4Yxm+Q zU#l)P{;l5n>goE(y0v$GzwO(-I&PQ#*F(p3e=XJjzw%&T`1jXU`~HWmUN=p9|26Kf zFZ1`U`yJ_N_4Ud6UD5lm?T^?WHv7uSxV>?aH-7(3552!ORB6xDb>XFuaT+@%_Fh{%V&d{p(!)|6;1n*4NjMT(T3`xqGc()~~p? z@y{k-UHoV6t)ErRze*Y&jK%0zM9)mj$=SIK@9*I#d5 zzu49EyJW%q^`51#FU6#qMp>8NeI0JU$oT8(uLaRPJJ%)sf35mGZqwf^fURZBax4YXtYwm@g zPY*BgcA9&kV2}6hHQZjyugy3A^F-0}XCPg-%ckFwv{X8*a;ae`Tk7p;n{cQBH zkNxJgO}k|GUF@ss*VmprdzE>B`ib(Y^LHD|H-?+5m3^;Ur(8edSm}*fPHz_c%3FVH zVZzk(pG=NndsnX2Jk)h6^HV_ULJMy7V3RC9(4nGQB3vy^3mpywSam{R0vTjd$jrXufOJ@qFNTuwBkb7ty?h7V)Jz8Cud%M*lJ;&7k#8M>7sPhT;s$S zjF08}jS{Y&J@ifG@Bfe^@r=K(-+yAc>RsQ8*OO-n1+87b{!WW;4eQ#aoZGmjT#f8H zxbK{x+>r;XLqC1qp}9}yZ{+Ns%(gY(x4#$pROxoU{&i&PHT|dy)AKhbo$On&IB$21 zg#77`w^OF^y*zV1bn7bluFtpDh$nO&`Y>}TkE-CLik}gn14Jiu330KpG$k&G(GcTe zb!kjka7L@pdh4C<-+#?p8Zv3iuBkgK_4R9e(vXgzih?C%zv8e>kE!VK91M_^l6n3i{G7@F}F;<^6_61|G4O{ zoz*2(&*y&m#}u~nPbqx1cTVQ%p6K0MxKC6k3kt@rF|qvo;(>Ulz=GfD<_!G(QXlut z&omO+qGqPe{!+?4vU0)JrC(;OvM%#s37=Kxdhh_FhDe8&5I8nAOwqC6W_3C^VL^x= zXxe|Fp+=9EI>+2&d3kkn{fY$R`@WvbRD1fq@6V-0KVw#eOAHtPgk`rKVLKF zqT-|rwGp4CZU1Y!dbiJ*5x0P|f2ZSJ`Dc$EgqF46t3J3>dwy?b?2ee227x(x=C5WI zY;qD@sI;)r@}3ZT2+ZZ$oNR|2Rzw&HakjEK1fPR}J<;Ai zzpZ&uzRun=>+al&Ej`{6WqjbNeD?mB`&T1XuB(23m=-#F>BeIVgGAL2?6p!YuUh|$ zf7fTGH$PbZ73Az)+!H9eVNygqOPZ3{%FM1-y`b+F9M2mp7#J9P9A^o!bg>qFjQ#rk z!~Fk$W^_zfHGlPYZOQknyJ8zl6wCHrF7K4^)^nUf<>dN+8uiEqf zb*~Pyj&Ci~n0EK%t^Kcm|La@%^y~YpzgOGV%U@SmztgtpUAykb_}6!)-sMo)x#NGB ze8j4itopB~#!Oq&yQAB3`swz0KhwUH+bLGdF45hmmLIeFnt8Uki)Yc@yD^i$>V~~? z@U*(pygz>Rq^hGI?N_fi%?y~Au`*xnZN%H8A2Vy?zZUM^e|_oh{dS#i|Cep8TmS2O zX;IY`-@Av5ey#O=EfT&sB+mX_--o|m=Sxp-om#asZqxd>mcJ2iWA^J?D2Pp89a_3- z_xr7{MfR^gU9jCGe$mrk%8PO%{yfcpm&vLtwe)M%uAOu#CX^HR@y`h{)Q ztGjki?_BE|pE0#ucYR%p&Fhz(y1(`)zSdTsetq)g|6hyuJ=WW8@Z{k3M=sXP!Qofy z+P~hd`g-eo{bI-Py4EY<6BNUz|B}A`>ier7>%TVbt$OG0n)Stv$uedC)praGjQx(Y zgjztSU+N3Kjy?VA^RMf#{{CCC`r7`ex4w5zzYeOp4mxH&>($nkwb@&*zn(I0tN!Zm z_g~F1+v>dOQMl~o?e(kW*S_wIdYj+(_t*98__gug@oU3Ze^=kNZd2Iauyxl~FI{(S zqu*=s)qmrDg}sZ@znXnFE8bZ`?Du}>h>fvTyPy4v+8yivI`q}2%Pzn6d|Tgj{nh)c z(^nl;UlY65|8<`)=U=JS7hm5^nxwb4>c#J!dP~%|p7P%M(X{+^`0LeQU+PwW&HB3T z>eNeL?yaj^TeoTV?)4G&VVl zY}`htdAs(^v`y9fuqQTeZ`jsf+e<^U){DNrzbH}s#a><0s$GAxzh>qM?S7Q3C#`>d z&vf0{XS+|{J^gyhydCa&dq3K*^?a?p|6-ih#nt=%N9}%R^nR+`>g%9gs}t?kyx;fa z>*){Sd%jj}oApZhdhF@W)2~_Xb)4R*XZdCBFKz$VLGh*ai$d>AmtCcuyuIDfw*LLK zCHvzZYeh|BU|{TUoF&Y1inaFiu|M~}f4b5ClC#K9{r~#^@Bd%zzo!59_y7ODqyA=p z37_`+#k^_$E-TH${-(y9y1y>o^#AMZt+Tf#?5$GzrT^@2we0Oh)-Ps#t-ro>Yg7FC z@V^mni&jVdt=`e4ls)_6oVSuszUcm*RH^-5^zqVX9bc!$RxODSpMQPH7sac4pI%pw z&v;W4UKBHJZQRb21;=-Ehim@}eYv(~dgsy7*O#k~{(7pvZjqjc{*!lqm)3Xwt@4gr zJ^yOcuC3i)uS}1s*YWyWrSvFIedkW4*xDHXdHYV;zHwh8`X{zfvi{%S>S-Q3{zr8z zSshj$sV#C{wD{MZ*;oJnoEGGLH|nY29_8$7+S5A2K81zEf4#osYr2(j_~Kt;PhN_y zo>ac;=&yryol|0KkK75pyXZ_*)jR*%kh{{6Y3s7xWmjE!X}`F%>TCS#>o3;c+W!3a z+hw;E<9x$nHTB}%-dzv;m%aJyGntI(+@~4;vfT5I>)H26S7c?}a+cTq@2~ED@#>3r z(*Ell_uaKzqs+_J7BDa{UUQr!-13UGEdJM%E$Zw0e*XXAzW%_1#hLxBmXR{<=P>@f@{p>eNHFrk_*3{!4v7JN}n>y=h;4-Hy=j z-`3xmseS!*p!ny|*nfM!{@Pmgux^d$`u5T{^J`uh$C}uUT?>?#HVAuf?;c%-b5bzHFm+`Rj0f+qS@KxA=}pR)eL8Q>cJnpz>*80>|F`aE zUEHgG`j5jG%BzQ@>;KITuf1QsY`wXP{(04+UthL!9gdkXZ^8fBS0B~fz8n@+@;~b2 z->n~O7hesoUC1{j_}t$opFED=+8I~%r|wtP?k%c)XL7e3JO4}j-`=RTbz22r+$;~j z8(XEkTjSc-TPCqb?7k{2n6dvy#)Adr+*&FZ=QeXlEy=&&`s%U%&b`(5zr?JV_v!2P zg|jyO-nV6&{0W7Ly8j)mR<94&y1Gm2Z)x3n^)>NrMREIz-M9YRDtP+oy71RQ4xqEn zG=#WVni>N(Own=SYIAlBSm2>2A`EKo&T!FE^giiWq+|pm~?|D61f8|v>)vs@5``_t@Z`s%XG}h+KLH>^L#q(c%UcF{P z$ETOGAAb>WmfO0P@$sCk7F(_Mt$exQBs)jfcMZ{M^+`VsE^yt+Wi{;AK?w{*= zTt9zZzN+)0b4BJRzqud(+{T-6aMw%Ok5987RkgQR+eq>?FW!CqYnB!J#q7BE%(LHy zE_i%Du-^M>Oj40k1@nd7l9!qya^CrKe^=U6{?X^WynNr25Y9UDtXZd<7jgg3zq^V2yW{Cyw`DBf zi)^m9m18~~Sy}kHc+ED?sZ+|L%5vR%S+oED|7F*9yqtf%#MgMeNhi-e-+yH9*>^|X zZBokg?=Jdq&NlM>$$5u6K0ioqe$V;f`sS?{EDm^Y&7Po!C7%$B24~_^4~h1qqms%+1}u{QC-8w>?jg*{%5vCYqll3@7faJa zM-3k>VXlrti-mVST+e@g%ll~Wk54||ee88#@@#g?R13a+-x+_Mf1db1)^0zih1~D2 zsxxa=$gkP;W@2AoP0SvtuD{HmHKq$2zIM%v53EnIJHBhpiGaUJzwWO1yf#z6?Qj3} ztwlci9fc2N1j4=V9qo+e)7!W`?I_2DkN5unKNR|H`%DY#?o)ivtEW0o(I}p`Z{i)c z7sp%*0*c%;99>dosw`8q?Dko8h5>xVt(KAy7t=vUg8&^>UN)!3fC(WULZHr(p+JC^ zILFNQOS9|k*YDGxYyD%-=X*O>U%QiTx_WA9mCWij&) z?DW%{uC3>~-+bm5xV!pq+M3PRPVeK}&-q5;?#JVCHXZLgk60zGU)XM)dG(3hznao( zt+)CNi(~l2cCJ<`+;Q{e$<5y)V&;Z7tH<9BHgV?h3Y@pXaMQ&5>CF1|TLKKjCou%- z?Km88aIa>#PSc6AuCbR`!@=t_&X|DiGHKYb#7tC(wP~Tlh5!{2t`?`HKna)gQm<#; z-}`ZE==0dGb+unZWAfWx76w|#IEVcB>-K(+dfJ~#|21suL*+v+6;1czpK*EpV^^`8 z7t8YM&K_`J(TJ7Xc|N{x$);r|#9vMIX}@(!UYh@d(T4|HS`OVhviEn<>Vx+TKg||6 z@})O!ZNxbfp}?|U!^N>+g^yGEx%D!? z_HEpC?On{icTU;mT4SQ_K_aI`>HWhpkYz zxEf)d5cP1Dz^0=8k!)|fPt-bqs?2?kvqV@yBmDX5>qAdo+8?uTgJ-s=@zQtK^tb93 zul9WHQMc4w26$wd&8Sh~2vvp6(LZ`B!kg%X-&Sj^4G}cSB!m$?Wufb@|su-PPf1g3iBN=l%NX zYw_3D>(|%qtlQi5dwpc=kzZf-tyTTD_SfmHdDj-bQ`g&@bp31g{q^3jWA9#HEqpiT zx}I{@wb<2PUr*nfwd?C4t4D@1LEmp}e|4#9_t*8;G|s!d%#}ItH|cND>Sw_-lavq?5kEE|8;7w)Zztgd&PnZjbki}qO|L}N+w^l68ZYF^Yb5{eS4>Wk5j+C z>aNZIj0tW@k;j{+>^2elo20aV{>_A``o9F9UbJ7gKRpSkwG^DFPk7i0E^^=>`=(u?bE(ehgFd!No<=X)5i zWOrDW&_VZ{V%hgkD%NKGdh~08dw6+KOzg?pD+~;*XB=mVv}&=|Ui!J}nfm>|A31#M zcK&}c>ucA#wNZbo_Ew+X`MT>v#Koe?7q!;?jg2{d^=$0-;`&9q@87?-Jwhw=n)R1? zJ6}Eh;`MWNXxH8>|Chh^e%(7sGIPKF{*-?w^*rb6&OGVoS)tY#u3%8PALV z`fg|S+Hn2Q!&9r)?Tvp~w0dRr`luiCelPSdTN}6e>HpvQfA3a3RlUBpZmZM&t6Hk- z%hu|z_S|(;ef8h?E>WdVn$1s*Cw_~rTKYv_>qxlayY=Vxt9;FupZDx*)~DGW5tHBf zzrT3qev6Nf?8?*gE*R#l&N^~Di$7v#XM|nSnfudPcg1Dj6}}4UQ@!5xHT!G!*ZkOb z*SGH7_cvy}jL(mnv?6cYX%p*?1#LRJzb$9IoX*1~e>P6Pz9VVRF85j11r2YT>eI@9&%nm%6@@#5G2E`_qH zDd`MXw=YorS$ZvgeNJ=9xrO5YK6og#$_U6Uf8x4ADXVhA{{0W8tliIY_0IkM>WWJJ zUwI}nw?*+bhn>4Q|G!iGuCG@=oy?e8*8l1*SEgCpt5vI3EqXE8RONWj)ngx8_wB!+ zs2VbVW%WO&SAAbTvX(XUt~qZyd?o_z%jp*T1ON-Bx>Mj`Y%rx0p+RZ!zD`(st_b!Rfza{JxuN-oO83 zXUErdzo#_+ek;GPxrwvm>DJ$cB5~)7pPl;j_^;^o*PzDsySLSkl+0G&>`>_Gbr)Z# z`ZDOt{l1>*6IXY~|NedYb)CYxXwi8O*IA`BURM1u)8M!06ie|x!n_xEN5xKmeXdbp z;f(&D({FwKvt8-$>qJZAz)za$-SRr>PRDsIu9l3JD@d!#lp zF7T~8rI6slrvI|&Rrte)mY-9cxA%(At>@hQ>u*X#nB2S-{7e1z7ghZg-*H;eHUG-G zs%KaJt-t-`mCxE3SJ!uo{^#r8e{)4SukuF|K zj{oQsxUl~HTb`Kzs~Hy>dM4{VJW5n!9nk_D@ew z)4iJYVd-vbb%U7Il6|XpPwg;!*;FjqX;aAV`ZhD{+h=)ciC=GD{+$tKuO)T8cB9>q z_bf>fdtdp@-PE~Q{(JnC)#j!aIqDC6i*(EHmzKqU4dyVlKil85{-yYB^_NYPx-F-F zempz7_|r>yUG>c(-!AXVJ*oY`T;)O zM+KJ)eeHOBXxgf)E;k#Gvh7)@5qk6TuP^S$tDAp)S^Zt`eEi8*f3Nw>)z3ZsZUF-W z`xeJpqO4x5wf|ao{@Q1}{ngXL{eOb4pTD*JWXE}5lX)Ap?!S(`zCQK;!>#XILVsB% zUzb~Fx9@MTW&Q7P>u~G8@jtcJ#dn3Rdb0I>Qnv8N+S+&dYrGA&oz+qO7IFLRlKN;_ z`=xurYu*hm)PaoAh7Fe;s4E({z<7d_KJH=EdOnGP>$0s zPq=G~9mhju|CsYpQ3=7#UHvY5LeSlXJwlOTYh{1i z?@O7p;g?J{VTm`chmnb%r`w<`a1M=?4w=J)>WElIaU1I z6WMt7a3<>UsziH>*@7srR(*pWmWENjazGasZ&$1k6*}XvYB4t z@iyc1GTGc}ibW@4eh()>;39SIjV-CFpc>mE)pVfsao!w$`nw+Z!{r zWc~ZE>qQpT#(#;h=TNb}#9vrGRO})OQNm^(6)1~WH1uA5-2v3sJb8AaF z_4LdWmz=7tf6Q%5%MM*nSz&(6`Sc$7iR@28(&9eb-;JMs%O?EQrK&yG1O9EdVwtr< z=~MnYF-NtlA@SEWlC28`jvXlV=B$2IGmpDYqxGzc5cmJ96<1d=9&+CL)5dfRz=CQpEdWml;iW-}`p?^Ix&>X6ot8`tkPusic4z26Go~TKcKx z^fQHB61Mz`@16Zc+jOMdReDU1ZS49KcE{+fp6PFvr8zB?^>-FVIK8~fK2PMs@6RW+ zWbSVG|1lz^RsX7jflk@;=d0bj{Nw%>&9^`8_IA7e(i|2v<&)tZV|_-t-q_^7(|&$DV4C;2$*ijG+lFQqZ?P~8(U4K<7bi_9JWtY zbm0-IGVzRA|MD`+QkHuaC$>s$Y7KjTaPsMQH^uh8@Cxvs)3&H%LHPvhE1QeTWGz3K zosRnNP&-}k-m!_Scg^?zj?A-lR&%KMKmAln>imr%+O3gcS_=LgkDZ9_Kf+;OJKyr$zK<5R z_FqqBN7nfMy88a^y^C%EGgjGdZ@ZXRcjQ3QJl;!(qYvNwQmDVvKk>}MhGPY5r<_^! zKcQ#6Lx;A(;e?JDzB|vaYZN%MnLd%*lfQ4Ta^(51|09*Qy}rKd!*kVMt$$9!d52S9 z_Vc{`qf|TRdFk6HJi=3Z@=xuZEI;Xl07qSatbaV8($4F$@Bi)-t(vp?d3;LV@3#?q z#CTpmn|^=w-4jdAH$K`LRXi!qfNPqbs7B8Msjj5Tuky1NRNFtfI$M&(Om}ic{Pl0o z7M=QjW5=S;hRHdJY)6t^7gX(t7M{kldh2qx4(=Y=)A^@G+#g-@e)?(aoeA~+ABzrt zxqn4y(NR60njfKZ0SQ4q>*u5{InH!xLEw6eHXQwMDw(q2;ss?&Ph_ zHLtYqe0z|?+5X|VqID*n4FB)7YSx{J;z@DT_}BN#iuI9&%NqX2Nn!hpDtc?rFTM27 z@y1c>xzjsh`GS}V`iz_wOcj66R`hVg(}oukS(Soihd-9oCo;zDvuifqA@%I&`SlMA z6IXunn3sQm=WJf-uiyP{oLZ94-@UQ!Q8{$9WcT8q-uG7h;+~y#!#*J=Wb4tmH=mz> zj?u7tT{F*T-xHAo1#0;swNfGP4z{*^wMf{_v@*Tz>G`kUSKT`K_1?qF8w|`3DnGh; z_ny8$_MQLVBwsXYKYjVPM(yeA#M*ba6}xP#qB7(Tl}G;la5LNRjb%cm!J#b&)x>=M z`RD4E{8&G;?folfEyv3zzi)oM(#XJd*1V|gEJqICVZI`xc-0c;~nJ`D0JF&b0lzN^@P{uXs)76_Is^#J0RRYO}_C zl}b6MeeZ%dm!9jmYD zt>4SCx;6Asb+PdEwvLOFSFQha<;wdjcMX~6vs6~dZ_Ts(C3WR*=>9N^$zl9H;vW@O2xa z$^Sav#@^bVeOLBEmwx#+fh()lyPWclf84qK)z#Xo*Z)j4U!C?g^7tadufJ4QUzqnd z%0F&>-0rx)sc~z6{q2b_Tif@ec72lgPImpH$BmRfrhmB>GXGP`rN83$h4!AmsQt-5 zzWdke*`Zky`#>u^zMj6YYHQZ&Ey_!8hn;>}`i^r}(&7qp_2++Y3E#aYusU>o-OBx8 zZ&TIRm+kF}WYvw@_|bNC^0c3}|9&4TeciI_>EEc!f7i&b4(GTJI`I3b{`%1MeLp{nqt3O;kMCIyYBjHoTsGztT?pb$UOSvGAuvt4&hdmIhDd%=@~!!z}yl^8264 zHDx9wY4h=HH~SnsZO-EW(ewW&pAOkDwK>CJ&y$riz8sQUHK(QaM?+kd=(QI=|{R_xsMM&B^dN`Ec?>Bjd;u#t&Xc zaV`D6dwTCO`|eM*r@S^5i8gaiO*v4rQDpOse_ME_EZoYno%wrELy ziq4i7@An=OOOshejEod?$|qZ#=(yUj?wKD? z&$gXwe3mOXuKl`i=GyF6CZ$c6o~Q*jn_BGV{cw2d>Z{Y=G$mep^DQ>z#mn~f>hRB*_cypnsY!jBP=QQj8K?3?r{4NjW%sy}`>NL$U5(l= z)*tiBbjv-*DKc^Xo%MXl#)V;at8TFvsMs&gQBU5}$WZcb_eE9h-LC8Y^H%&>YL$F% z-3BX_O6hC=4i|1*pLp*64P~Fx+hw2Zd|tPQtp90K`l+&R)6_A)832{ygF47qUn6zqtRg z?ps#tc--{`&s4^R4ZpXsK1xjsv9JC7)84+|p^`+Fh3lskzwaHIBmGyL_ij~LWNh73 z`5T5gn@;|IyDdsWEng+P{ZjbTlb=Dq z(|?WgZ)x0);?HN^btpC+*1fsu=Hk8KU8lmg^!zOQo3SQqkN&R2)Wvf$xULFo37>o@ zcVFHtj^l33xm9tGb&oxHzxUFrYtI+>x$pV^aK&s9^YY5KQ_q@l_vSwoW_?*+J8Qxn z=Wi-KHAe(q<<{5indi@>vVY(2%L`?8XHULx?ftH;UEkA=r8<-vtyFw6%YMUsIS%2j zkIzHmH~8}D%rDz!&B}VprLty?eeJratwp=_|HkXD`}ekdca_+2u`u0|sp5CdR$t#U zPkzNl=hNYA+v1nr-#g{u9OqfT-S1a(f15tL@%z~`OKh%8zAF0Z$;V$EYs!=_ehtmO z#M-2`Y}#AXW!E3{otp4iWm^2b{TqCu!fP-8<23%rmlNJnzVXRF-Y=6sy*{M!q+$A7 z;R|MaJu+8*dNKdjzwK4ktF%9yJ1>59UXIjP)96K-(Rakk3vDhR*wP?1l|ezz?Y}|I z^?jlYA47!it^504_W!Td7ryV*o3_LGYxmcyDc2@)WJ%jC($$%AvirkRm93w@{kn88 z_pa{h%^zHE+&%m2AIFyN*fVjPLgQMdnp=Or$u6(>v*zi^j~Zqg`jZ@muYCwrD6##N z)P8*3mjgxy(`JbZO+96N$<}6GQ*PDjyZ2S5$8FudeC@T@Q+@}k)nObd2@dsyAxelBWXkS>LL_!vTOArGokAN1)$|T zOT>6soenx0X6UGLvoalWSP`Hm$l2nQBs}xDZhdU&uDy5jVy+tn_l7=J;aqs@i?8y{ zSLnV@LZUJ?dUw;d*$u~f{yi~w zKbXf`p0@huB%x2@A}fZXW#Q*RwFMlv#65R;~?M7|5@@wWd_urL;u7 zz3bc7pS!lzZ1F#kcgOP8{eOSwv7OI9)bjc7?n)az9;O?6b!Ygid4K(2`iiH0RkeCu z9uNN)ev`DP*K$rP-kMc$xc$qTt-8hwZ*1THPG?C@GS|Xu5{I6FYI zl_o0OtWHM*7KCJoa<=$sZTj!uR~o%~MeMu4V;fd#g{*!duO7WU^p%}1JO7vbLl?jF zAn#Sz-MR~hB-Pk*SE>~=8?*Hdx_c`v1R>s+#e_t5fb^Bzl6Kt`5 z{kHBUTU4h-yg9?b^5js}1I^uXC85#ktCv|ZR|W{hqPs z^*+_@dv&$-9Q@jzpFV!r|D*2b^;715|1AF6&FHM))%8>BcC}8v6#eb9NtDo%O zU0HY9KV+YOnCyGPwm5BwO&+Q`etuL!I1}s=Z!zktG=0>`S88+U;Cwx{GS&) zSFdWn@;D*pkM^wp%gr^TILc3mD=8(OQn+qYf0^^I^Q~_ZziqQ`pL%^NjkQen!PWCM zdPRo}E0*q?_qXc%KL2j+u6s|v-Cg!4e8taBM+UKr&N2GgybMkso;#)$UcKne?xyqF zvj2LDy12{RZ<;GL*6i1O6*aNwR9!T~0|o}prH-@2SV6mEqs3o`Ub^m9^7s3H?@Q}; zzu&zwIrabEAf>8}zc$9Lf49=gYVKdq8vE<(uWr1vz4X=bZ1H>k_5WU}t3O|RwUle2 z|B(p0&UH3cg{mZz$@k`&vEh$}h^;qXJ^>4-cn^iO))m(n? zGl+d+h{E+>{x#vBYJWbe@;>D@%dRX??EGq{=&x&n?mn6lb~Mi~dQnx}t9R?K{(g5Y zR?*M8$@!t&(fzzNW*kW=^9#SIuDdApsQ+*3{p|SFXKNpctzWu6T>I*)tNqu4oOs&2 z_ic&$%U$=5C(AVUZq)biS65$)m%hIKD&=p{{QqJ483|5B4jb0C9F(6ft0H%8hSsIZ z#UhW4KKu=fcfN7;Q*hbdU*EJvrxe}t51-8SOX+>}Gc_Bhdk+P@S$TF}ocp)x@4Nc$_|AsLSgHH(SNm%14>PLy_1JKV=Bd`+3_B~az`aMWXqyCh3e1Si$hBF- zl=JtO_$|F9Utd~Hm|y?9{%fV4f6V@{Mb*D|6wmA1WpJN=?}>neJFd@g*(3WP>GmOZ zt@QUb=5-b3kL_z5|K{B4f6(?$TlBU1_lp8m--Fg#KlVO&WrDn<;`id$x(7YaIC3Ah zw^WI$^ZoX#P?$UYFcXW^{#^6>8=q7>)+t?Joh!(dQe)V*XI}UhuB~xRfB*B|@;_?q zzx;625-#N@N^=+A-d+)XqN7zH`_$gR7tMxh3v3kT2Km2g`C7F%=-awie{1iD-mJ#tWz-DSf0ncJ$K zv>x;L%Y4dC*6(`3MYq$RFaDn<8MN~F-$R@GW=r_#AMfb$UfP!H|98&Z4B_hg!Nu3q zD~%p)jW6Z0uZ{gb`QEv}jX46bcg#%Pxt6xnB(9tEWkci28HdbPE?NBlAg4@N^wBtR zD~7hpS=V%O9q8&L!RHnlv;TJ!Ux0&96T^&(9Q_Z%Zrr{>pEe(n{9~ZC zX#JFW9aLez|ZvF6v=D{8$~UhR4?5G2#v$~J|xycDey-GjlCFS|I@Gp1BO_zm_jHZX* zzYz6_+3VAfCto-vU$vcdS&-5YuYc_>+m6!_+-tW>i`mDwo%4uHO>1V_HBZrV^6xeC zbAf!yt-AMpjNg3sY|S52p{-9o)m?vB z82WMTfgFyUHwSkyPd}vdr{nSCiiYo87uFq%YF^;f$8v84|a8?*F)dPkJ-|OkDT>(7qUsf_w506MknMnd`fR z$7YMgVZ|=h;|H0gZYI{W@%CxKujSKKJ+`hsp;R#C=7&4-Ph@R4cDVIL z8ELHlC$_Wxp!(TgQ;ovymV9^EfAPeI>`wl#FPV%V{@4&F{o171M(VHm&o4opJ=L#y zIlTQR9kG3_5USbT)AM)hjHfqVD$cq#^>U4#pspg*nw6>^{ySJ-bI$s8A>m?#m&75J z-@lIv=c=Fb-ah$z-PBM27k)l zYgO<0U-4hCa>2jWr5DWEHB;rTsTOlNPk->Ue(i?!@ynOj$fy?GkN@(;$CAZfW|_l} z@98Wx9z_P7zfZpYE4Oj=jC%Du^Co0?sAfgJmuf%zS4nif`qli_HL?!PKZ_dPZfI++DZQCB?eEhs zKMfq+np)QQ?kP1gIpzI~_4+GeL#^m#PdqqH*jTr{*q_S(@QQz@xa2*hV7<(((>F*P zGEAApvok;b$+vmGSVS}?M}K%M7q}sLY1>VcG;<;S?)olND zgy!GucL?3wzxBu3orXW0%nqMR{hU+zZjy3krkk4%mw#sJ7bo6@He%8&%5K%i z4_}=4@sFNQuIGv_&P=6WJcZMJwOE??gR1XZ}N#%ny#+Qg>XN#e4IZ&HXUD ziN^Ost=hj_c>3I^^YNEbi+r6o@dq4Va!eepFDcFAq)-Cx9UFxTpP=j7AVkL9hH;(V; zF7I-_H(g@&+KP+=i)yZ?{tGEdc2r~mJsuXn!q-*4eGqqjc{_U`t*_~U<1p}^g^y;fhl zwttNJQk$>)eriw2o^wHIXZCGW3=>XKeSPBazg4SC4qRq4cr9$!?EGl@cKzk%aZUQ+ zWjpKksxR7_x|*qKd+Eu^$N%k^{WCiI>4Xsdck|oNX}B*Fo@*7B{eXdiOT%%NSnDd* z`i1#5cWtJJhyOag>+9>Ur?>X3y}mzYf4KF>dF!uls`>x@`mJmJ`~P3nWtsZu%ij() z&);$H_g&u|wl z?W;(&c)f~0wK?-)>fgltzTV>16J~ua{;FNNK0N!1W%$ImQEN-@UpgAL{r%VS+VA)6 z%Nm{TzYyrKUE6nIb*&^@&9=ox?|ai$Fa9YIE-5rGPX@$qXUb*qxzjmAT zv^&kdY+L3r2TeP;_>EE%L;053?H7No*m_#@-`-nartJ(}eR|dNX;)vp>sT7&cxE5} zmt#9TC3fTm`2KGZ*(tC0vbE@cK9y}$T>{J*^qYrpJ`+vs2Vdg|T#XSUXDeLel@OyB1gt)E}l zZ@!gl_Vm*G@{<`+u6L`OLhCB^zt&xUdiU4$`n9#Xo?o-YUx$2Jm?mYxrL#-%roGPz z_8h*y><$O-9_dMVki#4(FVv*>?c1|M>F;%ym}hMWx!*e{p_p;!SB43OHX_+PUr{6YD7Cu|__?ZJ&ux*NRzwG4q{eR}o-*^4hlc~Ii-ucIMeC>L<`ghbX zG2ewVcIV`5O8s1|Eut~o&+mh|a>`fXH7C1HdH>p`xa-K_dba!SXJ6DDt2hZ`2G7UgXw$k-kqx%vgWhPX~!L|t9^b*eRVdC5<2x&B-Ui!u2p{5!i!>>tYx0T!{>+w)H#2zbSm*e^7t2#>75J^*?iz4jTW+ za7;UVKX8e2jnOO*$yXY0ULKtF!*FKh+j-aDC^Uf6JPk$YIHS5>~WnNDH<5@*14>CWTP>|F5UcGw{>lDk(e+KJCl4S1A@aUKF`u6(u z&Hi0?AHHQg*7MRr!SZl{U8B2ZKFg~1*Ztcy*(diZDBFCnSJDbu+~S++>v(_ZpZ%db z>bLK!7JbY7&7kx$&$;7&eU;jdod5o-aDl`Aggr}t-wFHvbITexxAjd$LM^5d3YF!r zEpPv1fB4%XvL|u%%YuFV^1d8D?baPJew@x1xxe~-`=+B&$`5*aQWd)TKHZ-t@n~b5 zv3ATdx5)noHT@mG{abuQ*mKt687)tGw#FGAq@y zMSmSUD7F8!)|R**_r2@IFWT$>0IwaoEtPj0^smww7>-S}{eI$vn=hK_}NY_&{fud42J+s8loUVcSL zsPjYShnGPH8j6RFEZ-=`-rKL?&?9|)^6lGeZ%x>~XP(q*hZWBIrDqDsZ8_Y&(`a^h z|K2MCou7Q(AMt10a&(FK{~wLa&;Ly_`o;d?&DZ{$z6EO*xW>JrhuO?l3VbH3TR?Yr^M_Q~ZvtABp~qI&f;t0V8T)j#WEtG-t4d!w;t zfBcvI?QM>7-=Ez}dHP4lZvGbT1*TeUJ*LjHUV6X(q4;3A-nmCz=fjTqTyojBe|=ct zfdj$L>_1xHe0|WKGym^G^L*LQ#h1Qb;e543K;CI;W&Np*b!)2sztiU|sJJ%y^_i(`cL|#|ZwmE4J z>OVZU>^A*(p-f}`oYjw6qkEpP%}qc3^@!)MqE7esmj9i0eLEbNx8d&Y|FI96Ckb9o zo8$6|^|0rw_- z1zj`y%`H1^MKTQ+PnL;qdwA4%*Plbr1E0FR?@(NEcX|4c&!ol)(gv6#l`S<5a z+#hk-Z#Rq{fBmt!AlARAVWs4S{l?y3!#sV;bPvqP5dUytD$FQ9d>pzT_~0oVN3Rrmh0JnapVdvY`=`TK(dFM^yOY@5^4@%-#zkG-P* zBp%HgyZ+W<@Q^&sOueHaLg*>Tm!WA`_En9Y~_S4_pk2(Io z=RCK$wBf|6R`2iW{o&##_d7qHsG`4zW54F%+n*Wxmrbwln-L_mw!S*s=S*Gv@5`&&uE6 z;-U29e3#qgnf7{hErPrT>(=#7UUGHi+}nHRUo2eV67YNX+jsMlO$Fj?*5|!@_v8Ie z+cVYEIbxNby?gg|cdfg#=erMW>#ikNYJcyK>uA{VewCt~nv) zxh?y@@A$s|gIl!6#gwJuZ|lEBrFpx4NwfZAm>Jele>YQ5IrP^@_N{t}Yb4{}^>Roq zvzzk8@A?z7FbVD_H`8CKRNLSCtrq1upT#KY()%O6y(+ubAI)a2Z`iYB%KYp48ZN#O zIdj6kSFBsKNkBQO<>Bt#-$kzo3p=m1Q$5IQoqA&WqAlM8O&@d|-}!5gUGQJN=60jY zE3VIe{9)hJ+9s>$oj>2$^;uMTMBRSSf1d5nk^VXL2Q8BytG3Qv}IVy`<){`7inFtF~TQdoegln|1Q9Q+uX#%ItAJYq}`4IPNBs$eZmA3=CWqjtC<``gynNYt){5TT8aH|6#u{|J2f5 z=F3=GA1GEPTSpiPNNXf0K8~^A2?#CWdwQ@;^WW5eitE|j#5d%nY^j)@(?99L_0{3l zVblIcv+Le(X4Lw>`gD1xT&swD(@OmZQy+c4&;4VC&PBocxopOBw75(}S*65erA*ib zZZ4jw5}#PV;D5Woo*jj}vlx#Qs%|KVH`!ppwNSNvvdSrgKJPAOIEI;S4* zq+bs;#3LUuZ0Hb|F_^fSdExqmM=5P`y0@B>KAgT4aKg6ulHI>`p7ra!wAY7EzOHE% z_O@t!yx8u4hn=`jUx`!C-FCa4Ls7ImbA7vDg8l#HlY?I0&R^!cCen$8D?Rk7r_)cp zhp)0UU#>TKwIPRN#>1uJQ-g1PovD6A{A)-jgG0cBuiZ)2XViEN3X48Phu|KeeHe?`;HZ=VITSngXgI*3XTZb@kL?lk(IL zlWlr@G=vy`@6cI4vumU9tBKN;Q|@_e)HYaon&oVxXAJk5;8umN2fa!nf7e{-sF~0^ znYHPypPGoxWY)%RhUpHmc_|Gnmfq(rQ#L;qIa+dCr+$C${i}}mPhJkax}Y~cJ2b@g z-`;n7ReyiV`1C1IUVqXhryJ@VGV)yss-2P{#Uj zM_yc$P}7G)Mt4j?EAF3X+1+Mgl%llcW@`H58ItY)w0=xpboZ5;!Gv>t8z*;sW8-9d zR%?}Snk;J}88q$6l1qx_L9@~yZ_{mP*c*56?xDE0O+K@p?cV44SK#=}xa8ym4Zm#* z4yj04{(d8T>gdXS!4>TnzwTF$da&a7GX2Az*-Go5PYBx*sH3;?XUgxdN6H250@9aV z3Mk*R?5>b)o8F6)4v$R=O%Cqz{>?g#Pxzy!2=8~v&zcKT7w-I~mc-4TxFX(7)hSWu zdwJLIx3cd&t1B$7H?)X*Kg>-y;ES%Uz4U zGAaBC&=IYu+?^{V`PnOcRx`tNmW_^`?BeSJ<=3yaQesQ_^7^+(1)JRU{rYdz^CC?6 z&Lrz6CM}X?ZdA2cAFzm5OQ58$!|?pe`}wc7a?gAjz3{~D7llX8KV?yJG85ScZN)3P!UV8_V^tln{X?Gc>%RgOpZ{tL^5hsUL_OnJ2a+nVoJ*C$M$)ATcQ?atDB zr}suXeR!m=`f%a5qRSC^-&1Q+HeFfxyYtB3i}__+Z(R+%lX^cuT{Pl;lI7AVvhOD@ z?hI2q{4(jk{_=0ZZtrSmy|hoV+45-fvoAklzUS{=f6G*(Ui|zlkyUnF8L68;g--00 zm{66=d{O&-^dC`y>|?qmyJl~DqP5`otO-&3uf<>8SeSnG!RplJy}!2>Tx?6STi4lU z`~Orm+vzp#;`d9m*x&DqQoVjdr9ExA|GnZt)u(^t$;9L@TO7a9ne*;WtE!tjD-tHl9aOw~?U=vm|9uDR%FL`- zT(3WFei9P$ee&TSlUPLeZ#(L=Tqn}E@sG>rk5^&Dn|L6Mo#br|0SXktX_SVVz$xnX0>!cy~u|~Cd4mU&FDxAs% z1^>*uz5jli%If%skC#g<3vdW}6@8`dh^YM6rfhCz-7azIzxVo1B&Zn`y0fv-bH;xZ3Tui}6)Za|D0oX}71Js;(>4{wJtQjWbw(}Jzjl6KF1&J;ZDji&PvvD=PJAu{bu4SztzaLR(@ zq~zSbEyA*XMlEPkiaT?%?<*Wv|Bj)FVSigm0!`A)ey>;Ig-q72vP@x?1ykG01?b(HzJb&lg{`vp!uTdvI% zQA#c`ozNuU_xh>ZY0n?Eld7Lg{TIa1?;^CiF0cQU(o4O=o4KYw?9$)&$wiUH=D|GA zw!?{ctf#-?+OcC^Qbms7;howk6v##E+H3WHKigsTdBHJ3of$qakNUFhZ*_SdST@;U(_y*nPxov41R`@Ko?QxC9V1du z8oVSaFf5Ye-Ta5%+trV3`>|NSvQ=i+Z;rBSi+Yb9z4Vv6GQ%xDeD&X$TEU+Eznb2g zX}!_h6C3qqnt%WQ!@uvBuT9pTw(!WM2US&y8SLV3!@l>&g}>wUsi~2EBxD$?{6Bxu z@&9Rj*=tjuoPQ|f_${;XoPFWW)AcHXaEr_31}k{0s~P z#L~@*()DL6WE4ygc(7o>r*of!Uh3`TU4LzV+~kX?aVuNybsTWly_+^6An-xK4~_qm z9;~Prm!EuJ`mm8yjr4Oj1_qw>jA`BYU8b)zy3_WzPjUQ_4}t`>!$v@ zruFsp*W#_|ub%pc*Dcu{w>oCk>uXEDu3x&{YwMS-pa1Qw+iknmC{fp|_1^ps4b9Sh zk6R{OF8UMNIg`(%`SbVJpDiB7om9#e49J`q_Rz4`>1L*>I7^EzJA3`RAKuP!8x&qo zj94A@x3nuJyzBnavaP>1mTgrR*!t`GrK|T}$F9v~-7sUnoQj1pOTSN}>yZRUYRf4BU*KDXq=*Z)WU>V|&XUaH$u zynlu2$^C0%t2X{RdS`ps+LW*F^}lGIU*P0A_x#nfi6M$io(7UGb$0s{GF`6(i-(H` z<#)f+)0uYgav__jNDzLDu=;VBH@uR1&#JWu}4DVc_)+tlIe3$bTf%_F! z>!#iQ`}gGOpoOM)i>|H|{+hRR&HVYF>g&zJpGj=}Gx`6TgpSm?_n)RKP4-FuwNLF= z^cKC<*Iru8`~C9iubaQGJ#A%kG;-hM`ZK$3<%D{pnR8;FHoaww4q3@?g72=0w|;AU z3LB%}WHI)K_i{SEIfb_!PX4g+mFJnAa#72d-_O%_I-HSP5`X&i)kFTae|#2CzrWTy zJX?I#^8B-pwG)o~vv99Iq9ewkDYQSJpmI^abK1AVbL9Ry?7YEKt$zLLMxJkr7R>x^ zzM#)5?8?5qvRRLw2XmbJ$QNfBaPQd5yH`TaAB3IGOB|Dw*e- z?X<}5{QS$pdVHH6T;-HFaP77Khovf0PCer;jOF?m^R4C8y6W{|hhnSrgWiWVw|Tm` zcyeyy`k;Q#^7PYB@e>BHr5N8x}uNN<^BaMZh9Ph+lW6sOPue$`1Rt6+U|PYRAKV-emzI z^IhFz*;N7)Lbr3w*dMF(=;NgChrS;E6*hgY)Q1PxB!Z5(E%`R-=l;*%cFCAuo;Y7h zUs{ay)Pn`ua~Hh5V)nsQWNFne)s}?+s`0DYIFIBl`5CodL*V4zTPyBf5~^CB^+wa~ z+HS7t<(|A3u0J;CN}GRBQ8iIQ^v1>5%~urJid6hXt}p-J2AqC2NddeHiMR%_p)60-+W7U#^jTY8eyachQl z_Me)$H;lCSG#+2&_WmEgsW@TPOIP97|G(6puF!3BpR{Uqgfx@N>wfnBgA#N0$1#iC zKC&>eV8L?HJAqxiN0UC(ZZF}}ak~2}VU>oV+o#g-Wp#zjMe@GsViO8JcZYwIQ`Jrp zxA4_owA7S0}+UaY;l4-wk=lyS5 z)E<0ZyRAU;zxkI0{R`_SPI$NX{DZ$z@!i7J?|vS3o7csjYczAQ_OtkRxB2F%XYWZ} zp8xw(cWPWrweZFp{m(Y$OXl@`swsOXdU3Jh_O`Xh<2M}6{QRG*gH!!tZ}IJ4f0IAj z`5#r^a3*Z_@w0KQ8IxZpYkjqox?%aUMSGJgxBoY(XPX2Azhy|?|9@y(&XtOnIw_m) z@PEna{+}+i|Cs*kt53eZX#ce@K7RcP!~1)@rUtLiaQ@3{q4=w0v3SW@j+s*<+t{QZ zv_5^myD(6#WBd0XB@3U;XWLis-*l2}&Yk@$PMr4*I8v~4*)zo_KmYRv1Wi_`j7T-^ zS^D2cV9H_^v4iu!rEXt3;obV5J(oC24Sv-YycLf9oVa;y;elfcfydLOvY$TXy)~=j z`lm@|x56gAtLK=XmdST6?B9*_Ew5w*l0~|!JGa?oONTGf?D<#qZ;9KR|DVJz28kAm z-_riKvu@uL!T0n1IVxFd=Oi<0XK5Um@V~uF-pQ*e$WSNiy1|=J`RLiV*%$ky-2 zEltUOn!nIfbg2pd*?O-PF*9wfCJ8ityR3DU#a`y*86~4j`hJ2>zOUL{erfXUuj)ER zb^jK0Ii*Q#|K`7ER>5D<7Gw3oh*I8P%~ekP&x+|7^{bPRD_xo4ZpVvX$-mCekvhwI=ht)}Z)FU&%u54b2ioD$P~`n<0b zdwaHjXq9@#vVfaj^6g0vN)L-&|DUF3RPitA)9?8#PEz+-gTFa?2@4*yncrpU*;KSl zwdF@do$V9-?F)8nlQG)gW3c-2=7Z6k6&@8Pf2*393#Tn!vt{xhhqlE+ZFL))nbtjY z-hIC}x9rNRfAM}V6>d)YlBWCO;NQAC>3=^Z7Al`#_i@qBuRl81M^t6YD<<#WS!e9~ zKiSs#xWBZm!N0EZ0)bf*2!*D^hlib4sHVSLuQjjqy+R^S>dg)L z-(*xS{F-&EjC0$k`;%XSNdMmNge2`f{hL^z2o9RuU$$Oe**o|Ea~-J(RD{ zJpRxB-n&CjeD=+?@cDP|cHXat`Tws!Ha%1s`C4etq%xNU65SJw7CbJJ+dC70&T-Sa7+^XM&e|;Q}Y&Ji&f3jhn+GkzsZFX9=nQC%RBUhP~x+pK8r)e@z!@j@1#p5?C-PB2Z`LXCHrvDuls`IGo|JLf3ZsiGy->UoX-S2m+C+27U zzN(V`>Gyh5-IexRw={ihJDzrC_UmUHJ5x4n`k{C2jj+6|QM3NUuo>#Rn|ji`QciKa z{?=TUzdSCu*QDFSCW~|bU&*=kLhkdd`3F?i?b;YI=i^$P&^fox zG}+9z{ke7VOQtzy9*=u74eRW-T~7Bm%-?cUu%qP6eD?MG3fy!){FoZ$rSRE?Wp{zd z&c|t`f7stA*Ew{~DK%vly=vXvJ1xKDc>DQ<$C4L%Cvs}dIq3bP>sr}o*88dR&p-Vu zv!`}P%=x)@=i~)@+&j}HpkwxOI`8SPo4Nm9n>|1OTDOouskOUzum0P2&bsQ7 z(dmwtyX8Mj{Nv9bzs39=*CDkzO=rX6zo~Lwespfz#kETL-MiiM#HQ`FdK>4xsBZd? zh5i-^79X^#GW9nuWqJDE`)8&v%gU#x&1xR6T;#X)puAnxc9B@)&=(h%?2P&I&;R`W zD_VZJ3%gGH)@ZU?Y>}CFI%J_-^wQc{{%H>nrJs7tbE3C*e!Qb%i9y?w>aH8N_NAmHW8v#oC|M0wLcz^It5# z?%S<<_q+GgnM#*e3yWUPdz)2r{oeM?c`;8ut|~exXj!adHNer_G!9T6n+n`&;{8uY0YxHg0M7{eq!jHmrfvZaQKhwA|f7Z{x7s|ffzHTct zfAReDCwFfZ?zf#?%2yQq`r$_fDTWVrE7mb5P8GYrXY*|TkNQ>r%ewb8D6YFC-+q_L zp7$o>-TV8V9;ka>X8HPvk3!T5v0Q3uVyk`d%leh>p{He4A0k52w{;$$zR@#u z>iacGCp+0PbIyCNwF+OoU3W`;#50X4)wdZQFe->lQ4->7${W6++CI|mUGlcQpYwzdJ(*X`^EEBX;NZ6MKYNc} zyBfNC(|S9Bz}tWBXq&udDPD2dKXS^}(--zd?EAmsgNeTWo3bRwcsBE+Tgq)0{@nWJ z%&WhCH$O0D@4k4lN1!_@z|EBNlEP(cr$ z;CbvF>pCBbY}@cHeo^r8jMMwHZ=L$K|4Ze82aIAOJvt(wW1t)k1n7uxYcd^l*bv|& z!Og~Wuwh4sk_ab@%g;%_@7FS=Z7*6VI=N`ptktI<)NOAm+Wz<~zyIm)JNK(ytB?69 z{whmzJ>Q`>{+@5N)%>bzbaE_yL_T1hmAz{E%ME;+u9&`iAf6?syS_H?Im6VnJU!z} zU#!w*-qX3!%~3DCwKTEt+ROH0yYF6Mm;dl}@r3iLEq1l^(BRn7`c>Nc$?7!IBu2x; z>9uCOYh7oo?s!-KPgi86ry8f(>`mP3?=;>IICVaF%4Iccp?CTAg`aEP8~gV>e7Ht~ zb?2b^ZoPgtN6NmEZbrlzj(#1*V6WqPkLvIC9l{cH7~z(_l7DJxya8A^`CTm zlx7twFW$vW6suP9GefS{w)tZ_cQ9E zCP&z=zgIS$k$P@$zCKy}*`f#LcVC|B`hWf`%PYfxlEs?Sty5#$!fUMT+RwL}FFh`D z^+LS-POa?c8^T@~BpSqRHS@XGSLS7sViA4g`k|PYSx;Y0{`;M|fl&bz`<$&!i4F&5 zOb`=cZDveb5TT)_-O|)3FyV+^*4Mq?zdzTnELs{HtNLiw)YZ*V^2gum|NVbpPQC4i z{Z0Gbl%K}ds(oyj7CURJ>DIG$+NVF=e|MoV;cNT+>q%eCW^$?>tn0aUK>z4ZQLleu z=iGmBZ(J1coHyrM%bwH)mcGIN)kQ=cyW7wB#+{2?_~*R;;o0x@vab$5Q_1^xlG42n z#`E7lr)DqxeSO6PR`H;qLoz>j|4Hb~c+arc>dhRz`MzRHjoDso&*3%AcF^kCoppS} z?FZNOH!)@fa@nS+7+*a9TxiX_H)WR|-~D%^Zwjl%2Zat{lWvdM^4}jD6oBI0OGKEJ znK9X6j?Ppr@BwI3R0JhioD$2|yx#S7ZGK;|g=gs6(_O(*o79~@-PitNSN+kBd$~<~ z=Xd+-E}3tauK1Hb``Oa(*FQfze|$l^2}j22hhqP~Ge<98dCH$Bd)pSBJ9$vSiSM3StYJI#BZInS+;gTc5ARo={X5P0`=m4f8fKhz*F53CySn@O z&AT0+9!&igY%zT*^Cw%;*I#XPkLDVCwJyy!^V47Sou92(?7MCD zvQGxZukU|L^=GqZoN2GcT6;sV<&}NW^gr>=dpDn9XH*fHpd%vK+RB)CV1 zgB1Z+V^;{`~#zUdz|zS}apVLyx2`cb-(Ff9Id_SE*O}m-f8c@=wF> z*K(uAzk79jKQB_rFMM|4b7J-`dI_7u)PbA~Va(*EmrU)KKlP+9zRm#mCWRp_^>z~?a=XE%JF``afa_e4!( zxP0Kdjp@g>*X++)Uv>8MuJ_roRR?QkzN#pA{PJsQ>&Z!}HqY8!Ei;;*C2U;CrgG-Z zr8yyM?v)+8EO$}+^sfwd1_sU!$5|3=T&$&~;ZZ{GUSHiG^<&=p{r`6V-~an>{Qv*| zzk2Omz5DeW&7uxz;Il zKCOKl60`yiGObC}&28}#O7YA+IaTHJYuEc9MB7F7?wEdI$LecKwpJQA?%t*RY+-Cr ziTlZO8&8Ssx!-!9Y2$K~;! zDr z(5Q6@8p)ULq<>j%XfJpvuQtp7&YzGg36~aa|IpZWynA{FzrKDM_w*^3X3lec$*ID( zb+YUJ8C6`rzB;bk75E^bit&`3w3`;Qd*eRIjT`oOdFX_t9Vk2Q5Nv(ac#m7s{>yIn zAA~tho89of{N4K+n{)Z||NoI$kfbmpDsxjyxPkV}ZK1MfwoJ2%Uv;IgYxk3N`PcUe z#caH>uKv32vK2}aTfcqCpAw(zvuq=KOi4v!-n)12zREHNi3+M!@A~b2ub=DQyYHo+ zX4?JW3+@hNTX47Dyzc+xtMk35%*lPVCiIKfmpNNs-&ndnoa>+3@_6%eJbWUuObOo~ z3oJXd?a21FgS}cy=S_E$3|*9eW#RLkZJ#bUD?|S8y0}Ak(VzYGs(j9c73sge?~VGJwdrrv zyXaNd!~Ex;{IKQNdG|-E;%(2Pmp@XNlX7fhu}19eBP(4xih8Fs_&-u++%$)$Z`0Jq zQ$=0sBL1OK9KZG^ANlp=-riqZHflv*64RdA7qkBA?XMD*0`4{4$}8_Vaj(35FaGNx z;WVy~4?2SLGDO~esSj+k%X=$%^rMa8oWklW!4jU^^e>kgB)jbrTxn(|Wx2=2ZE&{EL_FvU6sg<67|1O(--upYs zOPnM&dv2ShdU*fxMIPCyKlR?NmDeoDu{pXs%HfW}!rlinPBJy#?z*SJW5my~B=lqJ zqn|5$LiUIMh-;kEx%G$Av_oZFO<|7~+;so=pEX3hulcT1qjq_0T?ESY){6WXo`uq>BCu?l|Ts4;j z9X`2A-}mIp?@rSXKM{$#w*BFTc^^;c2r4X%XMEZewQlvdjZe7OCN)arbQ-EY7(Onve8*Z#@x5~WrhRB0*yAnVdJd-vV% zyO+nh-46IQW7i`l>74I>!h1RORx7pq`>Uwx)bKfmo#CbIRsD}R0R*1nKkkL%43g~a^x@816reo?eh ztM~Q6u*9Y!ul8nz&*sN6{9`u#?@yeOyi3_KzVwgz#{b-hgVr^EzZ-K{$4CBu)86F5 zrHel=`&Ihk$&CY1^BOZ0qZ9f6#83Lt?!;;?{`vl*y;WD%MY+|zKP>Dv%{lDKqnu{X zvV<8z#w*rpPcVJ>CHBPjzW)1hT>M!pvI1q#A6B(l8~IQ4&W{4eh^eYI1^%BLf3mqp zRo2UXSowIjss7#GMFu`$e`S?^-?yw5{{R2x?W7ck1Aih{H1N!>P2f7T>6(`Q($tih zbG+jJxEQ2uVq#soeRBOnDQVgNPlS#Y{(sOFEE#ca{`4pNUPpf^vyajO;Ol1ulK9|7qFL1Vc!3O@5!e8$x(f) z|K5u_%>6_Ap{mWKNp`C?*w(2Y`u&gVbU)9p0RLwzPV=1&m_5Dpz{D;=}Fy_yN}xb+Agj?=hQ*>BVC-)o))mQ%RUd?mCHr;2- zFXLI?T6(p*tC#-l?Y~>qKfyhPWxZ#t{$~)3e>(l3Z@OJ# zTzAmVBbjwqp0%IY&!Ti8*==s{WTGU*ifQDqiiW%!t`pep-Khg- zwJWkv#rpM9R+g;T(D?>+?b}~n`nBbruyIcAlHcpqj(vZw-Ep$t;>hNV_J_MAe^vdR zCy>unR&4pBmPbc)z0Ul#f4~1&eM~U#r^Lr)^=m)O`y4!9_4}sc2!mf$&CWqbuHP*C zf3temoffuR^PWvF{LXEa$p2CO_d^B8HT#@)9OCWS6ZUPcLiOvFx62#%_W#g+pS#&W z<5>BroX(kNHZ9B*F0M2dG}Vd|e)oK;%Dy6<Cy{X+2K`po4(+|0`F*9wL(3sLvw)f}LUk4Qzoc|)aQa66<>8plGI_Qvh~^|j;X?pfEhw_4Th3}3(g+R{apb(+#MErXN&k}-(R0J?QivlXNn)0&mHaWtQUW9;;?I;_UQnd(?^?yY)Ze(la4Ye?-;D(C4ls((Z*sx3sgV%!ZuCXi;VORXOF7j6-=Qr5{^_Mr#Qr#{f;&3qZ;A>Am zzP;%U-s}@DPcOQ)@z-}3&AhI+)mLld)?PLJy1u-lYTv6*b@`1V<`3qm>6G496aJgf zqxN8t+?AViB&*nY_4k}u-E!o~Zykkzs)D${jiGH#W)C;#PPukgr9!ZOV|U}bSu+z{ zqEc7<+`Z=N8Y@1g6c+K_ZL6n8AAf)4!aR;1O*fS{`kK;qcRPKf$`ouLoIaf?cqgo_ zj-B7Zz|v9mj~st|4A-vrvfLH*TwOe3{rc5Uc-j_jIqLr4+=TTj4=XXc-rwDCE*@D{ zs1U~x!@9P^_JNa@!SB1?cdxy=zcym&>h)psuWP-`UbZ~^C8vbelhE_>`VWoxO81D( zVAZ2xfi1&Q$5ZDlH%Wbp8L^7hPBRs<`&z zs(05{glUJHul9CXAHAsFWp&(bg=1VS%KJM~Z`S0COC0IaSi+ZZJwA~Hwx~s{JzsiiZ^`P5cI)&0{r)4!V;-{H{~*Wp{c%q=yqz_Doy`@W_Z2pF zXLo*{er=Ebf^%LU+j`k{IbYs?S(5Lz^CvI!hiAMUXC3gDddVZu{fGI8#D2Y;*#?I` z?<(aen=8Bcmi4>6^2`2D{+g+`w`#|fxO?mhVXO=d4`afiuxRIccr{UE#?GOkx>a1hy$MFHqp0pY`b(|D5Lc`E5HR{z`I+l=rq8-JQ}VKz zQQMZV)qktMzQ4ZYz%;RIjGRmk3T#TR^j0l7Kf{RAW7Vpy-+M*ZNVPM!uskb_(-tXz zo8PcDh1yyUZ2lO5c{ERpgN7gpybDKS+@5j7bb3seZpAvll5fW z>r*Zj@29u=_s*)`T-V#gJK^@P-+!YwJk+{cD>!5I2XB_L3BUaQ9rpVs->P)V$Kdoy z-gjl14<`#oIzIgT^vL|${XAF2b9d!SgkN$vIW_D@=fi2zzBSs%q69acd;NF)5{cOQ z^|KGxcO2DU)-C?(>Q%S(;imD2C)T$q z#OBONTDj)C>}me~JO6&Oq?Vt!TKZ_cNYaMtr#*YNSNI4#-MzQk&Ln2qtj{HRdyYO1 zS{-x9;`+9|zqPJKzu($aAzUYQJ#M>GhkM-zJ>CVkua`QdgIQ!YF{+Nd@xPIH-;QR;op6dP2 zHZOa9b@j!FX>m{EKWN$c+y8x~{J@=sq2+zn{e_{w)Z`bPzxzw*>vN;+{3~PIiw;gd z$g%&>BJ(t>AA3W;_50ob6R^A2^ePL-oqh4&4>B74vXHuUKz6r+q203uipmpD_43T{ zGN1fs{~w2qJU^J$JZe9lsc#p5|AbnU{YQ!1X}2#uzP@L>rCt4#T_2oJX4?Kxy7#j@ zj-#JG-YZz@HxVF+Wmxk;diH|DJdt2mH292KN=smSa0f*U#n+FJ`8lJVk~U^8hR_> zyU)&kP5x(_UjNHkvSiiiFPlyotc;rF7RhQi<5yW|M!V$is?MuNsEw5WV0*v zZGJO-@>$O+?bSat=FMW!D!$IU@}q6B-OL^PHn`a{HI-Uuy*jOtY4=y~{q%IDQd;b}|Db_vY8lKv8szcUO`o!0Y*Z#+364@V{8!L-#S^M)W^G@k4 z&KI-nyuTdsP_}SbJ7g&64 zf=4Pmrdi*Rh~#Cf+gK;5`|Rp(#nV^zK{QB+eTK6{s(i*RQG$$?6 zciiJ0FQvK4>sHbGKa(YO?L|4(?5cjhOl$XxWh@(XYm5Zv99X{lf$hqd;xim)*#&kM zguk9%b92pi(W=jnOLkct{>Ue;sah8ksAm!H7vC{IaCS}g={@)Uy|=sjK4PPowe^Gg zbsyvBN;FmWJ&SgJweaVjwi4d|QM0eMpIp23iHYv1$G6m~kNtfzKW^8ac6~c}ZB<9f zk9^VRS_BtAn;_p;WcO*o#P-vzY+>`;Ypy)_^zO*)O@I7^Lv0H#qnH-dRUq zO-?&)pHWk3o27k_=i9Y|l^hj$PL=;BY&M)dRXi-d_;1f0^J#1EY`%U?>Cw&)KbwDL zv$~x~X}`h$i-B>{0yQQE9k#}p1#Xea)!aH)E!@RqC;e4i-54Uu%U<*J%+9-y=WJc2 z9q{h?s;P$YMNj^%{q^zmcyD?y;~JF?a8XWS-YavM@_%BYk$-_#h6Uq>8F24^>MHk-R{UtQ`T4|nz7|TRi)wu zGo}dtwZc`tt#NC3eWYd>IP4Q@zx47^&(AbnE~`G@+A=}SBI zOpAES{(j=D4F5~3KDwpwypUJXvnc&j^}4jD*f!{QuY^r|d*vGGiu?aOw&uS1JH>E% zeuPUZ%Wq%X8S(Ep4m0jtxnqr(#I*SjT68#0SI;>bB_1z*_t*C9@Kw)$)`waDjrtq0 zkn4Z=s(osmuj|)WY*?}I>fX0=-_EI-9_ew;_icg10vCSSa`~m!_Jv}1!nDhm|DST< zejyXvj4;*X+n?Uv{YZ-6iswJ$0*-H;zYG3tP(Gm`;9c5voAuTTu37Kw4*Qi`aCCJ~ zxb1V6ZMt_$NZA1&v6&{i(?5ormdgKSI#-bBA$M(8WZ=>#Cj83r{8Rizg4IQCUb^>Q z$v$Fha`q?Z@I5)~sSQ11tNMZuOj%OxR4ubUsOb8d zA0T09`XWDkz4tonzxr!^Uxyae`Z!r$49yAd;d1+tZB(*l760@P=U)Dcm}emB`lxgr zThPy--iJO#HeIGhLc89u&O3Uhf4bCUqqr*-SqUm<_Wx(wzj{u1NUr$n{}VYu4&JiKQ%sNqFm6sS8wNXiMGv5-S_SAk}Eq` z@t)_>b=63nQ_6hU@#WX}Q^(KRR@z!xu{bf83WmuyE8boBeEYBOD>6ev4n-kz^RcYWXW{fKtxn*C89=I{Rcp1F{z?e53e z`;|NPNgq3KC7!up&i;cE_AF{r8nf0WsED~7_@(`*VppT`1h>fvyEu5)gkIQdEO=(| zb5WM+Md?o;oT*oStXNxf>z8f(8`0l~Q+Fsxd4AuLV|K6YMnPHdqdoKPKU7*AjD?=`?@&WAxtSZhs5+2FKeI@62nn_xsP1BDvdfN~oI9j34?sUqAacM;uuc z{#*Wyppf6eLeGq?C36J6*37-Do1l2myeYuXXwj=xAh2aM>w+GkmL@0u{*xQC*9f*B zxS-^*hDMlGL-~(bM&Xt_5X9DUqHN(|kf_M(>z(;nA&| z3p38kTc3BH@qPc~-{m)_ioU=4;9Sal|CA?NR#&m?ITAG0%U-Xzv+AbB&Bs>%JKsjj z-rx0g{ndYQyRXJ==+}R+fTQ4mhv1L=Jx7?iI9rrtnwPsXzZd3pi1~U!{od(4uih>1(5#)xX1Zkl{Krq1 z+oev{2o#hEc_(>msh;WmlaJzmX*#za{#rgUPBOQD^0%(IwwG5V`C1!}x1VN^o^GX3 z@uA>f+NagwtCoLR)xy~+uls#{%H3ag17Ba+GhtKMG7x5mk)MNaRC3AqcmTC0^=7wEW`~}}6JWQ=_ zImfJXtZMgP9_JE~>9+BtOG#bxcQezE+h?D5y572AnUHvkpZvj9?E3;kr-+#8vVO06 z_#^ySLATFKgKYsqe~fQxrT=va4Ve-DUokwssrLP;#c@yGoUc6ezPx|p?Pe|!t4dk`P{s@-PCXPy4C-SN{Q9&atp z2A7Ts!7$T(ua@(O<;;Jw_n^#5h1!n3_!(;rr}o8Hs!ac|)8y=NVavE?|L>;k@9J6Z z+?0}%Rry|WV)hGPwi8Mz>1hvEyq&y1{jG_kiJifzQxm4!9hJ2;VY0C*=MidVkeU5s zdjG$WV9tv_4<7NH_n4*pwJ!5b#Xs}cmOC~Iue-^x_N%n`E4S`I*Vh40=G5Pa@ibUg z!JM{r{>Jj_56$?_ezWR+F0}8{zkf)yf4Cf1i?w_n}mPmFJQC)LTM^0a-!KC8(74ju2j53*6RN(R!0_HURtSytF? zg;H)3Tc?Pe;p7;e>Z3{*1K%y=xPD%t*`n}{5o7hN*q;Ye`Xgm$d(At||KQi$r2Ahv zB@Q+34*P8P`jBX>(Y)Qy|9+bG>5^mfl!w2T{QWLqQ+ZbC!Ay;&KcV~@&wLL2U4KP0 z{X8T8Hj!oGcm2P+2tIuMv9RCjwZ`f)$Mp$4b?o2gB_*ret;$oL_`Xv3iOAvPgj3m% zO0NmbX1|yJBQA%t`qs_D=SMDdCZ2osu>amYG41?i60aLBXmMQ=TweD2f9IXg@2)?T z4lVv&8d<)xs4Ctz+pPAz!?LpI+m;D$sxv|kJ&*b$JfZ5*#kTTCcKc4g{(rN-;IQ4% zpq;ErO!5<(?!BM>j;q2@L3gcOwBN+1X7f4(t(AP4_cYJF^&JC%Uon?ydt^nN($PvX7%l7cz^p~q1WQ_*OLk- z-fn!(&Sc%|zr^I^?XFixi_X4(-uY!xdA-iv`@gOo?OxLHZEx+`%zsPYUwwZu%m3?F zjh0F8KR8y^Y|biZa8NxKx%}9I25sK>=0E%6+iP~}Ul7^Vb7HNq&T)bLxBF%OKUlET z!+PesKXY5=-+uqfXW~DFrK=|Ar^LqH78Ei6tBV7lzGx z&?0Lv`&v|s&z)pv;e|{|pZzb%eeOQyGU@QOu#3uTOB1injNiT6;B|Q2F^-G#Sc8}S z+4PkCx@c1*cigE(oIjSWPwiSMUy)>U{9v?hNMK~ameV0WuXC)>m@p}F&G&EdtrwRE zPOrOCv@k^VNFmqzH4eMDIM*#X_|0-!w`IREW^XY=arv}^wsOjjwp^X6~$2hS(n ztm+n&-dO_IJTg&q z+gi6?Rb*v(V`k&brpAzUG1H#=-1xDP!FacRzul)R(V~fw;Rerd-du3D@NIHk<0KA# zV*xdXtgVN>mUQSOJAZ$!;`29wRf0Prk0(Dzjj`+Et>y928|OP*9(vW`aa+88ZQO(@3c4SeuO?P@ z_8d(xjGKMCe|cD*GK1Nb%^UBAZ_c`^+jwz>l!orMlkU%Zv(vaG(xpy5S7>Tb{80DU zMs}l|;-RO_pLA1J7y9iz{qRLZ!=44b?XT}WsL3+sf4j_akuTFvJ72TyGW%=}3QkJw z@|C;!_VK+ruEy{GR2)b=-Ek+>M7^i?`TvIj3(DSene3~v*%rKM=GXl{Ty*62-MwF4 z{nv6YSJbNK8&A~!UbSlSs&#uK-udPI;doTuWH8%cR-7oirPlW5;{U%`_dRoscE0p!rsQ<>51sCZCOB0# z2uwDVNDEgI)8;Mzn6O&YzWyPjxB8#?)f*g5pZ`8wU;BSy-S@cF-^@OmcKOF0FJ85` z`swOfPwqx-@ZD53Cn3Y>!pEZ%WU?ch{#@tE7JB}yBgrBA($8D>OqEy-)a;{q{wfAu zc(i3t=6Tt8DZ3l0ZojslmR`o3(Pr>ft-k5b?1k@5w!V&E{cr8Z($ehk*P*Yz?Ae&K zbA6i4wGS6MCiPxBbxq(s$B9Odr|Wk93k&MJ-AB+WKotzP|brzWcB6 z&hkZuj++an>^JPrWARN-kN*)TQ=yYkA!qT!Qd{uJqm5BPf14QBKlv20M_}TOmH9_k zZMewt@uW+kQ))=5($Rh2?i74D@=L6`@q>iD>NAy?twkZ+sk8q#2@4g?zu*5je9|3cG%@ zuvn%2PyQKperT7hawt^kWlp%P@PCW$!S!3*-|a2cU7OGBki+Zx!I+zsNj|e|cLZnm z^exw!K5WWbU7;x7qqS<)s;h;i+>9Xy&0^;?>^Jy0G2+B84+UdC-ON5kZoj8Vzu#}y z;`g&MlHj#WX-~e+dHD|iLSvx|)eFQ{e2rtS-M8S&ck%z+6Am9#=UR{=!&ct#>r$uS z(XZEkE$H2?Jb5Db3E?#nrlA& z|G}kn{jJ|DY`vTAsUEtlD=^Xf@$?UmbzWV6Gl%8&)xE{>Kh@V6C;t|8{i^qTuWCie zQl-FKHM5?m{L$3efBU^uo5Ja1{D)n3SxvdR@%H{B0ox~El)oFl`|Fx{@t<|s7!(vd zRA1>wq%ra_cW4?c7Tt1rKFb^%i&c91w+>p&5t{IwJN-l1aj#{IuJJo79}G}XdocUE zjRzn5!9@N~|K`u=EP7M9&_e&g6cbU62nJrwHH;1&8y3#*l$zOP=izt6)O!DmqaR(j zK6n1l%iS8tT4j8HN2Th-yB75RrUP?$*UcRSMhEg8#ED)!g78e66Yc(T_<#zvkLV^VRWY z_ZXEN&D!Jq;N}Gfizf@1mNvZq#Cf#7>2Sn?vdilKcCvNH&sk+(?6=hZ*Ya48)MFl& zhd-BEZI|%XT~;t(_VCmByPV)XujGis|^( zTDeiyNbvg8i<_nAf1bz2A2I*y(QjMB)=z%^f7h;O>#`1f<1&AsbjVHhdc_j0c z?0+Novpsq7`pO?G*P}ri6{pmAE{b*TdZFKHyz%kBFaeqCZ#fKHZWM-OWI8QXIGAww z`+0){Cqty1FJ=5cRebPgS-tw%zY^D6dDz~6_!fQg0K3wG&yCI-Tf}5 z{osN}^Rs(<_!=FYEVI_G;Z|sM?_iH(-sie%L9r>Hgw}rk;^~JRnrAF!6FOMKdiVE= z2R8rACSQNPYO(g=sCJo2$9WR3L^r?T`({;ot7E|~p_IKJrnf$SeVeavlhdR_J14QX z{Hj}Pc(<{?zp450uhwAsmpdYk8ywRweRb|2%acn-CzRe-dAzI4RUU3ot`M#8_UijnJw6yy|6#RM!oLVY2I6R z+kdHt%qU0{Ul>+??*E3kWApB83HmWZA>Yc``L*MYBLBM1XV)V>W~|}-c;n2{E!x^L z0`g0^IoorV9DQD_#B1sCsP^@PxXX`QavMVuPt1rj+~D)+(dEy2KbL&h3_tY9+n-7O zf&I7j*&o`k-SdCCcEOut?oP)`Sba;y?{57f_O|6i-I4kA>$~o+cJVEL{`BvsgMyzP z@t*Z}T=BgB^7>0=Kjv?^*r<2>Fh|$6B^%4kUS&0zlbN zogP2a@81^3 zzkKzj>59#>>^CIpX|r&$pHR@@GX3dR@Tcv%m;9X@k6x)N7^q)7nY-qs$WQu#Ub zES6B@l)O22!q0_^79HPvqHQCik${bB-C}+LH+!kcJDbiwZZXKvIwiJCC+>h~?o8|Z zU(DCch-6;Nuq0Y~mYBu=r@viNH}3g7(S5o7!_`}g0!_`Ycdupbi4WgmU}q64UbsC( z#hNEy{&gYqvH7xD(r45Dy=geNF3#suri%jmUcqtH2`h8-F zwanx4)Bm4|oPFQ-s%9QXziyMa!I!U<8B4nB&R$!!p-({7&ftCftk;?0*H=XBzLPWk z$3B~5_v;VsEH2yn;X&uuXJ5B{*e|+xv8KVgb*Voj9&s}5kCW^E z%U*x>lA&}`_Qmq;r7q>aMl>)pi9{j4@G|DmdVgl*cr5A!D+ooVj!JNoL26GiVm z7Yp0S9hoek*ZYMf>HUAX&rME|TkKBjhBwuws@m?GTGl4H{6oiyGO;uHQ>$Yl(^acj z^|yFh%s$4~e6l+8UX((IxUKh`cJ=eqw`qy4S@T_J$H&%q8Iud9-Sr`~^MRF1jhMdY49y$woVudV?_Iop%a=d1Pf zFMr-gzcc%~^@BJ}7XR&7%d_Ifw8vdFca9~ie5OX$g*F7XwT2?!qw`%wLyByq-UPr@1&a{;=?9<6*k(m)` zrTKo-p-o@!ZCo2z@wm<;`dktpqnyYL4L2^c^|psS<1(hck%wV%X?<*_*-4>d--$N7u!d5=by#JnMzG)c=$W} z&64EvlXsu(>RGC{`orseKhCT;9bdR4KIQ7%?$b={#iCM9)3)j`=lx&Q5Pxv{U)>pR zl6o1L0#)Yq9f?2B^C8aI!1MHGi}e@M_uN#vwdIFs7iR#_m62jyHsjot{qg{ARC%>lK z*c10IBx~Md@9rN*DmmgOKTZCTahR2#f62LDGxu0*s)_tLE3M{QqyXRfQ$EbqC(ZXi zea+RJdMQ!EQ-QU*PWY^$fa2|aUbgpNxbFGadX2BAoa?`~o7x9OE(tCUy{FbY*1Ff6 zy>or~zStjr$G0cvEn(R8yGs72$m+-29)7Bmex}MjGyVIbaDkJy5hu3fea!Y0Des%H zh5b2~TbOa|*_$D$%Oxy8Ia5nSS+40|!$uJu5pGVdg$^q^l#Dn!&pq3HE;X$5^_s_e z>oyk1+*>fYad~a7^%;v}XXeCTz4Z2N_s8?)?VBHj|J&?db!7dg>82%qf6hPCseO4{ zDZ=gQFZ04jGXvP&!>8r-r(T~l|9Hm#h(CXm13xW!cE=`dZPQIAr&?Zl&1ZJ^(uMx+ zZ<+92u5RB_f2N7c*MISq?K)mEPx$0%bEdx7)vwO$^&FQAv}^c1Rlx4?nKv;eT|K|~ z-iGo|c)V&uv&15mm~E1m{P!eZs|~EtSJm8iW!0*X(zZQ<4mY1OH!#YHOi&Y%W@S3) zXt2OULRgaJP@uvT9dWLfCbe9{bJ^wlY-6q*S$*zn;w9lAEm=G(zEYl&0c`@H4r@>xFI)1E#-UVr;9b?e$-2w+Ns-`ddGFS)|B+u9d~XfPP+1N9h-5% zmJeUgrrDW%*Wayazwr56Z|=}rIz79e@iD51^k@llakerhIn2~h<6`Y-OxWS!BqZ4R z{N8t&fB%z>)`?-H|ze=bJx*v#sjCoLDZ ze!I5y|Ef=4rp^2K;(9P39$;YLFL0bC#n#1I{L%hf%Qxls2O5<3xc!gW zr~Z2XzkmDxzs=tNWMBQNue%?;>Wy0;e>GM-M)7{BfAN#=f4G^>r20RIvQSa@vRoi# z-cIE;Tt`{ec-(Hr?p^MoY0TUcVUVZwy7NVn9oMw@e;4lly~efq(prVh`Wabq`B&rK z?T!Ds{ng#oXG_msG_4g@Rp0&~Y))F2_u2SayLMa+zcMTKu5Xt9^kWPx3W+u{IWWMU-fRw{E0O1zs$VNIlAsn*6#G_PZn@ExUerM5h$>|XOSbT!Lu^z z(>;T%qNdI6QrDag1WguASst-%pYrs}Tr3Rp64uzcZtFB(9{A_~bf$zaf0gy7XU>lD z`1**+@$VVO6E0C{Gr5JPc?EPIK5bUJ>{ZrJcJ|f}iTWa{ubi*(N9xZy8{uv6?~52(&4(ZC9k~S!>ptrvB4$U;LGm{l|#xNk9N^x zZbi`jb*qwc%D*q$nmzfN{rdwwoF@dtg%9r9eQ?J?g{ema-!Aj(hiL{sPZw zvQu@FHH{_1*m5a>iC!K_F5f9Bpj*zP3k!aXI+KK+C6 z!law2-#$e?;Cs8_U*3~<>$Mj#PiG9#iQ=r#ixE2Q9bRVGqT{7ComHsh>#ATkrfYu_ zfA8qd))R?!JZQATNR>4%@!FwRtCnSGsI<;sZ`rS~rTeVdbRKrb@3qJNR;H@w^+q|cDc62V%p3FXIUjLo38vE*Yawe@zoQvKUFN@k?M-cSj2Cb>Sb}obLvBJngA!S#{r|2f^HhdXb(Bn^v%KnJQj-ov(h=o1-GEc?R1# zIbR0;`HY8;^F<$t;aJJ`F0y<@n)Tnc`3X}34Yb~fet-T%cHJE-L5W`~|0hpgI@{x; z-_nb9Zl7;U<(lRGZdqSy|6%1)dv}{D8@xrA|35UhPtv9DnbE;!$BUcyZ(rTI`|^Sb zHud-R^XX4M{ZlWZNX~z{V9I`{WtOVX`&)7y*1hpjJ$%mObnfq`$=eSP({Yl9)nA0EeerVZf%+>HlbXz5&CwN{T!L&xgcYOj_0+nZ-M zb-nKH^!T0WQ4-A0!s-w-|6ohO^myJyuXjlC22}Vw{UcxAzD>H9c`<8-569jNC>_2}B*u7^=q~Ty_rJt`HRU&7 zq2c2H*JS?oI7y9v@$vWUZyU^g)+iXZ@8tVmGv;V5Q~moVNAkoJsSO1hg}yi0@A&I&**^X6BA@K~q8^m?`rwZEIL;e7va@-cV(juyO$jdc zh;2V7wEBpRcfVx~k3x$`+YJScirlIyN5gj;RxR1d>9+Rg-j>IA@9WN5eC$X}mhu0o zOc4uP`&o_ROdY+r9gOd{wH@Xtm)mi^{=-j;G{)dz9+(k7yjM|EhVeShKz6cDe zxBe~VdCA3gt7^Euc1eYMKi8_L)k&eZxix$5-Aws+!mp?Lo`2RvvxACL_&%}BTRJPA zZ)#hDac=tFbG3KRY3zP&QBwcSTtIUxeyX_9TTjZeiiqo<93hV;{GA zb=1?S{SUYzWcTnc=xEFMc}?|Vi~ro5h8h8ZjZ+kYCMyU!X{9sN1*aTkKKP^NfuG%} z{pVuSo}Y=)-M@7HtNIg_9NKXl+)Q2$2a?;nSpRcBwFrnlFvrk1t! ze4?e(|1B(8{L3*=WB&Q+@fRK3-Y$)v;h1w{`^TLRix0QY5fjviU`zH}F1z#K!B<`j zmzK}^e`L1!^yMet-hQ}f58rCx|8G2_u0BuxI&H%Ia5EO^UroYOy3%epUolV)F+ZNP z{^P_);YKg#O{uXKa_)2=7Sdx@m8#H7y8e4n&IG5jRleif6esZ zY8E_i)!o-u%H;g?{~D_ZZoXLSqpzCxn|ME%aDUoQ_WsLXmvDI4sN3I)`_1cVr|!yg z^ke4yzXo0f%Zqm#cm1q;A9pxar8sbEl@y<6kF*_!$2a>r@i(p$BMvV2nZDYH^T_%? z6E4>>6zU<8E=ii*+X1yih zX!Y#S1%KPjn`Y<~7~F2DJ7qDGgZqwSQA(LaQEkxK8iNP(&o5bdX8yjo{d@jjU-cvI zulQ=G->z%sP3YfW{WWQO9dENjNA%wR>jN(CtU6Q4`%tg$@gJQu*_}J~Oyf>W(!RRa z;m2uSha(M(9m7}D?CD`M-{19xb=3r)@A5CVM!K{;?AS7+o{9PCVWaTBYj&t;UOMI{ zU)NPW)2?bcKS%Y?&(2yGk6ck+D)U~-r^X=Q(#*vX?r&mW+2v)m?K-vjuw(qb<=ija zeti*DXxrEpa_-zg#-Ee#mCQI+bs^6}@qZGxxgi23m*HRN2fG}r4DM}l<< zdJOk1`4p=wyzihygZVAq$Oqq))?`19wAb6t-Pvn&N2%bIeEAtYgRkGs!lV{hZrj&X zq1x=nzqk1L`Y#_9B^CO7JAQmA|C{RaasA1w#c%)2yY+SHa@Me;U+y3Gs4ldAd-&KA z*Ty$IEK2Vp*xu_0GWq>n#J)zLY~q9Ntmd$HzHlZOkjP4!T`>y7T&vf+@YKN)JLF&y$N+ z|0>RGV$Q~Y@a`O+X7$zQ&uGlJ!#GvkQZoLs#m(!}?*Eyep~iKvusO3=W!c898fRv) z$FXsQt(YD#rR2${ZXY@3LjkN_;@bS(Y!UCjytVz^vwu#BYsPM$8LStR4WxY@)_mwT zQruBBU*F{3`|@+Sf23aLJ{O+jdRzbHVR*fcuKrvb_ix|7Wrmqge*X?|bXcr2g}@xU4bZzi*H(HKJ=c6}-)PJW*n z6q3QE>?6P8tNPK__tPIrByF6to#*DSUAvSYn@yHmCjL_4?7k(x+HZLGN`&+5lF2Te z`SNqhdh-ibDSLbNo>7+&I1t;hkLzARS<}lEPnK_(Tz`rG0pBVO*#*pb+u3sFub8Yf z?|FzEi}qaS&&g3UGJc)@bx5gD_O$P;kB=N(-__i`aeZawi>YrOwY+}kz2g3bT`z8T zUtQ-v>3zV1j+ZkN_Zg_@ivKm6ws*rxkCKMD@1MRYU#;I(_`&DyEAve{C!{yr^0-^? z#(2NG>VnNKy)%);uQ;;2=RaeuSd_f9t#zLK`S`Z^TNRf75uaSH%RRR!^W@*R7Y^p{ zF__tQ@o%X4yBalLgDdqK2c?a^a9bH&(tfujDKGA!aA8X?+mV9hW)1h;(|FxJ{fN3V z@kWyEW&4Hm9Q|DO-cwXAI@&)W;`7s~AHHkVyss~mzH;M?#7isYi4UGWT(Pq-$>y<_ zh~e593w9>YGRd(#_%G$$tp62zCeGS1J*?lC`Dfo56?y*8wjVh%V{h%^dV1k`SHH&8 zkAGGw*t7rIH`_Z`efq0Mjt5*4Urq0@`SH=UkY%yzwF<+gv%YiHMVE5*9GiSMQmxU)8em)KRnlYkPk;?UxLyEv_yM?>g0$FSVp&ht=y1joM3J*Ghhv zcGEZLC&!DM*PnFsRn;;7e*MX%EbrBdq>|>f;X78G_OWi<`}e1Ph@-jiC7xr>g+-Mt zoclN}q_d*m&QKSAW>_KqRU`Ci)1`Hpw&&YED*j7dr*%YNoA##%)^&e)9vaR$b8l5% zPu}}EPLAKWJ_Tgu9a?Mfm?#Tw_S!rE`sH5Hw>&KG^pd%II(?Sq4RZu?DFFVCnC zefiHe>2K`)x>tuEOldzfp^%rUfA_B@y?4*1{}=R*jEsN$uE}%XYQK2}C%IQ&Kit_r zQ~#jm`M&`d{=R$lw2Jj@+^@eP+ox^%!s$aM#U(9sthy@-_AE}9#021tAOStIh8sF1_2kxSyFAQSW7d!Csl2) zy)U@F_s9MJ@%8`ze)((ue|5O%_g_o-|NqZ^{LqfcLG5qs$Ng)5SuW6ox+>bTmCnCD47SY&)U@_7Po7bl0j$kkvic#?RVTR>=sPn zk2-5^-B@bTs$BV{^M2ror+KqK9G`Akz+=OHo+bPSU&p79@7ret{gZg?TeC;nrr}&q zvD$;`+FAeDgWhp6H6BV`!goR|UsP&p#q+1jyZ>?~FA6>Px;O54$w86*>sQRb`uEq- z5U1_0{;$5ezHVdaw~e8H0@N?}T?%`dUvpr$Lf^us9qYDjeErtM=d>%&Dv9l--0crt zf>{d!5_|hD8cgZ%7Km?Ksic;8a6uTG)rO0cSQu?Y0!|kFTUaf*IH%^2gcWaTom8Ja zn<;-@kD#0W>0ifdR24XcRzH6(x^R2V{Edlhe2fyjg$m6Z8uw{CmLDiEUr`qGG9%gxU7U$DW!hNGeF(_@Y`ErP@r;pX5srl`-+OD+)N1b}uHa0BT*nhkKx1GFgZyImo ztqKP{Cl;}CR{_SP3Ri}rLi>fK;kUjUCC{6^?@?~j+pJYT-^&SGG={u%*}VJ1Zv77y zZBEPFc-eLvAeooJ;0*u8p6-Y{ z3*N53{#+wu!TiHKD$ftzx$)2U>5^COMH=((f8Brne)*xefN38$2rx4}RAjK>dz)3i zy+Y#9`-%vQ2d{usP49x5xgl(p{7+W@`OvXDU}2$oathm?4pHX+3RU&T|C+t?d-P$y z$__R)u_*2Z`m;LL-TGPk*YWXJ*3Ow_jcIe^+m0|B#O7I^{B(Ioz&EY1!W|!eFW7ce zH`iO|-Y>4DA_`ybN2ut@by%}Jk-Js)oAJX-@eY-WHS5%`|1p|Uwdtfsy72uUIlnHc z$DcpE@%jn{kqOp;U)T(_H|j8*SG{s~{prcEsyozfo_-v-;OHi%P(_zpL0xJa?ew<4 zdf&f$;|bsO*Y-!OKRnCr;H~aI>EEXGvYwsJ=eS%(Qe3rf@n6*lg#+(YgLk^@(fqn` z!MvNh*|Hs3%A{Xfa&TShJv^!M-;FZgBcHx&?XSAR>g{Hi@vi>TYXd6}o8%oALPR@w zSPp%>I~$VlOniIZ&-(u(#hqUZ zSfdnWk0=!^a2<6jO>klnFMbQQZ= z3u_qD{|M8T*}QFsbwdlLe%Q&}5g&f(-h(qxo_S<);7*t)}IetGq|;M(W@wtd)F+SbE|kd z4n^I!4SglQrOEg~pU)+k;6GL`em*8A0aZXSH0 z%~~3J-^_Zq*2&O>Id)u(3Ae6KXEgjhLuMLFi|6@26}GP%^R~AAcp+MUNl@IeC@Wy0 zbT`kk4LdGv;)oWr-go%1b48+#{EdQ1y#g7Rd!DS_lGtdz{mxe@m+l?Y)!+Y6xMgtk z@MoTK#cla-UVT;yQq8;Lac}XbTD9vP8s|b<-r9dPeihjC@84mT)(W4+0so}=86G}7 zF7IpXzt8pBH1QWxj(?2z;#=?kYg^M%`H-0bmLW4&IM(J0-v58ruKG{q;+Nm=ZA!VG zwe3z>%D*j1>wEX?IoR;}_Lud`4x6rJwZENpGLOqqZtDM=uYdS{^}lTElj-IDzj&_o zkGk8(FYM7>AGUf`^#1Afe`b9!Y*LB}6<@<*QNntn!cnTb`>N}c&r(b)_Q*5IAM4<~ z6gh)+(l_ZoC&!)yhq((Rr!cTEb}r!BRIJh8d~Sj9!LLlg8#bBMMC}vPVEOuQ{;EIK z6Q+l3uCKcwAubtg%^*)L7&HsdshpF*0o7y$&|IXQxy;rTdEqX&{`x^_@tNZic zf4+VsUinwP_>WG-sFmWYLo3CB+7~?y`moKdUF^iwD$`%lzi-afT@Y@4YeE+RmfYjpsHS(V){Ewe=K2AkJ)9J7K`j3}(bL9JTP0MbZcQm9gbxO(2 z_cH(f-92WrzQ8Z~-ZFLPX8pBQ+x9y09Ldx?{ox_sqEg1t_*D5rg3}_R4L=8~?~LD( z)A8^bQ$f=!_uSbo0zSs}Joy$4eI0#VE1q#M`Tnd|wGnXg{14oNxBSzE_>&gMeT=^BpP$5Z3(c)mIP zBf9P8^>qiYnqNO)*LAOen`iYr)wmqR)vH}ssZZ$)UwYEkKXT3O-`Dt<_4M_>SH0)C z6&rZm=l_=OKUyBOm##DlCAqCwBFDVtHLw2hMSt5?NQYk&co}PHF)8#HaS^Hx*fAYU;)1=cn%o`P$Tct@ghhdp~X!6vWBJ&)QwN{$7|vwU0e#>huA*170Ho%HnQ@7BaRdfCQm z?Y_yD^7Fs+P02FW4TcPgQ&Mtvo{Z#I|FKoH)gwd9`c(WTotI_&OwkXxZY=E0?XjwQ z@>IsMrqL`+&_qMw{*MaPyi1Igr+OS3UoSb>xZ0u2zVcO5KmgzTGug%(3#TjV?p+>x zTZ4;7?5@TdL#`*r@$2muwiJ8H9qay`p1gFP<5O2<6~mgVsk`M_T{b-VuDvxyKH)+F zN2gWG1cr!Lu`J&%)O)1P%)Mw_IJuaqQul z-Gz5Q%)K*Za{f==U7o+3^Y^F}Cgw`^Tz{y%Q)JU+K69@lte>j>O-O&!=6;oJ>ER0s zP4{m%ls^5G?ITc-J^!?r(BtL@$EIJ4oN{c-EUDS+FE}0YTf48b>5chI`|J-p3!`^! z;xs#R&NXk#{@;JZ-~F^Y9C9howbksh!EU1!|Egwh`Zw!fZr_%xg{7@00?wS?vGLU5 zHH(gUT)%D?f3-x!2mmM?f@_@+?TY&Ykuy$^*}%ZPI9TYdBIoQQdRt5+895$k?`^`Fx3y+4mc z?)xJ6`I&O~pSthN+xV*wG4yaAcUEw(z3Z)z%#gl7AdC6V^nLqS=W$5erXQc6+#35Z z!s`+HKk1^~_9qrhd}XrOwxDKGrq7}B>XV-f7BgPla7o~UeL~;VyNa*4*YAG!=&fo> zd41Clm2=O(-`wGHe?~$ti_k%lW7~OTj$cx(jcM)OFhx+LO+@~Vk#MZ_zb3P^b3#e+ zUs_7iTi%_TRu9l-s&}RA>2Y7I@T9 zc!&PqOJ;w6^{o1HC&pjXzHqm|nRPXGS>8N$%R;~O&YZeD{QARB`(@>dn?KE8*q^(2iG9f3`w>px&pKTG zYFM=KuK&;153hd7?c00EF5>Rj^#|o@l5f-mN^PzZeb@QX>xXh{h}17{xr;i2f^&{< zUnuH2Q=ruS%Z8Act?5(WeTY<4k`_+fQRKGgeC#=i{$}-^i#XI-7P;hrOHk{hj=1_4&YF`*-TQ`WO8O zdG9Bz&giL>vh($K7LkOvz8=}K-!t_u>0W)tnd#%M;#%4yaj7XdX+yTGjr_OIH3z$delRcyGC9tYW;?}NTKdBK>V@9fTeBb4{tvoYAMcvH|KIfeum3-q zdPi;V{iuz%RPuhm|IZo{{+khN=C4DuufD$fJtS(`PU#~pMzvo!TbmLWHga|N zhrQM_W~kpC`#fe#%oV)^jt>v^W~~l9b6xKJkBEi)1>|dX+&cP4kmt8^>4Of<n;WVk{)vP=l443kVO0Ld-Eu|yd#lmvIpz-O01NE(we{4=?Ix{y( zD*S!^>*M90{y0=P2t-&WS52+6ex|VAZ`N*4Jtx=5ZGSHL%=~sRrFQWHvxOJr=WEWm zC$ru1(CwPVUb0Our9uqf8}iD2*>wB%?gK|e*4OYxe~o{zeD^H7ckhm${%Em&wffxf zBFBlbe-#Vs_Nsebln#IOv$l(gGqEG#5kmya!Aa8}Ts{4VWygh!LTX8jN&+k^Ds;Ko z%KMrxGR%C)8gPmI1<#rC3k&QX9+~2B!6^P?;v=rY$yQG<2S~;7$!M`=crkEIoV#0j zQ=Yl2!LJ)>D;QOt9Qu~#bGEno?1%S>D<)e-@82qE&!}gUC%5sPj(uZa(4BKDe*JL{ ztJ%jS;Ipqnj6tx4K`_xaXKG6QLSBJ0uw>NJ4cF6`uhdUfR8b>7-i&V__VKH+{b6UaW(sQ6~ zE#qr0gZb>{Mhrh^Ee&&1c(UoIy+fZID+kL0HKu?XyBvvoYbQrzG|JXT;9zNH(Qs<`L`t0=G_D0p6O6yCt zQ*1Q)ydH4i+0}zvHjg=g#W)<`(N?jsXX^3 z_w)VN)+_zs*7Mt!pP|pYb-(`4xT1!Gk6b4+g?=-ilek_$R8BjK`zoh z&iwDK6etjF`{uETnfmtkx>vk^ZI!hWx^iqmU5$8Hnc4jGi|3ZVKl5|K>6c{-7ECR= zESV9duyy*?_xy)-Ho0XiyYxu(`mgT89(@@JOC&0?j;>KZVE-d8wM+fQt(l?|{=V~2 zR^gOi9{RPdO#1nPJKcwQdsFtl@{hV>N8fn^BxU>H{ zU&)ckoW}lNtMg0K%x;;b*EZX~zAE><_FE@=jdIA{wVq82|A?|Y6L|5vK(U`CrF^wy zy_02Yc=g%oja!sC7Nr^e@KMt|Zp6gAHsA>3ft*eJ*1Bp-Ki26?QMt2i2&rU#gN08v=g5m;yRZpZ&1@Gk!o@)BvdN*+W*M9!rH&Dq~F!a7)_bj zba53why2_By{D4yc?a*)~B~uJpUK* zLq9jb;mrPi&TGBmuMX`sSXss<-}WUW?!J%lzOB8p@1!>G_i^FQn{ODr`*-Tiy+&zg z7TpiD-LgITkD6!x!J9=@A|X2Jy0^A#HT9j{b4@P6h3m_l!p$j4+(M`J@!kBP@NCbT z{CjJUOYTjQa%^87T+i!QlaOp{IxR!B_4_~hvr7ci9+qwVu+-7bI`P=l;}K%l17I?8Tq4%aq93OM}uhnk?Q&lrmq%U^NZ~L*m{BWy#Z|9MtpRWIt7F;P_ zub?LwuGQEoe^h4Q|HMrXPpD*Sd{q3llIKp9Wa+<)GxvUY(9?bWwTi(sQH!-MJD(oj z*luw#>*2KjRy*y*s|3~d*S`DEG0Eudr9J69^8Z=H56s(sFXRy}EJn|HOq3HcKu)yrJ@bbK=G6Ph#hP zN->gM|M%F1g^fm+=jA;3Bi?weDeS9=l2OaCsLwvn5_(o-^ff*bvs&vK>V0Q!*#31L zF?+vRJ$;|Fz1IB3>h&8|&TH%}c+Md#H`l1fPVmQz-`mPguHMgLli-wj*i5E*LtH|G zhqT3k$2t#Wch~w){ylrG&y4s-p@xAmm;P-$_oL)NPe|mvg$)m{ERdP&DDR`NcspOn zN4d$2r>|Z}Un)Ie?x#6bA3vRX+J4T*rdVG6dTbMi$n|ft>et-fF-uZFeWgvnYt9?H z4`$YvYJ>+d$7#uZfA{iZBmd8Xj%+s-{a?SH@GX(o;D=knY4vCU{Rc{60lygd1(jyh zroR7?u=b2#^1{OUr&&!34bG18$7arVe%rP1+njsm_Zio^7IsX2@!a%rPW6^!Ro`_| z&rA_ERC|B5F!ciC6|c;`goWSU1=%Go6EezOzG3#w>fN!wpGP^^zUFV`o|5*`t*+;_ zymREteW}S&D>f~EoO-NT_~eo$+V9ubDBJg(y4^VI-SjK_`K;nMH8IV%r8vDVJUDqgbAT6fk()ho7+8`OFvk{_&IFK~tbP-#<`qvket zUd8Oce_1x@{nKB%rs4HvZV!Eb^TTcHUo|c`{Pz4$oB!wIrKf#~UlFru!-`;|*}Q%_ zYm(RfOP#51GmOm+@vZG5olVjhQner4RYUuM?RT9@-EV>-MnPEqc|5 zo2$!LJTv-rH(Bj>?yU`~FVr?1-&A=bV{6IVtsAA+X7S~}{{NRx_*nYWpC<%oIV{t7 z-+mx@S#djy|0em~#D>Ppi<$Bdz7g5- z^qUW7^D&mQ(UVLPH79huX_2_QQ{eq70pA(AuYU&02%g$);G`1c*Hr20da}L9fA?Wd znfSokMZt5tG=t-my2Tma$lUi{R}%JZ_12fB0;#c0OM=37b)TQy*?wekp7^0V zhH5IZ{hzyc@fg(2%`*tScukV{sl~o>kFY^7D zbFM$X)-BoB*e0xLbNZ{OX5X7LX+K>jM%A9=u$>a^cFgh}cZY?E`kUi_r$+>R+n(;7 z&(nRW+G`sJ&kep1SM>*(Jk}aX$E+T$Ivp2PxA=Z^$jWn>mUcI~yjn_jIDQLxxO%eg z={F72R$I2Q_I#RhQ^YU-iqH|>hkwsYo7Se!5k6u#bz0CLQNfkZn&zoDRTTH^-}5Hj zEGRAI`0-hVHfBNlJ}ileXU{R4tYqeG=r2@wbNbzq!uH&sMLDOZoz%J?-`{gA|M=RY zFaGI>lm>nO`XXXuWWgEb$Cjmz=8r;G+do+!vFdl!)tekrN4c{9-)2?$SZ*vgYn$iv ziEmAphyJK8{481fc+Irmwc^JrW9Mnsem^#+w#!o||JUMMKLt4==GjUFD!ClVm(6sW zX}&(-&aSn019}b@?n%2W=UBV+9p~C*FZV9_`XhT*=ud4O>lXd$yD5#ErhfN0JNbEj zNo3EwpqCFj6}NJ3VSD{NT=`R#={jdQxB1fFet5IV`KUbp_bT`HPurhtIhQ8#th(p_ zQbn1=^_tzkCCdXjyEmjxvdNwBZ~lpm+2@XanD~x8)WCXu>C)-P_I{i;D<~l#Yx6YI z`CPo-zqn#=vCmiY&Y%9#g`aEB?+;TFwO;yu7rj{!rkJv==)97oNK}`|g1&jOWje>- zmZ}G*m6&VqtQFIm9do|(`mSkf4{FZ&aA#+ERnzOebKl>6@~u8T;OcGV$MFUYbFxa! zZa?`K{rT&MrFQLqnIfhOZ#7%A?fZWB?qBNX_BlDf_%F#TIZxPBZ;9{!N!)wCiLJr`P_? zP@M8=>*P1`*Gv?21Ya`+ThvKMpLBiC(qzHO=y2eSaQlkT6>FUP4|V=fI@2W5@5sg9 zY4E^}lgT1_oA<``?|R?unDxc<_0^MkKc;5dR?m(39kXib`(hqrz1iPcC$#8a|FI_~ z*7fL_R=Yq)pDSk14(#R0cH8jAOnmxo+196zmsGxF;#_{VD^Aa1-Jwt4f2VoODXMHz zOlWWjX}r|?=&U@SX{KAWzEF+7>z~B@x-G4dSC)%x(elz*#QY)Q-pWVrTu-;L#}=(< zkGeHG`={m+Kb6Hr>01k5o|*73(`w<~u&Q!BU4!Nu4{io22YK`@epBj>v^*3 z^e)UzsOp}X0y-kLUBH)j!$FDrJd=V^J-4K zlzjTbPGe4h9{1&qR=&|K3<7-x4h((Yr-uG4DHKK}pyk{srB zEXpQJ*Ps2zeS)*2LQ3|@+3)3zW!&F)w6!i1l+tjRyu_w>pC8NRJYW_PhF3uOnBSo~o~l_YR6bS-xk*qQ(Fxv*MbH_@_IhFWgwByg#@% z?TVu#`ziT@Y15b(bMyC$H#M(boVrEjm+KYLlTsUMa-OYRUiO3Qn4|dX)jNN`tt;DE zx8`qrqn&?@>el3)x>21^|7|R&{q1yeM|w#@S#~&607I!%mcKu%%)SZNql+u+YUTti z3|9&(-W)UkiqqTgkuIN3X4IFfyn45F{q_s9LMS{MEN!~Ww6JGyrlWJQ?p+g2Q} z*~jbD(4r&2yTHlh+}q?+vr85T+OTezb5MAeneid12C3Hclp^6?0X7wp*~XW*Jl;|l zUml}j+ULo)aCYh<`3_f7m4fBO`3=z|3rP{?P#r*A?8JIiJ&2Fhoho%q5qeOv*sw*Ets;5pUHXMj|a;xE5zTlm8}uzQ2BY~(YMy(47Jn?IsYa$ z#(zA;Q8dZoX#ApY7p}d3_HO0GU3;s3OtroIbbZ_ZLEWmtY>Unv zotYMkbtD|xW_{~x^e>;1&=hwx_|W31wagEH7|an6`@}ZkIS*?-OYeG{`op&$&f2hK z`6lyyEw|35#VfI`U7+uLYR>VNPY>%GOqto1B>b`c#gcVxdyeMHym(Jb&R-(udP*-n z-(l?Ix_^Do!_S?`mJ(`Rn~s{il@{U+nfXO8^Va*2b20Vmu}tSUco(;&{!pl^GM(qG z5dGtZzGBYP-=)V+l)WqDxw-2+$A-0bK631P+Aqu8v*Nt+@Wg}b_B#va%j$_;c6EC+izX3o4TLT%l^P8`$_@Dg*TH#+HAB04P}2FadK2#6sET^ z=82#)A8Sj}H}0M@ZrApH?A^TcRmI=OLCHxy8)S_>ls;RNoFp7&Ve&q~rtNy5RsHAg z7k|(Ht?pf5a7d@&_TfU-2#2dp$D?9wC-B{=xFz6ZnpSxvXHy8bOO8R-%5!H*5_x|2 zR7L%hRWla;%_#Ij;-b`rCWo#4dFr?HpZrn&&Z?@I%;PEQrlY*G)8pUW=YKT3yk&Mq z-u*fG{-&v`Dr@cL7+x-yTIxHuTX4S4{{02pI%i}p+SswwbMD5;=Bp-doOeii_P_t( zS8hgx95j!cmZYbh(HiUhLQ%-JY(hDU^TDgRyH7QKoVxVNWs8RGXFE(U1*{X@kRf?y zMU9(BpWfp80peQ=|2YZXy#LH;$79p8@m{}f+i2hMC|z8rvh_!q{l zuUmgN>i@2<@BO~a6x)4GU7(dQ?PYwjlieaO4Yiq8 zaYAFG-hneuL-psqsGc2S<0rv4QS*4ah~fof*WT){`3w7`H!N{6*vtR8QstQx$6kR4 z!jt}4aquVg)N=p#j>uc$_ayxI4{i2KUC-vb1^<6{aj#SA?o%C7w-}DUd3?_EZ@Di^ zPU+N3Q@rw`Pna97d#``ue(&B}$91-KKld;*-RpJM`c+kpYPrLUF#R9N=bxMB7Hi98 z{O@j$-@!9;NmlHsz4yK=^JFdNNIhP;BKu(9`DD)CnDr;QD{iJ$m`>sA`lQX%FWvTY zZvDFY^z~~dER1ozzG(AB?Iymx_rLx;raMP1>2JNal0wPjLnlrjNWbr#ymdiDU5a?< z-Tz&E^|PvWKiE9|qL4g~mz2xHPLAJeL=1J=3N5e7O(-}Uy>fn?3u{}-`X`@$YdWWv zmy12zlF0hz-;Vdnb1iMVVhi5uMQfGH2;9B#C+B|nyR%h-zT9WkbAPq<&z$j!b+`GS zUwdqFmIyz%Q*HPrxn#qe%c83`D2po1iku&26ts2E{CQ?w9LlS{*p*D1d|&@?kD`qA zTZ71;x38rSNc`x(di2BA7x|CPkFja4=6&ncG0*+Pbk68j9#PHqiA}*1K5?gr{&l&i z{iXRIfBKV?f9BXei}wG|m&G58znaQ>-8B5em9Vuw zK41R+*>WfHPko)2TJ7r9pDJz3zW%Z2@!Xz~pvK?%=%^a|(W!HGv#6{1ai+N}*~I_t zk=jF#$|}=WCQE-DI9kNSvX=i2M~HV!*lO8c9<~J~2c$MQxV0YmV%i^ia_XVjZLF8} zZK^$eTO-fHq=HTH=))}q0Z+7QrS=Rk)o0b3Jox|c{lY)H(^&jdF zrWa^lmee~lHO=s^&~o()ns*-=9((mrVY!k{Ghct~H*txtl2>-z)#aMG=EK}iP9C0b zyC2=ic(i8ToOQ07c@|q$Uj1^|@Xx|hvA@>af3I|Gn7yr`i-+-*`~8+JTx>=v^>4&# zRB!J$Js@#(&ZN?Pf$XxUjb1)J{6+4xZqq`&_k8NJ zo$ndRZrpuaTH`~!_0-qOoBiTAI;@y~KmT@UZH@nm{P};yCj70HIXSK6QR?rq>j{f@ z-}_kk&-2vVm71TbxBgIhzVz?E!)|OAHZ7$&t4|6QY+JjJ^QGm{fZKmxCa!VI72E&w zC_8_>ctYWmRSx3**+tf1Jw|pZX+=YB9QOIl50hM9t*?gQtnTnP*J)JNvR~YhMod zb^4RXF%2V%VCVu*Sl1CxM*jQ)f=9eD-W+xsK?Z{Zl zFJbmuGw#ZDef{a@eBb8W%)GdA_jIGlOQs#FnUu+rRa8CUdPQ7C-=0&oi(lMOaG1HL zO#Ww^GLvKGyd@D4~JPGi+Az(e%*;j8)h|Bs=@ngLkePlKj{N1>bWx?9{ z2U#~Lyo>={i{}q_m(8fy!Y6$ZT|aL=9043 zj&j^rE7!}q?eMyMXUG2n|83fp`G>EVXYs7hZZE(8vQFvK#_b<>#T_zQ%Wo}S^mIwE z^;edrT$5fs|M_``o*fo0kNE%Z!(pC-2mdCs1^sY49>(dWU(UUC?XN#gFVai3U!AWm ztMoBG))rp0R>AJxO4BkUCFTo}PZr82r~eb^3D}!^;k=%>+^%^qUfDjl?_4f+takcO zC)S5c?F1gKte+!)I7ssy*B=ixT~qhWjkBuy)SmuE!PRX zaAK7WEWh)^{nO3tDCb~EoO8t&=E{b4cMN?_x|#}PewC@u1$Xa zi1Vr1k=Uy~*B?D?wr!T`hAaCg zwWx9B{n=7A_vvz5#pe|pduG1MPdRCNt-qsroBYww^%nkJOFeGX>@!gdTm1gw#M!Be6qwq!v03c-FuR$v;7V~ykA+ry&&>%{H6>V5$7>Utp3mt;mityR9F=i@A}hwcFkv3&gGXW_j#WwDn*J)R-~^|`Z+EXMS{!mu!vEa*wf~QX2DGVaoK`(;ADbN?(IhbI)yG=Xjm#rOt$jO6PApT`stJrP8P6v9Ht&wqL!& zaxl>|e3OId?0pBdD)twy3Kr@H+=#c!+eNWZPrfAY~IrhVMZu6MT= zIVbv-%3Gh{XER>oyl2m}=II{IYwv#h__~m*BH>kKf>g)rS5fC%bu3;w=>C_wvcrN= zN@PNg2RAR%!G;YFbY5_?9!hkWa7ORe{q5&0&e=$<4D!9}DlxgUYajc8`u~69{PrC@ zoWF6OVrAY_yZ4_jtADZkomS!X*^awjp7%?8&Nch;rQH`Mi9O&u@wevrRX6U?b-eNE zu43EY+J@ddzv=3RO8;E*V6(=3+zp<-Fa4ITtLmE8+;Al|^p%-$|Eb@eY-#4p7Ed}; z8ZY5>Y0B~qjAhr^*jv|^{W{(7``bRTfITm^3fC3-uQ|-QcHT*?rxn-QOL&&e?rL#l zs1muW@aE_Kr^Ou$$~|n3cg24&O)O5XphLWhTwFt^(FeYRiM{rNFfCpOgM^eX+x z)DQL-|G$5If9b{fzY|ZcmFu%jpVoIjeSOQkg1cwFRqwgszx}X$T6M|IjYX?l{-o@C z@cHkGSwFY!v|TZ0_t|F>Yx`Z>m*}1O`?^7V_orOTl^(U)#TOR3o#xXL$b6MG^X19E z#^;W|d}C7g_iu1~j!e*9{~vyv&KfF4Br&bta*=Ch`emk$KbPZ*@@Bl(T(fU=iRou% zpLc9ugCfrrt|)Roe%W?U7ZIzyuvB9@b{2M283uF(JVY$jK-iEr;&Kh87>y3X99i z)|eV-URjeA`l9A>NL~N0(knwyyEC)%Vvsxw|nlIX!iRi>1uz@$G%clD~cA_krn>!`~P!)x+lM%e^ULo z>;GM^;`j5t{!%}+O?Cg#pZeEM`852=YTH-rzCLfiq2O|>PoV;9noJTUI$a*7et9V| zZ=cNa>bNLZM$`XQcay*6Zz-v}cly}|1{b|KGi=;1elw1m>(-l=DspvlWJ-c$`ad>WF0K8p$hy_M`MR*2?y~N* zHCH}eb&PRiD^*(&vs?GE^s#n{g|D;I^n!msjN1NIX8W&KqLuSAsx9Vz*N8WJ!2b1j z-Mir2e<$8KKA%yN?!a*W`2?o>GO9Y8=DSv1TcvT|fx%uicACZBFXHFtGI5*~n_#$3 z{zyucnfS8ZleY%?CaO!CCNB=YaNRXc! zywbwYcool-vmKZ2H%+i^nLJx=MY7Pmyaj>V3c0l=_ul>g>=pC8MFHFQpSmnxb^HHv zPR(Bqk`j`Z`7@-x&bw^(;N+c$J*T%N@D!yoYNka_xmgvny{wOb-(s zZWxG3v34~k9EkAH;ATDKXfVOWO;qu;)T)!7ieX)=@5W}usxv=-zv91)^pCf9cb&1~ z-x=~XIP`ySs7ilGbDiI8Zz-3Zf4*=pbuth95qWpjnWFhI#k=*S%%93Q#jQFNweB!q z-*>Tle+-RsbWCn6wyosb&am^H%-i=9>Vi9;{)n6r$Z+4Q&P*;#Gt(i7VY#qG=nf0h z;*i3czmuBcN~eiQ{`O_)e7!gKme7w+a-Uy0+^>E8bNh#lSMwFT7wmhk*89*wsX{~a zz0cw4(_=&YP5$g(6dM+!a?P;vh}XTS33|V>_dR@jDM0P_C9hrVr{!4#a&F(e{;{_1 zf&~KuPrTzS8Fnt#@=Ibz@7I1!G7m55`u~4lhHujRYg@0a|F~ai_x}C=e}&hi7e(!A zE17)j#=ApY5~)4yl@odOW^R)xyk`G9_wDu@-a+?YyjUEONCYA4Y-ZiW2_cANDzs_8>{E>4~TXAZ3l`a3aO$&~lw8_}) z`Dk;}w4$FAeN?WPy?%Astj3|hp-?&EW5UODzbTWJ7$vVk*mV(aztl$_+18XG)y z0@S{XajLd9d1l%@Pn2Ra=v24WojrB3;?w4R(tZnOo%WrjF2TI6`17nVsniL_{_dEx zYgWXkTBG%oi`HmtYCQNt-tAt+>otbbJC8?~GDe8b;CSqRRjlQtw9b(wwLYwJ4?Xrj zzfeC%A@GL6H=R_qRn}|ftbTnq{=VFjUtcS$qD`7M8_Z~xtPs0j&A=aZn=yQ!5##BD zPs~p{?q``P5OObst$Yga>!fM#*K)rpE8^)|)w?=of0)X?cfVtn*2Pbp5Y&~W&#_+S zjs78y-lK2-Tl1%dx;s5h;A(C%muV1h73*WSWNbYD>|#%)%|iv1hfOSOH-9g7o8)S~ z=+WyQ1#XdDncI)&C0cc#GVSxZt9vCi{Qf0o4FQqiXRg`@>d;$>~_s{=wcRzc7>-ns!`b#=4 zR{N<7r6Dmwf%5)MYu5Hd=YUc4%f@u}w#|BxZz-KYs-3QbrACGJZN*DDv z*y=v(@@%`Dj7`5iE2r<2WO<&zF?}u1ouh^OB$KZTTF%RzeyXXhCM(NnVaa*6(zeDh z`P7T|(_YrAedy(%{(5ShWnuC@lj)`2Syx|+zrK2AeRzbOda z8&5N9keP{(9Al%YPXm+sWvK(#zAmgwO4zxpy8q9-`1&Z0uYb$--uk_>Ht6bg^H={@ z6@Jt@`fJN;F0R7GJMM4%)*q1kR8)oYPQSvo*9?;Wm1!KMD;LOD?n>^om(fcW^zvEo zwBUokf7&nmo_h|Zj28~@q^6xTjTZfMxw>{^?4M`sZyH!1j3wT_f@if2A1-?%Wxeu}XTmyezYMvT58r*cbhMT4RdN@b!SVRe z`wNZs$F1|{tX^XpHcd5dNtVpMzfphVzh-^#+&f#o_2d?l(}!YtyI!8wpZ>j+C-DhG z(6N*LhTA7<+txa~+x~CC?9f;Cz4x0I^4xuu(z@lG1;4(;ajT%uPK%%SSNw4BGI<#n z>i+j@hrqAk1;-=pj@R5M`v0SPahtuX<83{=t2LHt{nvgtSx>0q&0pDEucq}dW%Z}K zY&Yi0?>qZZw74!q|M}Fr%?Ve^;LUo%hOLTFd( z|6^avW!vtQJKnS1%aqc{dA({*L&Mcm2U%VS8(yw`$nZy};_GRz+0Xy4ia)=rE9GRp z{=9!2oTpb5{wgWnwC|jO*|PZi6E+-of1fYjaCc){vi!;$o_mWD_Z^c~yYx8sYyHL2 z-)e?$Q||uj4lU$6(!9Z6JoM!B_l!ou_f`Bpd^ljud-_9=u2N*sb zU|4g)@xX))oc;m3eLJ3Cn3)``^_=T|Yia{yg_+}pFU(>8`Vukt`bu=J;ghd-ydZHdvQDSZ>2H(Sje4%>x`s3S0~e{V zTx|dK+U3puKPPsl{{0*IlGj(i3y9$`KawW6YWal{RDQ`1NCYey;gH&J|qpE^YVz$u~U~ zF|bJxi40%$;bJUvc|bqgzgyY=7OdIv_th%-^!5J_D*Cr&ZE&dma6EKRslmS2>7TC} z|Noa%RCCdLRjgBh?En3375A^{A1;yjWw%nz*!=&Y?|Ydicq~`9nMeF#vQsNrC9DB$niqS^QP&$Cx^PP=HCAl~ho zI^|gN)_U7tj#EPNg8y@KJwC+TB-hLsU$rqcYR&(@vd6C`)kp95z1ySe zF|#mw+3SOf%e~9F&&N9yYiOOmSpBKu6@OEy`d@8-#2vW1 zRiWYK{%Ohwa*~|BKWI3-%z|a9()JS_-wz*PJ-l{0zgB0^TGcS4nX8g|LVL5eZ94bA zQt}{539vaFWwXyCJ zf^mH^482CiA4ECq=3Lf_yHYAX^L|F|lD^L1c~_q;yC0J}`P&^yzlUk0as9iOvY^=Ee4zQ>$K8dIKJmYKb5+n!}1o7-M7Y(3U0P`U8H&ePvs zUHUV*{>Hi253FDMUiRiK*&DHu=O*9weva!;fBencZu`E=}M|=KeB!~jrF0p_J#(A+=XV}Z||+0xwY!^gM@RU_pAFBq-@O0bxO(Yd+Ycv|_SS_Q{u2AMzjQszld?=5rl8G9 z{{6wV+$IaBh=_l1DpitNH0?8ApDWMqqlek-_ozxO|6Y1wPm_kp?3;YZ>s&Xtg6BIFGQM0q^Sb5S!E%-Zh8kr8mu>gBqzgPO_@SMbcj&4_R&|qMrbO->3e3ujc-% zE>l?c*3D?=?w!aInRi1si{bwT*ZD>|?X$dWs&@ole&PLZ)2lswi?9COX=9OD9CqX6 z7X2SNTgtD_nj7X_CR88hu|LT4N5V`yr|=MsTCacesyh@?=N|vgy!qUA8+YMe^~apt z*S2l@-PPT9L~r4-=xeWo-rLrzPLECfvE=T;zsvL3{-kc$Y~%gu*R4Y`J8qukY}Zx_ z`eb=%^@)RM+1ScZJI6x%bc~vYnOy6s9ro5U`K~kyuS;U$?A2g0I@%Je zr1j)ZTEQXFlDhW2@%8ezv^`(9XRrIuqL%UV=% z&ntNUhtLPt!_-s%UjLJ@LVoE}qeCCEW=56<)II$i;(I8@*-s?LTK_C-jrprhhZ~E# z`lmP67>Ou;xW9P9gDVyXuD{vP@G5)G7A_TTnZ@FBuJbUj&=TZb6tM4&)Ti}6|0Z9E z-5Z#-bYkW4Q}O@Thlfv?y))_6>8tVLlP`bp{NpZ>eN$*cWnH7fLB}vjIf2{_l^-@X zWNpidJh_Nn^@9nUvCW~Yb)3@}7l6&<2xGmay<4lUB zZ;I?w&lys?7M%Nbb?@FR77cfGpSep@v!5^X&uo9r9ChH3DDz3?J^5yr8QrfQdh4Bi zK%@036G{~FxZyf5!@nhMC|SM$wwFk8pSx@vr49HE~s7FXSSPR1EYeWj?0Jd z@3hY{%3rzq^GM4U$FL7mTyGyzvb%ZQ`_$j+tvhZ}pKW9uHmTK3emB z+3F9WYoqLRRy|{CXa3o=Yjx(Dh$H38bdz62E&X8hScRdFkNeje_9x5s@P4q4}8aRx{Jvi%u6ALMP1 zIb1CuhihD!N>w z;=%GZBkk28hZz}`<|5lfU2b%zv0T~X>s;>n|6-m0kw5+~;$v5TzqRrH*dvn(;t6kKmDmb|LUDh zzrM`dShu%4ZtuIjaeJp9KX0NgVa3VuFx|EEKJ(rO+fCkn`XnGaD>yUZ3Tu+qN{@e? zYs?Ho`<$b0WmqL7nB;7{>rjySP(u?d}4IZ}q-#vYI z?p&_<%rX8>YRH$_#%dAX!ZN3RmH2M&@Z+kU_2i4-`|I&q--SQ@TK#C+->AP?zLN3J zPCr~I^}lz9fk5ZsuJp?eX6{Sk8oN8>cAx2AuaYTp;_y5DAD1NE6gb5%H!?CFKF?n- z8YVjLSo#0!r}r(KBs6XL`oB*<`%Ti8@KxGmZG7n32DvrY4&8rR{z}d2Ld~>0=Brtr z&Ak(MVXkN0k?+yvN7q(eU+tOo_wJwg?D%=lJu>J26+CzLdGf=`Psi0?+wiUbSzFJ` zku3P^&nlb#|A(D!7<`Lff7zofzUsl2#mx6_MDNXx(Nef`<#d`vo1tpptVu6QJHGwj zX5V*QCE*JHo5&Bh6PkTCZF>4S)aclcjKi1D9kwjHb>yJ#*UV`yrwe}9sRsR#d{gu2 zjpSxCNlot!)eq(`X5A5_d+B!W?LEFL{ygka&W>zwB3W<<+Wn9hF;8-F_z- zc66cptXZi`_gTa#o@H$3<`ZJdxmsfOO?-j0;YP&^(p**?p$-?<+s=DocAWLx$+Rwq z11I0#O_XNb@aB#CIeTlfe;i)E%}U z^TU?y!aRvOr(W#ek}%!sUzU1IAIBfl+k{?$tF_y04tDi>(BnKpc6a^ZM%V)css6|5Ef3FeJ6IYXll#I-c|G{xN5 zq-;L{8VjrPJ?^ge z+h%1^m~k**=8AgNw{piy&A;4@RJeNi{v_+f$n-61mwx-ll{jOAcbg33t!=J#eJp%N z5%q0{wyH~~EM1$O7#DVvdFzqIC(lW|;{9}M`u{EAD|vn{eH6BD+WwezU+NoqnvQ6# zdbNE*L%7|BbWs7|y(|C7G($rt6oNUwz+9?hh{MwV&$3cf`$Z{#RhZ z{fm#GsAq%Z?S94eW&abE^B(P3`%h!l`A`2R&zir!OlP}ZTut2{&)r)$3!e6l7k;|% z^Ofzt^pkdNUElfb`ugeno7XcZ1k9Lj^*!xsJ|n|NmhvMzRL?)Od{bL_?;3 z9M679?_SRtCEe%Ef9~*q)%(A9-+wr1&r97eD(N?O9}#-xBl~sFRQvu*mRdRL)uKG% zVS=mWdvA(OWqJ4DU(Bo3Ngg?uzw7<7oE zOPQoz2k@5f6l~e0Cw4Z-dB2RbL~NwtBNaTolp9$aMX7RXwZ3j~RbV+By!s39OI0I)BmjN0#|IGdFgXL{}a4<+$r%L|EZUv`vNuy zPCgLNI`Q+d6(Vy?&g`iDq4P#ZXc4o!PxP}OqaD+~U*?#6xM=3>eBM)B4^)&Lws!Sj z*>_91%-}%T7L(%<50Y7)ebSlq#&>DHTtHFLKE}3ZAMYRAH2K7fSwael-}0_J4m#6s zoot!!^K&2jtXa2=xGz8aXZld>&hZ`Jm2M=gdi)^7Q*1)ejZew{kKPNA{`0~proKRa z!Hqehfv>LLblY^|Uz~09+A9CJ{VYL`;^Uhnb@la5x!3+WTGuWA;kbm>0^a*=e+@5} zfAG91>9U_y)_iMO_L?Qt|D?q4UbVUX{fo`z^)bo3+Ke*Cw&@*_&|o&RIN@H}wf)`A zpa!;z3+;%&`Wj~ z)E7HG|GE2L-uz_scPEOcX`T4Nn_HEb79H69HQFVa@8R~ymJPy>_B$P1=;|aZ4`1$cShc{C{m~FoK#)M6yG3v4s_oPYN*WF&k%p_8}eKVUmf5KMb z+4Wb_y+ktqw>;Z@Z|D5@-|FQre3smvd{g%SENemWA4>0zuRkA?A;o?1|M~PMU6=Fk z91Y9)AN2gig#5eD&c*BBt*-t*;f24#-mk1jJQ=4=FPhz{P%3}--BVAdW7hpW4xKX! z&dt8<|NPyVgEd!!W9w(mRlLzPNtk8-eG6u_3yh5dmkutp?S4 zu|)+B?2k{hJ@xp*e8!Du_Jr0bvWVAsH?hsIePDlVe)n(J+uylT+kRFB-w~;3DA-;5 zx0UZ;*VXlh3ROa`v)vWrNPhL!Re*^_L15MK8517N6Q9q#Dr%<-_dDwe^SpBQ1v341 zo3+N}XvSO9lTS?V?Fsy}B4^_2oBF%!UH1Lw`tCAk?)4NIzvaJXT;F&<(%A6y*Z&q_ z=PV)$S9*KT59_SI`$$z!@#cKX$Lfz~dr!XYxjBqa$G^O|N6z2T^CQ5zhv}O!$wsV{% z%kITmR3(1*lyUjbr7x3qX~ykMnzs6D_sh7oNzcFkO`iI?wd%^WtzXv1uAKEfc=_w= zO&ls8`PU|tG#0k8sAwPhT4_-tw#rkNuwCYioTTZ(oL38yc~O9hUtC`-^1@6{9>EqaXT5rzQ+e@AlgkF|X{M9VEA})ia%%4-^EMMve z%Q@z3&so>xD_(az%o?44Hu#R#G7G6Miw{ijJndk-aC5*!uX`_9)#eoa&=nBoWI3An z>ukcjIQM10n%8{{t+wwHy3L^KaaW_{>>f2CHW^O)x%q#WwS3ro`psAA3T=L-{i4nG zyJj`+d#AeDj`Q%rPmkF8_c-6Ho|s_z@KsoF&k|>+o$jtHXTAS={q^+X*jSg zI%`KPKmGsI`iH{o3vPY$FBkHA_p@QOX!aj5$3WJk?d98xZtX9Sn4eVm66R^PpQ_u9c%a*o%z_Py}-d>!%hYiv>I zJ?)ySubxcXwfcwiqyC6s{XhH5f7a{}*i?E=L}2w@&a0sTOXBadXZTmIbb27fcE%<^D`BAfYweuYVX^WP`m)H>^As&2d5tCqi* zWmDVLZ}G`0+#ly|;dI-#z;MpfePz3MKi;UnCVuh-w{=gyuH&fQrxtxgx9a*44}bfU zA1qnBlnbo$+5Q@IG`!<#-*~~twmqw-=V1D3y(K@-<(IOUef_<&mL=zJ{O+&sR+gx> z>#hpDvTy6J{jY9K{ZucutNKI$F0rT(LfOZ&Z^)i~1zD?;2%a`en>bYbqB%?0lu>S?jcvU2f@} z?kTA)9VPoe6|4Qfwvs83Ys#dxi=X_8^2peD`cA$778my2XXdUnTjKIFYt|K$8cmy- ziw~4Z|8+C%+7rZd>56H>j~5vt=N~<3h;#a$@cPr4^##)|3tayuc!YK94}(`xtNL$g zX4t9kt+Fjw72101>8+=Ad(D^_C1&iH7pKRb7G$^fQ{#=i4|%ez2kwN1hUYNnFTC!5 z(6CUQM?pa)&Y|~q;Xkbp(+(+leeut4dKa^H%Jvug{uqR8RX_UulDppN-@96RJm$^k z>DjV>dc_UFd(TXbHrJXntT}XX;~BOr-Ua^^H9c1dY+6#U6Q8}}&5i&EE%Bg?THen~ z|Jr_u*yM0GG5`7NJA0R{yChTL@ZEdEPSJm28KYPxlZ?JxH@o#0Jp5nnv zVejt0W=}MzlHfbi@L;iB`lFdY47g;^a7-u==&M%#+F*A3LiOZtr>8{7wXrx}eh{-^ zTE%-g?t^X<{%`JX{1QGCwx)o)tvVK<|`BL z?!UPYgPavqB^&K0XCB#SGu_{R_53HXC&Dk(3kKcunfTG{;sx9HO*1|8j0Gz{D8`EY z=Ct?zqkYF?%gX%1N0;VS{tk)874`f5)Kz*Z1|NTC@Kc zpD{GL^mAM7Pw(?jc0F6@ZTfu$U*p@WRg*4C++$i%p~KOy+$po;g&Ze?jP2n&>Raz| zIo;@dz!%T>YM)(l`2G4t-}7v>Sl3_7{uo)ms$+WEcIVD>t>F6 zO&06*vKE$X`D6M1me;o$1$|DbKYG@M`FShrN6W3%x5747oQSPyI{P?)O_f z4wuZoz0SYr*I8Nf0yEti+xN_$e?djy>=O=^{L}Nzd0IWIS5+O4t@!_X`hy9cZ+@9c zDcsujWIs<-;>o%GS05f)xFB+M)YDT_uKaH@jLUj+`~QTO!PB3MKK=UVX!+gM_O`im zb?U#qKasxwVbBx@Uv}gBA{jFLs}$Q;vPsl$*gm(mQAX~v*Pf)8Yx%?f9iGG@yrbf% zms#iY;NPa^|Fr$$^*5gUTN@CueUtjPU$5@}<4BnPW1;q4W`isKYrk>|DEdr!89l`) zgmazU&sPDjpPkxpOQqlG$0mjLr&S;Kd%v&KQvSB>Zl1`or3TMrSNq#;bh+imW!4=s zy?LLP_sfaK(_|ISHe5exo)CFWyGCVqzu(+v<*IvrbKiB9TRH!~`i@Ca6^A~2nj9y(*gyQpsQJ0gWOLh^67_Njf@ zdi|l7oUyZ-;rAD(EB~)O-=q|Crsyws;zu`^_fqC10&i`Zf|%6R@0g^z?@CJZ-@W#r z|M%86hXoDSPv3v@x$DQH`*p3148O&S*S>a0e=_$h@5bbc`qigj z$E*@;Ir3ZXVdDA5mJ`2KQad(Y-WP8zA)GS9(Yy6nXUMH*|Mnd%`mtd4k24ZV2CuGr zY0lt#Tm2#Y^jj-Cfg_zf!rZ#Ce>lIswmdttX7iJvyZ;{j-uGx?8x@`ZkX?x8@W8>2Od5{0yZ=5)7 z`jwSe+?IW?i?W%&XGWt~a<@q4x$Jo--)k#6ZEJY;Fplxms`N$Ie{5>E(Nfo0JO5UR zbYXXR^WWD=AD@)+cFfu_tJ~#3#;1OnKNH$bMIJ8rJ}py)w_YTE>5rbW!qU*)sW16r z-tuU)^*+omY`hn!!##J0RM+-~+|?Tj@1(Bzx%p-RPt+@C@12v@u8GXlNZXXfee~|n zfL*hN($w;I! zs^Us-^|JQs=G8lzy-nI5=bO5^^2T0`-Kkxn#$Qv$G`~Z>=ymw1wyU|W_g}nxWZyJPzr~Bw zFYyu6BBtZ#J``1O{a?+RYwkmn7`}G@BjbSy=tn^{-}3)@0pGHBz6|upR}@c|FicWbMK1GEI6_u zKkGJEYvS9)nyb!M;=U3Ro{Z0K%RD*$O?O&vd-e~e1*}P@r{*-j{mkjM|Niacg|S`t zcOAXkbvLtYe;SUifldmG=ASYe7Nn=mAU>g zf9*vsvIe};H=Mk!`s20_4L4V&Cue!Q^Vaxau|wK0=d0r5<-M*k-4f|immALWvf42$ zd}DF`$C4}G=U8x7&p7{-^+W52Lg_wJgFvA?d&+pc$|Q)ln` z{H{OlPwzTRay)ak{C`IEAFT~F+L2uwy-O<=Wp_@#w)&c8ZR*{RQ>RCWH90!8L>Vgw z+%!=9d*?$}jKdPOb-yE&Z#JEnX203sf<%F8R`j7N$%g)_eqOVbQ?EQq-_2WCTy$;E z?|rOm1N@hME&cRwd)d}dy?C$Zt9Jf6_G88>rA;;5jofukO$QGaFf5a0^pRs=k+9$v zaQ!LlV$S!2A;U(EpHD2|yJ&vOkr_2-*x!7K?SJq@;p#POr|I)H)~R={ceD)Gxwgi; z_8TYv@!G;}Rz}|)+RPaxLJWM(2S0VBF;+*i&PjMw^dxM`98ukypI=n(mo_-4$a=tH zpXP2cpR6tl~1qGxYAZ z1nBI1F|qXgwC;(9+l87x-nTlvBr1hBT4ip@YmP_p!uLa8#9WV3iC_0$I3#NCx%GF} zGBhv=*z!le7izd*f8d0D1XrR{n&9p`Y!iM6HFSvxty-lA0{Vf4g^CA4-Q2xxe{AB< zn8cv{j8`PjZrve)N44`392ZQRD7Y+n!879xp>tekUs_OlW7*_z{wF)8Tg}Ph;4j|b zdzCpVi^Eg0?s=k7|7`CEKMUXQ6`NQqZ5RLRU|8nVEjO$T7e3hakhiMu&zy^1e>P8y zn#OMWbzfebNNSB$;eR#v_bNSclYbXBhrKY)|KlhVt-5rPSt{4D(=FNVcaL634l31L zG;8MnFI78Vg;sBU_kP;S{pO)7GD6sY*eR;t*qd~Wu|}rfU~#=rfk*LlZQhNC8g;_$ z+YTiv8jEen66S6?xYheW=(;M0EQj_jHk=IX3|+UGWVqIu#Ld5YAVWf6&8wsvT)Mg~ zcXEwhCd)opA{bGn!64Vjv4qjbWMa7erYcQ~LuR!N4T}=H7=O(<+Pd^pb?p;jr(OB# z_urZCaF9snVPg!mc<_3UQlf|Z<;d@Dtlf2hqpIFIG!ml$@1usTztN6=~?|hw%g}y6Zcm5a_g$>sx4Ca z^S=u{n7%Nzzh{Z_8r5L&7aI#}{ImWFtgZaNHulOP=??;m&BCFRKdF?k{oLrJxAXIw z5B5#!h0*Tk9%k6q%-O?sTJl`OIR@q(LQO7`IS=Nw>UQ(Z?f4hJYL7wyb8q&7|91Uf z8Fwv+|6F^zN9VV#w3Ot%q`U1m^P6(6oW6d;V8`)>sa3^?+i&+x`u+Qtses|kdWR;l zkdMcX-gw2n>F8O>M`!O9PT{z0oBvPc_`j9w{N=yYY^@dl;jBN2b?NSRXZuzy7MgPX z)s9(TuOzGtn_$5xZE*YEl7@~F%bxi&I3Dg;A|iOGY0-@a#>f8(80`MMx)yfwl?3b5 zGP!vSC#Rj5z@X(Bv!r#-F57o&Uklm)7AU;dYTfBlWwU*r;?7<5E#|wQ8wNh&HvUnb z>+4W~#0f<~M{=_CIKQm3Di++V>x7x)&onu3UQQ|0;#E>rt@=4_NTyzc*D#~hC6U)EP{hVDCP zKF#i!vGH%$Z#F8G`Y!}36f@4RX)=prc(?z}#`}*x$678vaxl2Yq?288f1+VX`SJ4m zJJc=C*O~CG-+6^I%i{FJSz57TOTXs5zWTazV&?i)+Vj`fMZKO{@TNtA)hqP<=i=1L z<)6MtZJt_pT*hE;+@25?*}qHAZ`SOq|DmVs-P`EDqq^kkjBD}x3nHH!yw99$^|{3F z__>Odg|SNvTKgxw?E0KAiM=9DG1-|#>MS=$+)6{iZ>oQ`%n?4e_B8jhs?A}k=0`8d z?cHs4%`4f5JMI1kEp3;7nZK=H#CGkP7PS2Kqu}>y@b^P~o$K}a(pPUzq|M+dmx&NP!%%tk+;k^r2@Xlmy`hH@2sT8xat=r=Bi+);p zh`83Ax&4f%c>n3IyV_KGo28ev{4Pmmt5Fr%pP&4TmG`_^UvpAzfplH`m2d0TX6?+p}-&D=trWMb;mz(Mgvb z+(VpVQX-L9gBaurLI+p{~{=D=$SrcHicK7>_A0ygrm+bdc z+sO9GxPOOT-5EjCt~0jB%FC^1n+vD!=b!I%Ox80^;XP*u()fYIW_e$pI)kwsBuPxz`4}bN+ zMNP20q})1XlgZWGQkH-9EXQ8Foqjd;kedFLBRV}()E*cv+Pv&>WcWvG?HwoICUCKS z&lJhHf9*-w-$~#9xIa8LW$%@LR(^%=q&9WloRrD8&3zUppJmPF@_qhySTeLfJ`p~g z@SE@1qEEpO6dE1QskfbSJ3FIXNAixy-((I=RxY2I*g|!d4~`4da#A9arq!H$wNxds zy=QXH^V7#)UM-Yj6}8rKoz#A)Us1|_$E(?MpM6%pGcoM(&$FzR>xCXJ{4`-!2lJxTGC?@j(J z7+SBW_-6Z_P2Zod&%dKzos%biqkH9=3IF1Y?j8!ivn1+e?w|Qa;!l?uzdQKOW?K3= zf6>pLqJOIA@AxsD#a=Z`@xIn&^L6%;dg9)aQ!?*7XR+A%$4u_kr+?}*#G9B|60|Q= znbxo?*j;tmWF~fV#tx>M4X$;2`7^WLiY)kBw0i!Rai@h#? z-d}w)f8IZTra#`CJa&4QUwNyabGB37J?YZDcyUgC8y(JL2fj^7W~}50QTUTmH#{rZ3V=YS>mHV1yH(_ihcd@6hOwq_BV-R_=& z;@cg6i{JdgKkHO$(JQ$r94xnNThe=$)y({8=BZe}=t-U9{)@98KH0M7w075u`}dMu z!VelVWFP-BXU*5g7Ob0pdfnb|WViL__1izaf70n9lc?|9Z?r?f+B)SOBDum3?5aVoZYfM-XQ6MJ5I;k=0XMc$e@*H(Cj z2X)^$w$|s}=4NyY^u>LoA#aj8hk3tt7%Qf zit_jSy}y>&O5{!3eqH0F2Iqro>FU;jD`xVjel=Nh>e`o@wKi|3{ha4AK}zYT*v;R3 zt38*!_fBa$zND4cC9+7iBBT5L#ha$TXVrS_;b)Y4!+iVw*UCR%IrUO`z6zJ^^m?YS z`~J2c*~%st7f$rzoEG_HR{TS!1bh1_2M;ic%=~AO%yxRR?xUyXrWYPRaj87qc1Lcd zh0~{ithmgD!kvCA)%R`sn6sqXY5GLD{raNav39G2)!v=6Gl^?Gv-`@9Hfvf;{8KBAD-xcGR(uUv*}F&^>UJxgX;zuc;- zdS=N3fk*O_uANtkyRCZa`lNRy4-{3V>gLb+v3}O_OG@&g!3Vm16-kkL`qy^S@)h18p$TWFKJ`{MJ$tg$G$Qtpyzs;`_D7Fgp7?l954(*2lOu22FR!Zq zbFiK90Hegrf1(mw5*7wMo3?7@bEh(=ms}AxE_YQL85%8xG8fwPrkVNdDlWJZx}=)j zC@iSV?DrN`QxPv`zc+tQvRs<{K5k)^?K{z9ZvVbosGH6RO$=)ByjAyYdFz&`o^#Zy zRHJ9@H!u+8J*YV8(UX18A2v^AHYz@J$NNeW0=>b=RW7q583#j*EB9L zC`r7^^_g@avq#HJuY$nV4ZaI!KiK&F2)B*}r<&X2shLlgto7zJ(Oi7(m@b3FGOiNt z2}eZ^E#fK*eD3%0$D2Ql3~TJCeR>_1CEi=bobX;wBgSG?1$V&}g`MnoJzb*#F)LKhUD{CXki_uh2~SK@VZxCY;xpJ@IUZnCx-vmcjEmK&G5J8m z2?=f`rb7i2LX?y^I%1wztXMT!-!xEbz0qskqgPg6xE{asPEFcM6I^+r7VbuG631Zm?if3Ym~2%*ERB zFwr4mf|v+vFJrQW$0A|waCx~}=f&P4H0<<{WDXjL<}g>%<_PIOJb`cC6Rufhq}oSDq#D&eKP%ew03YWZ9j zIwclzvf`Y?d!Z#U2VA17oG(nCBJ^RuoSNVR9v7+NzQ{SEJiqParFU(zimKuaO$=?F zCvF-M-Ec|U@rXK8M+lprNX&(gN*gWzdzbs~I~uw_wX;xO%U-S5T6=+GuJRm-h@4K@ za`PpB{GDI?j6MFk{{8>Hwz#XAU)SD!b6PhqyIVn*)hp}y&)2OI3D+)dvv?R`ylsu% zWg`zxW?_Yb4M`vPr!QjYW|v}aU|`_abDSl|-o;u}C3L(n{9^t0Uq{tfUtL?X|NE7l zMXSq}`2T^0La#6>DbMouc+(z1mGPtJ1UH`9;5u4_MkX&t70UH|G0&-0hg+-hcIf)Ze1@ajzymC=rrvJtjYU z+k5k%4>NLDT7NA4=z3=HA&K>I`@^T-iha0Mcnx1`$dMoB&5T_$m(ON!^q23Ob7O5- zXvK_p_ijfW{>MM;zW%gZ+hGoj$hprwASTr(R+@Env~ETV62n0fF`^ZVoV zKlZUQRj=OfDE>3++NybftKO~JAG7M&t^=N|O^q7*HtD=}vJRnnvqKFulPt1++*!kX zvQ^`nvWDG|#h%eEN}O&OG;$jQNGlEKcnLA2zc)iJokjZh2(u=dfjBrh$r4ygT-`3v9lX z>u4YA8~@}JYwBSQTb4uZO-Wu1ZYuZe)z>N+?GJyU^jz!Lu81Yete->w zOXuv}sjo9{U8mr?V`;Bet8g{QZaZ|(Wl7SuC;zxQ?;dJ;)p_TLa=p|VhcD7^*Qfk4 zneMcb>w}CX?@cE4eGjg$`1kR?W=r+k&9AmUKfH)9{#E?9zOERn=qUab0k>9I#?9z; zj(DdaWw=(cV#=oU&a)TxY!M2XwDbH5i<2B4C-=WqVbtnr-~aPcY}$j|_t9V9Z>ih4 z`bg-;yH{UdeRVKU+g`u4_WMV>S6}{q*GgF=i+}%{`Tz%yz{upM zW^Q*aR+WBf{^#`MU%GvO0E5nm1AINRUf<} zWO4Xui}FKZb~DT-2b>qbGUa`Yma_Uy4-dunk*)y`m;af+>ruqn7HdxP)AG$fUszK5GT@@YTb_A!6S8UyD_I{d=3A4+z3#H2 z;zRQXX#x8VE=X1up6?d5H0rW%#^VbT-?)C9KNS$Gk#frHcExGe(UMO5>KJkND%8F8OADT*d!1+WeLA&XVJ- z!RL;aw%7PX?AM%npmk-AfJ5#p2JQM)m1mpZwd`a#%v~-1bQ;=l!ty`1Iex-}eF^i+dP$ zx;&`PvWZ{scIWmC_XX>J1gsIT)BBY$@u%#kJ-U()cU#}t{_vOKZX2^bi}tSGx~JvB zzKVqS-@jR&ypaswcziPJ&XoNZqoTv>Htp>^{kXiZzUd+Vt}dxg<`cJ`Fk8e#vNSgq z#@CsspJtv>8$~ zjMbJ$6m%>{fBE$IqSTde?Ld4CT_l1SJ%7$LCcSNW(UQqK2SU?$5s&fK;4l+Rq=!O z%y)cy)@CzF2s$N9?RvGFHI?PZL=~|UAq+G9MVQP3BvpA@tG5XK{{ClvXM<2}_`A97%7uEWkJW7JxH5$j_e~G2etyPeh4)u>o{0rc$0i6q zPOJYA$=ks0m>{vCo9TdYp_Sc~yXY)j92LE9<}J$!X^QyIb=0 zR)3$Kw!19$pk?mv%B@?LtrK8bV$jg8zIMv=$h9)R*dz5W2k}okmt|A&aq-6_BK=I;OR-{Knkf%jTY+UoSr z!M)Oxzw0lUFy-{TvroRPKfH~t;=F_J&)MG&fBsurldwIbNc?5o^gqfMZU1dZ{r_us z)cTdLpR6lcbus=^`dRyp&(1SF72M3eCcgH`xBn(TH2!b8$n{@z>g>`r`{VywC`mP6 zj{Ki6YxR>mHT!L^2{o$MM^@jGT+6s{-|XM4Y_lSJ?yAWhJ3ZyXK}qE=ENoh( zJu?4)+7)lobUXRnxYmm)K|+Sk8h5?fI|kf4&kw_y3>PxfgOLc7E9M|Hp$f`@Tp2-&iH( zwd;>y#D#f*bw`%{{pGOZ+Jow~ajES|7k#3xItH@)tSb9FmE)?XqVd@qKiMRlPV;Y% zi&&bzc$?D^ZA0%RwMjer zIQuh;i)ISfE|P9f{Mhqnew`whfKKf5J2#@jzh~W8{?4;A>ifwr|2S%%3F?*H z%c(g2WA>g$%zP{|YwKB5(`w}BUwQnX@!$FnC*5vJSeR+O{VDubQ{eIqC4(2Qf6J%5 zvT>RGX8q|H^X+MB%5A-OnI`5=i`{#7{V7|65Vt;qq8ZyS&S=`udi9}jhzWo0qZf@m z*Y-}zeAPH@PyAG^h!;n{T(MQf zE4K%~FXySaet7+5gY3T-iIX$O%yZYj7qx-MNPLF=S zS@7nk|FKOGm#~UJ`k9?y({^l`ekE@2 z7rV=k+!%ic-2Z+v<=n)DJn6Hg^rob5OS$t;S}ae~b~EP!?Iwd|T5l$cPLg&gGxeNN z@nG?r)1P)LET}S>@#KBtHwih%6}PU2$ZmMvl_45s_}8FTKm3=}_1Dv1?5ta}`u&Ob z2L-!a?~B(w{w{lRNB-{pYrZaf{qJDwm*sc+cYK^?yGY&S)4x+f~5l?_+<8!j>G zaVC_pCi=ZNy++WO{jkT;imy`)-ml3`-@w4YRqi-Tu6-42{X*TiHRa-^r8V)d#s7X= z`)hm8>he|duhu;OcD3#Q2mk-mzrMe|Pc6~wcWnRCmp1npU00rY@I%kUZS|^ln+EyG zA{8@zW4|Y;2YOHFY509KyLDOnWd`4Nb}NUtS+NhNRO(tWI2uSRw3w^D;BL@KVd&sJ zrLKJG!Io8GwkdzVt8)J6o3rED93d8y2OR+kWs-8MGp|dT*cV=QHrspp;F}JE{JhTR zzyGyh$BigDUTZ_Ru4X8*@uJ3p_BUtP2M_N_zDg5Gz2e17zJ z|BAOek|dn{m$jw^aSQeF&#=9%yYHX=Cik)h8A5Cg*ZjhS0yzyOxQe$}hJwE^^%P#nR^wGu zT5iN1Ii6KndxEYnv<(*8Yp%-^C+_-h-;e&StH0VtTwJ&?C{E2umsu;))BEGk>+4Iag+JcCxFjX;@a?{aN!|Q;$v#() zpW>hCqNyczjIHgIN87c9J0G4raJt{oc2!6D9tR&oE+NS^ehlAYGt!+L#rJRfd;ehc z%1_~Hs?zU&>&<-dGwZ*j)w}Fz*C$GKPA+|ETU}wSIYC3TRhDU061T4LjE!HS19T3C z8buT)ZM6S+SD(w#(XEN+P{Hh%oQ1`$+gp7?ERv>WSESH3)Oar z178Vw$t^R^Nv|sv=yK^ldMuxN)rvJIw#@wZ`ahRI!KJB>%`Xbo&D#G$rPKZ2Bh7+I zg5Un#uD4#HV0{1h|2eCF`nY@d?2F8b&@q{sdis2FZiEqQNXgUf#amgU!mW7|lRBDq zcZc#;X)~mhn*6)}AeQ~kVRnsG``t^g>^-afU14caaoL74PU~qm{tEWYub5n+(64cw zBj@lL<;l}p_-1WCdp1=dS3qW})+?8egE|wmB45d?{XO|@2jUIY^`2vuTyE(v$93(Rxf{UQ0da|EZ1<cLjO-G0&ZS*t%E zz2wR4aA`{4f2TWLqW?b!eF)LbKG47sICGuw!q~7aJGeS`?`J-K@cZXNJ~qdfb+3>8 zeBQ^_cxhd4vVnpLZ{5vw{%7rilHt~~-nKqdOIiJ`a^|wX6NCOq_U+d^;@I`)w$A#R zST@bMP0r4o3e|tMA6TvS`MK6@`O*XXAEgIvI{5XkaOkH%pbia*SH@~Oqz$_J(j0lHDJ-c7m*eA>oesT9l$wLWG zEkRx;>Fe(0Ee#V}m?Lj*a%W&r88$LVvA6i!5 zHhfSzkzRd|sr0%#%M-kmGdp?oGpwn$5Jm-~k>Cff$X9}lvRDAgHMwrX$D%%|DutTD#e+FfbJz6q>{E|HOuHF@;khCPyUgSy~RdE)kl=vZB1J zZR5A2@%oGQs+jp2uRrx@GiOgi!_4Ds$umt3_N2@Tyt!>s;PV^H)_>OTHF)f1o}utF z_WpgtJ5#5q%>QXyl+rE#ZHMVC#VIpa#vc>sn6*4FCSS5z^!>@Ohy0T#%<3rH-hcG; zu`VyIoQ0oO$%~!R*X8ck%U>P$H-2BR(EHnOx2iw?yZdM$U+c8mMNAJvAAOl|+f<_d zb^KnpnMJ?;%%2q*d4KilbE>sb%RjheoM^53^i{EWpNZ%6+_&E>CUbbs7N0l8>T|Z& ztIMqg%uK(XI=9bF=H7XKZoK+_s}%~(`)v>Ou*rOW{P4)ULazJv;(Al%uNTY+_FS~_ zzFe5>hr8Ie)=xq(or$p!-tT@B`EKg{=S8)_buCwX@3(UXU5?jiWmy(n9~S>U>g)QmXL3#~ z{ae=C%DVeIN5|pF>#`C1eArH3-7u+R(+z`tqUEAu{|kj~9=LR|NcVkMOOO`p|Jm`a zQLA-7c()6A%Ln~5eBH0m*0eon&5cW*r9a%>FF98&>V8UCf5*v>-@OZ?SG>_<6%eN*?!O+E*zlb*I! zuknj_&;0wDgL_Ho+B5b1^&F&S zmFfTXt&@FO`Sl_1dYzW)s&LMr8WSU#>!%+t>+I~CCOhTl9ltpp{(btouXR2hteW;W zkK;(MlFRLb#wUGGJBZ(X$x-p)qo{rN?Z0=Xm$g4tUteCgA@uau+VA(@TdMCma=bff zk>%;ftN0E*h?e8(h>Q3Wnk19I=^qCx^Mno!<{1GqGbZp{cNh7wV>!oq*$EpM7>J`(yy@D_1$9-SJQ((j9%yvYlz|}OH^{?@qzx(&_H%{Wde|+|Ic}9)$ zreuko^-Ye9O|c43E{jgr{UMmv6jbTc#cw##Z;vF0SQ_v1cKKyO$w5pX6f3l*@3G+D zFVKCzD)X1%yR|Vd<^+9S{cop!QGCXv7c+CS+V%9WZTk^(?3hDo$i_=4+xYVHe;V!C zusrJV?&rUbT1-C86L6YY#2~VVwQ!jIzvc-GKFM5v#FYP z;!8H_KW#d}a<*)C>>tmAHabR!g$ktBi%BN-YIv@+_heag;D!N5l2hB!;1B(m)aB*W ztS*{9j$+vKLoBUHBDD92g_i%L^sCDP)toyvY_zGpwUlv7&Q00a`j1QdboS3>T;jo! zWgaNSI#X|F>DsRsc81@%-NGPv?LY&gr0;Ukn?cvovsYWzKhIT8-}>XVJFoMVZ0V4F zJ$EKGUe$6}O=HSF_`uPHQ;Fl%M7i(T+Fw^Ksxyr%h_KLm!?mdT3?w{A}YJ2{Uinu~R#e)qQrw;6Yu)-kmf=vA6Uk8mpABbRi zWc?`iTkF+8X}fo8>q76Y_FVNazOuab%GXN!AdVx8=2W~;O`5l4y5GtNQcO>K5|=$- zIMq{ra;u1Y`JUtxzizSo+4V7{zP)zB{lANXwM_K_PhPybJd?*X*Bs`N*9;`pi>seSqqJ2S+C%rBk~{nviu>-4Mr*VaG#`m^5Z z;F=9PEmW!*u6WI5W-#3FVpqTpP7hViZ&`;G1pnObw1{m_&Nh##NSEcwaEbQd(Vnz`@hUBXd-Kv1R<4CHbdB zM%VV9|2qG?X!li5reE71w*GIe>;FzBkNKQe&!+BBQ0tK0xTqueO3G&$LAy;$H}@^A zFL+*Nvi`Y*+KMY*o#*V|6(YIn`m24sm;Z8ZF=Kn?ES=%}W&wi$+vdxkyfS3W4_fsY zWC{LIEr0lIefRc)IVHSWE4XW>@!h+fHG%)3{xx33+VAx{w2uEdxasw_L-KQ$r|y%Q zGvVzM>61GGe(~JeWv!?tAI~HC`cqw&n#aj3mEQ0ab6MHfpCUC|US0dK`d6=Z@H?*G z+q^c(z5EdM+_x>#NqlwF$%37%=h9V86jlo)Ny|ShnfYAEW#QF|z|j07IU9eMemL!1 z^!nuA-`5wF{@Pl8Kl|>r`i4C$szO`68y4^^l>M-N-Hu0{V(Ou>Pe0t@Hn44DaBf5hZ#1!{~*2L=D9oj?#oYp{hdX^{(#cE$Un7=Z4uMWlL{9$9)7Xa zdxL~Yq2hn;2^JUhqi^Jvye#Z(sZUTl$=g&sEVYGqR8Pbg%rO z^Xuo=^h_VA_x~89#SYr7Yc%|>_tb2<<*n-pi^IN#%=yZivRR|yrf|}J+lnpc+z(fs z%nlbmu9_KGVE=2n+Oc3!y>-{KoE9Xld2})9$@j27j)JM*_p@2638XO@YSuqF5;;Z2 zLtbKbk;&h?*QEH@S8xcbz196~rzl~<84^B}N+RMG_gYd5Dw+>y?Q@!xVH?wC!*)`5a?&KtfDhGzF?(aurq+ZXR?|x$W z$3-E`D`y&ca8Cd9@2|)+4mM-$S4tZaUjEYg#dIrOt^WB><*KGf()KozYk&CMT+eKD z;{T=@H(rE@{%aF%j_RJBwes}U(7e2eUrm!5uYEf_#jZW-Z|GM3V52QfmoM!xa(?^z zt$20JhIPV8EeBU+e~9dC{V@H_{yQ$xUu1q}g}(UesCM9O<>ux3YnO${U6$0V=jHj* zI^|{M(QgND=KIOMnEXPv)8TEP{M{?NL;`O857O42<)aw!fAzP$Azw@4d;T9jEI9j! ztng<3zdu4eKC3+qd%r$=uI~RH=cZFf*DqXj+<%gpb6%)ekpYWn^S!2Gxm7~1)|{Wb zxsq*(fCQIG;W4(=kw5B;a=-fj?6(aKOI;Lf*tOv6zZW}N);gzNy6uu!VS8AamHAjq zZxhr17=;^)mvGjeci~yGzB~SJ?TqbVf17SzIu2PT%*v&;R7T(Oxh9^?#nu6UX$| z#m3njUK=~xRZ`6_P2tV4)7q@;%9WBJuFn$v5ja{qu`|^UCLas9n6D(1%-d zrq^}9+82L){hgoxw%4t7{q|~2+Y!r~dwxu>@E5-%-siJgq;j!---3JR3x9sx{qvqh zW9!nh3HvP4zdHo=>Lxe2zyHW0`%zml{QKP}pI>b(N;tk-VA&$e*d3R|Ed)18)u`WD z#JXsz$Mwi5=PycVlx6-rSRwGE`RT@;_SY2DX50>V|6I#>Gn;{;c26|KU}dQK4DF@n4O5)lb;$i}jvnwUfzw6) zziZk0>w4)UO+8!f|1R&Z>7Trn`AvAkTY;wcZ&cf_ta&ldzx>}G;i-&MjueJJo-)D5 zZh6+%x8>i?)zHC%?Yd2Zn*5nz zPG{VNW$v(NPQSOVMyi7I5aTk(P$|*J`ImXhegD1PzxDr{W{x}FHFeq#Zme3Xn~-F) z_`Z|Vm;F)+_cadP)_wV~@Nm%)ZprE5`*f#V*>}9>tlCk|HGvcB3-9!&^DjOsv3UB0 zS&!IV5B#iH)6tq#+2~qrJ0rwKylFGseDYBl84V_ByE!#X|L)?%#?X`NBCKy=C5)v@t$TP(#61<6aUwcl*Zs zLX!eN8t_GCZmG}Nefno18*gvp{V12U@iOPazRoW?xRcX7|9@mkzjk2G)>$8{Pw_{D zXrx%BhW_P#^-lWphSattF)Kt$dX6!CwOo77fM2He)4%m`-S^kJ{8Fu3WBzMzh00ZC z*U*G+*_proC_fH7`7)sQV8r_mE}046?|6inEAk}m_;07>a{pzFJM-38;%xsN;#npx zXL?yTC;wEl*sSc@$xpBRpZs&x{rzXpx;*`RQ$^oaYWj;MwnxG?atrR9U!%cx;l??= z+}_U*jSB16<|qZO`?bt?hv}88HO?DXzB8_@wN;W4>)@`Gwf$InJtHT$>igBIuTgue zzVF_dIsM|?pZpe%d-hL%b^XQGXV*LS-kbdUpMd0nce~4rZ-4rkr{uA7h0)%P7d?MX zytt%iTh7nNXFoe{DtbOsU{ZV>r%T7&L+3?!WVhY=V!I~n_4Oq?75cn;@7Fg8x*tAn zm3>gdy+YvlME1_fL49d&8ro+#lrT0JvK`QW(RKgy_67z9t~-vitW43h6Q4*BUoOYAPU)`do1;^u9iF_XP_rud1m2|64m%VfC6@hs7?>yb~f58XvVUz;MQu{{l@jwokQwlcB#SwCCMU-5Rgn zzit^TwlX%nV`q$E>wj=>UyMVs*Xpj!yqm2eu_r@rupW@v_f+jl)2^zOHk+;FigP7> z=RH3A`A)Q#?vKykR%k7G!Q>+}k2%WordFAIcEkU_t-HP`$bQPPDEASw&o;fjJ zWP4C!+Ke|3r`^c$@sQAJuXtF(Je!Fjp+bx2#v?JtfJ=3ia?+Ybd;eV*W3&oC&E$3U zx^Uln=cVQQWA=wnzq+z?_1!;r*LrN*BylU|k3*R8lzkryeLb641se1k7aU9NdXSNG zVACFj7UP|cBIgbj&J&0fzkSL3Q5*Xzm5=9jKD=Oh+~UX8)Mci`DX`+2bwz_m3!@hE z+0*VTD@-mO-Ru3DCAYQtY3MU=vCT5F8QOxJDoWR*i}HU?O?qp(j*DB*+%(F_w$R>k zM`Uc#cBh#gd>+!~Z5lrgy4;gG-(C`?&E-(oc!zrl%kpNHFvYFmSX^6svx&x|HE9)3TL(6|YwGyEYw)-gf%mlzCh4+*`8o zZl`{Xi&kRa;@5U@%m*9}rh z>;-cPgZr~@E7MK6JQZvkI!!M<*``?&`Z?Kko0-B&GqITFO20}5{k@a6uQxfilkt<% zeOU&vLo<)x$=#q`U67^cbNIKI@(V2B95<*-lU1@4mF&^}EX6Ti>%qGu&NDT(-xpywJL?NdL*EC;P+&q-7+8M5N3&_Px~c z{k~`F?amM}u3QdDcDV~}d>@`&Wn_E0Wuu!=g`%nRy#*_bR_g6Ls{i^yyH90RMf>q9 zvKj4+8lN6r)8G2+{U3vvB?1qOZk}WOw=(?6Bdyf0!ix_!E;RqL*5Pw1!*mtWsXY_lf8$)gI6*IFzIe*d z_nV?pcHLg^^{XH26-(XB19LVvDIE$n>Jkb*{8e)!gXYBpj&h6t-M#;(-9o`CKFKg@ z-JLp{D4EZO-eH&BHM?S;JpSu1v3*LGjX|we5{^IP9E!G$}MXAidzppe$KBF^66%L9H=(;yfrM4MMf#Glzd1KmeZMf5dGCd@ z`_7eq6G;%T{Gq!$B<_8p*Q31us(%z*rYASQfBLmDC4ITug2ha8(%c>YYUPwI-Tg-3 z^6iJWzU$u7k2|;XwczTgJ?balACy&i_eH@ma@y`gY>V^WAJqE&N3-kXxyl1J8yHd! ztaxD~a?RVTaps=g)#lf|0#C#(j5pO2oyNeDa7$LMah>5$b%Xnl*KIvMYlo|#EHBe~ zv1#YSILd;TOHRLgQT^hDhrh*K*phuiPs%Ua{Kiu|x@hyhvomM^F$!$?Ew8`gqWb!m zzdKZfceYm@{GRZ8`rE^L`l%b{lx%MJ7c9QqWE8H-7(^q|HEI%o~zK3&yY@I>r4_ir|G9ERV`;si|flW34VE_fJaNE;|48_tGQRj-J*n4w3!7-N{Bt zK`3nB)fY3%Z|T;4zp3{7-5$Z|X>0yC3aAxDKX6z5Fz@}T6WiS~64t$4{bKXJbG4dl zCp1|9udmv_dsqFBVBJOgj~A)Oiy!@bze(p=`lT=R>yw7;a_qy?)Rd~ z|M%N=?~YdLuHW~yLe*r0UiJ2A_YS65SXr?Z9o(LObGqV=tP7=&GiQC`mvV7=s1{-V z|G?yL?-mCBWA%{JJvC|hzQ}E*HWsUnrSTP*Ru%wK1R4nzrzZh-y^x9lj6Gfh@EGxzf=9P18Yty9#a-AIG)2C-unN{qxboj-_8BEMX#-5X^(Tm zF)3AX)f+lj&lhTihPK;F72SPx!C#{M;fHN^u1)hYiq;BW_|tu@uSaNekgECVF6S1e zDGjoJF6e!JQ>}F~bz{Pp!>z4qCI_oxQ-6AR9~X3t>g?5$+#Ay9W0#lmDtqt4tL~qE zB>GjO*%nr`ubc)ci$D7dYNAyX?(%g@3Vp{HZAwpZy|SZdm`> zu4ymWyFY&3X}gdo3hypXVwOB;x_U8-`;WF+T>=waj~+90pDwuK$I8V|oK)Qo=Efuk zwz4gJFW=r5vSz`QPL=AG#jmEFo><-UQssR>`JwPT4;P+`+Uq7DAQzEv{zbP4kNj1O z4BMx_(*NsU7fNt4nzVMIsP%~>$A3(IB%@c-Cc5I&cfCaSizN-m)fUL~_x0b4Y-P2y zGVl6(Z^5NA&!c~`3VuK25O%I7o3rTC>GDV3d3WFb*>dMhz1eq{-)j3m_^vv+CGJhz z^y*W$Rc^l44wOCX7qesY91(vxv%evy|9*RTbLEkPA9LgHT$_Ki`@{5yb4++TekQGq zpKf|Y%+7E1=V{-Xeyut8ul?D-G7GDnwp!1Ri5G5FWm~;J#x%^w*(u=3n{^)|a)11u z+qL^>3g@fVJ@fX4?X2t8n(wN-_x{(UIqIvKs!WnMh3%iy{qV@;-*ti>exj@P&sy(Q z;~4j4?n{*iE#<2Jw+L{kh`Imy@gwk^`JDOvPg)Jk?H&gAmb}@1{>S%KwLd=hJzZeZ zzU;=?4o?hK{ zPEhr(^{uj76^t5JCa8&Uvoa+*8YSqs2(unK*687*qR7eObT;Hv_O%|B``eQBK5M#r zul7$|y+8WnJl}uu5C0YaP<`{h_8`t`v2?U(zl zC7iOvk9dXlTusly<-7;%dV@`EZ`R9;GnUu?V_?11yCVHz5_{dVtP=OUx1l%PPT749 zHfvOJh>I6c1^fv$+3dj&8os)x*xx$KI#;eBS>f{^Y%-y2>KX z^M%$fm24^5s>dU{b-fFp!efWCQOymEN+CTOqC%`qiH-&r48&wudqDSBX-RN%F&%Q4 z;L_7ucKVssyUtn9SFJ3aa3||hxbu%$ogZs{C_nyX-|=5_bzfL{^nT8z|BfHO^jv81 z>zBLZ_gK9Ce%{EcWg>4z#qZiX#&2o^7GA2I`CLck=;}?&cijrfTpoLm)n~UZ=c*_B z{}|q4Xc2k;+>%-Tg)5_5;j*tM3J$EB!jOM~x4ZP`>-(SI9Nb^-Q20hZf`P@_X|-Re zZp*>H#~2=2eyy9DcPO6Y{v(c<{j00qbOn9*w%X&~+r3uLY(J$UaC>*k z-%B0L4I5-~xYs7~s5s~+_e8CkoBhl<$o6sC%3Wt}h5ycEZeSFf`A=4@AZDS_d!c73 zpBGw`1bR8ysZQxzp-uO1nW%QRfT~*K(;P6@E zd~oB=l~r>X6|PKh6VVoAW=v6t&=BM5VtN>`VUCv|CyPsB?(we2dryR>t*<(HO!cX| zVNKeq63h8_{=1&7tN+x0vHtty<4?k3?>E2t7yax0tJNRxXU~_rRde^(r<>eQ7yXkH z=DSx@zuzp*UG2P9#+_}?JMX6cy?LxAJFZgt&eAohGopXJuw_^mYU||ca^(wKUvTfy zFaJsv&rg#2l`gaOo#K*{Q4hpcHnwOLef{FKzVz7DYbQ?3v-vlV!{>diy;k~$dmE0f z(=pX=3A&xH!n4T7?y>gY-TA)v3N`rHVy83D4sTw1?9ZCqoE@68uep{iNn3R1!>e1> zyJ8M#_?_-pdr?-xr$-*2tnkPxLgIYySvgHiaqvE=HaMU#2fajo#^T^t^-ul79d z?YC!v-~Z1(x_#x_vQ_mz;5|0gjqPgbkeWXLl-PR zY56ldtLvA#t!$Eiez~6Q!=t_bK6CC>e;FQtuCUV*QEcsXOjd~S64PpJWlY%N;vy(jyfXUKN^L4vgWnS@`lot7ye4B z{7N!#i_$oL$^ZVh1`9@okSQ8rLagmf2?rt!#H?DG79}kRQ4^AEjSLTe^-*MXaH#0@ zf?Jt65LzasI=`@1cdawZ!JvNf~Lo2vIm|7hfu51E`>*tnFh zPTm#G{_T&asod@jD_-8ZB>H2s>MnsE<3GtcpY}IQo)ICva>w&KujQ{7#e8sm-dobJ zOZxGWBz^OjY_;>I_g1`*ooljdqNRey&w~oOZ+pJZoO^R!2J0K2wlfP&78b1gbbu$= zvMT%i&#J(j6*s0h{Fz*-^J(Ylo3G-TTHc9Hxe>#eC~jFGlQ!j6|H*e(w){z1$j#%* z*x{*sRAN!VcFDyL%NY+aFmQ1>&QjpuVy$1O6MJ9$|H=Qg|EK@||NH&dq$u^R-~Y$| z|1Y%u`>);m_g@dnxqGd%M)AH&{QA1?J4{EE55#hOU@;N-p(m2h#QZ#KgD!{WVW#dw z3x0Plkvd>i#_CYJ^?||h09)D37nbZdTdcMJgC-YC>HA%i`|JMldL`}D|KhVhY~$&x zL7I{c^S0Kl?fW0Ux^C~ey)iRwGqm*G?z!EFago2HF!f#A&wvG`m-N0VzM8typ2Lwv z!$r&Ce_i}z_ZgG>>~-cp`WS3e^H}Wu0~QUwrrF+0ObqXMX?9gR%~|5pSz`MnPrZKf|CO;KQBjeW31a5?|VO+q=FhM7#838vwHQeSee4t0yIh@aMZLjR$Gvl@KeWPrC&f-z@mL_i*=?>nzLJtfwx^nCnocnZ3 zc-7auzHs~a@>^T(t-o8f{p@}F6KkT&zZ}e*@^9CgVh&~gt;M%r{$??a(@j1ip(Cet zHqAliZCA;6arsk=R27AtxT2h-v`Z1vkQWNWW9yK=h zNAKae%KUDQc&uM~Ox5a+Yr7vRYrpTUDt;#Z(O4nyGfTHk(hX(PH=GNq&I>;C-QdV@ zvoOMUMajA=@3~~P=FGUKkvT21X~Fe+_uF^&Xn&WUeyaJN)yXHV>Hl8tzQlVe_``}5 zQMx|`Ppo?#;;~-sMNfHY)awg7D{ij*=zadf(q|g#BF{cVcrX8VHnC65UoI|QJhCp+ z+Uf5B^Jez$rZ&cZYv(_>KK)DMOr7LA4C)SFLOD$S$>v4<=>PTb?bWs6s!(PIZQ$BUWW z_u1OIHkO^-_gF?N%F4t@&T2x>DTY(bR+(*HAO zw#!OIwvWH(m-k;WN$`?-ovU)laQS4mPW$_ZCJ96(3W#q$Iwg>na`@@Ew@o=FJWmqm}rq!?e=QOI+uxj~J+ZbFDNr?{l~+ao5<%K~0=wEpwYx-PG0 z(A`ldGf|OEUj5v3=e_KQV`5)Mx9%-A`)kLg_xJt!(y3Kq$7}z@%3Vu9<%zfP?sw@6@PI+YHO9v;~B4>s_#otwWw%!DEx3|UZ{y_ow@bml}%kC9!jVFM?b14 zv}+eKm-+m^RR+p;icU)0_-N~Tf!{}Y z+rB>tl;ieU?e+aVQ|FP;(o1^`8k&p`b~a7qw_tpJ_~4-f3_Sr1$~NrGJFXV?%y+BG z^=P>?x9Ldwhcy8V>Rf!x!G$I7o;F->sn4=wY!F^@Jox{gT)bl}3VBzv+iH z$8mbfn$-AqJe)1xB>DXK&0i0H+LY|5S`bqX3-yGd8 z*M?7AQ+&Xalf|X@xyB=Wm}uMk@r%e=ll7A?eETaM^+)vCnG2qC`MznN z49rXaTN<~a>R-|N53ia38C>6e_wKTF*FT(nAFpt*r}_EUmwHi3PuXm+F3noUCoof@88DGYG?a$@SE0# zCKjfs%ZK|z?!W%j?c^xbRLP}n>(USNd1F27euX5c*f05Omp?a3;l}7%Q%Er64>P#4L|X0i|EQezI?wN4yJuWoaJliljcosc_D9q2o>M#(A}7SH z)%jZCwnX{ur>oR|r=_$?u2tRXvemyfML5x`$=~Ss>EK?~ox;hwFT+bsPe)zQRMlYm zoX+i$uJ-cPf>Yh~U)4X={t8_lqO$gK%}zV}*Vkgd|N1ho^XjXw&-VUaZy73SIwx+= zlHLC{wC?wKwdZKjyjFjmEtTggbpJ}aXK}Ls=_Pl%<-hFXxP7Zv zxNLou+rpPi)?b?aQ0QN_`rcFyLCvy1ub!U17HZ+(Iir8}z8brbu)l|F_AXs|@am`k zKYM)-C!fA}>!{6y)T{rTg{`+#tiF0_^11(inpEZYpERrAS0k4({eQ9NyxoNbt@DbO zbiO{_x7s=5Sst@~e${{0E_}88hm+04f97wu#xF8l9~kO&=IQZ7-6wuN zKPO$d*1wbc|3|qsS9WrVrfTK=UwGGB=abOX2Mi3{>5j7$I<#18uj{{vi?@G$cfD`c z)2OHB>%zld-(Ow3CcJ)q+1|39;cNG=Qhua$b$@iYYSoo{d-uy&Sa^tfO>cOwQ@Q^A z)&~vg`}PPMUH%bdGDGb6gDoE(8D?HK3qH(sw(r1u%?py}oY$UgI~HUfwDOpE&9Am( zHyJkJOul^_o>#uT4?SqeocbXAtkVLMfXllNs~$W1@aE|gkE>GeeyLj+A9VEB(Tg?$ z@9Nd});sEdnxFrEYwG`^*iWZhU1m)7n$OuUwCTzjSijmvJ=J0*sxo-nst_|avx&3x{xZ5fk}G-^-%wq5z{#wpAB zw0>;sZuc^?;h!oTR1nq@w?Ha8*gD6{?p2jRfBnDm0|yQ@w@%#u=Q_7cl<>4I3;*mB z)(bO}_!pfLac{k`D%a$Bo!7(iyvg;m|Nmw8>c3zkGmG`d zbjQng4Z=dcZcAd#TD3KV|M@Mm7wd%_1D6~)v-CO$i6BVZXj>f5iUC@YUYgS6@BvJ^APR$>~RqlpH!_ zcqV^+c$9vuqV3C-?+vCzEVdO|{`Hi9_`ZKv_lK`L#-@6}W zmjvdpT?)0A{Bm)p?d?l8dGXr3VpoC^MSK5FH#lTH}sH~qcgko=fE z+*@_#%9qh4ohy4Lnua+ENq5cq`|k|bb=Sm&e1Z?{niuBO>NQ)gZI+l3C!#QK!?YzD z$zi_@wfI+w{a<9?Vk)-YY5LZx{Z~6}Bi4p%M|ia}^_!Y{@$5It*4Siw`|i`zaPh(fss4i$jUK z*XD}nv(JCN&wpxl!IMneD{z4Y5zX1O`+L`79Eic&077)Ws}xFRpG7YBWoPuiaV@WSw)U7cN1S~;d$y| z<>l7&V}5BmM`R8$&GCFHktDJ|K*`EAmt8z!4dbz*7-+%I_R(%V&s2XT&`uP6eU0;4K zdgXdLZ$fj3TFKYcAGh~7{n~N!%IQ?Gw&a|BGWqP^?GANko_zW?(bS(ZLu*nZ4=^`p!0PtSKu%&*?J@otIz&lYw$6_4oe z57xWfnsZ_5m*;Al9JNowyr(35 zyj=alnI)}SNjvx7O3uHj7iG=QhTpCCoqT)!)mCvu?wAXULd7jzckMsOKl%0NEP`2BwO>xX|u*J{+K?l|*&`m42$B0K8tuRj=HR=I3~&qj8skL*T5 zZf>sHpVXDy&uS{(kSDqVRBb@gISQyL)6-|E_OSKj8i8 zbE(^GvoMv$xrtxY6<00!{bVc01Z9@ESuO6Ktr}*J8=nTT3ZH(KqN!)DCm>w7=4ee# z0e9q`WBsw+J#FEydoC2WZj7ELcIw38X&N_QC$sLjR8qGoN_xtTx6|hxiV;4*61V%u z!u}GmC;QY_8vdH@^ru|4CqKR~`I?u;eU{2?yXSO;{0!|>oxg5Tey;TEZH>B*`hC&hI~K(0h5xmj2B@lyc&1mJ07`RsDK;M`L_T(T^tUz2Dz{ zoqXJ5HiwhVqwfu}Xe zlj7wUDsHendlu@{@NR2Aw`kN%xBvD1>Lqio`t5V$e))Ef6W6*AFIE3m{62VVlHoPOx^v_Ds zzY8Liw>YPG7{8ykCi%>n>c|-lQ<5HU{j!g1`}IjDl;xG!YgfNf3wZ0QQ`9v?7xQAE7sP8 z{gvIXxOlPf+sUhbzuywhb#=Gus&8E8vnH4RPES!R`gQKb%%Hfx-u81Rr`|j1`F+;9 zWaZj-p=Q!`zd25aY=7P=UukpYe&LLMueq+%WmUIym_G|z8oG9Je21FN<*UZ3wf|a@ zJYUxqJ?`(aINANy_VrxXp!u?yd-lDb5XoYl`D50vllvV-%J+9qKbNYu{8!bkx!T8z z|Ha?^mt_0j{q?+cou8IJ-f7Q${mZ6*DW3N>ODCzF`*VzCck90N;~sHQ8)mvr^skpV z<$uz+C`vZ&pI*Q7{jYtKugO++=e_*@?R))&_C2>V`u8v5Rui_Jpt9)i(++o~$-)0; zX{(hkWnkbaaGa&c;l)}Me@XmJ`kp&;b$@Mtb$@m2%1c>O_eboyzE6AK^>hFKMy-3d zH}=={?0Dh0y~#WCuKo|-&(hM`a$?2G*ZONWZIe)Vf93lFw&~|p_HVm%y#C7`JJFn! z`7ami_}u=oV?v3LX6cvDi{JbV<0*Qyk~^OFy41SA5pQdqHpW%{a`EbY<+48Nexr=v zhXe(!!z_zS=4Uodxg7stH{XXj=Qa7;W~Tmo5&HG~rXPO7<_0OF-OVnKeWamz<P&2HZ94}TrYY~w7Cn^^i{$-Hle zQ+b3Ku1NZnnJwF~v*zT+U;liSqK_mA+*=iDd$}Ot|C%__KXKMmluv%Dez(-+RP>MC zUyHxK`yu&#{m(PUKk8jm62JHPxws?$bPg_5qbG6M*%@4l5_~@&x>J^KscgL-d z+r47f*X*6LQzF@+j(>ZPde@*|7wL%cKcert@9Rs$WzLHKiPO% z+}5@Ld6iH$PRVzDM`oJ7I%!+S`Ff|q(^Gqcl>Or$W<6Se^3|((!H#>^P4|gsGoS5a zHT#>yI!|5s6-B@PR;|;%wq&<{>(bb!!-1AF7EJZ&opOCY-_(QlGgVCPznNfqtLo3M zyY*~Bhm8+4vAS%T%DhA%cj2pRS52PXl~@<=dF>#J@r=Ya`pvM6U+t9(WLK$dOZDG-a@xPBBRdV{XMd@ies%t8m#V#a zt&_eQtZCDX+v$1u&eex=X1eWr_3O8ysf@#3Zrkbq1DUwbw=Gufoy3so5EQ&J@y_4mwQ!etR-cNv=*Iyw2-@W&sRe^ZHvYq{c0f>{>k)l z-sildDSmOQtiXJgvf%6u{71e=>i;O%GuOjtV$$3H^Y-fm9{=F#(IRIt{p%NxO?B)4 zJAJM$>-<^QePz>K=Lu)*S)WQ~aM!-BWBpjZVzW z&7Y(`TmRzQyLaxbnar-dr@|*v+`s&dhx)5^y?^((#r~Mq`(CIqPy6WAoIkba=Wg}w zQ~$rvvM%nap?pU_#~B`e;?h$haQnRy}L z$eGPobk*M<)Q`FFDx~<1d7=5Q?=RNgEuH>|`#;C31JnGEbew;1^m1LDn&d(LIRfUU z@^h;H9{$|1Vt42>iFc2-K3e)c`Ekg4@BIfQZ%vm^uGEsa7bL?ud;a{_Gfr-`xUzj= zo%$wT#gxjO#R3WSwl)1zJ1cj#ZoFx}`je4mc&_H4bM3yvFG@`zQfiRuAeZ9 z`?sO>{nK|w%T4NPgg)M>)P3-sW%G=#pP9LFp_9ruT3)KmEoW7m&-e6VShD>ZO_%e2 z`=Vw|*I)Gg_0=Qm0~S@ke^kBRH7{xU?vFRM4c=An`uE%L?%k{3IDXo%KcTHH+hKc> zH)igZA8EBm1Fk;(|0iJ1&6c|k(sfIp?-A|RY+Kc4|I7QIYn%RpQl9G#2_ENO+bsLS zyu{GmCQQaD{ZmC`68UWtu|O*|72{Rnz8ot;A_Xi-S<{;FNnC5 zJpJq+Ume3~(k*{quaZ9?(RO8Sq38>CslOH~RnJ$e|E)|qmGN$}XJ}Aiz>TP=drzl_ z)a?(t6l;CTXaCmIyVUpJbJ{KakE3+*=Fb&pgQ~AT-qOC}ZJ2HE*MDb{HvZ!{w)ax< zhmd@k&EBh6Paf%he_w6;&D!O<2L%nLB+X*>xv(we*xB5~ODgY!URcDjN||y*|GM;j zq4uhObCRy6Z~xV|VCld3n^|qePv1SZI9>EdBaATA;vuM-dPZ!@` zlI#nKiL*4ldcQGUdVi5qhsne<$3Cl`?%$?y)23ub%*~rF!AfHG8^ia-T5R&qh_=yR zAMv%c^g&DK%hO-G-hXhvRAbNiZm#WVO(Mtab**s)PQU7E?dC3Z`0co2%?|eLRY6(x_q{}Cn7z8Q@~6<^c)o49 z$9DSck8U}+YRUBK9U5BeO{%Wn(C*z*`pA3c_TK5wCL2o>MHEi^zIkPlR;KF3rYgUm zi|&4L=2pA-x%r!y_I~&J>#--Fs9s*ZsIt61`|8st|C_qADh^HlQGWW~n$40udFm&> z+?3^@%`+)#~b;U{&}nyv-R%B zx{IG5EOF(1n!oPNdgth`mAYP&s`mU_7dx5HRQ}cXvV8{yPye-l#1?tK{4L9mU8tt9M4^qD=Teo z+kfrP<7A4_SSP{2z+>b%OR*!1wSHk?{Q9!Bcjjsf-8yDiOG<)7y zIoyta@}lG zll0!W=G|-=N2HUtCY;ub%a*^${7insA)UB&tB$?M=%FF*yh_|`mN_NkhddezMDkK(!X`Q>+&V!cH%@n?C(-o5{4uV?kH zG2fIYb%s}2>i;LwQ$w$Ymc}$#Uov}s?dOG>RlT91D${nq<@H`OYtqf!uWuIY|G?_i zW7T1`{pGGZZsDi*A6OHSGV|U3Z{a#B>kpOGsBB$XF?HSb-SeJY`xCJ2?e4t0kMGwD zs~rjVh_BV2C@y*HY4!<+g)5yx@12tI`egS1uJ(n0GwV0~&b-uWVI3eExI(Ul^=6Cj zngxw2TFb+)*?h^!&9$&OUw+NM=j03DxO>`v^8}a*+;W4{!*=1y)N$Z^Z(z7R6e1PHZd#OJL_|o@#tqx zJQ|Vo`Qoya`LECYf9M*Cg|^fa41T(t=!G?%G7Pwzp~Wa z?5^Gs7qjV_O-H0n1!EfhU5VdDRziI=#QxD8i4bbezvMS%6e($%5KY-~H_58LXB^Q&LlU+KL( zc!lzV_rKl*>J$~1=_}iF$i933@m{NH->fft7I8T_|9&SjC!uZmsZSqPw2S^Tl@zg= zude&=@5zK0hBEV9xKtz^9lsv@rts*UmUwVl66cA&iFT^Yyzi&AZvFCDHT=ke_IB~E z(+*d*%4KvhPoA~SKUaSDM(?his=21LrF7$4_Ok8LwJL09 zXUsFIzkYPF%k-*)9Z8$5?kWGt&pdm?RMbgv{ex|rZ-wRl{@~;I^jhw2wNq|>_uooZ zIEZ&FJIMZU(p-kMcaP7R8ocb^&T5DKPBIcdc?#yNnrY-~IlC$Tc2?1ErE4aQxk09x zJ6i0Q_xGrEACEL-_8;dkV3BQ;C_6FN?t8$=1pcb+tqrw1@?N#i zU!i^LZ2jInI-dHqmJz1~chCQqeDA-A@9n;2*}=Asri(0oUkm=xe=gLQ)%^D#wWQC5 zCx5=*GoL$7^l@C&HTfLwgUfHNZH>Pl5oP3J{@JUk^Ls$mHE|;qNz)ap?+D*jT;?+U zzVL1R-(I5c6lOG?UE@%;f6w_9zb(VhDVW(m3$L{Ei~lF~AxG|~%I>dKucK!!cDK=p z4{&m?wfgz`dE)A5{wa)a7zc!nNO*;Z8=Zjj^S|K~taEhke?7A|!bC$ztlq5P{Q8#nyQ)wBUr}_ZCuwdjpG1|$%NYWz z+jVDr(-w@Y(XF1g-Lk(|^+uXd{lvXLyuasv`7li~`|8P8dmmIVvHcOK5zyZC_4Lu} z>u1N6uSu-!Oe_?d5WpuH6!7@j)!uop*^X3fxVQM>+Q^qpYqndt|G#)@ZQ|7F{|_CB zQnzh*sLj6dYn+Nr6Z^lMw+ohh`?g_yMnU?DtnjQJgFFB4y;Ip^7qawX4jbb^sn!@UF8zn_cvet#|ODAi*>g`3_@7hoVD8*ZMiB}|7&)@ z-@WfV0^Ta<-8j3X^O2EVXVI@iu?G8pD1X(J3Jm?XHNx3xnt$Bt+tZ)kTd{Y2z`pI~ zGrzBW%og?kXqTSLj{0rJzwv!_5*{FU6&`t^qo1*Y;p6mQI2e(XNu8jDw761>-5&TW4Gar6G~{XB=) zn>TH6so-|sc%Sq7t@+NgKh@sfF_B~9V-H68%Ri@w&dt1k`P{7dmqFZfY?r^6X05Gb z;$30;e~Fem@2Nd`3-(-pma%>-=dRr|w0SR06aQnC;;m|5tkopAB`*)su?jhsJ9oD`^_DM%JJi4dl z*1o!o&*0j(HCi!BZgsyTUiN)0k-GcJEAi=X`-J9W+D&E$UixJn*wFVWYIcOAiGcai zpSJlQ9Qj^7l92*Za9Q|^6|L+qYeW!@UPWWcd{$@pU$sEfXXJ2m# zzYIpHb zxAqJ7$;3@C{53(=ar1?`l{ZT5f9p=pc%J;%x{3Sl;Xf0$Y<;!GdE)w`eKL7_*%oyA zWYt&&xT&i~)n8w>wl+R&Z(V2jzFURgzHFb87ACy(j=dH0(u*^a8Q)E_F6TJ#;Qozz z^?UO?yk!+tUre1dZ~8T>J5v(A7wUdZJhbFl?S>y^Qx4zAFKa5-*#Gy?OT+vfDjzQ& zi*Hp=c|LdYG?~Q~t^HSTT&{@e;MkRFk?M0U;%VQ9|Jz!Zb6>I&bHC4|{;+=knw5rb zXYQolIh1yLMp}ISvNnN^Wjm`H58mp1^)BV($|H5D?{7(#-#v5g@~PM}yDr{Kf4QsA z$~CD%)AH@HzeV%+#gtBbu-o(b@!!>spE^{ZU15)2^sSHQNW0#zUzOim{)xXn&C-6y zdG7k_ORlZ^&f*&Sx&D5Jd&-~Hp8`J4S@L~vVa-nJryZ_ zXQzl%^Z$IH?4HQ6Mk4%feQ)}v>VFG5cJD89VyXRe{=euYsf*`)Dr@%ZtG^eFNH~7* zTL00~zY8C2PS_Inh$;2&J;Rq>O02te72D=#z2DHetA{r#(!KV4PlSp6qt#ROZ@%x? z^fqg`V&eAuU)HXDCdmFj%1xQ^^y_`1v%2$kc|S_qKS^blSM=|Dhtt)x?6wQsEjzto z^@l`TyXUXG?(<)d_`Lhh&h2^AwrI2LsXu*Pttc-nUc~x|SmvjB+LCK)@18&ZuH~z} zM-#hb$<|-nc7OlLb*uht?Z4N{_g~xob>D}phkEr=W4a{&Pk+CZrF`%0`a6yb>|g3H zI#X@;$l{r|{o8}rZq}byV!f*uEwAVh+p}NOtGe&w`?!SIDJ(UAQ^g*=`*ApEM(g*- zM}AxGuwP_#y7=n0%l_9LwfFV?`YuskmG}Aa-FcCwPgs+rZ-e>_^f12m>jrDuxUq6(*+ePrg;uR;KXte&H=cMBOfPsPYk>e~S zjxN^Ph5B*pZ|%RdPx){5*HvGOqNDa@Z(V3HxzGI8GG z)mQ&t+Hm&tr@kK^*Yv0VQ9LO5)$+~XtR1VrR-cagTlF&(bp7l5t0!ike8=H+@1b($ zoGE;l^-mrCQ~XijYw^C;!o5NZuChv>66QF$!RztsEBb~~1`a24qe}kEKDE!PpMKt2 z=#Fhu=Swbz%g-+zeQJ@pLPx(r`1;L953|Z&YwP5d3M6>S>V03$v+HKzS5vkBiz~KA zd^BeKGG|~s3HmVL{wZ>n^QiGQecl6CLBSib+0I^PAJfBAmvs>)R+cdWD%Lyp&rL`7Y-`x4o* zTqsteS!dOp`R}8+O7FN<{+&r^>cp+ZUO! zwQ1e#uh|#tHr6F2?wKIn$ts!K`?JUBzVEa+rnNcWes`Roe6jlWTF#!Y^=I5Ru2ub8 z(^9Q}Y|gQRqQ<&q*{*r*$s3nH4m{XcGe_ZzdiZZ{6O(ZcP4i4%@;J^cz+_3$Lq&$4fdUAOxeMnJCS2v zXij&{^ma+sbvu8v->TWm9Q5w~F6A#(Z$0J;ep(x@ANBow_^(%gBUT9c?yY!ovi$xH zmFY^GmL{t|UFPo4T<}WvDvR>`IeEd}Qfa2;zh@~+%PI*iWcj=8_&Ft?gZ($Y+A$m! z$T28e!256Cm;1Fl;v@~keVDuE`gY!9nqHM3xqpMQdAK3(>BZean&0HQo`3pXexZEE zG21KMUr(fF9bdR2_OqI8-Pr8h3FTXo`&%e2q7u8DQYhU;5p4xkuXWyw~(=2X0 z;y)NEk#ue!SIW7x1oQZXu0MACJ^!5N*kswCsfSM$R@)woz5k#*%yg&gb3vA_3$p*t z@jU$-&$2muTUE4fu_MP9#k-r&9lm&?=lbP&UXR~*=3BKFJr!9tOQrVz>-)BM6K!vO z-?ZMduKZE_+}E0!9B(rs7j}1jF#Y%}+1E*8%f8pV&1vixUqAULUDKm2+8QsCtbg=- z>X8tup!&tX|1>3Ux7lBJq5RmNkAjo>7s#Z34dp!eSE63w!Gq%khd<2M5zRuSr*4zW#8>Qz!WErmXjG#O8gk4mbbd zGEa7y^>>Y#aq3?p_6sDwNI6w}^oDYf|I&SJ@53%{mU_#S$9G&)dYjqwxHmXzXX2PEFAJsXUi~YvkXz%`nf({scXwX6vd?^PU9j!H zj?e`m|DLD2&A-}zP%k7M=^8Mz=mZpduKlW#JOnA)PFL%G~t=wvLwf@D%O1pjYA0M;c{qtA&{skYM1dpX2 zaZ7($ard-=f#mGHk$rrwhhO;Sc>hrDZ(8oKe*J=(mUdz0;gjFH?r~dum}Qoem(I_R zthT>HGb@yI?w|f7Y$)(8`oXG6AyT`4@BSxmeSBBG8Nb=SnJ3q8*W57i;j{R^&MC)Z zr2ehDA;NzlDW>LO!@hmDwoOX^+jLuzDY?(f!7J?9ZvA^d3_dxm=l$!gsAliB@k8d1 z{QgtdFP^B<504JJ|Ksfch~C(>r$0)krLJGU@Y!)j&U=4sZT9m{Nz0NK_F5t=t(>#} z{srX;x{|DN+rMxBZ*uxM|KuZr$uGBMWZIok$>`Xld13#&og1FpKUfgXcj)3S8`J3< z&3yieP7rWT7JA)sZEL0Y;kWa>|70GN`Cl%jn%!Cxezo>u@!4;jHaDJE-J4kL-=w6m z@BZHVu@8e{efuto#$Uf;@b>$&ISM=n8Ll@u?|#|-?yA}S3C+q@(U0ya)Cs@ooEKTA zXa6vLXLaM}2f1@=G^^@N^zsbeO(H@{-{p6x@Q0XyD_Wx1TJUc@@fzEKmX|6uElHS-@j>a z*`zA?H1F2(f8Xm5I!4z{EPnMl-g5rDQ!Cr;{a=X9FF)-Q->>?3!hMIOKQ0`%m&x!C z4s&G8vURur;N!Z8=}zNiT_N?!zdzOr{N%J@F>+dKlX&;tuds>sxp)7z7(UJ{?)vWe z^+(Ml*HpO$sgJ8#uP&2+a>@Bswb0AnpnuF$xS!qo!FEIReEiw9g{&;?mpmFiOnCIY zxJuWh(k0~X%nutq=Q({^zHeApo@c-HmE^5x)U;gapoLpC&wnuDsn{Xyydaql>e!)im>0fkxj90#1bjN36 zx|aTqQ`~>NPygHZcXR5O_iuda_r1UNPUYRd_lKCL?@_Q27Z%+e{?4rL`{&q-X2;q0 zP3>o!WnWC1_{;6=&F`;@^pAHPK7Lhy){Si6*Q!UphD&Z=6LS52SGT%d()Kk@I)!oX z8YPncNuL!AcUa@G`@EN@{j;Y}PrbkY=%nAuIPduvZfIR!eNjUA`Xx;Dk%U*|n`5&?W)zi1x`}Xxs`ZzDVsOsuRuebNV-ujv*z`;5F>B+AfO#hfG zS$q!M-)6wMYQ_7vNAIzE?+=aAvbdG1lH;IOm+;&D{ndBB%pZji`xqZ1iho;+I~bA`j=jGI*KxlQ{|cfT;>DAwY?pn6WnSo~y4k?S3Shg$_w zSN{FFyQw!wnfdV154Vy}HBB^Aw`%yfaRT4Im#;7CYag~}yu_|$cJ=j#HCk@)8@Ip^*A(Nc;%9+msRVhMX#QHkkO>a_1jw$hU4Y7WmmT?R_xZ~-1PP6 zf8NtgQB`-uC+z=T+STQ{|B=6~UC!zE)Bk%2OqO$*es8DxX`@4XoeTaxk=pX5>aXSc zhk8BrcBb!}!aGUuHZixoFo40&J*OE@pJCC z$9V@Hw)lvi=1}-MXX(`L4Idi69SgsEuYmQY3)f$Ud7aYHALkqv_L#0_o_}Eajn_HH zPyNbr+{XS2xcT$?Pk#g- z8!vr**&?vMRzzsZgX@*bW(|7-Tg zO?w@B{q^;`@84;-RVfwU@G_leeBt?gHtF5dWLjRIUjF9)1W#rrtEmhB*jDp8-ZXi4 zwx%k=uuX++ZM*oh|9>iVU1i;WJQUZ=@BC=W z`dc&Ortg0iHT}5q+aJ8!o|N3bl~MJWzbSHM^hv2D+duqq)y~-bIG#7}>hUGU(*KIq z@69-C^v8Yu-tfJ(n>hCV`@LWH-v7iGc{8>go}^Z0_ggo&BxApt@QGP%YCk6?9rF>N z|3}z2LVV*J>18gPzitq=XJg&+LorRe%Qcx_>b>0`^RJ0NZs;(!8Y;2tuF0B8dFNX-@mxXYNz`9kHHmh9+zHQRwub9rF})Q_kaDr z4^KItjhpq4t>%2x*SbJf$=cxW+v}F88l4kAD0|y_$Fg5v^Q?~7?OONbgG-)4()2oI zhvdiHddWWff_ix7S*T>c5tzi@KchMK;qG_41S;+3|HuxFn#rkb{clhE-s`LFMclZ; zRFlH&or301I1+g?yy-{&Qt!!DC)fV}(0($`W{tQ!2g|j+cQf{{5B+of#oh(`T<__) z%NZV-zF6F527kKJA-g|&>^FOul`!{u)QcY$@_JSM`C8V*%Kx2DPlkJnzc{~tfs2c| zR_}k2I1?kiee3M||1=dx>v#9r{aN-&Nu&L#XeP_`qnv%ysU)#}HF>MwH-}i0v)+zC=pZxGn z*sQ%bIlSNAkNW=NNA^TV{qoa3aUH!^Uw9w+X&C(PZ0?HvEW6VTk2@`8t^4C z>$?4ZzuLVg<*V)9FG$_D^~sXyAEiQ{U48eyR$29(`h_3+BlZdwtPb;uU;k#`{>^#U zo;(VS{}x`WTYddgi>&;;c;~OlOZHZ6)Lp%7-^(Z8Utcs}U|{*;I7^x16!_MUwP9;} z{=5>nS~vZro93^L*Y#J1-p{(fdhWlC;j+Ju{y+Nt|LgBr_t)27TC-}!EdQ;q_fM@_ z(faQC_3W!pO-fV~e)9DTS1jn-o_+n~ugy>5cCFigjq7WYh5a6{M(byX!5z)zWjPu{QjnItyktPn!WGo*O#@` zH)ieor?#&9kKy_k$z`q2R=d3ZQnfo`$=0l|#b4h`My-$9=oRzpss3tV@xMh}Jg5Hu zy1sk2Q;civ>OJO@o_?)dx6bMJ|JScLe(=AGzN-I7>uBk!TYuw^e5$Em?|JlOX~?|b z;>z0Ve?R^E)g-&Y{l&nr+|LpK)C&6uw$$xy1ieS-t!} zLLJY~2^Bpp{r&0EUeAPgMi%=77Vnq#YTT0SV#@e?zq_$Z)Rpb(GhE~LD22_E6R%eM zIJsli)$c!9>NsC6lra#FQJ?fbdBgPI9}?G2Fc4gFe#v5oj(Zj+#`TAeChn3qeqtpt zXaBPPf7$y#crKe9`=h4oEW^VclPf;n3Q{TbvgFa&5^&`0#hP%L7kp{SpB``L+FNh# z_MheHteBrqf4+V@y?(}|jYW617lppq>HWfae&6oggf~Yr#FFF=zmG9dJ6@)`&|m%k zL497u1v`$KZ2EWcW1Yej*6C&^9H;;976|jT{&Kffz4iO%l=BY_&E|do6DV*z*z|+%b9~oI5oV66`zM z7kPZmn!AX};lQ68C;LDB{QoBV{by;(*dJ@E_7pki#(aNz>HXh`^?D|SudiOdb(QOP z)atm2iTZUd-XDU37HyN;Ec5h@%D?^UAC3OlJ=y;HaDngVEBp2fDqf$leX97$sk8pL zElb{CSkQCt-kLv~{(G-zQJmjo^f`I;8P!=n#(OI?MUQ7p6l#CU zdTveg;=L8x*1ic2-z@I@zrAhtdhux&i&jUh{k0~3wb)+Qdrz0W+hdYc@v+z2Y{mZv zU5%4uM4zu@{_)`G*C+OJ0`g5#vTKsJBlSs?Of7~lB6@2`8_@LFo+aLPp zX`Kl9=l*^AuMd8Hvax&<6>YO)u1Ox{I-=i_d3pNT{hw!_+JCfipH<<%ADQ;*9Dlw# zZkBkn-_x&USLBnaSsOa*W0rmr_;NwL==a9=pIvlnzja4nmvD|d77~}RIau(bJ6mm^ zQRC?!D}LOseLw4X@dTGUlTT(;|Kr#=VPV~Vft;Ps&tJXYrEcaVrrH1LTN?vY_Cd*~ zt=muk+y1()CU_cOg8aXeg_HC4a&}KyvOCiDWxf9ECx&$q%D=up-TUOz%c!5PpMBjQ zq3<=_Tzz5s-taa1tJXjA6wS$fvMIgv^w+CDl;8dRd15nTTq&qgU_Of4{Z!zRe=B_xq>cdbgvm?s}+O z$(-I3hmJ^c1{MGF-~W&I--O+(>#zQfd*>hi?0RTvp-i{mrr7Z9C;n~9a99{qW_S91 z%<=v7+RufOOCi?98zdSU5~NBS3Y!^8GUswJmfj6dT(|9$M-8~JhZ zX3;xzCjDJ;At=+<>u%??{%VC!qC2`@>{VZ1w{GP>Wy=_KHoJJZ1ql zQ?ITpzJ50%d$ZKstS_qn72QtmK9?2KWR~n{TIav~HRr4PVyVeLTtcsi`1toduucyC zEO>2UOvsIk9#iVey4<^3bp0fm;`hBaaB^c#c)pGImrZ5fmsg=HYqJ(*pURj&`zxE^ zp%CoZDpID)De5KiB_KyAb-&=OHO|}zqwx?Q?1DKIsNIz8TD5Jsuz_4E-X3z!{gdfDdn&B zI}7f-Y}xyk@mBi!$5Rc~A9PQ)u6X>4XWs80P4hM1oVoedqV}N9>HR^6qqqu}dF?rO zMBwo66B{bd#LVknKP^9M@eXBKX9;fw!!0jk{U;Ll3Gxh%}fvj5|39WuAd|MB;2^H=(p0#`K zhU6#n_F4a%7q7mY)!@c)ub$H{GHwb8$ZGj%U(qi7(Q8#J8y%jtMMsNIs9K(R#~gQ7 zxma%9_1||s{qCw%w^da0sFqH}#eYW*njPT&uGFM|>h~tLZ#wJj-}f+0UC8i8cav{n zc>1c0<30QL2YBdyPnsEx`58CzR5w(;F=gFt@a1ft!Fr81Ydyex&9GRcAs&4R26tHTz2di<~T(*-s;Pre5;6mxu6%#ck!X8Y>j z!+4V}g-52zJHr#<_3!oX?M?@{t~x3YR!&*_v;x~m1+bN zC-L56Uw4>EUY^x6;F)quNWj|<7ye!6U~g(YG;4Nq_8S3J3F5ebCZstDp<*eCvU`TT_f;i%IkFVUsYU>m-q9^E>0dGAmB z?p?Z`zfS(X-@APWZ}g>xM+Ryin-4PGzPZZCNiz8V$!0DScIk5r{oT!99(}){e7L6m zPNL!c%IfSte?Mg9hMoF#kmXlf>C1rLRehTm%YWN^=Kt^W{}r1(oa;YIywkR`T<=^#97B}TX-@cx1|6c8o>)g-Ar$4ot8-&RmeS7+G z=JN8gtjVlxH&T=Dp8R}ReO1(hp4@j+%xtba-1+gFYf_wBQGIzyaE9WGO?CGl`5w2i zNc~#+xZ7=|#rvHzt{+{TH@(+u!u6fi`%^SOuCJYNyjJOO?VnQt;aT_B=lzO(CdxTO z(=Na_Kc>?6an1a`wGU=2n619QE{wl!Q}zGb-~RqrpEW&8^sO{Y@%1Yi7YgS+d!M(6 zZ}!bS0s^m`_ODP;U1@k`o_BV~ok#4|PyR)}NZel2^kDX}+8wg->kk(G`>XvYcgAO( zNej0$HJ+c(J#*7)Py0BIod%CBt1qA4oRu^CYWC!ryqRAwiCxbQiZ6fEoAKegXNCIe zbz$aZk@i1}s*3*KU#k=QqI&Tkw~DHK{r`^D%Z+XJuXE-9xA0Eg?7P=5|5sGVU-IYs z;mZ7^Nk4w;UH|$#k=xpf+cf<5;zw~4o?hQC=EufU+jh_N!xABjcZ=Wr_E+yM{}%Ut z!?gE%*Ih5V6neBuFMhW0i~oy^^)F@jTfLX-+vU$_TeW3E*Y+nu@%6FI;&BQpvnz|g zvy|`GRZ?@9)cfw(UYFSK`-7yuz592yOZL@Y!4tEWercJhv}3RKq^bAC6cp9cw(Z|! z{fbd$=09#JLy3cvR?hY`ns-fRqj&eT`-fRiBnUF*-KjM!elqv|nY%&X|5cxgVmU0v z#mF8N@%i+<$~%k>vuu*quhfdQsjI(#W5F*LwO@~x6koa>_xoxQ%g1{fi=#OWS*EfH zt1R7<6y#?3`})4)%N}s(d9*uCRC4|Ch^KjH^m%1Q7H93LdApD8e#0_*j%j&dyyc7K ztJMzg`j}ttYgRsKrth2^41dk<^lW-{vsR(4dvU{QmBwjP_s;MA$SS&2qe%GmyBU`s zIb68f`05XbtY4K%k;QNJ=m(4Vrk`-S|LWh*MLh@RGMg(IusXLaduQCC{^aHLWz)DB zHkc}eG0d3Y!}#`RiVUOF%zxSvIte?2QqOC9oqX=7#l9p-R^pSpVQTwo=dP2#%FMHM zr!Gt1eQNrPAD@lSPqI95x!2oJ&UdY^Oz!Vv?_BFIuG)OGRPN=!lQVmx96C)Eqc1)9 z8o2&=&i(D@CuRNK^)*j@`hzdY&QH58-PH)KpSSqrHl`Pa3R0RYK3m**bYJ4bl&y0Q z-ab6}`QiC-Hrh}4Zoc4F+i01$#O(fz+f2==epjbEE?ITpbi-BGl5h>FRUafT6*UD< zJ367e=th51^ujrFAMM-!;*EUb{)E>CH!Q5pG~e&~JEdUnj+76k+j#x66W$)#n&zT! z(lzOQz)|DXcSX2mXWVG~TgumB^-D&=JK!xlqtcZLSt7!`Oa~ne5_}@KSUVgG4d&>G zbF?%mDRjhKkJ)ah7V3R6>h!AD`!>y!mbN8g}pC^7l;8&x8^6CJWfXJ)%2mj2iU0v$L zS$0u7Z?)f>R?%6aj7Mz0nFp|asE+!%>e#aNb|Kd%OI`b6FD$7ccfWhvhM0JDo)tXl zTB-#FjJ5X^Y!6*bwY#*^>iRBQm4fW1ed5X66@zA8V2y9xC-+~ak5Rt4?~G^5k)S-m zQy(XteA+??vUfDRt9x`eZE6e{5?gU{ny9p&`b_+18Zk5HUeaiM6RQU_pqE zt8hz`V!#9!zpvBJhR4jh_cXF(YH6vL-%8C3cJlVutRBzG-BSJb>F50&{_Y3Y&kt=j z*z(i;&ff6SYQv|$ChtD4UUTjj*Sz)%X>vO=cb#~u`iI-#d9#OndDM$P;-8{Eekh#d zV}5x-udKH(UJlB}T$|+fM*2gccNA})qJiw?TGGPtq0Lar08v=ApxLP|F7ECze zCdjqEbj{tURmH2OPg<9HWzn*wana758M5rZ?=N|}`C;MV2Qe$W%93u1A6oJ=yJz?5 z-TUoT<2Q?1dVl@>;={S6?3-KV=Z2lVxuEXg&)6kZF=8<{4qwju|LOg@)B^w7@c7!B zAJX;+{PGSx88A0wUsvpZ_JB8UKDW!o8noLlmCHLTDtbI1dqKlf;f6Kkya$gpGtHVj z^A=O3_kBADb1%`jt&EWl89pESYW3>QRy8eeY~9qOU0x?%E4%QN#NrvtCuQ-quqMow zh>ZMr?w620*L9cWvsZs)k2h;5V3ZM=;3Lekg{JooOjkv@_Llr z_;2RCOIx_!KJq{7;rKr0_3m2-{Wbp0o7EKgX7_GiRoN@?Od;NR0zGewCp+Ddy3p}Z z+Ubwwr$y5zZ_7z|z^EoNK}&?ImF0}Xh5#QGX-+Q{g^sz!XC80$*|S{cCa3qBsWMtm zEu;?LxwL-%lRu{K{5D=Znf-jR#@l60SGPYszI&-n-tyo6=56b}PcG}7wl{WHbZ`By zw-0MBZGSya{BFI~{Rh=YF6oKAIDNA;QgJ5Ry7mP&`VX%CGQXYad>~}eo)_Hz=PfK2 zEegNx(E2z?`oyOY<7@`@dev|7@vVE5{&pp4U#q--b4z;jxw?D%=G|}OIm18YQq4Zr z5B|&J?{7XHo#3^u^V^|*i61p15=;?gou zd8@s_%{=s6@UE!m@8?Fm3NPELl3%)Hjc?Zfb@6LLkF2j-=e@Gkc71^RdAFkX`~Gs= zpWt-5M=^WiZl|Kvd#~<}oA%^sxFNUxYL#o9pKHHn1fR@!|7GvT*wT!R#b2hrx_bKl zMxkF{w#G$m4BJ}#Y2n;!ZAV@HUHU)Ki|>cnzUxbO|C3YA{@S&Ek9)b@_PT%8F<)QD zwE4ac|5SA)?a|2%%{uSpCrW2+{;#Pjv8X+5%Aws$oC>D(JYU^)H)*{_U%pd`wrg4W z)dyRaF1q3F_D{Yk>%rAG@5|TQ)=pe7HdcB)`|2nJny~(?G$?5G2@U1>t zeCU6Z^ZD6x{~d`{*{9w1UA;$U<{v|eDX+@bE8bat%;#TAH;Wig;kTd@l93q)*Yz?- zobiv7nm6x|;El=bJZ7`c1#xmpG-n)pXMXYCy2bi2^RGuGD*KAf=Zsu4pL2gR$HR30 z1sZQ{mPsD=$PYaJvG7qf_ZeH|m%k)=*#(%UHnEv5zWOdw(N&nQ|J&Q%7hKG|TR%E8 zzB^hH*nYEi!}{wYFCBbVet3Al{6}q_-P*Hud#{--4@}`%^!mm=tJfP8q{Uh6jKW;s zaeQ8>H23d8wa0e5?oP5;Bak}lYchw{KlwQcduO-r^f2$6RyFC(PwnM5_*@RR|DT}d zEPc=M#^Mgn7hBbGZlwQE(bw-jJ5fbAVL|a0^Ev6Ng)%WJ{pHuoT3@TTKXvR@q`#Yw}Xvvm=$dE3DGLT1s3p5Z1UU*f-JCVB)dsZ;WT_F+@%>;N$-pH1E-> zjjW2@eGF+oUmMi12%md#JzV|!Ef?eWlby0Z30ckTHcFX)>f`LM;gxpv_nf7zzw~l|$~H)=T@osP9er;9$U z+AI5%hxHTl^&YOL9ERoGr;M0SKd|69b%Ud0Ven2fAL;LcJ=0<@bWc9D|Kz9tGj=!} zeWAbKRk2{_SK;079|u-nF}!MYp>2!MQ;+E1pZ@Es2F9+x;r9FU(Rb^2q)$nhuk<7B zt9|*M$?7`O>$<+z?6h6~LQBET|H5+JRcuw-Zx69$e|;z6vamjJ_TKAXB=Yt3UtLq4 z-g==SY3}F5xc9qP{I7rU;%WRt_33}T?sxP}_<4Hrq<0F}ssgGvU;4$ydUEaEyLA=& zUs!6jU)?FBfAZHaKM$`9QsJC>=bELJ1IiBk4l+vLYh1>f)u?bwR?&9ha^BnPyk!5B z{S2&nTl6>kk)7IGF}=E~{YNElme(In^Qz%n#Imp9ghAW1!{rZ~HubdJDwprPdDwO_ zi~a1|D=SLY*w0stuivz9{)M)>DCXqi?~giNr6k#U3SWd>-yYq1WZ|)yW*3&Pe32jT zkm)?pwwfhHETQ}T@;}>44BYjPUHmP%xo=C^q=kifGO2qK7~Ea=n{)2IRkLsUh4Pj* z@w|k0)6W*Dn@tRuyMBd?=-KK2e|bN$x~6HlnE!b#tLrB3m~J;+MPcfK*JnOf zA64GmAECOJefNhC_hZ+4KG^c{gOkzu`&m^Les$lCbg$2tc++md|CU`n`MW31N|$ru zJZ*89PpSOd7cS08i<7y8Hkh%`F*DLwRCu`GSkkU~zxwjHkLzm{jV@g^PuKdzyVGo% zUZcc>=F%#5jk)3bGZdG&3eBteOjr^`_WaHU{otma=v>$WL=U0mS=RfZ| z>khRyKjrr(mL8q_`#r1antk6pXT4c(dba9=N1W-lKjyl&$4`7xeCz#J=E3cI?>5z4 zf8^x+c=~0Q{rcJe*34h`zS3S&YkGA?*dG<^2%qzN4?Q!C{d1&L#P{~fcU9`&?XNG| zzh1vOUPQCu)6+v~%YU;;-+5*{AuE2O_^Y*3+Lo-#|6^`u;(h%?a{W*1U7x2$R9vrJ zKS_|kxcuqgyMbJhHQU3vz2_dgKRr&YZ`$>{4`%e&+}I=UsF-mmPW|1xzfDf5a-0Tx z`Cm5a^KSX|owaR6Uj34eu+9Y~^S504neTU6G?Hsmz2IJpE{@toA3uCNR^vYT<9}Xn zyEVtv>#qoF$3^;Ke_ZwX&LegDGXu;u%r`0e96ObqQ})0h7dD}H7w zc*|q^hUxD^xkYA7GkuryGOdAh!0`ZbT4 z`R(rKEGow(bI)Z|>zGd2e)8MtEZtSE>a)vttb1SnF;?N0BKJqVwZK2-0Hg*R33b9J>%qGch&Xo zzsY}8BKParfA?h7Txo0nw9ZS@aqp204eizMzL&p}RP4UD?`HX{jb|0vTeXs9m)YuN zA9%CZcIEt(^Bu3aT@I>!PgcKMCTzJg^Op48)pa||)|Qw5dMdgq~`YnpM+6UwyUvo*~%N^8fn%CCA-A zmaX~u@zne4G5hVqENkYi*1x{1H1FB8wK2P+CZAF7yWI6-JK+vUsmJGr`$@ z?<-kxPMdGP*xLlNYp*W+ziRdMWmhX5tB(JTt6m?k`BP^0O}mr3FSLEdT$@tgrFQU#C+6BB7Tx5j*0B74d6-%|oD7G+2c5T5ki_dVN{D+zO~c9cDPBKBy#+0A!9yvyU5pSC<#*c-Ej zY45|?ntOJB-{`aGy-=LmGK=<;Hq(P8KUMr+Wfxwom*;o%XR!-UlKO^+%#%&nRvQZ@ z&Dhzd`1Rml$GYUO=9f#WCO2eWE>4py`fzd5ltg(R1G_b?%WJ>M3N`YZKAzJgW40wf zp~dq4mgKn5ADkD7M@3rEV`cpEEV_mtbL%6^Iy{T_xvN9SRa>`^SHUNo<8Uh&;M`D zh5rZbA7{?BH=6Z8TTW?SQGSZc{eL=7lU0AT%cty?UdtJ9_ugU_{S}{G4sM?mq7?0L ziAzcNbC1%q6`zjP=(~k=aOXZ|`6G5te(E*<<#+GT+clpA_Pqb6wGyP8w>j!QeD6%N_C{^?F#m_?e&Y0u$Pcb&ejoxP@gXJJs%ckkc3dKJq4#x*6L4`JFi*Z95J{ZdPr z+$Wt@{X5_2M7%H&uKAR#AJ|o|;UFSdcKt@l`Mr17{4)<)^WlGv_x?3DE?0w@5Rr~Si>*<^#xaxbNzjFjIR8@r?fEos9fX{iIWe^X70FL z*TlrXZ|?gSyZ)wz_*s7~u3Hzj`u_U()dklBT;~1^yVuqwEg+htHf1AoW6j(%1-EpL z30-!QOaHpqrN`g8BUyTI4ER(d{6?^k+P%zrLD%l4eBwn%$X$uyrN z;m2Y(_%>Xc|CeQn?7Duul8B=T)9?wGwnp!uboaT|H^DVb%hW}7} z7I5&})(s(9o?3-Rq$3)CFJEVT<*8Gbw65#B_w_yR9#_`h__RUw_}gm!=wokKva(HC zQ_}g$PU!CRj$ZnqeNM`XC7Ya9JIqg6F+V}4X5XZDIsYer{8!>Rrxud)UUiOUH+&&Y1PXl?s<_5X_}{lBB4t9P4pseS&)n6tJ{cmHZkycpU32)Q=KY`NlYeAd&AIXBuF$GU zm)A>Ae*KH(>G_-U?Kah9R&r?9S8eCd!^N2>%iUgclk zdtmY7d%;<)x-9&grhh0;$eS)+%CNX#?c(3QtA70we#|*r>O^0@=*gGa4-U*Ua#<83 z`YLf%PX|9wbnByTjmD`ytJ2<|u6@#eZ;W|sVIpOuPMJ;SujMc1F|Gz&C6kYmp%)Ut6ufBE%~+iJSe0t+4G z<~&<|no0Pc!h>D?7j&1(r+iA;nId?1$$96CKW$#76tbF>OHO^UdP|`UWB(>k?afnX z?)k$#^_9egrocZn?O*@%Jlj-Xn{@1niO*r@kZ+r&b94QewR_*+>hJH>&u1odT+8J4 zxq2bb{-pZqrw^Za+pF)&Ugq&(y4<&WN-_Iwe(VYTvYu~awSHarzW4r<4x0+>Uzk@X zU~c^F?L%qXe!xKncVJ(Jb(^K9l7 zg}jrwcyL?9x|`)MGrisJL`6+jZ}dsz@6$0W5p@!qX0&MjnWG!`b~*A_fBY2t>!Ey% zlEth?!7NR&chBsbwKppB-9^cF+$n84k8yvydr++HZ_3B~ml2^456$eoe#|p+bI$*J zG1)2)ro~O4tg2t&;;2`&{NawdJjq8t=eDH$<2>@>?!7m8bsU{}@5;0*-$kjOe3`k` zQP?_5JLM8f|Mc1S>)SW2H98nB+rztNeHzQ!qMxeynulsuthD_;>2llcmG-y&^fl+| z`M)VW%AsFA{U%qE)%9ODyA`(0KQ5zj?sv%Cl=~jp=O0XQWb5n^%QC+0w`{ikAy&J9 zZyDE3#opb0ueN+?e6N(u^e2kxRi7$1T@6YUPWrP^{O-H##k=bRLVU_AXS|X-U71~z zVk!NSH~HE7g=aDkl-Jd5w7J%B_0#+{S{ z2(6!gWFG?q`(ejfs+?Y|wP#;=*SOdJzyJ69>HjIafB*mY`)|bC)c^nW-M`FT{r`Wh z?yu7qr@s3AW7faUb)l;kg{{A~FZ4Y}(`PO{@z!|P2ty$gA@(2fp{st(K6W|h_~lhG zzYUM;?wa`T&GQxSQ&yC}yP^Ex(?rWj%Re|e7k#{%^)<=y{_dFFhg`1TzOBC2Lu20S zYdbqvzs)`zEF@hW&*${WBGV@8#I}CTjB96R&9Ae5JZt&sM@|9@R1^{#-&r<%vFg-* zwlnj%;`FOkK|gCIISR3|a@85xOz8i)@9uhKu4X6K1#Ijk4~qM0B9?Z~XKj_%N$t_% zlb^vnyY@=;!|%fHVuc^F^fp-^?!RF7h8R)XqJZCj78>)7TSxk2fjG?7zh<&6W2i zOk1?;X!5`1g*Un7t&}#Y9ctCH%)NUjOuM@4Xx`(V`bk_~$6`fNx9$n-IregmZdH{KIC%BY0X2`sCa)|O zeJwn5Wq0TnYq?sB8K0c{6;^gl|MrJH{>qy$?5yg=w7`4@MUS! z%8H}f_T(&MM{1ffxzsB`r%Ua)Mi>+tOco!rat3Ai+ zkl=eOy*D4-7|U0FO}hK*O<-Ni_u@-Bp~Y2S{>ePu=6!ts_jNjR3Z6|%iRbyaNL`d& zU$<8L_eUqg$Upxat{+smeOKXcy#DN03XW$IyzA^`h1b3^`nzAUV`+`~pD7~Nf`=a- zls5F9y>I=+>Pf4j_5@vBU$<{(=$id)KTKb zr;N18|2rky7iQd_^TCbnW$XXe-Fxrtn{oE6eUm1q#?;>*RthL?{bjQwh-;c<;(xu{ z(SH=HpO~~1Y%Kq?`}WUM8yE4}|2kPOf6md{P<-KGt<>cI|J7H0{k3Pha)E=0>fIk= ztY)02A2hhP9?5P}*f3$js$-8|)qjW;6k+COs40v1I{A+fckS!1P4R|u`N z%RKzo_6I8#rY?v(KP5!SZTSnOr(a*mY&FyRo!qa(_H^mTUGMg2{C*o(qxiC-<%5^< zB72R7t;HMuPRVi3(>aiTP+)uZsXd2g2N(U{`~HJ#n59|#ggu!nYtpt(4GnxaJ0>c= zbfv;RAInw$RIB3j8`eM2{->|rX3w#p|II0%`%RzT{WxIzx?lM5*9VJtyr0_J%znhq zdiu5#R_DVm`lSErU-N#l+T?wE#HNL%y^oC8`B#Ylj87&@c`N6P9rg!PKV52i`~TCi zlK)pf{qtX~|1ti>-Ur9^qQ0N4>t4CX!g$(sqtx|(mToYdaOAb>Lfbf%^LAw}@t!}F zBxjcy{fIu@)~B7bCfgt(K&WMZT=AOS77hQ8-Z0$U`?0|3u5ZnvO*)*?=J`Kck9o`M z|Ia)1V%?#I@?EoDcTK*pSJ(CY_5CUB=l*`*w&%!v`zGPtyTqP9tYvhcy4}$I`_rDH zcL(!4O()i`54ih%eea7K`A=H_f>5zBPwybxoc1`|VpBv+edbSrpHI6MpPw*ZcJ` ztIwt-Hvg^uaI;kLhQVtQr*&qP4tLz2ufOM!I)(B3zU<3?gx};HT$Y{6ylMRh&$H{} z=X6^e@Bh}eGiHe-qyL&xqf~dPSEsM(Bu*;gR^DT~biH&S<6IT<-`Zy`-?ERKKK-i3 zj*0W$Y+Jix`M>I%1L2+6Gw&+6SbiWv>3uVVI{r*$? zbmiL9jq6YE{qaKa?yIcHvpmg;{;4>rIPWQ}y3fIRWPjAE?(XY5oMqmx*d18aL{;#e-H~OOz8St0;)%`U=KVN@ZDIqQOAuxI4*2zZ8U%yhg<(qLl>iUYE zoObqCt8N8~M8B(9yXNAz#?Zh2S?bs{Pv^hNvNXP|S-E**it_sywZ+d5-7hb)5k?{*9y`Qh&q0`CcCH15DtAN@2 zy+2o!r+$r_vvbu&^Hqv1v0qQ@Ix1}5aOv{m@)ydxMZOp7cTB%_QC#c8k?DF-`_(%W zl>>h}3N$o$2sl(L|2-kfD^L^T`YT4djy>%7P0j>equ|a=&?P#@9cQU_X0a9(YIy(u zvfauud*3hhUAwkcZM}N;*7e$N8-KseJ`(yr_W!G&_0iwf#_d|A{dIlMy3&<-@7vE6 zUJCW$c>R2G-iDh_P)NZfA!pzsa-+;*XJE*kYjP$5fF8{LV;V3Yh6@m&zol-{Q5VQ z=HwQp{5r}f&~`9-_3_uT#|_L*iG?jUQrW7qb%mH8Q|gK=L+1?X-Vcj%cigr$JF=N5#GT8K2EA ze-y3s@)1ki+4#ue`$CC7ot=!mI*;(=lCPclkeS$us-mB!KsCrfrpFtReeRfU%9Ggn{zvlxboy9 zCOHl&9o*+F68LAOd^~yFU2?g~($MHHQ7f z*@ja}6fe)c?EhRL!==ccb>XQt`{#RbLnS`kMdwpLy%IUax&U*U{pg+L^uIPk)F%aG1YBO_60jEBFISd5ILYwV()7>kHOuWM?0UOm-qxbm|E!x6^(L)<6cD!Rm-5!F z=k~^2ef58R?7zL?s^5bayZ*9SU8eNK?q7acj((@exb<>d&$zlA=2iM5KYg7B$tve!a6QyTfh6LSNthu%JLJ_I~xM`9JfOIN2rI>l;rf1WH8h zXLICn>b}jLZ8he|@)r!y%_SQPG2MTplJDo<4rpm*#+2a2A%KZA-dHEk6rv6%)`Ty*ar)S^Ce-_)akJ~qNRrUKAfopsAhx>j% z)qTxc|6RwbI{ib9yFNOtPY(HgT_!~Cp-#(Z|CyaLepY-q_-c=6VDNdF@Y;_$nnnL# zzY=|Rv^(y{$ye*^9lEuq>puP8)^9$wY@_J=-CI2F?N)nPW8t_(Y74u`^%sjpR&#Hb z*`h0ydSPLTP)nNb`S(@!hyLWvnyC2o$FKh_3p{_+E^QVwm6vR=|L|XO-H*Z*Zmu^T zIqXkb9`KsEX5QYa-|Kz<1TX$}x>M1-sM0{d+pUHU;myw z-E@7Jf=kxic@`Z0Pkug*Z?Zc3{)_O7_v28~EUQnBBRFFU_0WCx7Do`|t0I z9jcob{P-Zvvi?`p``!1hadU9oU-mxbrijQXzr>!0KR4bu^vmkHqNTR}n|VTWs~Tjs z9KGAWtK($&{^X`MM%T*>PdctyOEKnetc{(%=rN~79KVr^pX&hI~K z7iPC`^POLEer(%=b5$3Y%vCU!DlGc{iTjk>8y@R=*~`l~j(xW;-?lOQ$N%g*we>Id zmcOrSsoJCeaOxjj17nTU$XzD?;@=idHsVc|Wk305f_c5-^#CKG;BbYe?q5F&w)b@( z@1I(D{~uR+%BhowCHAZKl~w0|O`4qg&#zTd@pNml^zyymn;h5Ia4r2BetgFD)a1P% zR&!eZJLG0&K4VVK0fB0sGq%xs|GysV_PQ>S9ava;^zj3>%I`0Gp3REXW_-+{bL0K# zf4iiwxh>8+%Ren(>%CL2!=x_T=XB)j?7M56TiDy9wQ%;6BtftTd|r=<+(;cRGFJt?2O+ zYYwx`y)|Wn@7@~?pY`KvUte!}y?3Ya>9_?gb0(+PcL~pmet&(=H_RKw$H|F!ft=U)P;TWOfS*wENfgTej=Q-_&2G`}1mbL#s z?bkMY|F+5K#qr(y0@*^}bMFl6@4viX;7Pvrr$2GUUD{iterY(&-uV2~R__-|A?w4Y zT}*o48FY2k>#wKQN?v^(S-<>nxnA~`nxA$1tCza`Sbwtc_59sSF24V&81`b<+cPnY>VuH0q5s_k8pdHiLu*N2a>Nbitq z+mofyKBYmbzet8LhEZVc`bTlz`<>(Whc=&|v?R>)>Au}A*XNZ)KHZ%FK_!3Id|cGn*xTW2QmyC0joAt8oxB|-Sqwge4$jL!-fk6 zVmz#UjY$habVMb2oDv1K&YW6Z_()SI?%k3#dwRX}BK|aA{}S)K>gV?lc3ZcHl;8cO zz4iPZp|$dTS^jH&ty6H%{&Jw?goJmzozT~koM8VYYu4?b_jL`!nM41*&hYM6`CFh+ zbHm|0lW|GL%xMQ-7O^(zux!!wQ&AA(PyO!5^5=PycN71on%QBQtL|*gtKD7LZ})oN zU4{0d7)9&wI*yIIPa4kXE&Y1;Uxn<3^|qg(ER9_9;AAg<^b%3i7fPqI zy^4aIuD-vv=IY)bFTO-g^pk#a>h$wpjNk=-Y9h*-OotmbT+k8a=5#s~u)sq}h@$wX}2O)bWNCZZ5_*l3j4!9`Csm@zv*7ydgbHQ;4;6dj=vX|e}1z(&hFXr$ID%w zK7Vgt&R#$3VQ;zW`8oe)O}b*@)#x52s6YQ?$nEt8P0M@UOZq6i{*fm#@m{6Tg6tJ> z8#q@Tx>GB!FU`o=-z{bAC34O1xv(%(h{u%c%Qv?x$vjAOvYyo5ZsFzkRb$)9We@#0 zgcw-Tq}B!W+p%AFy6Rqb+T2LF&4O>qyNzF=0iAn~*@~!|?8PDXXr9&D2||YI;gf%a8eRJm0fR zHCxtSF1d5JZfAbHYE}FHV7_UQe-guO&YtW2t(RAye&peUi;sWI3isW6>5l56nKI`O zKYacF;hC0C_D5=^&*-Jet7lH7n*h5pIWEp zKM9BqYn7bg5vy_G`udHn-{K2|mcM-T>&#Wf%-gfr%k%TUfc-haM?{#Hqh!tW| ztWK?o8WCE8oGe~n*M9g~B07Cq_ugkSP0w_%7gvYU6nQbucT8~g)jX2 zq273fon-pcPj+9QY|XXjntew+^Z(^nGRNmR-k5k{*~;Z}swY*b?|vuzXy!@T`RwM= z8|U9Q&z0slZh9j}-$yy-`L%F<%OgAgoeuB#{N(G0MJ-xlygAGKPc2YUm@p}WE4(T_ z&-(3(>*tTl7pz{yV^_nL!R&CS^^M~BjUQXqE%Pt&I_YxjOh!w-b<~b`4#{%8PbIqno46|PpMBMzGcd{mq{nH~yTuFs23mk#xuqqSt|nbUVqu2E_H7#sHa zLu}lnZ<*`P&OW~8m+*_}Y>V?_ZfEaW^66Li(&*ol{`+&@TX$c}oN=4~m;ZJK)=%Q6 zMRVqS*(z|NVQ*MwPN@+8w-=K;zx-Qv%jmq^^k3{^W=db<)=$$pEhGLU>Mm#YTG76l zKhi3X|2z9%UjIq`?0+BPPNX|aKE3s5!5Pla6)TwTa5ePXZ+#-_!kFvhe!4a~O|P-( z!k4KnJdgS=ZdsiZo%%H~>Z8}Uxi5EV`re3+sh$4)Zv}G$qngMJH!&_&m&SwzGggRk zu{Jd&ESREW&c)i2m@uJ5MNpXg+NU4iuO@v8eYz?pFYoE8mJKzP`#Gwwgbr!s_X|Fwc zr2g0WS8|N`Ey21kw=IA4|6x(}a;drV=C!`5nm)7epQ-%YT@R!dZN4Vbo1DJy@~Wwq z!){uC>b~|?@Sl4R`;Y9~&)$39JU#1aw>z7o=}Wo0UQ^*~E~9PcC2GytHsJ?W z-kN35+`z!Vbr{T+`k4JOyCYUz?R!_c^Op9XQ=jil z4~={F!F}J=$IXlfW3KHi)d>~*)3CtLc>1xhyVeEzZr^pd+mE!a(rsK?Dlb%?$adQ6+5JY92<0Vd?HxQ1Oc*1Uo_zbi ztaZO{*v6V4-l9+byZ_r5zdn5R$$Hseo7@5_#3YhW9ez{$ou^0T&x1(kBhlL@e+W3@ zG1D=<|CDam>Hhqsx4-#2GEZ}4XPxk=Hfw3RXxGQ+BPAz)o$9s7;;>X>miyQg>z>CZ z?m5Y1>#Ze87tj5)?>t!8a3EmGHH%X(dc7A}SiG#LpXYHRh)eO|mtXOx&b;PPP!Y&o z@^x)>wAZJ&pS!yh5AHc|+9BktfqsEQ{`G8! zS3TKj<@_u2^+oxYaodzUCrwM3{+6xPVtt^on(?2i{!NStX}QdD@siUb;~z%vzWeU* zOW~?RPRF|WCGQ?jk`{fm#VE~M$?W-!H@iPuy%M!rcgmshWaEin;Wd1P&y{|hntp5h zhsRvr(&p1rzlz?EYUY=&N!WO_Eh-}C+~(tJgcFlb+9*lSIadAjOZmH9VOIYm{;$5O z8{>X$wdTA_O@-6D-q+4=vcC6T@Ops9uXU|`CV`9>G$(F9WPa&i+>@ufU&ZZ@d-^w` z^35>uWlo=aOCde@G?q0zlm?#WrNaR>1LwL*SF2rR~O~h|9bi{P4DTc4G^0*|lX+ZdoQ zZxZ`trm1`64$SA_TKy`9&1PH3{==&@Yj|xq?}&YNVrja-aVcwx$oleY@qb;@W?3`~ z+NwMU0DeN(wL+!IaVu2pZR8CJv02< zJ`rV(jaQFLda+#&v3R#8(Y)yIsa#>V>USOP)z1zJY~Qb6=vErCbP}(YLrzlHRD+Yr z`72-Szr93iztr`Uh9UDG@kW`PzxnBZXp?NZsMy(;4}%t;wicN%+l6Plm%Ll*fr9)0 z)+P)%99PhVQ_2WzmrqCBV=0Azts6(J@+d5P+HtQ)ikyDkvF7{U4Gv6MXQ}z8|NCOqpPu|Eo89)lr;#!K?BZgNeLNSf zre&yp=eZ`6Ic1B*C*hv?zgN$CddPnI`u)92*LI7%Kh5|2=CSv6f&BTgi8F4reE#~0 zt=;8!@VQ94O?CCHl}U0}+RDRbuJm3q|KIbe^@{QJvm<9r{~vH=N!zRqB~mj^JlX%B zUotpK>>%Gn`QBu0L2kC!(lTdi52ol!>~>$W|5sc2(XTTy8XGpWe_N-2W_R{KpIL|1>sy@x=XSvs>p=)iw8@{NPfQ=br1G z^YEdV`qOXC*H5ID*sNIpV_kNlK*IN*Yxw!q^1DS-8{(Jze)??c>7Vsm3!L=qoZhc~ zRHDD@XjZJ=6MOT!`bRhpJwN`*A}=+zcHh*C_3v3Gbx54-kDu`A@$0i`-}b#RJO1~_ z1oueE{WoG7zyDqK_S~{HLR>n3PhBu*pU7ffvd_fIwocMp|CI5EMx&?I`h88x=cla` zytuJJ?sHVudG`(0OV@b(k34Q-;2^wZd9)9+b;GfXr$rWLw7glXwKV&7P}G#KDekLJ zs2(e>$}hbtUj6Sr)7jSbd*>gyH*MB~RlBeI6x94|a|m&L)cJ4qLDw*a70bM4|J!@~ zOcAs83X@9{#iqfo+mf1wY$83>y@kZF-vyu z+`2j?^!ls{dHWWPEgsju!_4ixWZ_59?n0Z)$HW^y|rvM?9x>KR#pBnU{K#>rB_Si@9eWD=2HrJ^aM~ zwzyCK6R%{M$r*j!YHd#GgPtoNUo@TUC}H<+p5VO2UaMXwtFfjow(?vN^3>?H=c#Mk zHcsa}aL4(}hf_aolkQX+Yvq1Eq(1B1+r!>_f4J}GJXsxmP@rjY%0@^;zpqd?w=De7*ng6565H=@M*}JVPuSxA&e7yMk4xM+g zrmUY6vvOuUi=W`}ULtH`#cW=aGZW+Mf_OBh@9(ZEDG zmuZ@H${#w`{CGFDCvZi&+uG_Mr}oAw^ep#!#FEUSxorR1x}du%Oa8sA3oQP(>+jL( z^+Dpblc!tS)@_>b|Eo%p4Yy9Lz5bIgoJaq-N&igNncuw0GTD9ke{lgn!%u0-{59QT zUwuNu*3V~q7W^}!L;dRU>CfLEnId@a`|_C*i=K77U-EwU$DikPc4i*DI_v3)bgrh& zcb;hc7fIUQA;w}c^Ih%6IXk93y_V{jTA6T8#^v`nohEkamCp`8{F%%0=-oQ+ck92; zfBGdu<%37i?YQ`z#!DyoAN0@qw*Bi0iDn-6&f*<=gj>7%0>b967Z!6!eHz+R^`mWS zr_j^A8~x%fzuLsq-#3llQ<9*=@kmbm>E-D>4Qj4t8h6+DMgQfuDk^@u`szQQ>+koy z3vu69-4wfji3kVNhJ{XxE&ka4o6zaf#mZ#xaH?}#X-7fz;dWz5+Ydbp8E!b_@vUER zx-52cL&20#0pkK_tGgMcBzc4u+&O*ldBf+U$5Pk5 zGrm;$L)dSI#?skA7nyym4yRVW6JDY9hi_(iv-<5XW&7j)Myek9b@bQw+SPKeu5Z}L zB=q`6!0uh4cghq`UlyHqG-?07f@Nkq|MC3*t&U0BbYrn-*8vvs*n=+4&c8QCM0OP2 z44kqrs7aa0L!g~r+WYdw9{7Z3SNwP0N3Yg&fye zd26e~?E}lp_RGgsn!jeP+8Q738FXc}=;TZ0k3AeCGf%%`UBuA!h=nMg0U{YSG`9B^PYJbSx-`*QvEhDc`SCJirHjRc#(nOE0-UYWY{bLYg8 ztIsa-CO`ehp(n*18Xnaqq{Jb7it~i%mkm?TZkW*E@ZaiDwdIfU`i#xA*04DLl17L9TrE?yPw(oL#(*fT+pI=Pc-nwZInU>2L#-!ZX`pEO^hf3Q- z|No8>p)RMhS%d!VwO@TC-qy%vk59xo{+Er*N*WKG2)w})7W`m>`gVrGV8_C_kv04N3#c31 ztuNiLqR7U!FGi`k>Ee5VaPd6X{OK#%EEjlLdv}MF9A+vv+@kd|_He0G z#?@(}|0Yk|aQ$eMrf`qHZ&1c}r-+W3%0j6dlIKk?O5SG4xzzjGs{br=kJ$EBln8O} zzI%7^dqy@X-RFE%Vtg4G2Xv4!mQ_AJ3OcN|~vCy-C!%*PK5NBQjjPpdGD2>m7~^nQ7Ht?JXqJ&WQv z%KP@+iF*~Ccq>&_%}@Hn#iWag|E$-4JigCD`Be(vSH^mNlk2-2BO32)I-|V(@2Oa! zTCSjH_rGFcijAM;ID`U&* z6f3J8KhDMMu@w|K(kP*JaDh1^V~m=5-~N9JdG9{XwbK8e)EU&YUGZRN%)amO7arxk zuipQ`P50iL{5?WaEonC9Ulg{@Y`c?p^Y1CgbpQCKMN;!tKl1$i^x~the5VeyY?>eV zePYG-rRN+IqB$cr?f)-$xoDPM{Xv~4LQh!(tp92KYu<2PbJm7Qzc^Aq=hzjk-{X)| zTko?W?)6pIvrH3hxJw&Z{66S79MjgnUM43I9GuL-uVz}@Gi%bm<<6S(=e&Ni;`i|n zy)v0!MSeaLkaW)4%ssdJ>NAt_!&T}@cNSI}M)Q3$DB2Ko*rfk|y{WuI|LNQDb5{E- zTXlM|aXgQ-%_PZ-pDNDn-*i|>qn^cPi=>X>>3P~0v%j96|4{V$Av>-n(?@JsSEjt( zV^YW$w%oyD|BL*Vdg{y}l}l zN518($C0Ef`&MxrowH-I@s`Jyr~h6`*kmku)ob_PchBB6%48|E-&cRXx_7SXS*{cb zYp$$?U*>JT_Wwuy1QFA(M$G+x`@lX%8sz2lbr8{z5GIKQ9vPkDC!V0zok-3(TzZtwr?CCfJV=P@2O{=hd+ zZ*1g}uTZwLkUM^+<*3cGCq2b_+XG^BPk6VjTc3ETu+?!|SL6L8trLBjayK+MK4@^f zia+_aT=Zx5L6IklInO%VH@thl_xzv0M)?5K=Ya+d|B4)~dM{f~Pcr;nYe3VM(_weMGh&z;z3o@^c7NeB$-BaQ-DOLDFIQR2 zHrHX@WoF~^Ym34Z%xjNsT@$qF+`g(UyS^n(y#JSDR*6%vsG7#B{dsa< za6Dv|n4Z7)Q}~0;25shVY>r%EVa?rfrSR)I*)KiEs;ph4b~7GwjlVwoJp1eEPv1q= za?9@i%JP_Nw_@DMj*53z_J*F9`&kz%G+8FmY2q4YF_#arzl47C6l)%7<2 z{RvUYH13;a(=x;#olwx)6P4u`U8lY4>@4Q&%<#~y4R;D9rpjG>eB&0&j^6d{d!N2Z z&|W5+)svT)-5x)+`svRWNsES{@JB^K2X8-%P%9BK{ZUbxzC?e? zyBIHh17+(S?Yh0yi%#v0Dysc|!hio85vfS2NmrZd3YGOBB^Pjz_D!xNl=gl3H?@J4uT!j5UZmQ4TS+)L<+G7<9 z5oKfh36sC=zcc^#^rqV@gkMWvtzC4*UF%3y!E!@)^;QoK3)Qx-Y@3cXeEiEe`*yyF zg3ydzf+yejHT`~OfgPl_~@rUrL$*EG`v5zLn3@@V}J4F7v-XVckFST5VGc> zO5xKLKaVE4vwcq1@5#)*vGDU>|7k}Plf&Yl%n0CL9A-WD7RRe~`D#Ktc7K0&$XBvT z{k`(TmtW49TW{G_(SBU`jpPr%FSjcru3vxnd+*Obd%yp@d}fK#!D*9^X6>1OEB5*X z7wO3AlPB$F*|tyfm#kTnuD)*aGR{xeXZ<_*>2B5QVw=71G;jJI|9ib?dPfFZ@zR(d zO_uq6dp<}rA8D1EId3mFYoy7J^=hxy$Cu@tm|3?l``Bc?3RCgeWwWK^5}R6h@*EiE zi-~SpGiBOD_bq}I-BBH1{PvxXx6PjX`1jI}<{93>IhW^5vlOl0Tr)p4`#Gn^#ih6Z zE!5EqtyM_pf8s+Vb6wg`yzr`-U@8v;-1mRZMI}Q)?J$ABfvSN*qX>WMoeny5# zoUuXhW>!R(Yp_=LBH_QTVmzYH6WwnvVA>uVa4|FC?vAZ*`OgcKDaSrcblkzfz@y?g zOP%u+YkBDlpNR9*kA&;y?X~w7jVaUr8+G*VN25aRYZ|jV&0F@LJ!p{;ah&^A`1Zsc z=V==z$3b;*V8j|A{O#_T|8wg^B}VPP;v$nYea2? zL$~<#ip0}AKI;}1{8^+~F2Fa3?~HEqihmVZznd;E`EY5CjF1acjl`W*aWgD`8tYH~ z@N^Pm^g&ybZ_2es=>iNa3=Sy@404QD_RDYv2rx0kBzgzBE}0*=vo_&Af^QGm5`p=H)mzl)rNf%mk{eBds{__~GmWyTZ{o=&RG(EjVuN*9@ zTwh)-*`fGo=OM0(zc&7hp1=F+x>xU>$=+DOT9NOO<3DT1|Gwt7-QOGEG@QBO%{5`2 z+`fnicY}Xo4pW-kS^AQC^&{toI7PAS7kE}Rr72cecB}ZG;KK|d7c5x0gydhHmRKWo z=UrU<0e45;vuUcGCW)<+4Dmnckx43*#LuM>@T9&&64-==G6Txp2N_6;7Yw$|I?lJ`s@E* z6)qS4n(=q{Z?*Uj0uTK8T4XH#ZK|kPz-D)-?bE6iGB0?RU6}VFS$|!if)A@r0Y?lw zL;noX?JrGpJ=EnI8IQE8G({(5t+8*M{_tP-(-YdWoC0R;kDB*;ddH9cYis85a2_>M z>gQzq-7l8DNAz#s=`+WaXBtw&z z>#vUYzW!QMZ_Pc1oG_`)mZcBOm<5>*$wj-{8l-N~iaw*RG3Vgbwk^oQmjl_TCJFe}`Gl7)qUN>6hl_o^wR;=*s^Ib0%0e2W(p2@bvxy;o5Ey zvjPSiIUD}t4G*oBu2@uB`@QQOn<2-GXtm2Zzs)DVP!Q;vXJVtZ_?%0DGK0IMf z>HR$q4+?LT+Z!98ZF|4;)8o%sKWneXs6J*^zMyu!$>L@I{#3)6)A!y?`dLsnJM*28 zWsAtn`u0Wh|5p_LF}5qx*|LAu>x)I}^n&XCzur*&v0wkrw)4|}<_O9}JUxBBzoTfw z5?OApxW6GD76OX}Oq16e2ezesP%-=}XgJ}4TZWKNPW-w1mG$3x%mNvuB)mDc>Y1Hb z$yxoAcZ!63BGa-WNsGf;|Nq2pcH)cu|)!9vxdbqW3TFup#9IyKK`;Ohz*|qyi z)cQzYqf z!N`e?N%rENGKP)U<-07tzTh!i`0werww~Sdj(-eZ6MufquVu5(sF&1m9-NY{;`^({ zqi5>nbw|bR=NL%ao$xbIuy>si`%L3h_`l)}zY14Rs;vFr=8}5qp|rd2lz%6mekw7A%seSSX@rYu%DAZ5%Y`s~!hSyLbNFSmEI*7vymF=d|jB!{;4E1e%( zFWlN`^y#PFls`@)y4rXDEm&9JAo$Zm$Nc||d7s#-SGV`wH}}2M`tR$>Z|{~|emJM~ z>C*fO?FZvK|2V2myVAtG{Pn?-iRToQ3wjr_eU#&{`Ow$g@Wt1}$0+ZEwEmK)2}Ral znv6==+FL4ZH8$>=TK%y9MEV*Rw9G zuN6`#Ni%49u2?2hcWiOu-!CeXBz zL(HaYRd2?kV=Ct|1m-=HUR&w$jrZSP?>)jdQq64+`&{stXew~hZ^Ln2?JtfCIW&s` zYEpmN*B3^-cC60$F#V(HntfScpDkBBcRqSn{q-Fxy4ptJPzhT~+?GB3k zYt|i;ob^lj(S_6Bi=Hi+n08z$>m&DR=5MABFQukUNjaqIexqgQ-F=&!l$Cw{ui-x6 z!t7xz`Si-xP}?8#Z*y+_aox-~!fK|P_P>>WAtt5mn=+Oy398ZKV@)b|IHMiC!|qe< z2@#`+*LxYh?OV;2!fYB8@x^A%E#bu*wv?xO3hQ@s#1|IU+_*9`tU_b$4E@5=H}mVx zy6xV5dBvtn%$uX{cyoOSFk1hmY<_fj(wXL*ddU^vmmhJ@skh-fGB5f^#J2gyEIY1$ zNEY}%&FAe)^B-ALbGTA118e5HE`4Z}YvsJYfA4O-sLQb%oOgWvZ!k-Ed)y+A?|ar{ z$0nWpvwDl5X{q!1t@9bSU&Hb-1!7K5fs_kU%a~|8~{s_`jpYX`6kgrnl*`6);vfn4U6@L*w z+qw3Vu3DnsmRFN6`EJNbboG(Rv)Gw4au9Y0YiMs>;;oqvyRcCxW{>jBhJlJYZ=H&G!f6o4C@4ug8;(OI^$vt~>=dX*K zJE3)xUeu!i_wrAd?LT==77AUjB=4ae{`h109=Q)Qem~o>;(ye&X=*3Cil1D6#QpGb zjbG#C%+zzbf30eN97y2csj)wBntvmo*`H$z`)>wDW^kN-{dYlMtk9;vRrztVFG~8? z7CaX(KRvx_wb|`|oC}w)y}7Gu{}Y`I0kPQMP0+clbzxy_6Tled!m3r^~2r?x^h%u(`VoYS`cE6(^cx1!BpBB3` z&)9~4{29l-+53L8)U>j=|Myj$*6ua=IW^xP;9s-TtND_rbv!0qbh=YvI<>xrfq`q9 z<1F>gSFH65bG!sz|J}OKynJif#@O&nU#pHr)c=2Vm~H?1eN*p${~L2}?XNHM)~%24 z<8xb|ufvnj5wN_xoNYDx=L7qsxCHOjl-Hm8_Tzhxz~j$1*4fwbxUzru{qcSx|2qly z%y5;?JgcVl%XPM0%6v9Q-=jKt|8d4XuJEnDo_-2=x=%Cy^}VP~OXFVntH1xX{nn#h zRbOA$W`DhUYpeMG?@e-TwKGGMSG9&u7WkjNx73V>gO`!*w%C!cMjyk9cQB=}F5J9R zcKPI@^UND{4mB-$wV8P(v-<2D*?rmiYY+V3YkPI9Oug;kX3fNHOLugKR4q7QUoW4e z8Sj(*JwnX2hgqZU>#wYyoy!znHA$Ug@G&-xXSx$~p_*0vlTe`4oQ{3PlKBTGY}_?n zdRqVEUhT>q&PN|CW$ru>w2p6<$cAe{=YG~i`~7A#(9k@jUBA1#`I`8L4M&UlxB~AK z1T`;p^x&SOSMb-$uV&MNEeiD{9*C8c04!3m? ztec-U==@*fVWi1)>40&6T4^LZm-YW2OdfpJKe8`R|24tw)Ug%kGIOhbOud@Y%EV^D zakSMmps<6%JGzpG?+ioKep}(Gi~7$@c>YjJ_4uQ`#+!b&Y)tG-Hk`<2wV`Zjp-r)W z?uFOJVx6|F4Qi5EbGV-=WhpPpzTmn#bp88l@vDpet^M`A^tIme#*3{T2Q3)elbKrN zZ%%$-z_w%!_XK6}=1o6$ie?FNx~XJNT397fuXlK%*t`#$c@`R8JMdt3L!|%lQzv*L z)V}||c!YbS;+=@>hduY23}bKBe+`RRxOe)aH(p zOTV6ew>Rl;>UrC>{tsGi@2@jIV#EDxSxd$9gU65Sz2W@uoza8E;BuSVVYMcu*z4c8 z*2(H;{&`nAKcx9Dd&dR_0TpKf1_6N-35`nu7d9j;ir|p;zW9Ps-BG4ENtelS#X=z$ z&VZY7A~WS!Y|Z9>&55@Cc7)^6Z}}HT*U4QrSM@BpwMj^G)>pxYe?J>E*)G(Ne<=Iw zYt`3}a%=(Zjc;sp&eyGu;1FO#t?ob^SkE4X*?Y*e+ryMB7tUEQKj7pEVM zoM3h@tWRM7(YPaRFRH$%Zm+E}HqYa>-r@3np}|c-&H1}i-d%pM=$EnPrHcA?uVO(> z-5FIy|E!!Eu5uVHG5__=g_l_-zfI}&_m3J|73=?>5(xiPQ+dWnyU^X}YPfUA|MKdt zcel5D7cR57ynB1tlixfJ8{`;WnNo88KZ-ASSl-mdF5c+0Lu>EB3$sl3i(MDE+OB{7 zoyKj|TJZ~)Z++FYEPlANkR>Dg$x5A5RbPdRW+nW+|L{+d+vy8e_6oc>{Nc(JQH{s9 z-B#Y)WxJoH?M|lY{Wfnq!`jbEhSR^T{h_Mln!H(c_uO~NI(4F_i+0o-gAF%tJ(44d*AY{U713Oa*2~qDO zEC#+I1$+*>Zd$xrqCC&>e#^P48nb=+pWe0qd}U=fLn21)Oa0cjaY{b|FHL)Y$aQT| z`Xe+euqkNo-T6KnPgmOfkNV)^RIjyjo7%gdwcKyHuV-95 zRjv@}u-(SLQpU4ey9%=`8IGI)9BzvAqw z`MvklhR8c5CV>aU?|c5Qym%XY7GnYTBt^X~PYoi{h#eaaobd+OiRfA98lo}GG{|9$zNX_uaU>l0HI2sBVL z%bTnH;Xpt_|69ew(sS3YN#wY?Nn@SUv_-cbS{Z6R$hOaqR2Lb4|fe> zB`VJr-qtE=I3l3pb=z>pr29p+GP;%>9%`N61NMGv+ETvKfH&JR*vpzs0nyPH)%w^eZWU(!mRI@Avv_T(D&KBc7Y9 z%e{}37G@s@3;v2TtA8Jz zd{4M)L%W(*fMjw*<`G`TzFkFCIhUF{KCS%|o~3#6XM|wFkH;$i*^K^%)C=lGzGW$@ zH`x28wtMMts@2pl}J^8kZ0tHJt^s}+rbyS@n4Utjhm>`xPa|9Z7ERm%2l z<-3-dbsgot-tj!~rMLJ_-kr(n^Y%*d#>KBRYdtLBtVt2Cb{MITh^_MKKpTSV^>h-iO~6vgK%|2PggbtTyge0%>%&9(d2+dobsO5e*ppMFeonG;sv<`+}1bLIXP zAJI9Xr}L`~%FhXYuWMR!`v2G08a=Z!>z}WmcVF=J)p*v`)BIyBI5uwLD6ctl;2gWv zxmAz58-ImcJ~d`!5;=5xWCNa?p%eal`mHCV7H{1I&VoWG$-!ze?qesKe%mPn6| zD3@qcW5R+E9TRRgrh|?f0<=VgIzdM*x2OmzZ4I7hTpsl-&nd`td-3emSA`<}e0)=5 ze($*Z`}_7+%_c0})&Hr~=JvBYPcqH7E1%c1Km3z*cl9<^<)6ZWi6zs6#5yHEuAg1! zxbAk$x8wi!v9auG7hBW+*-c+0TzGGk%cH9s=lo!>Z(ky#pU}Q?)92oUt zxaQ6^JXu@0G;7P+TPu!qx9VS?<9L8kO=KdEiZCD3L5Bko9->^VPR)S|SM&spwniTB zO5Lbgx%Q4eS8{jPWan*<>-}$=+eI$g=e>3Me61^UieGeJe;2*$?Vf+)GfW;PmFK%zQyYUcOol~?T-aM-N#0zn|&I{k|ZZ8)5oBEl5N$fd~%?&et@LY`VC|?$u zxO2siYd_0mBe~{9sopg1z4LnAG!~d5C@z3;4nA(`3vt{@8 z&HAVLUNhNnyH`kG{y)D+PJR8R;7xDi-gSFzUOang`F(kzIcdQ!4qZR5ZqLL&<8Da< z$DKtE9;^R7G8HV2jI^nq6FlqXrTI&fFDO>3)Tgxu?X6hO(ldwQ0i%Y<1RW7&R;Ggv z8v=AxxJ3^&Caefh6X9k#w8&vbh*Ns4YO%rI!&PVF!d7(d3@;2Su*h>h_p|@f_Ll3P zVo(2)-uj^~{+s8ky4U6B{?C4rf4c3w-f8U%V#oG2-VJ{h!Ikw!P3hk!)4EIb=kKmf zoHTv*k=w67JURUA{>PcRA9+qxW?3g&`tRf3dM1chs6b2p#-?)RS4>Nb>m(kUb?z25_-+%L8_^bcjcT(Oxezfj>P{f0cRiA%z3upOm+3K_;An=3ViquF6 zSJj|B*E{_e$S`V&OvqBv=4Cq6upvQDMVOUok;94!T8_r^kDs6Q{k8S(IWt%7n!R(? zl2xVx7aq-Scl%gkHIM(~tNLel_AS{z?S+ls+poM=d*y7t`q_CR?VH7R+?ggDqcZ*W zCzYZ-;iepq1OA79wo~)5HP{+(!y@R)!rQZd{qx%GcBT8o(R#U4{k_#ZJ8y>{Yd^lY z^!}3o-EW7lDtz0R5WglpTUlN9(?0eV{dW&*cZkneSN+THU0=yM2AjXjE^;WAUW(kA zHOJ5^Y{HcFDT4Nqm#)RE(`asB)CigI#)Vsy>9E6w0Iy{3R;NZ`%eC4y`Lla`Urh^r zHT!Dlv(l6ccXvEjdb4dl|F>DproV=Z3Wuy21%+_m7;-*32 z?#JV#A2W;SKiu_2ks)y3e#i6gxc?r0y~q3h%^(F~TPyj(+Vdx0K4kc2w}Ab2kM-R$ zox?KfOBcqv#(FM&*%`XLMEW=A?EIPkj6^j;I*s0~va`PS<3fSM$vaPaIs`R0yi}dh zbh?(yNG104M!QL`&V*-X2~EqM!e%w`wg>YQ#j2^Rrf2xvc6;$dpqaBt!$Cr5`}}|J zK2N*gw}PR1? z@8#vCbpro?ZLeLoKkEJJ{k`#1U$4KuBI&o_)%D?P!&m>?`|F#({jUc1wz8%ja_&rw z;oGaX>D`GqRncRy?qPv-hxD#3@3^?9SqP~ee-yZ1*DSk(f%Tvg1Ea#{1XI4a3U=>? zkJ+yR*w$K@G&3a$=?dK8|F%t3kn!4a>67*c|M&f!qjI$C=$XY$jwx~<9{i5$FMb~J zTx;inzP&N=ukQYcEd9D>*RH2Kv)=3fzj`~|yz%3j3v(u#%Z4;wyj`^N$g;x8oefm##Fcr6KY2mWb}0d)emrtkZMFCV8jn7KcUUo#IjG z=H?I%;@WVuu~|2N!FGw2$Ldxrp8P|kTO*uRQNZyuXMw|o^(kRDg;rl)FY*0vlGBQZ zP7nHhzB`}o3hg$r3|v3?F*9{a+s=_E%l5WH2f zP;`UA3l_O@m&w*&w2Z)$kWbi;&p2Tkbo&VZx5Rgse+IJk)%;tKki~f5UG4h7rdm#~ z87C}K;!K3wJbX?ybULMq8WnO2+Epowerx+$7@`>*eqH=)_A1$56Mx$7=s)>GFv)|- zM*S6&gigT>$G5LTr+;R;)6?2Fe@d0qXTNML1iWgi0pEP-B zaqvDfH*7!#~ovlUt=9IS8q`lt?eI;*bq^yGKFiLU4m&=Ry_z+f%l8(^IT+qy6R%k#S$7=&0>)Su#0Ti-dK^VuO2yLJE6{HODseRzCFlFp3)#$`L0R=hDe zvoR#2F47c^>~? z_%~`@Xm{Jf&xm9*vKc6U5J=Sh!Ljik(=skryPP8*7#F{YyRWh7 z@7AX}7A?Z>cO-RcPtCf>xYna?32*(u@RoZT))Tv5>23Er9;XmFo4>_!l9SCHwem|w zvx8FpUKHn@w?6;jl)q7*g`3XIyd(O-`Tbdw86P~7}Z=E+;io{D5x`x+kZx-58Q_rYshI8M#}@xNq}rTV2k)6%D_ z)8a+ zSqv<)|5=W!_uM_R!|j*5=l!nZMc1`OLf4;D$bEOS=l;%?AS?gJX%}xl{J`nSpmkQS zGr!+1!l3fg!Fv|Y_xFEO>G`K|C;XhxzW=PptzyjhA`4RgzleEhcIT(X@rS3+PyN%@ zuAWjv(?+*ge^Gs-$ld5)7q(8eg7+Gi#(s$ z*L#*d)ShyDfm3m5zSfjI3P&!-ak8x2dqZ7xXN!nUXFy&?%~Ga*^9c`NZvTFb!zx+f;uJkk>$R+YElm*t|CZgo zWOn-N^G=Wk8=IsInhy3JX?u5NV6n({;Y$=|sB2Q8E8Oq#rXQnv>;1-(tO-NJkO zP=0~4%V}-HH|v_*I8VN9iLy-lV)E26^xX+I{z|)FA7YQqdd73U@7GJ$iM=dlcRQ=f ztj|sevzpOdbYs0$*I%q%qh-?1xjx7tZy9qt8RXO=ZuO|99AWt znD#UL-&1@hSM}+XUDFevSqFW5#?f6~nG&s9qo{su@01rOv#-9o?|t>f-d|tlwOFXP zTUR1&3yT1`!-d*%oCbtKdmeNu3BTouC$e&!H-Y%>i&I()!n`8RHRJo7FeW~ z208`&c+q_FXZOye1#d1*`c`n&WZ%p;tE=TgwH6CHPUcVT)Zu#Z%41RT~%4YZNtFL!&=Gnw(T=M7F zx<8JiYEm1D-be*=ZsB-%|>3R2K9xK=XzFS`6yh9~y`KS6dLPFezGiCgj-BGPs z?0CC;0aw1i)_r&h|^kLt$Z+z?uCx5QV;NG!1}(rrtV6Ae^Y+4ABdp4;|pmuQzf<+y52ck=Rc>^!Y2@9SL5&W#Hp+ob>B<@Eb-N2S6wE&JYYO1O3UlB6-ScSsd8Wtv@4nGzZ7r|fQ8-ukxt_hw z$wLBpQdz#|W0tw7)%maR)2V2fAD!&d(zz?*<#hG8*C)&=7y6m-q1<7*chQ?EQf*=$ zGn<=?nI8X4woKffCc9QkFpP)Eq~YqlBqpEb4)SqAp3K}%TLrHOc6TJ)|03A+Eb8&Q zmlb`xzCEg66aMDR`Z)d5p5L#;W&deyyJc7-aLp_2XQ1t>tY5GE9XmLb_}*9f{aKo1 z{`AQw(@GYB>-TQ13{+PBKI37=@2ULIwWmx3T=bOa1F#`;YuyJ#YWFedSmG-i?{OHukT!<^S+F)`@jYxlBdYU3Zt} z#;##`$^5u&w%fWl_XQp_9)A3p@pvMStD!ZKNVz$J6aAp3G@bz(JJ#&JFly-T*icp=MJ7T9?2!Cp) z*OH8Mp}Jw2VJd;_zRb~TtVmj3@L|AFIxm5hh12X8*J zSlkC&yYFHB=i@en#;^4(UG-FW@|o_CTLOYCZcqO{bYe>6FMDgJ{VYjB z&yPK47;C@pQ+Sv0I;!N#8IgnPTQoDb3R#<9Q(nLD_+*p)1&i0P^VD$jz7b*E*Qv06 z8&_LZdY;8=PsN*`b)&xWSu{D#-E=MLV8c~?ooz*@AIz}hJYRe@WVyg+gN(3gwg3Of zFgZ8~olln9Id@InOkM82^3SEKrygDF|2lrHZ=Lk_%f)L8l%-pyZT)C#XDRNBYQ0v?;vZ`KdXJYA(x^vYpa_4`)Q~&;J-7JBxE9+wR>{D`@ z+A0uG;8nR~=_5nu!}c{-*0y9gxwF}O+GgKZ|L8GOOW=4E^K@#^@}cU7zQmj3!$b-msE_3G}oF*>`>8*ZAUGsW*))5WGWg@J39o##nl z-u&*=Y{_%>I-b{K?zq%H{#qJ-(rMbOm!`ET3v1O^pL`wu`lkK<*YyDr1`}Rw+BD1J z(X>B8okqdam2JYV^B;^kJmbP~!#5KT{CGE8dXGrO`)$v4Q&${v3wU+G=vws!BdK|J z_Z}}+)xGq8XTZ!~s@jaMljnKtx#tmkQcLKgT-WtVyQQ0yl)3D@PFz}yz(?>V{%=eS-bUE8s z7&$%o^xw&jl4sHaUY(dC*LWyx-bQWV)3c>h=Y?qXh&*iHW&UZo{QbY2RZFM+(-zqj z|N44EyUg0@2mB9&eP;W?WMrXa^j$opd%9@X56+29CwG64`+B3tN@u44a}&GKEQ=4^ zQxcd{u3ih&diPIxQ&jRv`P0Ad_P$Wg?0s}b#rOo;m=dHi~_)s5Xm$2`~jsJHfxsF;>PE&c@58Jao4jboc}f1{r~3)?7_e0q&@W6G>dJg_TQ?N*{mAJ3l^+s z?`Qt>Uy4aUis6@lkl^n7(Fd;ZGK4Xyxz{NQaD=prCZw<&TBxi2X#4bY0;~*sI2h(| z_^9r%KP0*H)nwL%9~pnhEQ~N?TbRbc?!5i#eF0781u?vr&)l%@nW8aQQsWWNwyJv8 z$EO`$MOL5pdVFm1|JS#qZ?XKZJnrx!VdmqO1DnJi=~tg)J5rJ}`s{BUe~Xy@ zoNt?ViDkJ}@5$h8T}thDTxEI_u5;?GnO737+7!sRD{$wQd)4pm+GxP19gav>C3~f9GGbJHom_BOn?rbmix(ry zqK#~h=EoWx84qk%i4<77{zuaR85X8h_4)UcfBct?V{E$|U|P1tbFOZksK!m5{>7F5 z*6@C<`S^ZCotbSDk442?y_-9p>wgV-(4@lf@Iir8!-@!otm?;G_#TyQc@&%cw^PCH zbMu??-ZhRdwO(caIy(KjzV5{tSF}$bbXQ|vR=e}(_JW7So^tZbZj@=n^KkTXAFtx8 z5*L5GKJ;CyLgV!tIV+Yfdm3f&>2bq>zoq_v??lh8l$jNtEwcSyNB9+{9+4Ft)A?u5 z{#4JjF=4H(qU?{f7}kc?@I-b4y@Vr`ZfwW@7)9JTBCj=h^BKX<({wlgyP13Z%)grj z)5Rmha{V(6zPFrNYp!qNz_Wa>$A!m?8~$%!DEzQs?r;;=(o=JT?TqXe|9*YrkjFkNX|;! z_fNk5RWa_`se@rhCEKzl)lQIS3iGMl{xRiq%vKgTljz=rY%M*eFa`UVU&;0A{gn|L zT~igOui(F$Tv<{2HRa#?fFGOR1-^Xz>CofzVd1Ohf2n)_#cx;W`>Q8SU03NpT{WZi z?DZz4-Mjbgt^U#&&~b71<0OfP?0X+Pz4}x$#QIyZO@e4ujX>v}e_On}Od4lAn4a|i zVC4mc$t`k*{_Bsm=l$c}C3&`HuX&}t{Fm%kFK#Y;BT?|?r`E@wsH;s*K6&w%PCeMr zay8j|-Xe3QJgXSb+0{7*LzkRqUSZO+YerVa3(YM%A}4!KVXAam`o}JXZGGOpOLq@9 z%@gw0&De1`Z0Xh=BJT}8);@d%5m0!}%71M8jET7mA*HPW~KyBLBTaz8t zf8J)^de8r^)3@f2cT)8_PCa{|cQS9Mv(f{}@OM`qDJY81neu;0pvpA$*u^(e9&Vnj zEKvDa!+EK#>z=DSZo5{y*9n!@8BX7sw`!;9|1XNu-g6(bnRoiqLVukOf6>|5U+xr? zdOm%d_&aF&I(aF(*=+y#c0^R%>?(*&lwNZ7R+X=FrsIZY%~$(4+l5`Xb}Zyc%4P)-817 zRa3{`@BMjuTCUIi{!2ez@cg^3CtbPq5*mB&@6MA-ZDv~YHtEj$y9cGVEr0)EV`UyAK$j&(0i+&wcF}hVve{=&j|~k)BW*g#T%ssPhY5c_xc)0{%yKw zmbmxsxnt3>21^tk*U8?tzxVFUU!O~t%r9G)E!cO-Lvho4&4u@~-<(*IJ8AtV!^PbD ztoyd;FHpIEYtr?Tw=YjW^Y^4v+mny$1H}~93KrikQn%s`iL^J1XSEEOd&IuRFZb`g zn7Ott9BCX+A3du4(9kGQnqZw0m8IM?_vt34b?S~bXD3R_e1BWAsGswq(wRr`pQ8BA zI$k})<@JlDsa5j9)sTA=xSH?F?f&%PhMtC+G~4z@+n1v2=WSTR{~(Kn^NybS$&6n* zY~mL_p4hGB?smUvxH>DKN_C^`42FPhUw(v-D72owe6DXx)d#RIXDV zYuW7kqJHXLIsNXBQ2P6gaaI529q&F^(o^~Gvvl%Co$a|CySJp96wX=q`^StXW1scY z8jgM6>vr<6&ZS*upJFCf{q9)u>HGfI`S%vN{8;>B@#~Xv;hvvn*OwnUVc@tnQ@T)f z_q)%0YwA~g@%3%{Y25qYrqK8Ck)~*;81bjSO5U>kH(j*zmdfg-UF%N&-&^UCS-&^F zR>Nbf*W~|ero?FL_ipWc6A)1I|HlsJ^BPC8U;l~z^sgdxnnvjAryK9yZ~P(5cP#v5 zkNF`Tq}jUAubj*OjZLx{9CvsP%Vk_?q~&@q%$%f9Fct3w~FUaKK^e* z+bZ{+OGA}TW-$m>yWLrA>vczEW?1ZOHD<U=SH>PC%BwAV#8y@u5xYfx&Ub^S^4vuUmax>-7BVOTNC2 zY@H$2;u5Dl*EPQ`Lrr<>#K>2Tkw!B_=WMhNP~d#fu{EmH_`m!MQOA`WTU2I>UD>hd z;QITK4|1B~*YL<+@3Visu0(Cg)^|UDPv`hQ>0@l@uRYWAJr1+AF!-I4DD<;3P267) zVHP3SbbOJNgd|ItswIo!d4rAYJ#CsHe?#8QKWn&3e8%C2=EvL5E?OR>zI$_R&L+=k zU9IPq+0~txH&dmeJziydX=(8+oSVooX7%ZdjyUmU%3 zl5Luh?t)dVTKVhd@6m}^lc&5$DrdvL3Ag@whu5!-UoCGk>D%648aTZ+?48~o zvS$AL8%c*AE?15d*E?n#U!&=Osur zKS?(}_eXBN{`U9p1pCY{|Z+qXKSpMWuQANER zC-?5{Nk7_IEISHzZcIDfdTKvIgkz)kl9-ED5iV>SL=HMm4OI<#T)5?6#z)3xty_UH z?B{-L?^Pv5@ z{XqH7rmCg}A4Ua^WAFCdW31h3AajFHw&BVVc4i~_uf0-y(|;C=o*i^ew)BX5+_38iK9=oYLDM$L{yZ7bq+X93C%=@p(r{?c_ZMFF= zVZBAtr{C@o68rA4?t0zsIUPbXZ#Qk9ko)e}yo3A=x97<%+P&jm{GJazf*z@L);sFn z$}ib{YqHMFZ~2jaET1_{W@zhncjbNNsQ=gGroZ>@{Rn-*>I0rrI2LcJWw^v<`(Rer zl4KRnzJvhX#U4dTd+(=P9p-e%I@*8r@50=B*Ec>|uxeW6n!7GHrUq(lU8tiKd})f* zM-PYad*+YhuYbO-aN=OWUlZ?~c+-|&Zy)gZuHXIp|I5Gbk3KD$J89PCli&PTC_cVb zdiT?jU#E73FF(om+VP^%n`rkei|LMI=4SW^K>BMq&`6u6KXKoUf(ry(M^C zo26~R;`j63@164K{Pca3-d{QNv;J_?>`lHt3+J5SnP>Twb?HRkv;Pwp%I#Nl;n=-5 zML_PkHowIGCqLt6^SPZ~`mlw?YR1i}e=794o|zj*eVBgSvMhATkIyg6JJiw}OSM$B z%KP|z{vGXF>ESfxQI2$m`gHx;g*orue+k?DxI22fzJ2PJUtjY+RX^Q7`S5yy%=%?1 zv9jfp5AC|MaQ6g-JHPC<#BbYLcYW8UiqiXOCdK+2t_AnLIw%;k{p-{fm&8mgHdIb1 zdTn)3lg-k`;W9^E%$EC_9mWnP|9+0!z`BNE*_VTPTvNO3wp%19`7z78WKQFcVEy@! z$$=-8yEcI@FmSV+wW`V=jYg(`&|}e)Rzd#f~wlDj?>}L`$FV9WcC%>)ao_9#Z z>m29l4$P1}`Kw2*|}H z6{Y)^eA6DwuAW-Iw)}o%^}Z##vbQ^Ux&#H9gdYBVcJ^%H#>GXITr8@7EgZ@v&ldk{ zS(9ZrJ>-Pno`3AwLe1(=90lKr7=;@tr6-Ck_~Md3@3o1kn5xfYQL6V3vHq2<@dxA1(klz5-)mt zH|YM0_0ccam?@V>9JsNwewFw8-Jd!wPv5jzeZA;~^Q>~G-?um?E|U24{DY^@mir&# zly*IgJiPPK`X@g=g$O1{KHfH6#j!Y9J$J|LFY}usOWNbqcb@wcV)Xa!)5%XvOkyMp z*;1#4bF&|9f5q!^#MRJeJCElIN>uB+hm^B|13!;I(5I94U1xJn{G6`Gd|plNf&VMp$=%a+H%?dnrx(Y!r}OD! zms19Ft%{$76f+?lZLcRwjT z4O=Ga`teO*=8p;OW(ksaJ|0?EvcO)w>lMp{4ws8J_Qm|vj;Qd6@wn*{#Vg5wxJ#-h zfZ_OlgS=mR^WJ>EUA8|&$lc1x^7|f9BYE}mxTejkm#)@-_BS^E#a#9Eo-fuuwLkq! zS?HkYcQ<{{daduTu9uc(i|&aj5$^KXcwaBEb7n(=D%20La&28IWG`~I%J z>i9w8;XB^CnODCYEEjkp`ucf7XL2|YRM(m~}Lhvlp|9gpt3 zD|bD+>)1)bv7n%pD|Bw)k~H>pYpSZkKOQCf8tr++cS*UdRhLw zOD)(c@o8W2aV~{V3y#*VEVFyg9vO7)zY)(1HNEDK%XOV&!uAUB?En2h^6ZTV6BK-z z^HcT9-7Z^ymey#n|I2qPGdg-dZ!ep#(si?z#0@$do=q&#l|3X>qUhDQjDwAVC7+Y~G#ksKk+s1v%Ui`W7kMBaw z|JMg1x%fUOd-+@~C|-D^^NO6vOqj8%Th zPmWeB{jo-+<@)uvRWIwN?G#f9|8@1&{-ghPru;p`a8^RNsH%%eXZ?l-i#^%v9=esr zTzCEdZ{MmfIxp+?iWlXr{U}rV+~B|j)+rtzG91?LOgOb)-OuaCZeGCQq)slzb&#v!Mkp8SMe(;kML^jp6zs_68dnHhVVGNv$#Xp#C<>I`pTMP zYW=;nhaZ0kJ#+8-A@zn8#!LUSP1#(SQ>9a~MXW6$JN@pzo>_4Q|JNQ~$~BQKLg`+S zguP1A)lxgRq95 z<(7T?+VHjUVw~JZY`;A4WD4-Oy?f{T*Ef&e348Zd5-R_udA9b$a)8behvnOt4U~>?=CksFbI`s5-&ava$Z%{3FWQX?8jL-r6mm)Nhsa zK-oUMdifjuf=iDc6?c5N>9t<|PE;4q;ji}>#@ojp-qYs9@zjL>Mp5X6`^L?i}#D?m8(fvRUONFr}nP${ndE&RgQKy6b>|hzVqE< zx3j~koohF{`&^lAA@O1Ex^o+LZ(I^|sfb(fo%|~ceub0Ww>y)Y4tQMmvrYA=_FT3j zcm0nR=?BX$g?h~V7ypr)wJy4lTYt7k;WFD*&m{hAyze?^%Di{0n~bg=kPO|u?&qhI zpQcuF?(De!;85Zh-txJ=y{S2+TDAYx=l?$vuCnLu{e_0*+uE3fVm6+iQ(niEFtdh9 zmGOdKFw2g0A7&mX;1a(q`jM|d(COG+m;N=Lj~Y@RCOpwF+*w-i`B{13X8*5TQvw)N zg&0^?7#OmhXeei8xuJ4m>iiCo^Gl8F=l?&azyI7u?LPvE%J1HmaLJ$f)X!XcEGvhD zYvUU^U!$56E39>Yg~Hi~5Up)Li<+|L4q=aM{>U)z`uDuiu!c9eeis@s0l$;^q4f&zAph%}@4r$dSu^;~u36ag)j+1Z2wfA>UW(?q}h*CJQe)9F-Y(fd4Qy<^`#4RiHCh)KGqEIH5sdC~$ zuil-WUtFPL{&Zzaon6gWp6I>1SPFN~WwOY96Mwg_d``3mh+nCEt=e$t;u5`u5!`F6o^M!>u+>+OSGg z@vdHUbiH)8=4MtILtQJEtE_@dOy9KJOEYW*EMGnR|9xV@x-*~F>#t8Oc?3piHp)WeO`Os6|>a>M& zyAHgVaPQ-iqx&zt=s4`Gwsr33{V!4#ot=`W))=Y1Zp3r)TH-|s?^sQ2=IhU4pZ z-)GttYuI^vBR3aIZ(i-C3I7kW^Vv*f4C5(jeeLEOQ687maBY2vyVS4QhnXZ2<}Ul! z_GfA;$IGgat&z$exo2#?e zU$vPMP+ib?@VAU>#H4xNQR`i&mahKux4P@4miP4tRke5TuZvH9h%Dabm%8{~aS`7yjc<6>I)j$NpA+ z=laY45ArOoSZDaX_T{95p@(y`nF6Qq^V-!Y7nyEcv*fVRCefzV9IvF>gx7Fx{k{EV zmEnvh2bN!|e*HMLU$e$p?uSliXItjKFo{|2!mO5pU*s~E3+0v_SbO?fZ}8pc67|z> z6g!?|>quYu;pVd+T-qfZw=VzP!=>=x+~ze3&kvXsecT#VF77a0LZq?6?#e&^?M7i5 zxwmW|>e>XghnKDUASCrW-R|Me>mD=L)%`#DPI%%ZmBZW*&39H62Rbr++j{%M;;Ffh zA2?k%4q0~X)brWr#VZ1a2dwXH8^r2e1IAN$@y%g9NIi^E=wJBWjG zqOALW(=AV0_O164I%xI7hkyAdwHjZ!=u(CCw(&n-Kh_Ty_}e7?_TbNC-`q{HbN2mB zG+O&7E4F_9zlk6I1oX0fY~B$n@$a!`?8(FWPuclx3e`VIDJSj8oqnFxr07zx?j)Ig zfv?j01fI@m>6!FIJzwX=`-4~6uB3*xUecSo^vkq`5|d7Uu)ROIfMfdpvUNZC_ptgF ziJe=_UF$CR^Re1Wi@na1e$S}t6R;MZ0Prk-^6{4{S$udr^|~Y{TkbwS4*wl&!TAcCGJPp*Y}$HRQJzUG<#(CaU1Y9!!opg0P%M#MuK(>kfqh~c!mrJP|5>eH?(uN;;TsBvRtQ#> zZS#N7Jk5H(e9R*0MVo&abX(2en)u;i;C!w36ZK5zA1V|wy=z@{?S$is{#~c~GZT+|ng7&#({*Se( zKR0jp|3e0;0&Q0Xf3&}S99+3_u zijU`H#WQt$ROOn%bwt7_W?9VaORF~@UpIxx_28Fj{};?JkJ}rsXm9`5Gj-Y7|2H(v zYV0C2{(bkJ{wsImPW|(+w%O@ntzG!rfZ`w3j;q9Z1dda(@Wt+SfzC3p%Z}#*5 zv&-yG-@U%bdbVZkl*FV``;5M7mjkP_--{Yuixa)r+pjM8SjCICCNCy_8n;r8bE40& z8-FJ<$9`An?&9<)F*9+TVHfVc@w^;{f01tz^qJA5|k{Q5W1tn*NE>rK}A zxjWTs{dzX0JTO1cJ&9+1mxz$ff~5!dn4B)1&&eYga>=9Y&5s`Aq_QNr3g)SeQ!-Kt z&UogPzh0%=yzcPge~$zDt=<%~jpbpx)(a^7__6l))eYVP4~uG!<~?7jIpg}3 z+ne8Bo4;#gSnj=(&;S1t)^g*h+1+%=kg19F;Xkv>M)j3y?SC#7rAE}gs!h4}e}6xx zl=$!IW&NmA;&EB%=@3Cfec8P2Hx{Z#PUPsL1bX%43s{Hob?^i_3ICh-%w)=Wg z{d-`I*Tzayom2nS_v=0Xc>7m`=r3kNy&6&y3( zKDODrOXbKSv8c>|ZjD)i+yNDZP7mjutp4rL9JE=ZrzJaM759Q$J8x*pDm-A+5$W&{ zQfg&67_cEgM}(V;>1aX66em}Knfol~R=xfBdEM8!)n{7P+e@CWyBIkqD6T!ZPX!IF*v`LW zko|X9mLcrqv{=Q^iCu}&_mnb{MM7j4bwVbnh;XtxIXP+sXo+#PGA(pi5#b@h&3VXC zU`B+Npk%9)TJguOXM3_%mq)FAbT#~I-OiWmd^rz%u-$8?cW>S~{S%cd|L(r1&GzQ_ z>)7Kz+Pgir#{HT*=U~t6qGd{_zQ+GpSkD=4)!uhP=5xK>CU35U<8{8S^DFn}|9+Kx zV!^G=LM{<68s_}__wT{#o9;=?3t|@4%f8`mjuN}5Kc&L)v%Vcy_{XlvExY*J4>DZ- zb~Q9bb5tOk&B&wH?iK@ z>9y3<{kL!bZ;Gv(@}){MZ@SMnGiG(YD^rTO*Ut=HUz9JD8R}B{_PmKsx2I{ z2O~mVpIeHFR3vry#w7+`@%sEpJ8;vCi3ZBgFS1!@KXJ4AYZV_}@3T+m^{T!buNNGb z^odJ7_K(j`R(AWzmp`tavfcGuw_VR;P1IfWO{azLeLS*Z%NZtdWuwK%EEp9)-tvn$ zn6$)u=h>^D=2bByT^EjuJM@<|KjEfP;k1P8j^5mt?;db zvHKN7oGof%)fR2|u>Nh8jp3ej^GbEQ<{!&dW=}h`&e-g{#(dpXlJA(yUFI@(vR~2n zxZf!oV=-IdTeN%R2}YOgmlsX_=^TH1>aK53OaJ9ZE;@PV5~pUKim#-f*UZPfCxT@S zEVfN|{H*Lb-lX=B9QO3A4XBmnoonqEb`ocF~YI7?C z!@{JKu`ztz7k;ZZ_7*cO+S#+noH;JJ0p!U^Vp)+Zh2BmNJ!|oc?I7>Ht#{mSFo^lb zuXuJ__0t4Kg`0oZYV1G!`#-~<|Hs;=_{?woaeIrkTZhS!k~7X;Hk~YboJZQkez9v@ zE$WWT2%S7vDP@w$j#m@E%#F{yf35su_wkc8ukzaFiuDQfkFmul~SJGJR<%%rP3Uwl&OzFxi5xM5kU;IasH zjs6RQ|L^SnewcszuSp%%xorILhu=2`-)g)ZVEyC3o5#$nJL=vUnI3cB+^<;pgz=1r zvt!{cc1DdW9j}Cp1P(1Wm~cczgqxM+P{WQ8JwfSXJMQLvd)|9ByXxHRMLRzFo;f}J z$MFrHzg-QDv)L^x^P@8P>BQ?h*XPNE_GfyVg>Br|thZ_M!2kcU1w0+tqVF zd1gh6m1I7PDp;ZY?#yg$uW9!t?-fW<+9B4}&ab`x?cp4@ZEgL#WcN03{Qe^=p~1ja zz?8aRhXn%zU!CJDEv_!sT8`5h@l##@ZT%B^Rs87Z|My?refjV2?~|W@e=so5Ix=r< zXsF<=gX*jAuCEYsJ}7v5aqW}Y_n9;j1d1NFPI~&i?qh_?f}rj#C;gZ6a*E%P+&k6U zG0y#tnYz^f1Ve@xZf@m#rlpeG5A;cYSa2ZlvQBYQVA}s9f7`P;=BBqZRJb{nHX1Bj z-c|mFHMg2=)0g`{*R5(^@|y2e)Yb407KfN+Di5eRCCHvxA$d%(=9bveLhh-3e@@0( z=GiT~9`sfG|L=EB|9^cw|7+ttYrWVtKOHR&Jv`jRK8;zxW`m4(Gn@I0YChA~y_?P1 zKdFhHyWM!%(PLJ|V>VTN@!PTw^sRZz5}G(xzdVt=wfgUpDbk+~v5Iku`F*LhR9}62 zg~$xn#J2Bx+UGU6DlV`p7%Bhzd&S;;MyKhn=jNtM^#!gU-IjSx#ejJS*W>D!hntVM z%AT8MHS^rcy4KCdg_yH$yn3q6_UXoEW+p;@aKxzwcG#Gc(^I#Kq{ke@f~0W$GK(tr0)W${M{s@SE((tJ!Hud14Qi zehdEnVbR(b4;dyLZee+}N!sF7L3Quq+l8oifZk2mXpXzAtNBtN-Vb(^AH}Z2{5qc_ZH2{|VCGv$p!( z&a7+tSNBK#t$taRAFi@B`D@nQT2F)i2E)L`{Zow#wh#_qS^LtLv{dZL7b&+Q0Rx@EI-E z#*{)H4FjfUW@}`hGfz0^Au`F6vo3AT28($D-#B=ZmNa_m$u0HH_xg9u^PvwzveZPg z=ifUIOH4WSOuk(AdiMMs{)XDCXFl0?I?U#b^eNb&|CGl`m1iO&hB+pPgXwJ zf8}e1iatZgv7*O!_olKkZ~5R}a!7qk^<3?F%>9)GlLPB76+Kpw*ME9aWnb&p;Lv}| zSF5dh8zc90m;M)4hai>%PC5!3zCWsX)AMdZz0UpzMYg;jEH+GEFSS5#@?qse83~hr zSZ_69GVeJ0ha+Xdk+#N*N<|;+y#h_T6I0VQw4`g2g5`}vnF`HxFZ*GF&P>vS(ZUM8D)X-3Vr4!CE?rg5#n|()MYPEaSAFZT|FQhXhZp@C!jpwzzzi<1I znA3$l&vU0=o5qzPKQl|I=H45QsAcbeE~iPO%5$JJN4=3QXn{`+YRh zRQ#9S?sxBEV$o39qfuFkRi`9Vp-_W1=pD}7TX^Y*8=Ef$#; z>htCZWCkx0+@W)#IB$uJ?7n5^o&6VIDd1E0ThUwICqAS6Nc4guhTe1a1=}}nV`4E> zJM(O=vjNA8smq_R-B4M({r!*2Vh`n{xqj5#tj*K>{`_oBVaMA3n|&82_nf@PzkXrr zS*K^(m9<|svd-U^FS(F4`a#33@H<+Y-dIfHsb|TJ3kdqPzqjG(-|8yfbc)NFpZ zA@QA{ZH}M0l(}PX#9Z4hiRD}3KYYvP-*Q6z@`jfl%8zVU`MmxjxV8U>vdTYskyF?5 zB&Lam-z_{W|HrB<|K|Q_U)`LOQ~STK|6#Dl_RL;B{nMNF%>T8uZ2hnOllGdw=jyZg zuUYgX^+!x)yWXe2r`Qr3K6q&Fo3QJxWzqczFP)6r_ct?TpGuf`V8v7U@D1HEZ%mKO z5&8S>gNe4FxPVVZK|Oh&B$@lzW1B|go=?4f84@b2dmZi#9uWzbY1ST ztvmCnZPiDO$0ry2I^9>F|E@@INC zbLwBGe=&Adznwczmd|?_=dG)n93r6cT=?HVU0K`vN7mMOy{i}cC+2Xfd;N_%V*849X6--!*_w?7A}j29c5)dpak2$$;oT~h#(3($ zyER8M601KnPUC zw*K*lwoRS9*FWB`b@}>-hu^*aW8JEMl6O|Cq|j_Xo1a&h4;=k)UP@=~l~p-wEJYrz z>B@c3-)}tc{l;&0J9cKz`@N{*gThaqrGGQ{etgPX_hG?`8mXJ_ihewB)R`vQ$Mn;u zC|iwz-$s5x`WLCc<&{@*Y*ZICW*mHc&WE#Rr^d2;HkWNLbh00st@*Ivm+05jNy3Nq zuSa_7zpuGoe)r*gJDn%z|N2EOFZ}=Mr&-kN*__|YwS_ju&s*5W=yd~80_Wy2K-ad`_nq}?%s4MY(rwn_;cwc8#e>Y$4`C{wj!yi)A z8`emMWymQn`u}C?@(tI741F)p_ij?Vrux`5P18niW`+dUof?6;TJE|}re3sh|10}W7g#f+B z94xbBRgP%xfjVPpiATKCdN=>y&~WbUv8WUN_S;?s{f#Gnt?QQ+!-*-0r z7kb(l`}n}dSB(4 zL^aDYGZjG|&qFiB_1$tJm7>mxB|0o+>oa&GFx$vl+Q2T7qx{pq z|35NLl9j&yA+-D|M|aGO%OdBG{t~<8_3>Hu9vAKP*S{<^i;ryj$zwSo(Ol|K^z^Ww zHSJ#|6tpe}w7)-@kZrRsNTf{TlWf!RBhsHfhTpIL^rP|Ap&T{AZ3|rtC-_cX|M_~u z#M!pmj;s=qq0Tl>=KkToYoFe8_2gHc%%m)fnL%gvyf^#0Y_V=bqKV1Lg=)^ve=xq) zKGN{QNJiquhrD9HjZfOAtNa#G5aIf8y*&7J{bfh1rJ+AxZ>jwwrCDKr_T~BYM!TYS z?Yr>p{q@*1QI2V{lCMAgxWDi>=M3rJw4!X@5%#hr-*YRTDp6dg5#B{qP_wi`uMr zucBW)RXC@(_}$HQbAD*`?|Alh$HI@w^V$Dh@n06N{?d7lkuy{Ahe`H~RjW>S#{^~v zm~ZG?dVbZ`csZSf;bXT8K*CPXlMwZd(cL0#s5IYU$1$E6n?R; zVqiXCb4YyZ42F71uKWxCX1%*^YcgOMHd{*7PU(l}7ST86||In6*ZLfc+ zh+LIb6l|}Gn<&NeJ#(qg?JcoSKF4^6cxPbje zUA?|IEuJOy~y!ZV-tDf(#fBOHSXpw)oow~^0Z+q+Z-nv?~YjV?sAQ_X?QuF@$ zAJT^oI@^So>+A0CDq3^s_)eC`aeS@44^tP$Z-0Jdo{C9F!aN1R1qZs>8qUQYdj8hJ zzvw(a`Wy2%`m4`$ zujqrXSsuy!+569C$8O_5QHQH>izgIrj6IrvtD@pSgxBQNU(a8w%IIqOFaLb^ZvGF+ zt|nDWqdimn9&bMGwLdH_1GNqL{oKHL9)_OURu zIGAv?ami0tw@7w-pEXUN|4HGsrrMPiALZUAr_JiKmQ83_aFuBv&$gyJoJ+pleLU~R z?Gw!*jo-xYUa-o0G(ooGSmAGsP3)z@~6uWALLN9^N*hRK6w9& z>Ui;w_Ui+du8!Lu_ctnW_Oj{M_ga}auXnk_Atex}`17pLf$8j%rGF-U_gOgIhNF$e zz|Onqn%8=k9~LYudP&qsXU=S2^latIK%K-h?|<*UBx>|Gp?1|&+xJuE2{fg1 z@kWWB<~bH|pY6_uPsXR$FiFTaW=nLhQCMmC^6#hT8tPUJ%Q|eYh}RsxbFXz{Kypab z_u7-$_t(eXUthLYwXMhg;q^JJ4L)3_4!NX)r#?aR6si&t+41E2|kZXR6huKMQm!~t7s$y5{e#rAGlV?@bwhg{A z(^#ZszIM8~y|ugYY^w6!FK0j0^^4bv$DiA4`)#ed(B6>Pt5LI;L@3p+e=z5|jovoD z=Um4>eXLV4TweR0dFAroM+#QT6jl7b5`0nr$1DEzPUq_s&dPaj2yst}zt+=yJ$t`& zrAM>K>52OcK7D)oZ@%OM6Q1o0te40y^HqI$&hgK-Ztr-G8^%J0i#(2>?mn#g$J$Wy zw#w!Uf1bqm9=qt3R{MKP-Hy`goqPS(Uti7a5gj3OhU1~^`|7=48z%Fl2)w^HgX==u zXHI61gVT4v`{OVB_rV+E>rF*B=NQPWaZ+9{$gj{-mZU!)sGz2^*9Q969XTpH?-&k?@ShW>fL*@ zhINA#bHc--gCR4N=jp0E+P!fNZ!;guhJ`FU>#it=)ME2OoqNDtwH7 zS8wq%=34*%Cd<7a>rQl*>^l}Mv3RcM$&dEavV78NWse^fx?}C*x#raW^{WqRs(pSE zp853~S82_`q9>xv(`D|j2s=1&zC+3FPJ?Uvpi^mb`zJ7Xe z`tqOW<@Xl3U%$V)Zdb>|>Dyi}5N^2G%ElpbCEoVvm0-bNTh)6sg!!e|8)lreu2Xzo zy-j23w~evSip15ckH7YCZel+Yv+e2aeam{!e-PnN5WF`l^Y)Lr3|*a!nG=l-Q%`4h zEpFg)Fnbol;l3hKll8>WGs}-1xAlvDE&lM)h2*Dk(U)?(d6SE7Eq{HmVSDZOsNMR% z1XBJ~>KC{36;%98xD)=0wrc2qlJ|^K8?|*VuKHN0L_T;QbuT3-jL)*^CxNPR$n0Vm$jXm1ZTm1L`nRoyHrs)47 z(YMac@a;T)Fur_RM4D(^tyY~*=mRwo7kQa;Z|W_!`gmox^hi{%o<1+y^=d@V&gwh! zI8&XtFC-=;pEvy>P_A;!yZ-b?R!z3!d<#8PGheS$QmQ=_|J94&RX8))wUvR zs_tJUVh_2ytJ=M_v%VS?!2a5eL)`HH<$WACQvvG7$m|q`}q=9qX(iI$HZ!@1>DTE8LfPUBv>|t%J62L z-@jdRjttkoXkCM8YeoBiB#KWLT2py4X8R<=womu=PY&LD>+=1H3vI*JURv$*Vg34) zbB?ub%RObyvL%GX`Tm_Ys8`%F-!jZBQ*+)yKIP?Kzxn&te~NpzS75DwjEMK`a=oaD zcWRo_yLI#I-p%H^sImCcs_&CO7n>YXTIW^ZxWp>LhMy&7ErAzUIfzOzk*TtK`*; zw*}|qud|u3T=$30l7mm1e03z^AKf;ox9QM4Uw1{S)oZK3cY}>}51HL$w7wtw#{4Ja zfPdO@eKzI|k$O5jFa8N$WEBlt-FrdHAkL~|$=*+F%?fjFM3g5z5i^#2XI`$l@NeAS zx&X}V;@3pl+JF6Dd*=S>@9Q{PV(0On&u`?`+CEQn#9S?otJgxy`$}!uK(-h%Jogx=`Ph&{31Rdzp9iO5t@+zD|EJ_x_8$ z-}atXpR)AtojEJ+Dt}juk2-l!U|IkCJZVNDaaI>*>oC=)?{@Frz4NmA&cqGVA|B-& zT=gw#{q6${47?82>byz%1p z3TegP4=?@T_~7lls3Ly#|MIwX>%$Ml2LJO9k6k0bwo3TE{i|;iYz`W5ZaCO#=o9OC z>3ChBe(|L_Q&uz_;W(gtm_?QGwq#chBiE}$_etimI%^**2rcF|dYzhV_U_@0XPjE= z4k+5p)1Cd{1)GCwsH^jxyZQVd!Zs}|IbeB)hs(g|k>E?#dGB=YNUdn;RoVT{M%jp~{I zELg~>pk|)IxBTt5GSx!`x-3&WD%GF%zB4qc-uoc)K~0WkjY)w^z2urTAGX{tFiy5y zE^s0IKm-T>r50{>3DvUWHC^{67Hkvq;EwRIuRkvE_a_&}!;SJnk^KhhtgTnWCTFc$ zdrN%wu3byQ|HiG2$c_Ix{dLUV>#HA$?alsLnLjuFz!atvdWNU-vP(LjI+%F3pO+zUqIBc4>a;tN6=j1&+5gWG(L$UYj2{{cP}+vK5Q>%{as8 z9@YDqoo#7uC;y(^zY>xhk1lP!KUI=}^+AdLJAWBQCH?<1 z-9FA=_)x4A<8M~o!oyosMCPtOw=;xgqPy>#UB8qrYTi(}Q+@e=*Uw$`TkMzY+5i7z z_kE6lb-v*nWADr?ef=uRi<6b}LCiKTN!1A)&vqS5RC}{|$7c1(kN#+wEK#&8+B5NW zA{%pgZHLF4?zg@9tqNAt|FSYR6jUj%bNZjpsK4*{+$Hrz+Y~weS$_I&xzB&`$xtcp zLT;T&y~mR8TZyuBN<1v$OLYzZxW?q{k&H#DOQHk}XYB~OZSD}Y+x^}3)$<-oy{ z_qTe<&myLi-+Jfs*cZy0Mcd%8lr(at4_tgzWrHt^Tm&%U-2thYL5oQvb*kwfqay$8`IT<*`qH-czU~1_N3#vWkET~DD4x*? zdn6MR#h|vJL%fE?!$C;6!F~0mor%Y8SkOmKkH8~yd|^$`HFmwPM4d_KOgzNl6d##@xALGulV11w8Q$| z=7atIO%wzrcNPmgSf{zuxN(KB*lMx(3=UVeh9lBU0*+mG zIR3UM@Ub&Hs-A0e$}Rq8v&&hnYSLDQE_sbA4F(R@1Nw?9SZ8#}i&jbOW)3V&?>s8w z@nLeoD#zXJ=gxm~lC4a=W=TBLFsdVa!@R`jhJror)-#$-Ndquy&5w{;) zJdX|B&wW4tpT%bTef{ara~k&ll(S}ve|=Zs>+uO|&!n%6=#{=Ba3j_Kgnv+y#i!cv z6(^%v1lFIfE@X{<^eB3(GH;do?qkOztIZZ%?~$K4r|IA1&<>yfAH0vvWWQY^v4!hI zRN=d>sg*YB>sRlOn%@1v_g)#3ftlk(nb*t{${NGF?t1^eRr|nELY$51WcY`@sb7A| z*cFt1oTwyQdQViqxx&<9#+(BTu@_mF$Q=|v9&X>DIZMMY_rK$(M!StC|7hgenxFdk zykN?Utc8kgFO%LGs4VO*JS}*ws(%}IQRa#DdJ>QI#n!VFKlwW?`Hgt*2JZR0_l7)d z^#Ao%!9$^R)ykkd-ClX(iU#jED$jA$PM>!3yXxr7sw+TOm=LIHUZc#ovyiKT3aJ9!V%=v11qTpMGGGAlqXP zI~zCA^_!BvHkQ1zIl8@kv(!q@`Ki(s3|vVEW-A_$v0$FL#!yYeV$lupqVSBEmsWjJ z)0027`yK3Cd~8OA*Uvvqz5g5T+x_1lzcu0Hd-wk=*C(hdF!n1{B)raj7|*q;q9*X8 zBBP^#gU159Io^lkny0R<%Sej+CwsZLN!(U>MM|FC;-}?&d+N0mmu2kV*{go}k6>j) zt#-on-3H0k8gJ&$y7S7WmuJuX_8oI)CA~FSAKI5@7n9|GUw(o2xy%I-%l28Y&-r6t zdn&7CcAK_%#P@9z9%!zZa`S)n!<+w#l^3g+{%HIBYWl$+&$_+dI)CT?p0?TVjX>ap z#=lqAK0W{Ex?`Yz-1^mVA1#{xzt!HmDsGwH`?x16)$2}AI#{{w^5X|9*6Ge51i<%9;zp^&XJtP;){b#d+_nn8|?RCt3c6i=&o>=hT!|S4Y=lzR+?c~Jv;Qt}iX-~uFQ#MNLAm7J%4S~Us)+W`P1F+RaR_vE;Y6C``!FM{V#Pf2#Ixiv;BCY&E$PC zP6yt~Km8P2Fkdw%ufN_=k|#+nogrz8lYxN&@7GsNQOj$6DmQ&-nE3kb(WtOsyhF5 z&Vi|4o^Lm?YpVObb^Te^(ktcLLmY>t!@ZpzoOp~^Lec&wKsTJ4=XlTP1%C~SDr z!M-r4T=MJkupL3Q*H>uYE-=m3ZnO3}JEuU7o4@H~f$^-DFJ@X?3Dh{~HnV4bQ+~PD z{eQpimi}0id0vR)T9?@2G7;&SM&9qGcb4n3dD`pa6zarf>uPD-by zU!D8p-^-72`#%5MRTSSIk|-*vKmCu_#zO*S={j6H-KQQc-LTT({CkJFO>d0t*A~u< zOA*tbx^it!^YpaPPi@WZhs>()H}}m2*V-+`+~B zYW53=S!;MCyqmjsZ_@FOx^vv7HOID}Ewo&`rm0=>z++Xm{I*oOWtMX(r%?zh0>%4KE}T#@AbX^XZ_$1gTc4r`MDY| z#U{^-KmDSz{$W#F)zP;}CHQ+ke67AEn z{i=nFZ@+ea@;kir`o4W9f(?BB2`dQI8y|c0e_sF6EenN9d7is=oQb=$@qhdwWA&YF zVV|ta)pV1-%VeKYIym8|XaB3%_lJ~^1Xl>|x~Vi*`#sN_MF;D5wZ!pepv6G^!xq2=~)~thiY$c{Gs-KPHIija@!3(clK`kemAdq z<@MyM<##5{S?`c@+TBTV(xmQf;_oDGt>Tfo(->XmeC%q?t3=MgXBTHqKK#xhBEtUT zt%Pen$z-Nzu$huq0OIzGkz^q+IX{O9}0XO_mi39C`now@RP z?Y;YJvTf^qZeP+YY4?+z;;MS4WyR0qYi21M7JN0!nf7$^g){qY>Tj3wb9MdEuivAU zlepmR>($3<-Unw|Y`wD3Y3;^YJG0r$HZIY0bY~6z!ec$FuCDq-^oxK^DZlq06jxI@ zzR!RE*Y4^Y27C8KuYap=`fn!p>OVqrX8WFbSC#o@Ui{xhNAGGRY6#6OmHEEX(Z5ZQ}X&esI#UHhpl>(lcI3H{@Vf@`IK3o92jD#p|~oY1+)< zCoHAnAlvJ_b7kdo#bCkTf0K3}d-p_r_5Zv6r^D43tgY+3RQ~q!pDRDRKMFD#yx&** zBjZflv*i&2ZpW1j8yFaP6&z=2cfDdQeGyf%?r*{St*dt0#?)q?{=c<*>-zsO|NsBb zjuTxyC;tDpt>>2H{|tgPrv$K_J zKbzEtNUfewFPnWJS*P)O#;1#7jkmuk1o3ceRCw{-t-LGZykBO6wt}zak(IA^i!S_Q z)j#?0|5ZMg|BnT}3~Dr65c)U&{%eu#`)6kF+InS*dt-u-#I25crU^|2h7ZV0Jcev~9b^zUgO?>}9+cH{i4(Eqt5`;8P`XDV%ye)2LZ@uQd7uOHRA*K1`Q{u(d*HGwZg|Ng8`wI?4mR3t}v`CZ9g zx&QJOF0RG%GJo18r8a~&)nB;rb#K)*jd?6zlXq=h_vCBd?^V0>{wD2N(@641I(PklxqR6?orLu6wKoJ72i>(75fwZeE_@(q58Dsk zPuGv>Eq}gYHEOS^O~GaAqHO;UDkK8n3K=tXQ~o-}+Y< zB%}@$9C}*So?95kzB8?$dFvj-y{1Z(CKrm&a{AeRuk0Nxfq%jlIsvo^5s!McIZKuO@7r^!+iH z0h{vaXWEIrd2@t zUU+-j+v0yaUte2#H*0TnjQjtERa?chTiT_6z3fu25I*+x-iO5=Itp^UjR#nF@PAy* z$FQCKgVaSnv$KpZSxvP6d(_5WJ*jSc{&jfiGEP>0op1NUxIHIZ%!`Q+5s^D&w&K-l z?}hWHsIIS{vByzgz}x+yr$ea078CBf`&VdP`fB{MqD%VOw6_uoE_RQi7?s8#X()?`u5 z2~T5;UTjsl{^hZ=cOBiD4#n!RO5e!5;Bf1zK%#zW@$dQC?hGCw+5PZQN4Z@yPjNLV9D09#sghm#zda|k6q(=q z98ED;lW}ZO+QEZnpI>G!{{y$?vBf}T^FOgQ&t5XGUMNPiX%3=PjZEgnpsTg$!9Lg7j~4-5_c(3>G7T) z`@cS}@5j5D+OH!P6$m`L#`MN5Yt7CIBUT0FUb9aU-0wD2GQwv9~{bU-rx8n zT<_)~aURoe&D-znye+(Y*Pji0_T8^gINTZXM{Cn^Jw8*#8CPpIe!19S6Y}oJ(%bdN zCMFjjJEEKZJ)N;_W?r-XqKB{TAFEIISD&%aWy+TgeB!_N8Cv zEV~V#qONbfCjH=a&;ot;8ndN|57t@QZ9Lj*{Qf}b&0W#VI{&zJ?j79y?r-(Kb?d{| zM}IH=y1s5ltkBkVCY%Res6LVr?K+yY>%p?XCZmVN0uS2d{$|IBu(`Bw{N8)l^^%%z z|NjdiQ?~ERSNiu$@Q(AlcYoh6$q+Me2#sJp$s3el6TyBm`%3zkr)->28~-;wddBx z3e9<~^?#d$z6(6fjIr_6et6rr&*-JrqJ4sa3(s!(=}|4ZRfOa9%ezyiA93QhusXWD z{Hv{8!_34Mbs)j+ z-^P1+Ozs{TwJ8V1nk=ik)vw=3Fp!=n!uIZ;mG}B5(k`DgO#GjImC`EsvpJyb>5R{g z_V=#wJy;)>!RxpEk0esye^KZXedok|eenz!FVy&eOtsHGlRtF|=*l+mI__XDH!6Q@uSr=7~ z$M;IcNqylyHJLA2_NLYEDR1UK%I_;UaO(Vv{{5$)p6__p8Psz-e*cN^-EYc6%dgD* z{>OE8*sSdJQ}XO5u{}2Qi?~!B(aLIgE!3cNFOyW$bS)OKQ)LUoCv3Vek$$n|?c5*s zN7)`<-ma_nU-eSbAHgTQ*4g6EwsP)hvOWHHef(O_T{>ab-y0(Ye`F}v?ftgy{)GhR zJ9eGCD+JdsFyYAg82e9W?U{_!uj{%WG-T*oOYxZ~*d|Q;`fOV8KaMB*-_Fe0b#Ch` zz0XfyxCt0leLuOiz`4A-zv~QsZ~9F=Fg#ow=ar*y?U^oZ~5t-zIyZ76U!}B6gF{vdj0j?qtd_P`e(!H z&-jL4+W#=dd58L+ZicfnG$-&|W|V~F_X#G=`+q!tkBfk`#>s;gtm;o+Ep4Bup~>pD zk!jZzfysiW_(OLnoqc>I`GZg48j`gipN6+U#lwCs~zR(i6(>S*gk_FJ}&1y>@c=}S+||6BBpWu>C*H@08=6oxf75<^&`DpbPuI`(~ zw|0KqqHh-DRQWEuXojt1YU8dO&(F^3%-0mq`n5ctKO!#U|Hsl@;z2S|7Y;crhZ#mu&i> zsSpuSE3?tDE<5{d&8@tE&=1^3r)}mOh;VQH`0rQh!s)M=H`e^qV*K;}e5%Mshs>}Y z=W^JCRi>2wHdtp?y~)roe$&C}62*odRc#!XJ}uu-?aVeQ+{<_RVh@q&6K-0c>u2Ws zZZqe^_m6JTfpK*&!#NM7q??(n{xBypw5-Qb&Dbs|YsXT}Lf)UL59)uHhqZ1}ztOy* zzJ9if{+A$TugcfoBo0YWcKI21|DHdKYx_pMd&`wt-WuEg+i7QdODyTZnbjx7)-Shl zyVzS{ExBLEP0f4NhG@f`b%*?qIMlsrGC3h(V>7RF!xtONd8S9?0+l!v4`$Z4*QjY~ z{CId?$SB85=3(j4rSm>rF@LSMp^=G&<7LdYn48_s^K93jx^n7lE8AiN>*rST0s9L> zW8{8+)JXYxRdZL?-lBIuk4^~Jl~!LZ@UtmNTlg*a;i;1*8U{bwV-_<#-rV)tj<~pE zZ*r%0uiv-6b=sQw@hkQ1C+z>rw&`zOhuyaMM<-ir^A#l4|8W!D_~^{qo1b@tZ<6Y` z_K@qu;!C%(|ID8$xMPpq>7sJkSAoSL*Z!t-H+63IV%oY&@PGDEM>)xk7N?E(#dg^k zN_{)?{Lsas*Y)b(=SIwEXNmwh1TRxExrutypWSc5!;uhZ-RXrFh}@KRc_J z)UKRb7G!;Uf{5>r_=Ap*j%j=C@U&&As*em^U$-|rXwKYk{r~6RNbdjtX6pKbnm1mz zi*0^AZMW<6tdi9c^1bd_g_X9PF)0EqyfX@ylw5yU@jHIj{FQ3A;v@EPaO(bn_heC}yW+vxyO1)$xLZx^$PCsKm++}9Jrqrj{XUh85&xGRHPL|Eg zb`!bETUDliJuYw6^I+Bc+g~b5<}auaNfF)e(xA#1#;76!+AzuMv)Xs?q=temDW;T{a)o4cI|8Wu<-r%$Q8AUdaq(G{uS}Q=&@%v>+PeL zFLS@&-}r^+{H>oiyk_10_OUi@%k$p<%=`J%6@Qd05W9W<&niS& zZC$_a@A)v-@4=L(R#H(rkF{TAW7{xekrvzqJv!6t=Es5>lO8nxy3+ z_9&1eoMX*j7&9~6_mE_v_0~9W#qG)oap%te<2Q@U!HIe<-e@nHy$4@g>ch{E(_5X7%`Jpn=K=d+K><2Hs&x*F2*6(=U^wVpf zYAx$G@we7omzO;XI~$R=;-Jj*=GK2pnO~SkCsm!kZ(Ukz8fKXu8j*it*4apvIX})M z%T!5py%F5ouzbx^uJvbk$Szu-p4gg@d*lw^YQ+mr8dW%+$OXy0W4zAzo++ce$y3!p z$(%pNUx4Gz14flA6I4VwS(z3&tq9N(;bvuKOxTg2BhJm@vP57-%e#vA;h(1(KfY}9 zWA5GOlD5ltTU@R9tTgZN&d+5(OKZ;k=dODf`b+Np!DaPjZ(W|h{K2*=ar(A*pO@y8 z1s`qjx%2&mfwdj8pB6j&aqIPK9k}#aO83`B<*~)RI#b4+`}Im}p7`tLV#)VsA{bu% zuKmv!x~{givaI{d1J;VW;tzdiKMU~E%Mm{R_~9-8HnVk=uirLpT63wYCVt6Pj^msu zr-UbXT3-udn%#JSQBP!wimDWAOJl-yaoEcj)(jYdqtgaX**;+vv)^e8sDXX~`D*JvJ+Bxxx{) z_zZIcqn=2ImWog-%R+|@6I8^wSX&+j8iZ&lakM%;3h0=kRGMviU*0C{@9yi({ZiMh z?*8_dj19TB$aLY27X@!E^VMzt+|O<+*jzTja+hOJrtvr5rHv=IR+RlHJ@}{o%Y!TV z79YMeua*pd%W(0wvQW*|S#sZ8w>Yjf-5mhULHJS}&ZxspkLv_2oKSv!~Zy{T{c|v-b11?#KHiqiwEMeev`xeSP=4 z$-?uj>lb|7_js=Pzq%FT(%hC2GKq{kK2CUX=<|kSOL@Zkp9_`!@c8rLx@d(XiDDqK;}0Ozi-;Q`hU}d5RuNhJ)z~bcTcyZ zhi_cap<{e-s>V*+=1&ics)Wwn{ivYQm~c1p?2|s>@Z;T1mv=c`6nHt=d8=jR#);Oy zxtk_t3cZtX%lI6k^W@BB&BaD1W_D&CPF(O)p?+~%w~*L^Z!h+$9C>$g&efoYck+*P zH?P`Q(kU-@zu~QXo2SCH3)eqvy7o%U@6B1w6sfE?&u_jIf5!Gz=Qh`?xeHmYF7R3> z!{oZs^#0=yor+eW26Y{$zD#;6U%{>5QmFE^^XJ3;*TSr?%!=0aLtwpDQL{5wTta-&r z+w_#<@ui*LgI9R*Ozkr~zkIt_*4jIZu4h!==}!B$WpnC=gR>Mu)vHS**Y9{|_~+sD z7x}l22AfZxZMR|0L3ZJ`=kvwc|5X2Ix0Bu);gq0oFhJs|P|Ri(zM5@67Vm!l==V|G zDElz2jp3_>W3GRcUzztiGM-iGSn%@dFFH%g%h&KbYMAhfhcc$S9}HchTYAv0@`C*0 z+$f)WLR0Q2ni;HeIPmVriaqgniz998pI?<yJjz9{2J z;$NeH*?D3Df_Vu$8o7&8B&NB0I%L?21|H^YRr$2u<88JHi@+TvC%x9kZoKOz)m#yC znRLAUYJk+7ovT(J)R>YOx3kVtYTBFCdehBsaa^DMYvae*__b4a?fLY*Zm;U+)#dtA z+(hL0KmW96Sjgnhw z`QJi8k=)4#qO3P&*7^zEVl*n@d^TzAL9+#aY8K5)oL&^@X7=Zeet%v6-Aj*tfmUto zFu1CuzP@Pn`Bz_Ghraqh<2Fx9;=aa!#(5U(k78IS{65(bTFW_gpV3S^S$melq0O_8 z-xizD$A3`M&%<`tU$#a2qOKm{NDIjNv`U4y=)d2Dp462_V%PoUGJZIHE)8`veVVw- zWzWvw6}zX+;@EjVQLaXyiRsOaxQiZd6psd2KfCNa`+ohxig45Tvvr-*OXB}W|FW&@ z-_dv~DZxWrwCnyfZP6_`lP~{?XP?JWEbzeg`6-_rF%N>x8#=k9!}AtM8we}-@<@j4 zxbr~$Zck!`d+&}yubR|^7ZJ*znPb>2nv8QKYNPVcaj-vVb}Kk=lyA{H<&2gZ$!pCG z7uTPOlZj7$xbh%tt40pT_t*2XpUS+NG5F^+$UDgtgH!vn2Na+5aUXZ$aY2 z>kFcmCOq4p`Yb`VuJ@p3emQ4^&Yms%&lPELi~gDK{kLSRT5W8UPMG$6F*!M5)%UwU zToIahf7j~Q%nP(QHbh>Dy)JT;H|S)@ewGVYCUu*J7M4zb=rvn>)!NW(P6bW2gOza) z#E)G$^w6-Pu`*Ha5$k{EEkZ{a7|#f;VqhxJXO*lF_`=Py;8|$zRmLMW`<>3jw`PVU z>X|xt?T^vFv|xJY4dYUmxYb`gRv(Jo(_iE1g{5YHwnzN;bfHlI`xO{8Cx~pnf3bxuX@hc zi{!g19k1AE{X72m-A}*vpN!J2EHAHLIq98e_CdR^OUvTt-2G7%Kd)Y)XWrg!yLWR} z?3=)RMRixBoMg!k5&b4+r84F>9IAQzosBtB#L~t1n&-Q`$|Fo`a=UttyB(sn782e4@}!)U<_nQY8XA{;>t?8KQ_l14KR>TW zY*yxy6L}^Jm46<$TJrvNofG>e{jXK0?k;7E{kVkZn|RLd{}Pom0#h~#B};tUZx$>w zTlYv;&ScMvIzbi94<;8q%41$}Jv?pU|B#TDRo^ZU-ZuNZq-rqHV=dPup|3Y zCN>^+Nck49>h9MU=k`0re-_F#zCPpl#FKv&1KhrxS+*?y9=PD}tcv-b?nmnfZ?0J`;|i(0aVh@ZM1wE3=cZ~)AKZUm`b^1( zfQrR(70-9NR8_IM~ra?AVAR{AlI^MTve)s?k#j>#_6_&zzre#zIaciq4K{yecSB1OX9wx%ZU z)*-Q{pQLI|Y&9{i*%DopfAa4my{Z|P3jgn)+hnh1Td6l!d_#}^^4tJ0Y@a?%TV zqK{|R`ETz!+_Yc|`}c#f!QuiZ{Fi(%@u*y(V7%5ThO4$d{Y6Z;&iA_QtKWZdQh(6B zaj$pA()A(5r`PWn{H$i}G4o8Ls&c&eBaf9e#rGQd)vTot>=S2tQIcN1sdw_K24^3@@Rn@M29 z+HaR1bWEE1;?=nW3=D$H9B1itYq8dw_ z+)?4VUbx=&u2l^gO`GRxJ&1mG$J{GWoWG4FA@{HsV^?O$(|m_%6GHB3F>9*!>*$ET z@O*!Qc};>Svtsva!56cRo>G%7Qhjwn{OteGKVqkS+%{N96c{jjHa%F%aehO9w$nWI z(BlCwHs!xsxAc3xVEM1FeCpS;^{;jsE@X6`lV)GFZL`CisE0Xre=kk@vi^RfLBnSr zy)WflRXa~VW{P2d!!8l#mvM)G{@R+=%nXb_7#SWg6g50AU;q32nLZ|E#@I_IN*A!C z_uc(j_ULc9em3ix($^{>42{&gpxBgTD-|O68Z+6G-nKQ|9iM1e03un+zW0$J| zlQ&#tZ7^%lVwdShuW#2WdHa3u{H}NZKZbvHdC^z@!nEw|#`i{X(rQj9c9kc%G zqZ0KB(`JJXC6)Sndze`G8a3y)pYd*ZedbAA>!e~jfebhK0h*l-Rc0NL(G%%%c6d8Hyp_5(r<1kVq*|>@^!3W?YC&i;&DjJ z|KVU9da$X3k2lbX)h{UMPmJT`sWJN>M2Hk0w^04~|5MGC{9AhAXFr<0F5YQd8n^Xs zaahmRTbtEibTLWJb7+a*;vkyJVsypxRs9?7y`7;~&Z?Rg*IY{a{^GCd_4VGhCp(}1 z2{=4KKtRJncv7{4<)++y98SXD!X)&dD`q$2? z|I(f5ceYy}>tLRs&)TdVr`X`|Wy)5u&smWbDWPV1e&4q|;yz$KNvi(?_p>FgwU7VB zG5%A3pki%w;buUXg_#4>wmb8ttDg)llM}!7({-(hpF`Q4mmZ(4Og7lnIXix;Yr1*J zJ{P}-e1_UFzE6!iXaU)b{W-$04W z4*5P;lhT7!x?PTW@hmGT$a)lg`D}7q)-=iOpDM3aCH}m_zUD^L$wZfb4z{+jXFopt zeCkljZu?KVQ9_?$V+%Rvey&k`SpVv3=atp>jgCf^mag2ByXegd0pa$dD_jD6D?eEu zZ%obIAi*hj_xJQS)^8t4#k3hNiWe?f?;oD((CoCSt?rV>$?Suk-<}GViFO$uJntVA zB>(w9r#wgLf=7;lj(R>3`m@^Ew3e4moLZG|_VGUJ{qGYm9bOz~^G9djv8}p)q>lKk zc)b5#eEbm+>+>sWnf}VeuShxQ7ZIbJsK0#E;r}bDc+X#-mAdJ6>UmxE6O9Wa)AQzq zt?IGq%>0(xo_cp>mQQ@%tN6%u<-g@BPh0)^ug!gZZu`$oimac7Z+R75n|u6=fOEjS zNqYp+RO~oHBzSLxrzE4^A<7 zY1NN8F`gWFW7?bUf}x4Oq}A`t4}TbAI5a{wdqgW{|mlO zdz84LXS=B6Ub%qV4Yyt#vU_JPDEqzqfayW$WA*zUJ2Q6N_xY+}-}}Xa*=%TXu8Yuc#5-ekkt%(eJfn<=|#K#AlBUFWsM?cdkQN>kF6o&v&g<0Jv8YExtQJtb0r^s3uAOC!HzJ=e0 z-exwRIkt~NmKn$TYIJO1ytJ*M2(7BDg+dwRa(`~wc`KaMIa z7vl4u&F}8;X@ zPC@nf#!0;%dcS}D5vi2P4Obk;%nU#ESic;p%SMu=qUH;GM)mjAig*|eMS?fz3O z`?qnh6s(OiQ+*ydbBff>Y1d_#{Q0^qR^JJk@kM&}Hd*H~rIi=Fm+U)YqGv#~xtbbj`9Cyj`nVp+k8qE~kQ`)hLc-+gFoqdCLt zuxXWwp~j=YV>w@4Os}3+pI3k5qWDc21Gyi}xwbPO8=9@(_IKaZb;+0hZHwZS;h*oY z>lX`)Y;pC^DZ2B*nZhRg(fnuiw>j(MZqdoV7TN~%e2+|+)3kl7`0KdnJ-63iOnPn- zrnp@H>GTPww!fc#D*n*N389DdHZHjCuxI8PHnZLHTNNI?wvzE2qw^sQdrxVC#_49? zH{8FQten#1Rwo_0?SlM;{C%MtH`%Ikal8&a7r1m03-@-eB`f7tsf4#$uCiOT?#_yz z+gQIS-#YK;u|;mxmviCMzgtY6WB-yjBW?R9`(?+%w4Vgu&^XTXaW7NHTfe}EjcZ+J zd}P%NyBoaxgXGHnf9LqcY!$6P5~>lO|E{g{*Pf;Se_s`xx-NcQH23$kM)9DM8`6)C z3im3Q^@t>I`Xc&ri8<@ugKj(*G*?b5eV@*L^=QTU>1Wrv7d>YZzql^n_`<=)hV>t| z=Tz`HcKVe4)}H+J&)1)dQa&}>I_CF^V>ipV{#U={w?jprk>lKAr#cDch2P$up88LE z{_-Cx3-5m29pI(Xvp=)<_nS)Vf=e|oK1eNj_=NBG8o{6T`k9UzD;*B+5a&K>v8+LA zy9QI^qII6Pw?CQd`M~qYE3ULh8=}Kc6$XoIu8-QEFLmXR?8+CWzfxXEY z&pKov`BPuPdDcw96NN2N0Zzwtn7aZF% ze?5GpZ?<6pPc`TDX<2_hRWA=_?kHC>Hr;#J&FRZO$-Q^XRNK#NkWHHBthHzMRE0FP zhecvbu3kyM{=cg2&|_=E_XStqPx@(l!%y|fv5d<s!P7->UJB@4jcB zI(vH7d&93W%xie-XV^L0h7Jp`6e6b=hG*oyzyly>I!hF3?rUf2HDDbD3`zlb;tJ66UZ>T_3RB z?NhX#7RRpppQ=CU>AU=%_v?>X=d7fE+c$r&Q&;2gc@TH}>enR~64uu+>h+)gQ+NOE zGUs>R`b$GRbOdMpTxG_#wbg!QL*#_0XY)UweBW`*=j4wG&!U>*8XA*@K28!ab+-Sx zI`LKI;vMU^T|AN_DD%yx?ZtkN@0Yj^3T0gCd#l>D|5m_8F=oblT(z%tO``sUer0@N zQPi~3ct>{ETz1c@$4ecrc1RYV6I);;;}RpS&v)hct(L{fUHs4Q@ZaAV5M2FTdYg6P zhquj(Cgf?$hNVr?|8XiH&iO&vwSf74wg#-hU%l8pv~+fC%GzjVA^(c^hsHjg(xVSO zg7w~BnU|+0`h;uZ>P&}pult_OK8sFPJfGw(7kho7{+&tpCk17*=}*7ub@GVhj?h1E zmoAQeJ^k8tHi=jK=es}lpZfZW%j|a$`w6AdTH?HQ%-{b%I-rk=3 z`k>qAyJ62C9?ErF>^RNmxc!P0DM@ku>1Q%Kj~!TUqxe#_s^b^`=Edc|Bfo9v;+SGzvG&<&oBC^ytp7e; zb3J3SvE0rt%j*6Lhy=Aw4w&WiVS;+G>zZDL=nZ!(cy5aCSez#s=CgkK>nZIfT6g{* z6Ptd1W%u$+9d&g%)2}~bjE%U;)nBskhH|M&(#juES_l7J`quJqZ`}ST>s|f!YVVGX z`kUqNr0{2YdDS1wj>6I}5-*>}A1;aB`E`lzBlg&f+@XT)o)hFMN~HJctDW~-7=Kc2 z=~tE=8~pzAoxAMDw%198r}~XV{@shVg8E`()8yFab z5*=sha(l7XFSLwb>%Z#z_5c5N)~+p3TU~zsBiH-?@Bd$m|Ni~{-_>(CYrn0$e|5=@ zJ~IyvTMJ`eZqdq=CjRs4`Fr>to)L@t_TZ)Z#ize^o&E3rPQ!ZEZRUMZn@(BQsZ4n9 z$85=XSe#Y7Q-b$A+tfpiJ8H~ou2nwf%xF6Mu;I{l8+-_gD*Vh#ay+1Kqowod;3Z*9WQYgZ@#nppZ; zwCbVyx}d*NPan9mS2k3Hb*Tp`JUlP&{fgb-;G|HQM+>`kxPuz5&N!0dx9iY)sgSLf z9oiEAbB}w>TxNQxGx*~+>%-~%u`>^(yiiyvo!oKj>l(2V&0jZC7wrl<5OMf-*>|~_ z>f3&oHrhPM5)%2{yK2$Vr`3Vmz4`RgVwr|&8Cjk{MLFUKYOu0QD3|zE?-lQ`eJRSccdezn6;cu({ChguI zH~s3lt?zpdaoL@$PyNil{_%y3%g?GfzaKdv@|dOc&5x4IKZiD^2DZgnI4}2cxZDxG zhKD&Lah8#`6H8gdvE>1~mux;DF~PKH(eovnFFF3%spj=ra=utr4$JS`x6ag*UNw2> zTJkDQfMJG_6N5TehmdaYSq`Bbj+dX0?_;oQpD0;t^5ydXbMh}H@2|YvUZb%5!0pCg zGSfE2_ny|+SS`e!uWyUi%zN;K zdpUoa#3Mf0V+U=07X=`iI`{a6WhsU@pm{JZt!i~ z$1dZ&aSiv9?{gc%D|7`jjMW-Lt{y+^a?nvyM7FEt^wGn6=YQws=h}DVYx&*ct4psc z<^L*W4L#6gF!{yQm2I~5r&0>5UwpZLXSaT5)s8!*hEX%xEKV8S;QenNklc3M!F0WV zb=0=+N_!l3`Nm9twn^!=vWmTG4e#G`7yrw)e{@`|=wIu-JVZ5X+QMVEFDw@NwsL3P z^{sys{|i^VUugR3$i1C@e-}9JWYdl^IMTz=^5xR?^J^`yT{tjf#o?asZnx{V$f-|j zkiC?prm(@fNknSG0jZlBGmWdAcL}Ccv@bs!XO`HZp*5M;dsRHsj0pMGb9r$5? zYu*95hu_W5J~gR+$Kd+Wdy!w~iQ#n+@v~!17%_RQm)B0+$?|MV|5g9=+pIk~ zmNruplMlC?$?*T|5t~%L{ZJ&y1+liH4J0{2*9^x!4FZ%zDLqRuJe1>xWuh-R2 zGUJ!uyK}eP`i|Efx!9LgdHfBkvv%~D7|5{%>{Ncdh*|%CaxJs`!PUCC>QhBHd!7d- zFG)Q*<^CFnueaF~9alb|ucg>Mz2oJMncG!oslKRNIsK4(#0#(aAKZMNb9n3QU*E(O zc6scg=d5sIRlXGIE0Q#u=VHHH&M%H}whcd6V_t z?&GD`Jp1-7ygZ{$Z^7F4UynBBJ>U9R){T?L#9O1fY|ebGy{w)YsaZBW_4jyuesDY~ zXj?REWn;lV|J|t*6}CpLTr%k#r`xpFdHa8|DnI>nW9h6Xb^BCOZO$76ec!`>P(j#n zfnQ+szDFC~>zNNp3otxrkP%^!OyF^zbZ~z2^2Wro)7PuMTpq3xpJ#bC*scB-%TjZW zpj{JQ9DVUMV*Y-H=vua;51&NTeLh%!)H#RUsr~xDw3MemE%=M4yGZ#@?7aWR@_mhF z3(JAVBD3Pn{A*^MnJ#spsQTqwro%BU<~LIoR95V<+i~{Ouhr8;b*d9h&-x^?Z*lOds&k>gul?s=Ytvihy+vj||mfHF+evK%j(cvdL zk<3|a|Mpy;!T5*av`pD5#Vi{U^V6ex@&4Z@zmHwja(yow-+#F4eKIjKa>u+91;LMusm*7uMY!zGcxlKmVZE(w?;Fuv z6|RYFS|4V4GjcC*W%GY}=Lz3;(ffBZw4yH5@5y812sOR`NIdobn~K|)R6$NbzXB`NUdSeiHPG*VV_5lA^Pd}e? z5H<7bOS-$nHF27LXWY9Qg;z`o?pqIk67X(scpk;Pt7%qhs1qZL4FZylta!Pk>=zkOmpRIwIMsQ%VcaZW2c zZgx)|>$^LF;UPS+y1%;am5Qmmx-GnsG`lOpjHhtHlWu=&ab1T0FaB?QcICUj-0~RN zD=&^KG+x@$Uz`-WQ?Y8-hIemDjA9n+Z?R6v=D%}{?X#zZht9-`dhU-mBUjH^#dSEx zR#aB>-0WuuSwwt%fWK4 zYz=`%?UkPO({&@6Jr!aPV@Lw?=|8!_os%JDrLJZnSwXZGQf+dhgR| z8!r8_f4`mghVT@x>+dBz{$)-uFu%V;MSS<6)BhthMB4?+U!(+lJiS8fd3y-kZQieJ zt&&c!<#$&9JMGRd`)uv|x2KpTS=2vh8a~oZXfN?9Ez7Igod4g#!0xGGZc8S^+4bMQ zKZ~{CdUf&pl0`3h?QNIoB^Yg45dI+|^Tyo92_9zG56)P2VO8XZr!S0)`t75054$ED z|ML2cgqGy*4;>SZ@ZC_+Kie!n{rXuR@i|N_oi!7}?_90^7igLPF>1O+t9sp=OUXaB zzCW~OJHx*P|5G_VXXhra{&d)FV$Hnx?+fNnesL_~kGZTw^cAVB0}oG~e`#6pO_}d& zz{U^hiT9oKqN3KyZCrJ~`M>a4Z~N1~?3;W_*)C?ivh$n&T5}rPZ}Gd|BzrEc30t|E<&sF$F6zAffGvG(l8I|su6|BoeA(Mq|2XHUR~ery z-6oT=eX+0iY5Sv@GV_eLZ#WQqd*S+>Zt}PP*4~^ZT6~#HSSil}h-Ft+v0KHV4MrDYzW$nYxxM`cEFG*&erT=Sq2aC)=KTRlGD* zb$&}-qWv4Ohm4$0GR_uGou+v8(cApK4{}C-Lrd?ii&ek&O6INBe7DCseE&l?{7}(N zP|j<(ZIXQdaYjPx`$HQyv%J;)_o;TNs7%;%xm?)=LFfM0M)F*Ju)FFv%X7^I$#1s2 z{{8#M$92E&Y+WhNR7nf}eyKwZD!=f#d7=M>4M+F_IZ~gP{Z*FyKsmr67-2YpSEoiIV zcDd*H2?zH>p$^`^w(R!}R#J_h;NT{^YN!5fy#=bD`CjeXZpp%9)38$B?}FC1NrkUe zYwFd~r#+m!YW?K50$TDd8{YHJ$WdGMnZ5K`VxC6km4;W>6+W0Hx$|w{+H$aSL-K;p znf3=IQtBJoX4ZunImdnYG2ujvmT!1y%&+^`I3752v>jP0e=$&G<&n4c53CPd()r~O zb8JV*v$dxx-(Qs58yj?e(Q~aL4%f8b*Ta&WnN(~`r*m$Ydyw;wRl%>#>mAj8OZ#4p zcz)x@A$eK-^rxpr5xu+puJ+;wCx3i@IO8Dq>*q`LH#r(Fv)bbLwYBnd z+$ola&xAb8csDLS{*{g8bnCJ9zgvH~Jll8m@Eq%d>tiHmF7K58BiwmVJ1ytkAM-83 zW{O#Re}12D)pY1X?UZ#BrhiQP^T_{wkca@s=bXz1hQft?Sx(^|;=5|%-~3qjLHpmN zKYv@^zT)ySj(Cwh%egh|{IA>=PJ=IKC2rzdca1r5 z)6KfavsgnrB)3bK|2pzXHM71_(Xlah*`BMHIJPK$W4yFL_VlH9C(m>^MqmE7qsMV> zuM7WXna?Zr4@GDw=rnaM)AhHv7c)GH|d$B-mev$G4VwI2k$I9J6_Hvx65Vb zJr_Exik3{A_F;YYgTA%blU@6NN(WtAxXz`jIc(NW#+tq_Hw6@WwygL+d#~-|I}(us z_1-6fBNgs{pX6~$e3{U4LCJ(Wo`v;^I#W2d{Hm4QXXBR(xB&Ygn)uz|+Qx@{QPd}q&w=Mhs zj2-J#idUR^#n@N$#4#wz`?Hb0`ifis18t5i+!?+)Y176<>#wcvT>ZD|@;d*z>#Qu| z*_nZhi{qQ}mUjduiJH}QJo?Y#-rJk|ckS;jF;=^;>`+o<O@L+mCG1Vva_|Y<*E1K7G~d7yGaNySDCC)u+TI^_N$FfB$;_wH;q;BTC9M`YOWj zK7O%m>x6a7{g3c*r~F!9v7~-QvDuUS>A?>e7ZvcgnSMlwaqZrnRFId%&$IT#?t?oP z{MX}pB=O<9TxQ=X^@Arwcnu64bDRHl^aPy@44dM9a7TZL?WX_D6>`oNOFx*j`cH4{ z+qk3j=*qaYuGfQ}mT{D>PM&|Y#prZ=T!+i#sabntPQL#CVE!I!pCfml@vROG6Pt`@9Ci8tvmk~TX*dUe*SCzyknDe743KYZ#;NH!q#Z{4z0&Jk*~_RQgwFP z9Q1fGL8!TK_hNT5sSd~9BK2uATF+0@{Zw6Fq16}=akI$kg|*&}7#p_j%Qv>v-CuQn zdoV*_F@Hhu=gX!|_ny4fdw7pGw)eWnCh6H<*=v}M9bQqW+-%^vLK< zNqt+Tv!c|Z{zmzIEaqnw+`RPE?ci;vHSUGJSaiLqz^&@XBmZ5;`4-jw|Eu`-Zqzr; z-{I4*F4>uXH~Ve;*V+UP#sbb087`8451egS#+&*fJ^nyTg13>9!T0&D`OB^PWrU== z)J1Ym?ljo;Xk+A#cJ@6M%|*}t#O3QB7HT#K;Br)`xopy!_~Giw=m5>>H}95v9k|lu zJEQLRgIljZOxn|2@4b~nV{MV7pz($o;lJPg3!a-~kRPZce6sfY`TM0W-W9H1KW+Wj zm9{av%qnQ9WUx1=VqIJcH#RNgWVrIPnOm`+8SPNZghRsm%A_jb-ayNe{o`oks+@v=d6TF z1~=aMK5|T!)pU@KJ!do3rtqJozgYaW-1Xm%CCq%@+IFpifBiBx8+6zVTn_y^vfxgY(vO+8vgh7Cy4k#;A-P>HIEC3+^x`242H^sh1$hcavlu+s zw@hFVSY(~hz_3E`kP}nTtQjZYu zn92P?!^X<>M#axf88;L>2#_*baPG~FCFzn^;)H{ugJ0-oeD>gDzOP@{JYRlVtxMtq ziI+7u^#!J${n5;lKI7t)?M8{4%hk^+h#vWJ>9_i0^$vqF-Iw|e5_e6UyWc)|H%D;( z{_k(zi~L@+(e~f^5_hYsQF~_9H~E`xwa&;iT-eWNQ(#fux-ZLkPes=9xRx;Im8X4m z-^{Cdx#DH+{RdypR)2E7(k|+JfbmC)Ni=gs;e)^`pZrL#Ofgg6d&f$A&(zIzFTU?^ z@Wr&~d+~KI91n}{Y|txY=Z|}}*T5^=bEBFkv%_q!tv{++R^6~%^Dj_uMnmcMwACf& z*h{}2U*BdR{`yX?C2!b~T{RJ=FLV-piwe%|(-M|%t~he_bA++#gzATVWs+MH&bSs% ziTZF!WAE!GIUkAX;?IS)Jpcb}Vxc_u$-32XbIm_`KPctOX8sZV!K#(LXZPKI`W1`X z9T|S_-uLd?_Qo2)Wl8VyZ-oo}|Hi4j`l)aK!LU;)CI-8>TJmL|#0jt~^3T{WePW`o zk4Htp+DCzV^Xxx&aIaQcnDXJk{|_6Z>z=9Sf1cPcmHkdeQ}z1elV97rYRr75v3Onn z@cifGw&=AwIc+V*40`+L_}{m3WXW7w*xm7BW9eL%^vUbLoSI>@qMw6@^WdD6`%Df? zuJU%)mKXfho{{tYgGHtKk9vhG2N``T*3?Mv2A?6w-;6I!~0!@(su?+-siM~$40K(V;UtIJ=T zTMF_-6c&sA>|j9%bCWqWgBy;F*Bq_Z#g-j2+*mFAoh4Y$ z-+tfM!<%Mn?Mj?>d!7l~i$`ZW$y2I*Y{NZTxjIOPfp)!_I}y^>b~Iph8BxQ*1L~>FF*N!*@S(KfWnM| zhE)oWO{0&nPoE_E)U?dl(t7fqW$PtX^u*nZq9+=J3$$~m^O=UdID0T4VcD}Qmwk5K zeI-9b{YOY+LR+h;V3T~C_DsgBjv0;nPnWjr)q5{K`N*2^@Ab}JEc&YLbtNDFr5xQ7 z$hhaZgM9oe4!2vncbZt8VuWXPFFM%&kH>Tm_r)p4BVMV^zkWk`YJJ??y27P;XDW{W zIW)WH^4^@h_%2`P#zRX?br{l?EYtJgYj1d{l=a8r{37Y=d>=Y$BUgL*=Uu<5ch3Cw z-&Z=@7gtot*fJM+J9_e#l4!Yz z>(4itnQ33xTcLC%xLeTs`(-unst2>H{L7F3@R{_p@{)^=dC-lV=%9jpqax>`$euk< z{0#2B;^5fz?aHja%|8p$jP^Su+cN)1I#jLG-qL>Mx2o{JhjsRb%!_MlC#Yo3cd7s2 zxL5SWhW^Kb)~c&bxBr>9H6~`d{fxd}pE;kMx_wae$EV}wWwIi(cd8e2Sb11FGq>c& zRNg=ivhv|MHyrBcfH?h1*&WM7j6xo&`^H;Q(b++u9baG=>k9hiOlDi zm5_b0HPWH{vX*)QU!-0C-rw<@Ogt+U)-FH#Xx7H%r#Fk7_z};!+i~}q9htk<{_~5! zB-!xbl=83O&1|cq8rqA$-!NeOzEpef=H<6@tj+l|YWf=+B_^dE4mc6;+i>^&#A$JD zbvvguvgWQ^7F4c{i%N|KX3N4%BuZ)rnT5#+-c{( z>c9H^AKvLPAFl0HKU!?~I4WqGTxTq;Bp}|uo`qEKu1tz{G{byRn1}Cdu*gTy0R_3SKyVJqkY4MlC zr3H?ehVsiDzL%r1;Fw;r#~&{K)M)v>S%>+X({yH}zJGJc!|8EHbo~d*!(REHm3zy|T)Av-~`cZahZJYU~UrSUcB)EM2D(aj$`B`gVmCS?-oA)eQ z#ON2L^I2+DTb%IWW0Bb_n?8y$KsUfki9@KmlBY)_2mo-{rkp~oSUv2%s` ztnSH@Q_nW_d{cM&bMw83vi_fgjvU#dpR!fgh1Pz*=O2Gnvj22;Z2#VOze@l7QC@c6 z``?r4zh3Q~^y2%jc1y+ha~WSJ-~0ddX0FVunR@r%2yh&jI`w#y-=`l;Iu4)XI-~B- zzk1oQXujT`eJiqDlfC!KlrAzjYL*eVt0eZ=6HBFEuPmIp^zYn~Q#zX>IziKDo%?(R z%gNsa1;US{J=^`u$?vV6T2)J?aoLY&2R42R{H9~S<54R|WamTUg{gdNj^9vt#ItG9 z|5N><+yA8gGH7HsJTx!MbH?V5=?cGD3cqdB433x<5wOTYZ*#KO#59!$cmByg{8Pri zVV&`v`%Ab|jsD&am@8ym_Otzn<*RHvRyl#0>93Ov+I5!(ehPCqcC7l6jAiY*%*M#_ zvZtqSZ94eR^^Wg9pDouHr`}N1e7W#OZtVSM8{dEW^y{byP%BH8rH5G)sDXeQy7{I&Pj{?ga+ z>znsQ?cAjBHGSEt)oZ->-&-gx^ZK5MdH|z}NQZ|I7jM%-hYc5WRJhrg9tJG9p(V)0 z>X8`8IZr*>e}3Q3Z|~Pvmb5;7^(U_Iyrik8g|xj*e%{gTHMd*lp6C3q-#u_Gcjb%8 zWzY5(RlmC~|4HO(+u8reTa=~PZ*Ti6ld*2!dail@NxZd+^yVW_}iyI>O>;y8D zs#B(KesD>TB{6K9{f6)z+y6|ty0!HCmHb#IyT>>AZr&=nKk3#yulruH+UGtt6qHq2 zZFb#!eqrMA1-FmxZB6@e>yL8p6xleT85-`s_I;mvan-%uyLW|Nn>sn>Ufilz1vYkTk1yZ)YVY|Y zrB$D2|GS)*psYS!xAx^AGP4anU8^`TEY2 z{sq76@%Z~}``@gRKacN^`1>dAE z&E8NqZTZ7>l7;yXzkPkN{>SE+Ti&1d$v@YAdB3+tJSF(z75^0%&h4o_WoPoHN-XbI zwDqZOy~}o<^UKZ^3Z|4#KD+PU>I{Qj&9+|{?jD&}XErIoL4;XL^6^RIEi!7oVHsDy z?z-x&8X3HNVSeP5h1V^b=M>1KFG&t}FIJX#Vmz-^LtInf;Ip02*GXMVQaY@0N}W0N z6d$A3l?f^?T%Aph2{U3e#JJd)4h1U2Xb5w3Iwc9ra8Xk<-nU)!=gafQms>sW4N868 z`z&kgu7{zJaeZPzPR*@qkh`=qu&<9&)Dy`<>l6O%Qp$kxVi7U$GeB8Z%>!7 zt$&_)?*BE>Y1?<*>{H|wdVed(cmE+j$&Md~9Ll0gh`2qLv0GvomUiOi@u0BkF0t!q?09)z;R0ecUgbRlN4zp6}->r_Q{xbc^LKixRuV`QLm0tKt#KKIEe>>RxaChOKh9ycIS47J?1iDz?iJCV&U{ss= zcdCR=%u1)D9?NZCdv1BXKu;{>*ul(S!uKr7^BMl~CZGJh?DoSN({p#ub$*+$x!;O+ zhggQ!o>kA)at`e{>EfYlT6+4FS?`Gj&c5sJ&7D#jTzvAvd5vSqS%!zxjE#G9RyI#N z^L@^vu$Gr}()h^%^v?@3daHFl?hJP^|S9$gBS67d0X=RLK z4Pxd#@c*OxCy&GDk6JqjII%l39-LF%Qs#KLQcLyz!;PArcOU9H2G{TiZr{T-NpkyT z_wT#D3B~2zt`)z&Kkl!a!UOlX-Elwdl()XWx^JTO*Hhk6f2$v*?Am?7<>-T$W1H%n zH#kW#|8Qt*>{?YcmCd|YWwOVab?dI5IWu#^%`CN>*0ZKc?6vuqa;085bxZR%tHZwq zogPj}*kR0mOQXslfcM~A4dxXGq+F#BN*r}AdT&xwrr?oL*J^Q?O{(_xHJ9~&%2zXK zOwd-`HK(cEz1r*jrvQ1OKN~Z`AEczVEGqK;(|2#C@t;j+C;bpT-s+TcO7_F!yD4XP zbWM$)aq{ziG0tPtWd*iAJIZG6`RiHF4$oNMz%cnm+YfsM?YMc4^C0`IhV4HNn#`}Y zVSAtXEbi?y%Tkg5hZ1@W9hOf%X;`(6?{oTZr{7y&U9Iwtuk|goUR$@)F+s14!I}%5SfDE5zL1C-(i+Y1y6cW?PxPXNm~O`xrfMd2wL$ zv^)9%K4(O)uqTGJ1^%#J*lByPm}lh^Rh0#=l6W^}2p?Ee&~Q0UJLgQaWv<;Sx%vAI z{2 z54oK+(hEMirZ2v|yX*UvH%_xMr)~HxaHsZ|_%;sCsI_5R4Z6^hure8hvwfk$P`P-zw@#@$AU;Hh0dj8!6O=oqL2!(6sX8r$lPEJ8EqP@JHZNaiT zGT!YDh8tFgf3z!Kcb=(j!2>nt`1$+ZmHL+WY};A2x^A!Pwbz&K`M-Zzy`MQF{)mKz z6~o@m2ljlHX?f+7zsdwuoe{e4%T zd_Dd3(7LSejVbEg>I~a{TwMM1lZryxW_2l#C)1lJ#cYs#Z^J(C zSXO@Vu8!LJms%TZvb8q!Zg>Bp`zN*f>(%_KHR0#>mi}9_K5Sps*Y(%-M@>FgS)3ey z#K1wj;PgK>%K+N~4~89Q9)=uXS{2K{%ea{M(N$K*ICCXFykE>U+wrz%Xlhq@#sA(vpMB3glF9uR>1o5uz0hFN zw&=Aj2e(-BJxpW&_%mJcpBeuv%?&M#erYNN=j+#`^7&k=sjQ#(*0iUhNx9+JGDZW{ z!WI1QgH)5R?EPUo{O{Z|z0+gidroEBBu3$D1$PUV&ldTk&--HzfA{G(ODC+AUOGWE zto!4JWaEN%NyEII-}Rr?nN_6D3ibY{aVPvFqnMWY#g+E|iFE(G;fkxbR8N}L z#;M9y8?v{q_kZlQU`yP^iu{$Y98xnr-}gHz*y`FUsrQ%U=Kc@dbkN-CiQ&(B-cp{W{SOqU-r2a=A$RhF6DJnu)n%$2Tu@kl zlDGJhwxV2OVDtU^ZjKVKmF(^}B?`Z`m{@dHrKUqTY||q}u@cGUj8Eh1x-Z|Y+VsJo z?}L5EJtMY_Q3t%bCPXZ~U%rO5o=4tv$-4!LJL2wo9}_5;cy@)LGV31ht=D9o<~{qj zr$@ratI*iepED)Z2vDX?zY7yIj}tGDKA1btKcB1C6p((i?65+ZW3p_YH0hOOa@Q zJB3wX?Mv?UoOUmMxZPgZYUQf%I==2<(D_P_HmT2R-DmxesCvQwB5;Mcrod;h-@E?B z_9gvQSd+Te>9)Hcd&$#9a$6TJip^0@`*!jB{}&&t7kfAUnEHNtP57_PcdPg7@0|Z& z!XM{pb_Ib;_~rQ7r!=0B;}h7K9LJ&%k~lfZ{XbJeMWK+lr}j0UH4*}%OdB^=A5Chw z!+7`O?i~%x4S7186HHtDD;#{>ESCjN4!h|ZE6;vVV9f^x0nP_;J_i_{NiFz!=cfX{ zPIuFek_oam^tW05Xs`dHW%^!9=uC{ph1H3l^(@y|+Sh$>Y%rMG_-pRW72j^U3CMrE zqQk8f$Q*9}*0)>hfcuX6pKtH&-ecwSL`QAf83Qk#0+TIbhl1X_b-p-#Y{&D>EF!y( z9nG^#d~@9GGI zyVWDv9>v$a?DOq`!N`|&B;1Z|5m zdXKo6Z<)!zR&uXjOZT&?Wv#V|FTOkY)wxWoTDxw~+7{mJ4wtWY%a~qSAu0RAbxl>> zqAS*tMGsu(1t|nhd2Xj1b}b}i+k3t_E<%&#-pHMJzCZhv^E}u0_mqCF=*XOE!`V;DHp`bl& zK115;Tw6Ee=ayxA|IhYtem{#(xIDh+kH8koU$IKtbH3Mfe7KMic>7;E_sy68j^8@I zQP9A}XXzi--G9^THl*pZZ1`bYZZ2Y6eBiU$2hAB$?JNQwcAJiU-l6e&@)Z^ff&E;k z1^Ai%d`jStej3a8lb461E%FkxTk4wMjM0yF>}H&*@vynSldFnX#Cqlg>u-}PdZgZN zo)vsJ-u~o|UrjF3Z3z;|J2D=2EuH@G%gW07FGX4(6_@TWb)MbAqWa;{)Ay;hZxiGB zelES6By9i2RsHSVUmkn!t#fcbn%o(q7<1|Fe}28i@vi^>FMXNkv%kK?XEn<~k#f;R54tsC zHx=JV`MWpAE={sz76zw0-ngXjGVj$=B@#nx0yg*D!Ry`~AK4>$lb>-m02;D(XAR(xf*WDwCYN z&hvAp#MCCfr5~0peA9C2-50IOQ~Re2u8y1RALqJ8wLZWjV&AW?pNnd*<`sLb6@1t( zbNS!y74z-|KRj2kxr_&=@93%Bb3yOGWQu;GkkP2OML^}qgz9s3l=Usr$o z!`T;W*41!sU;E`@-F}wvn#WK7{o)kda_2!qrL^zEn*j@^q~7~`x2`eqC-03U!wH>h z_=A3*j=iL^VYb^((_r_N*-Sg_*7S8XJrQ=UGJ4f$xWVmWTl0gK4Xnb#*El?{o|0+W z@?o0M{`slr6DBsA{Ez2g;!Bcn;uJdBkhFiJnE1}q58Rg&eX`n6E&8`_v&-wt@rvt@ z<$k=sRb^j?!6c0v^L=xtpINfxuU`9x%9wN(fwMw(v+U1DCp_61xzV#HOXTFOl#wa4gWMWM|n7!XHX6Rjat=hzb7Jg!5VU72>> zs&}lp`26e{1NZqpa(>n)qH1?dyvjDiqIc)ld7qJ`)9uB?#T z_+?_SO;b+q3#Nta=YLKAyhHqcb?a^u=NoqatBZRSbHbNdGkN}aBrh{VZ0+I+1#d!K zW~FI=`eLx{eA_{roNXVO%uZUq&COg8rW?Cdf5HhDHxW&ta)BqM_gO#9l~~ZJ@OZ{t zmXEzJFWmEsW(awn)h8{?zCt1S+ljpmVN#eXOx*&q(q8`mqs62VsZA1v&4>e7EAqKg4gcs>%Y3|+yAHc-?#;f z71i1=VtvlR)+aq(n8W&abBo08u8mv&KkP}fOmb)|y--^!AYZg(-hxwq9ws%Y)UB{6 ztlv0UQQf1)?cSc-Wy+Nt7A@iyc|A?6`uhu}v|X;>P`fy-!HSjyEzqyDE0?+Rt{`K@}N|KT{#O%yW(tb^U*Uc=xeviNW z!O$;fOsBj*tM|4-S!?IL_)|wdII`_;mc3zG-*WQNq5cK7lik;^yzjNyo6Bj7- zwQW*eJ~8rNF8-C8BgMASAn|O0nfD{1DWA=L@CFoT$9F&QP#Ds4)8#31{!U zve^Iq#opj6d5b=>RzE*}+4W@7j>4Drg~1_$NhKGnmR+%ae&(aZdD%%!Uch`(XkBfV%@Rf9r$056>`a8FgqLs2k1tnWnB#rc&!px3S1-NSC4SFCmS26e z`o7iP#Pg-6{fp}bG-NBE{?nAX=`lSsDfy(tr#FX{%nBCYV_*=z;y6pMdlhS;xbEGB zwJ}$-&Ry-l+P&|8nD*p<{bAB!|NmF*+NtNg`rQZnmKq)jk2ETzv$Do!e0W=m~){K@YR9&BRM6gR#sdN`(4bB;mbmjcs2_2MdjJu;RZuX&?- zYW^Fq(yFgDDoaoOyXyVx)!w4r?;nZpQ}W&;&&u4Ed6;39?KA%m^`$SH`JNuUWN;(q z6sx9JdO{DofR*#K&f|$oY8nn|=5GBT^YLMThk3~nrUQ#b(mPHRl>C^Y%2)Bje)>h> zgEx;aj(@1NJ63bk>=Z3i_T-Fq!@{+S0xAvDSROvMyWjqy?4)4rxp=0J89a=)!k%fZ z<=O94nSF`tb}|`M-v`tl!#^!oqF; z`Ns2W8=kpIJ#gV+I~B7$w(sbPppQBgJ=K3X*I&~P|GirK`rY;GuWPMx?5?e^GJk!Q zMPu#Sf*p_N9F+ zA|x1B2#LfAwcp@oSbF1uz10IP(b+PEhbBh;SgiZTxpia2nje33l6tmjOC75#2#QqK ztB-2xvTHh~Es_+!O!~Hp@ud}4#J#$vd^O=aP(Cy4z*LX0uH&yYEe+$HB9`wD|C?G~ za_Z>5ReOux#vYizK>m-y%DwAKeKU@92(+@9uI^JtFE5nK74aMrod0(>n_}q8{cEQGd8=tHBVM1^ zIc>|lG`DcIbnj2cw`%U5w#D@7t4~q;z2mmtx~4H}^^WN?+h!ymSobnv#=B?Qb?ILu zB(948E#dNI4;HXEHT|ws@e0RJr(eI*D*f52aMk7ad-eJ2f7yf{S^fQfw$@?Mb%JZ| zy}G9Lb$@y7sZYiR$J3Thy|YsyHB4rH{5v@{(}^_)H5(wv((ebcMe;XlQ8M@{?V{y#KR zJ8a$6)mQ&bTdi~3BClrY51*r^2U*`s>lo-Y&N#^O%31pKSEFaJgvT z!q=S=vCpsG`L^~EW9iWxzu@&;C9n71mUcfjC#GHfhL_3hTcIiMtlnQ z>XU?~&zx=L>q=x-xW4_7I?MgikMISzKArz~&ZExz)I_sH0H;qJElTPO6-hd11d+nVKXfBXxtR%HK%M)hOOKWko| zHDVPoU774QD|*qyS@z5B#GCrQew6@aR}dR)%(Ml@d`-)z%f9N5<080lDH@|C}S=W9pV zfL(!R3%Aw2d>|5fyk6&cM`r#!1$+Lm9x0X=opA^D9^}s!RQdJKDg4%so((#;8eAXp z?Y|iRzP@Qri(sXVwYtE0`3rBX%#$8?KA5I^e_r5!^+wrP-VeNo_~m|B{3^(`n;^|O zEhIl6(9!1djyU(EvqyrvPkXO;c$Z`Q@t(b>c=Hl#8)MYY_m@*y9jvf z$nZCR{o$jWhuK+&o=~?^pNh777c44em4aidrhO4Ey8lPuORu{;tH_kD)U5gWr+Et1 z1#B#guH64oH1+Ae162+p8wBrp?R#bLO&fWMsiv;mx`_ z_C-F|UtcXhn|)w{1WRgp+5DITXPdfrEEXwGGtYkgT*O#UTsw}3iFMV@YaBdW%*kFp zf(QA;j$YZ>Vc~ovP(hygfjYC?2flZ|?gwn;zsEH$6PB_`qQ)$F;=tP5y1Ia}0T$H@F?U zS5O^0g(q*acIeKa%kd8yX0Zs)DdOOHV(}t1N-TU;`Mb+5UGh(UUthR2cjmmUQ3|R8 zDkgijEL3TmA-m)CXa4r&psNkV<&{zsCtQoZeCek?PoLzU^rR;{5f=xyuYBtle|*jBq#hff z^6tAle|_lTeKV(VEGXQuPuORAq2kkl|0^w``gGTSSgRAG7&x=gIk)-u(v@ZNpZ@8LV%c6%RlB^+ z!#&=X-ig=)J8ybES>L1|^j%66-jRD>3L^db9LPBk$qw zOTO!9JzleDZn2WLzt)-KUR_V})Gw5Aue+}^|NZ{P9j7WgNaH-53W{I|K$m_1UI zcjn~40{q{WEnHaj!C`{j{x=>%y+WTfV$S_;m*ee>dcR2|u*IJtINtWx>DCbYCXN{9 zz=X68Pcg>D^Ih#;_5~h1{~N`->#+M>!hC^xb!&5`NuY!{B@mj!OzR)?@ZQz`lH~Cribd;fv9v z`sjf_)(-P~UpTzxHM>^wGEUp^{Q|YQZ%a)dscyR=wsBV6tH+gl)7&S2Z}$AKefI?A zhTkn44$Q9+{+TrY%@d7ht#KvU79x8+ijk{NPK8F|MebI?Y&#N zQx^Q@$kUFKyw`MM*0TCJP0C^abZU3zgx|?d{Iqvfbk}d`v%b!k&!25KT6eN*QeEe+ zgSXF}X$+fid{VmWrdvNHyuN>1bVowKdfGRmHBW-8_PuZpXOBM3rg8MtoTmnpF3PLF z?@(&@T=%e3>94}X&$C|~{{7fYdRN~R`CI&+yDtiUIN}`k_0*K4_^03As?d&beWX!(XK+hkm=-)t|etn<3MI9F`z z>bA~=^9BC{w|3O8x2*s4x8z1mA73u_hTC#RdyIE}zFug-p?vh2_Rrc=@ocPzr+qvj zz*2MLE>m3NG6}!889T%@-JF9emP$@BwTPE_dhq6^H}!uScQO~)wU|EqaOn0ud2jdc zCK+-Y(%)3iDe+dae&a2bHs$_#<_mG{+LA&SQ(i95-V||!WxiD9^2cJ2e zpvN{NE)43*Rt_KKaxo%zR0h??c00 zZWa5zmnR&V^ds%esqeL#+r$rks7ts#&DA-A|7ZEX!%JC>FF!utQ?k>{qF%qcY?Frg z{mYtthYj3*UP-OH&-0y!<*8Og*{^$XcX!^}{X_Z}pTG;b_WP9&r{CDRCa*rzX~#4> z3snILJCA_&#hDQ*d@S=P#QuA}D*MSFj_otP{#UfPp_Bh2_kLkt$9f-62Fd2;Aj9(k z;frK57QDZ-$R&AGW0UIU{Mt1UCj`Bp@0nW0VY+WdMwjPy{o|g3&S_^q8co^xv+u7N z%dIM|)|H2Mcq!|iNjv}2j{XQMC@ip_a7r@T`B%t- zmOA4l`=%}wW?$REB)LrE;mxEw<>C*fURId+Wv`6cj`KXr{q~>ywoKhQ<3h4gr{%h9 zW;sy`=kL9A?%Q$r-BnZl>ig3~9{<}r#ojEzPxOv@#XspM>>&>8vqVmdebevip0?J* zn6+l5-FKJ6$#G9y>fe6b8v1?KhZp*S~ILd;#keGx2I)%^S?PCT(!XkGlYdk0xK&R1D|@>{<6VPLa)&x|zJ=1o0Q ztUAhTd6Sc7F!S+F=euuK zJUZ3A|55+C*qfyhFAj(8ToZrFVBVkl#av~lcst&8Pn&S<(w1}YTJ`UinKXT|dmoat z-=A=iDFIA}s&ac@sVIpVvPyOFpBfRxrhws05wNybv{Uadyc`9XE3?ZRT}w-D+g%NkLWZ+#3WeOBM|w{?**eu#@ZJlx zg6)lw0o&LWe9ZcaJ5y&HKmPb;oA!gB+w&itP(0G#x+k#9D*f~kiDxcyYbvlb7xs@--hGXx@Ia@B5UDSN88uEmP}0`Eu`{1#PGP zi`_aXqWYue@XNUUaSuAyiFvQTI$zI3!n<@$>J~r8zURl}ej8cVe{0$$5_9$O-Pr;E zs{U^?af}zQa9DKzkD%+JUuh=pm29PMnP2o)|L6Ib=pOxGntA$yYsG8+eeg~DmbEXg zkF#{%`bZvM>IBTq7!wpIo}{;o}(* zvh4lF&&4*@h`!^u$zx9rK6p-y@xlc8?67~DJenWA8%8Dvo_t_`u;)jijYnZngMG@X zZ8Hw^-S$6!A>Pb1a&rG9`>3t8sh)irU;7{Bs65*Dlc!nbtk)aod4IOPc>6wL{zmRg z5lSB#C9JCQ#r&UnnhNBzW%}kVj+tLKm5F<^sg6I>UW?U!b7vYV)IN6OH|Ke^X34QR zWqx~iO9sZ=5}DAOHP>^}`_TLMcJVE2`r>=j#`=zS?Wx1%owG~=PE9I0-yHO*;(Zb8 z)^!tuROEMN8kLw&`gZ#1|J$~;{wlGxV%q!q4ho8$^eW!qKmEAg&&uE{0_6$^d4lcB zOI5V`f4|o+F#fXQ|M3sgV@}Wa&l4)>kF9^O$y-HwnP$(iU2_lnedu5dc~kdjOVHuJ zuGY6gWAAD$ndJF5^TA@qs8`xc*__ro$SxMtEB{#Gw0>{SlE?Y0uDZ*9eDJ!pUtdqTbo_bs zuYUVozqRTelWn&ATOYUUvHnG=b?>{s{%KM@KmBRH%FM|ubrW>6q8~6Y2)a7X(&s+K zTEy}G!nAkuKYfhn{G)hl)3aT#udiPEerslJ{QiqU`s=%&b*?(9`{@T)f%D`$JOV0j z1QiavW4rhvphPTYdr^$VuaB+M-?L0`l)myLuDWmT!*5POpXO{zG}9+y6WB4&f1ujwd)TzS7={0n&A+*!TtZi z1rHbcSenlo_AJpiOemV_InGA7!B_0RKzWEy!GGi<5!_To;!;U z-k8U8f?#a z9^|l^UgPNMGHGpl#Z+4HH*w9Bvew9zodpNkFXpYwW4rCq8*$p>?)+shO3kAbTzfye zNZgpOejxeRH056_zIY3NnY(FMXU=UE=TCRDV&86G-+VARDW1J<8pD@qZ#=E1A3l&W zjlrRieTtCgbRh$?_3;-BUhQghpX9aiN&BB}{`#hc4?BB8R-c~ZuwstB`0ecZe$#vA zH*zZeFR#_zc-s5w-Mp1jU)*GWul^e`>DfVtCeEFm;p>hnym)kc8b4<~A8%avzZ(BY z%P-D*4{eZtZu;xoziQj9Q;&QpQq?X=i#hk>{}<`?+5a9CC!|mIJ?SMCoV(gck>PYr zVG6%s*4Ds6BW+2=2V6Q&ZMI3B_mw#Iq0woDX1x>d-`M6bO*^_7P@~fGNO3o7wf*-UH)3U{_c@mSI_*~w_$1aA)SEJ1!bQr_ddz-P`o4kvFP^irw285 z{NM4u|6J5NF|J8lLs$L$^7BXY$rC(R&o{mMZ}+oC>Ef^VdxHK%?RU{Hy1M#gX~@?w zY4JU6$G94-u9~Dbo$~345&gdG2ev~ zZPy>@y}Khnx9507>i5uJ9J_D5tzN&i_@Uj>|FQKfKW+X$vd#LNu-4Wjgv)Y6UViPA z$uF;;|Fpr!^P5r*)4dhyQnP#aTh(!CK1g2W8`k5mz3=tKsP}t~>W=Ku{HXduQ zvD5oy_cFp@+5?mQf18@iL~5tbbG~Qy$wgsd@(J}Te7Cf^4b9k%Jj1t5h*bJ&)V|Bf zbc;%41#*3+aHOjvS&+3@dQv6R)kRkI%{yw~6N^Jli`{;0Ch{tUDc2s*I%b9~`+Wy|ZW*5JA6_?ACt&+A;5Bs0Jw~i}v zx4U+EYf$%sjMFb;{~ofu!SB?pv-mN`(jPBXKm5xUS;DeDRw?q+!gWjU?pI&5C%vtQF5>6PUHy^9cU~MQN6{7@$`Y2WqU6j>|J&7 z!*><|_4QBwmHh2n&;DEd!B*}yDhaNir0vAI_shF;9BpL&aq;`*Ne5eQ|1=Gqu()pS z)F+CuI_cssI7A;`PgR(3=0|C@l+%R}8@0%phs>H~7JLw0%(y{^0Z&j|+>QK4fT^WPM-syDR!hyui(V%q!PnS$Q{E|~KYEb`hkCL<+-ewqB2ie|6jQp##}t58XJdh5@Mw&)d+ zhbH*dBnnBa^p?!(U$E};>i0~cTk6Zjzx8va?lL~C_@{LCcFXRF^JSuHuMOE|h5gyk zcjxB0u%B*RS$b7|q0Q5F?p_~%e@&ds_sPHYdq3CcerP{9qi28YW6zb+3Hx4iR-aDJ z44KO0t=n>i)67EhKy_Rj|C>m!Y2CI9+$VW{loUE~_xPqCb<8)-H@^RU$;{kghs@#I zuf0WUv^QFr94_^-y5JBYF-5@f>76*gD7RBR4|)3L3I~^Je%zEAGQ&t>>G7L2@-55? zUP)gT9=%lBocvGvqKJOF_UhLy&u%9wA3DE;`{?EU_K}lp?Z2}+3Hr0h?4L5>teLFa zBeq|$C$F5Ve3-QOmR?3)#3pAO-d*M5KI%E*|DXQnWZ&k#CT@2s_tj&!`?~UVj9vdH zCVfd-R{G1jb@B#H-TwhkKkoVPSMkNaJ5%54>8`H45wcY2=jZDlSANYoVc^AOkt@8w z?$||xa|?9rGi(K|V$S3%cCIY5`~399$;0BQN*qsC=p4GX?qlnuo+l!4@1K2*tXunk z{-^Jk7H?bNr+!5?i<5WN)J?yxyQ#ka5Gr!>YGcD&bBzrLYz`JRe@F-xll-OMW65`L zeaQBXx7x`^65oV_=^lU=3AP2OF&KYo0pcE`z9pZC-GnD{z_}ez2IPB3L@q=nruR8~BC@n2QDmy~Crh-j{G=IpBqKRsv3EQ#XaoAfTR z`8QXugnH7Px(4qB2Nh;EpO4;CJK^xRb(7Rne(n8m(?fuVFGJKksD&pg{{PXq3g^N~ zBeD0t{a>v4W0zhpXOm+&)mZCj%z0TMsn=8LdE8vsgDfVk<@I$?*Qeh!gdgoW`t7>GUY+A?Hc=5b9G`qIdv%aq`s~x23j%n| zPZ%y#SofT9L(0m9$DS0;U-x05M4VCm*%ussqCYJc{LOw?%PzJkrqnUpc43TJ_~!L< ztR%|frsmk13KfToPM5oPVeW&@uqpeW+t#-!PH?|H`}*#`RVRP0pWOZG%GZaN8UMxC zCgmw@?RfNKeva+z58FzW*cxM^SQ%e8>h5%kn`snj-K=|6R;9-JZohiN5jC@wMsM0B z@)l+W|Nq5%_@$qN)Q^t*pNARy_?y4;36#X;@ozA_)Skco`{e7>GxQ?cQlG{&KUw_k z%%Otm^DCamdx=avVLIhB>-FN7d!y>>IM>~EvRkp9x%YM6yPr0f`VQT z?U&CMEmao%?X2?C$H+$aye)^tq1ThYGBa7dS+6H;OY#C-zI?@dUT4KfbS;#b~_w@x#*&GrsZf z5WbOGqj)TOW(&(fQ;Vkp=8vu}+I4i2=@z%Ql_rrJ*JL&9zL?O}x4X4MaUVmG(O(<0 zMFslP4$6oImhHH*TFdlr(fY`mOl^V*nB`B*>wnX~HfmHy=n zY{8BdOA4!&FAYu2h?Z;06u7e__1JllZs$NLm8Q?->vIe}k~;m3xv$85a9{GwVfQx) zqmAWrfAz>p6z;edytP=7H}&Ygi*oxs|NfYqe6RY)zHjC3=e}-UA9OzbT>Q6>ffp}* zI$oI^c%EVZN`39SMQ#h6jwbd@yZ$8ZcG0Ax7KS%|-@d>8(Uu&m3+)na z3H#4qIVIW7BN)BZz2$50M8EGb`vQg3Te)_Mrz~DA#cg7iJpc8Y9HRYh&rS^u9i0CVwD#{_z^7nl6-X}-0bhPgL`1Y&EZT}xB zN?hv4%iRlV`}bVvaM`}!Od_UA=-2lBldrA%{dSi3w=CNyU;l2{_pYAbWx8+T>JKs2 zpDMiT)UAb|URDtNssH*$8sq)-;ZARAPlhhAZ?Vo&JTAq*>H0>#;D?9ana>pdIa%vJ zOU$ubcYmmE+ADnD@+U`Db(Pxlzfx^RLN#qOKWZLx4Ar*%+ud{b?t9_I+aIfm-Myi1 zJ+U)f*|xs?{ag0VN#WkrXLG*F&p-aZsp)RZhkKE4f3zm)2;IlGz`?l*J z`JdV(C=@04f4Z{!eme{M(&M&=?;K6qnRwvEvgozDE}7lhW2drl_jJAQD}Ee0Vxp5- z{885IQLI(;6$QuLcaJ%o zS^Q7yOPQ^b?&pAycWVCZlZjV!G>RzA6{uzmU3-`?4;%mJi^0CY6C7rh>Bk`9gVCrumeI>t&lxJXtFL#(I{|^x{ep-tb$&`S;5nO_gbT z_gLqfRC=50k^1=}yQcpr5t#sZv;H=64@uTB4vty`767p zyJvQ+-ZsD9;EnsAU2n`XZEN;(8VavCsPyCy_w=u3%Qj=A#btn=>e$2Tn2@$fk-Zx_$Nz!&N`OTYUSYxyPd z_%%WCS>J=MmTdGauU~uT)$#h+)%E|s?pnsbwru^aiB_wx?KA&cCAmI)UHsa*t-r4O ztO;LRxAoTEU;CRks)=S;d=h+h@aLfx9)YQB>hJw*-p@LwGutV4kL1*Q?|DyptSG;6 zi&fi4|IM!VTkKX}J+yWC<=nMpJL|S4@8(}ywrPF%?scl!PyhPXt*Kl8>w0!{!3Gb@ zJ8xY7nO?t z>8n+e%v*Rj$0=nPH)`m`%LN={bZB_keC+Ugr?mVZiARD$4wM935}Ex?H_F@m^w*{i z6CwGBH(x0Kd3W!A0sq!KhC6=GBmGyWAMSYg;k?NH9xLEBYm+=5cLyHVg`5C^%b!!V6OA?yS z{kX(<@Mw=iNY$e3PS?|iLhG*Wv)@{t9kW`@^5eN{;YX?kojTXQ`mp}i#DBhhW!h_& z+11b4^&x+9htKs>MY7X3d+z-5TmNU^O`l1zk7xL{s@rVS7VWwdrkx@CsU_lQS({;N zkz1#pH=D*wj)Lc`Y#K*)R6I=9No^L9cvBNS-+S%j)fG)03CkwPSKhQZrTzZEZ^o9R9J_)Z`XkXFj zXw2G{+LEq+@5P)6;jw?26xXs&eY>ViKpCaJ}K}g6vnKcR#&$ z`H{Rof6mK8M>T)H(*JS$>#D7pdihcN!{54!RvlFpzIt)K$oek^Dcg9mSa<>wEfz+e z{8Knf@aSu|dnFyQF>Ez^eBC%c+$&+~G+rDcQN?;tpSSYT_0LkzPZg!yUobbu*K5u6 z-nhS2)$60)JwE!aH~ix7(5Ff)mj7>aoH^Rx+x9fs^ZJq2oeO7e3BMfsYvbMEy0xM& zHrA~?{9wVHz3eNTj%~Tz*4Zv~ZRNCS@&0`}#!LEF3VIkwzPr96O2WaMDNoAIwl{BY z*o*!1Bm_GOq})!+%wx_!WF)Y4di25F{nq-oA{ROzHmYpTWqxBmtzSZ#P3PYZsqJUD zc7ADazdc`g)uCUvW-YlP5?mnrXThFJ=L$MFdH*lcn}5w^zs=S{>x|%kJEFhyNb9y& zJxXV&F8>wBv9C;j`rR*$^(K>i8NZxeDdw6JwnE?*mkita$E&SBA6EZkYg-oQvhZs1 ztgQcCTeLXs#W_W6Gw00|%Y9{MvPM#-w){#^&HFN*o!4Hc)>q9^>Rr8io&oEJkH(Y5 za=gSMFHi40o#)tiNPKx2pO8h~za^KvCj8s$BvfX&?3A5ryxzN)ywf>P2GmPGFq_0G z7d5jdcR`|7teIDJ&f#r)vTo10b-+>a>#@3py?Zv*T-Uo{9@ixEzaUbN??+Sfn=Qdo z1zM9PzAv1*ucNka&)S+Frn38{uuPs5_inbx=0GC{%fk#4qMts`O$cmd?-tp!GxHDU zQyytOj)Gqs84{AF99SwR@P3_aU`^)f)`QIL%&o_mycAg~4?9X*{je_){bJt1yD0v} zWp&-Uhi`iqUfNuc^q|Ggw(^p|+{#Nn$F;-GYuWErfx%d@`Bk-QvoS(ti!@@k-7|gLp0kr0lY% zUwNip4qf56c-f=+R~)-_Y^tsv41M?bN!FQtA7|e>{p+5*+Jo8E{r6u8A2grdso1an zW3h`bd+p=2>FeX0yiYBjzCwBGqz7d)B0l`tvqa#?^J_ms7Iw~@x@(pr!~JUZnn+8b z<3DS|3*K(s`aP`3-EZw;q3AagGTo!DPqU9(_$Z*}OS#Xv&M8McS5EB^SvKp;f7S0H zK92PtK4b`YK0Utj)9DXu45Sh>KBpe!%ez*;hD)*YlzZOC4u=AHy944p`4J8_-0iKb zUFS-EczwJh^1)-P&4I72DlH{bF4R5pIm`7SAfL7Qn_EYOy!;XwJ9=9zV7#hr$4!m=+uY`o0@nEZDoC>oBhE(QnI{`Q}0h?eLTzN zP>1$s&3c7v?$n67X8G=0W#aM8er9vvD#Q1m=gafo*PB@BmiqONc9=`^>!zJ16X&k) zIkq5l{|;5PoeR_Fu30m6_vfiK6*)dX?X&yV%TDLMFC374{O-j%wRrXNpFZ#2zuGYS zt5iViIZcj<=0TB*+$?XuWO>};(kA-K_0XReI(w3|)7zHyF#g&9m;HgyZ|1-04i9bL z%{_hTd;iIg8?&DHxn0<{m7}Dx$~nDn`h)oyjZ>9=Q{lD9| z{~cu5zVU~O*sn~-Je$yW{_CVyDla|#Pw?8;xc48Nw|$Jtzx&Ry|K6UbF&(}8_E<_w zK5n?w{Y~&)oqF%qigIVOyI)?NKDqk+v<>fa^n}A7D9(T6e#x8thR<8AYv+W7I6Ajn z?(aR)+$W!7nmt)*C+Cb8bC*PY(!X-*DC3kg!ABp3ICx~Q)VM27J8>=cf%*DHhq$&^VAl||JrnzeTPw{ z2wSz!pY`GL&6|w)YbJ~K&dfgH`JiOh*8V&5wuT4(GY`l<|4VrN#nt-n|Ng9>=W^}a z{vRn*Bo8qh+aG-RhnkK+t#b30yRY?StRsW}L@6-Pe1ujmPaXk3NrAc}H z^g}IwGXGxE-M#l-oK?>6e@pi@l)O4!*z^CxgDK}Yq*NcOHP(OVZ`;*z;jzN0khR$h zrP}O&)Ld$NARIJNjz4R|{G!uRTTWh|ACPvt%fw)MPY#FuAAza&yFdInR?g$OO=!_^ z{pH>pjpYvVihek>W$W(9*zYe+2d%E2V{G=-D|bunlfU0j8ZdY$ve?UCtJ(kM;h$=q z(>>A0kFb56bNO(|sdY_1)b?0(FU)sdDNbw|<%d7GwzDsIYjXUL&~-ICJMP^* zzxT)3^WOU!7VP}bBVj_T^sP7gVi~nu>$#pTKV-%#A?OfuW6F)+zeQhisB#!G39=ti zpLcUfMd+MFH{MTId*vDLD^84<-Oh4BSLB8E0j~!3r#^||m=gYOG2P>4EipR~KYB zh-I~G+)%;S;=;AcBE{oOylsT=KklvRS2iSnoo=aXym|FOu@r88zS17^AKDA23Pw!1 z>==9N^+y+D!Bg1+_0PrrNA8N3R%GL`IV*ax=j)kk^&1aP^2!uByfm!Sc3Sac7PeoEN%Mms3`KP_j5W z;qLkJr{W^p%2)H={WWXH_oO>2^_PF$-LS0A`_M1(&AU0iZdluP=H|u(MSaf$e#R5_ zCuIM%*z<9nwNzlC8MpDDJu!P659A-Tm{XW#ePmnr%D`z}seh_3UMm!APD$WBc12j? zSkTFzfq&+d`(1dZ`aDyzD*I2e%)XL$a_ycCfAh5&P1-n*sz1%Go!`G?-_acIuyg-^ zxK_BYFFjm&qQ&2}u65qb(g}exUapr3aQY$jDC@*zKCNqM5{2!TAFhiE{?G5U`c+<- z?SH|!zwgH_-~N2%M2Bx2=9=thXDV)fIAhwMSGzluPj$Vow6~4fy?3pR!`yr4e#g3Y z%(ju*FUh@gb?~uiiu+F`pZY%ee=wu`?-L6D4V%l>I9=xcuMsxWSY*#@rmk!Gu8+GF ze>QRbT`AP3Uv9tO!;wesD&LW9j8TU_w;f)8l9$XPHon=;EGpMJ zfBdXH_AHTM$(27l_&W8ok1S`p=oj_E#6ZZ6)N zKmFOCC9Imp0p))t&F0KnRyV0&-Hg0S`_rq8b&u^o`Ec+1NzS?{KXoPsb_l;vc>L{v z{lAHR7k_4Fco*9ieSWe}uyV(q+DTmN*gb#l?A2}EzFfe?-DHN}-Zv55o@?K{skiv! zyrKNb)Op{kZX9t~uu(?mDp}*T288ZO#0pUF%N&{{Q04AEA_|`^{hUX?|T+4ttV=9 zr?Yz>xH8xCxPrZ4`EBXQGYX7KpyQ9SA`TX*oUYz!KDDld;a2G0eU9EyvzyCjf6m$Z zd5Y!vbku66n7_ZvZ2S^H(VLq zDsCRG4NOVgtKD|YIF#qHr2K!&zxk`yul}?7{{Nupo;S}6ew|xAFHLoE>EEkG*MFD) zI#VOlc4wBy{)%T7tWLu7wB|YbToin$R>QEAY4UrvkN{&jRfEEtSDrgw$zWf8o4LGA zYIm``^b3!xw`Aw{GTqqK@AN15e1u`dT#G*$3d%hX#XH|D@>1t`z^E|u?<@(OkV8e9 z=VIf&&AY`IaIUB*?J3)I-HWBA*?V=%-^@y=VP7v6`}3rH_42OwzJVX>PXB#k_Wx5O zPnzJJP1l0HFtqvxxm{KI*pjk`~u+-*(~{4=bN9pd;PBBbnLJaM7j%qbc-oSvUZ z3H>lbXO?DQ;)YLkJ-qf+a@$(^-B-xI=VNc(#$1tEc1dii-`y=sqo-h{5(6Mxs1{KdibkZ{oa*!|4-J~pUQl*PtP#+TT`jYVzwzy5BQ0< ziEmDF=)Tc?H^b^Pn;ZM#f_Rxj@A(*0UcG$7apqrX>HqD$YuCNl5?C~yb*1L*#LPFl z-p$LbtF7l!fA{j^Q(wQ7?Xl)%eOJ#_xn=*o@YyQmyVKeHMlOpvmU8n8s?R7?zj;%f zap1~jj#KV(wmW^}#`3>~;$ZNVIu~$}nnNnV_N~)Y{UNup%HwgqxMg z$zZ_=4L2^<7WJUY?sN`2qI>j%x(Z9aJZXt+b>0;kIL<(SRDbSEf?(d7MpFO#TKcT)VOuQ{RoLN9vJf(MK$A``TP zgjkvmI&Pe!BO=Jk;^<@$pd}*Aa!9Rs?LNEPSJPz=d2N_nEEv2h){kAe;Pk(z^V=SN z>{C9gGC7oQ`QPf2vjHbh->S-;qWUj5@VVYi#~;P>u2;Lwc1`^IPW-#J)%#`x2l?OI zPIB{n-T&ckS;^Oypv2D)cIj)@2A})Lx6U{F%9IUP-Xu5uxvo9=@2mbFdtyI5i4Bh5 z-d>(3H6`qH_s5(~pZvSlX@9ttv06p;=vr2eOeQ)0_uC}iw@ut|pl#-n2Zq|uPwu#< zT<9a>;hk3fARO-yt(GD7yVyTS7A~Ze=GMNr|#|h|0}qz zus@8f?M*%QFXL`~++uYn>r?JqkL|wy+{|UN=DyVIb%DJl)k#UY$-jSBN=|6mC~;MN z&qbv+Io1D{JF=raf_o!#m#@8ea)rO((+?*e z{Cu|8^sCx-(GM~w*KE9c*Dg3;uSaX2z^3Ji51HP*s(&B<^47bZ;Ya6di#)nqpHlG3 z_}c$~FZ1uuIQqKkb5z>PIqs0EA8$7JN^FpM)CFE&4sm+XI{szt*!qko*nyd)9R>KAur}Fd;IEoQEXrlUwrJp zb)l=v@7`zJW4?gJ@?e1~4@a1Qys6eF#-(#Mg!jvOT;7mn&J-!Gu)ZbP%_u`_v7b$b zsJ>Y|6UR*Ewj6=N=&5cieV#{JKC8C)9uh0n=rcWP;uc{e)6ZL$`$gn^zO(-0ynikG zEOX26XWd^JdVkl~`1&>RtIOloOuD{%_4}>K5d|6(V)`>L3q2}}{*x_ZBO>9xXG?+h z$9LuRJev-jZ9bEyKSf0&U*^+VqtxlLOBmReo-tsMx^!KJfn!p`0|u8V<YEp3(ld z+PTK0JGRC5aVa&Jww!#3l4v#oy{#=18B-q^YN zvy+*?gcJ@fr^5$&71r9xWq90QRGqoGv5NcD)@>ULR&Y32NzY+O>Z#Q5>IBo30q+J>$$Z{ZnlX_m7&KnjD+QD;xRzY7UKJzoYR&#v zk6j$~?mjc*QgpW!iF+aPueHQ?UGC+(HoG?$#jE^TTNk_B$kj>WN+iGN4HvFmF7k$U z@dxIx205s0mGoZmSt>xccK#3f=KC9jZ=PO%Vg1Eb(~s0Un(uMj8ymK^{MMG(Ys+R$ z6Y|pjv}wZg$7<5kFSQf}n_Sz=-ub|HZc*CB(+}K#JH6WO_AdMK+tPBuil*5!CC$CUfRaEXjwy7lQJ9R94(sjkwUlzV;_q~*(VD`)C8Ux;p@-iQ#7u{ZQNs2AE=RDy}rC|uf@91MmKr>rsh?bLOnJ1 zNO$vb#W2q}Gu7xp>-s#7@ACdF+cnp?|1WQ1dOT%!cmSX7S<`P4iJJsO**7?b3wR2f zvle9TJP!|DesSXzIS>0ixrDM++}m(%(&(I z$Fk8t{t(-}5OZ}0v-WNJ=4bMAlN5k>BZ5OHROj*Dw8cKa9bu73XBK18IYqhXt_;k~>5 zhJVs`ADrONYRJIg%p}3Iv*S=KThqZKCo)%OzdX&t&bop5^PDJsyBvu^an8TO`#DY} zU-dD%YNh>0-9N?i)5hB8>-#3%+p3>@>V0bUS@nOMoSn}i>-Gy8b{xC#_22rF@8df* zCrmImi9CG!;mf=g##L;E%M31wdr!I)pV8vKUy19Sm~gDujMby(fD1w?rB-m z|2v=MWwuLYog^jOqqVl;MMAVz(N>W!I#Jo_H-Z*R^{slb z?`jHf?THK4&NyhGVEex3o^|)_{LN-er%bubXBoKMPbf2szpsCN>DTwycU}*&(o$&( z+xvOph497uU*)XVDcQ6jLs+&#ao>-bcJ*f+Hhew!C1yX%vRU#hzT!_O)ouM&q`#;C zPurcJ>X&@Ix@3j*-ghnAs2KQ2?C|W>33t>VOtn~QcUXG2?%l}T|9>)1oo$qHJe>Ku z(eB-H?&C)f_0+^K@{}}x&{V9)@yC8*UD>rg_qa-YP5xNVuD&wqgx40SE*mdr@2CGy zb4l*qsW5%jgURpjOPM8o!@dzFz&V*=jnaT=6*rtcCwL* z_33B+$j1F^v=K~KRq^+iF}cIM|AxV;&yzY;1(P2($Qf(i3u^kjC&R4%*rcaBN*-_S zdTL`J-`i!ih3Bc?g9g9p|2PyY<~AQ-^87G=qP=Lxs}!3(eIJgwakGX_e0F8)pM8bx zA}e_{_~gv>jvOicY#zLH$J1|Z4O)-7nmJMq-2AXSmWPG^LZHJJhE01ISlOj|ST-LK zKbhEa;9he$$6U!4IhKuE)@42pxc>C(R7K^I5DA5(wT~Uym@6LDzrJg}drO1U68^30 z+U>s|kCHqsW|{qL$)2uh{NnBH28MUv3bAiZX4GEUnVh4!UENE@-}Xb&N7Iaj;jRMa zN0j+1pp2 z3Y_;R^6j+0e^9P{)xxeeMO6{w<@$eSSd$xwf(_>Jm$8MWV2VmNF6>A|Jf`b$bG1qDawDlvm(!$Pq(m_|8>y9&8sGR8=F}Fy11tLz#Wy{JJWL| z6x9;v%ulh}k@?=kcG-(HK~vRv&y~x~7qH@rUb<&pcRWLxQQOHK3fy;}ORI%4aNM`3 zov84mzfu43R^umb9VX|l>f~EST)Wz_1w|IQKa+}Ruok#{DTBF$Z~G>W zq}vvc1e?A;_q&Ti_tnJ6t_z2l461_Alq`J5B{q4J5{O6Ao;#sCY6A-_9_bpraCr=L1zg#a) zIkuRekF;<(H)rjdl0%xy9<fe z?XsFyG0&PsX5Q(C1`QvNHmv3CIrjH-UY1jhsQ2>oM_ZVtm$m(Q7cF|Pf|=vWwl!}! z#Mk``ns;05$F9ratM|{O?&tI zJ^N!X%zE|q_VoL!SbzJ~tzQ3l&OxbrhebEO`}%jr$9adt_bTvBaFqR`y2bA}KW|*q zwV}k84^}IDhTQ`1>6_ z`)>KaIdHeOzBS|7tf}UObA1`tq<%ZSH|xPBVbjQ+J*D3dhl|ge|K3K_A^S8xmqq1! zUCI06E$0O9zI*rX-Pcxy4)@N)KHI}pZw!~LS6CP8ZMHQ`vgy-)qgi$DoNgOV&%ZiN zq4m*6tHPXr{#R6Hvbc%H&UnAe>t@NjTf0v`Gxc(65WSz6O z6^G=6-k+R^BOYq7VZavuKP^zOaS66ubN;)&;XVSe*P$Nnz^EmQEP`|!Nh;Fw>#e4?V@1tM!MaH_mAtrm(G88 zZtLBtTJ`?3k#LZ|H@8i0TgalbvEOVI1e9TOUSGVieeH~fNGc|X%y1qT;&wXq6D&gdBZ!`J?-b6p;d2cQ8!=h+o zOXIQT`~MX$t>2OQGyQYRY2I5$1vc(sJ>PfDXN^0X)sM}NwpUg}NuOG?!~5ou4yD8i zih`|udlfd2{jp^3R`}NZWlfTv-oGcZ`!x+NOAT?WoO?QFIJic5Nvm-4R ztPg|-C6uh=+jCnutM4ze z_Z6(Yx-Ry~=l}2bK3Ulo_w`Voqea|nACh0$y!-=J-zHeX84Bo$EW`+T;{`W(Af@kFtrX}3DqH(f; z$J9nysEdvLfB)U(58j79n-dsY8r1MHHli$=L1X>VV$QGc`Lh(S)v$=Q>FDMB53s*y zk#J;j#MgwPC0e_fy*RyFBL1zcDbjm?{)fMSifDjfoY;z%_0BiM9VF&2v0z!dg#U!{ z39G*mtN%vt`ug&1_Sd2#MS=3`JtuKlJgGW#UUehOw{=t2vV;hk{khq`{8QbIn1ziO zSN279{eRlV<5t)yaeUom3)4-t?W_K%t3F=(-LrV*tFL*PRgU);Y0UasFkQ1t;^Wj) z_uK6fkDYBTl)snN`@u{xls#~5eg*I#A}9(yyi+4v&2ndv(!W`$LCphlye09HOe@bneYBlo%$7LTM`>fTn zjZ^36brPR-ApFARx%<)=tT8ZESGZdC|Hm3XQ``R=H%r&AWsc^nzWd!UC#U70TT57e z$(f$t5AS|8I?0sv-=SRe>eia=&(+gxtZsSOD$EhnnwXlcSe^B3Luyw(SHGQm?vG=t z%#TBQUo2g`X0fN>?^kUyHfKj44I4!$En z>i2hwx_ErzWG}K4IPrWhhwZ}C7xSK|WglEEeSG$tCpRQH{A-^mJfE5LdR1D_rK-4f z8t+To1Fs)Tgi`mc$!vLo^PjM+5!h;B%lG~N<8*NiZMj^BjoPcEUYIEe%|33o z=e(U=&IJLMuO>hBO?DQ@zb*QF|9-{I4|A8Pgg*Yi;`R~GTGd*ScX}KD%bs;*{r2~Y z=#8|+#tZ(GwlmI_KKpV0-_Y~TC+Znv0uO46@$A3fdnIoZ2j7}WnNHGsw^oxEKa#awwN<#9v)-SW;ew{`~q zDEhwsMZr1C$yYzc2!!hV{n>1g%)wjX@%H$Wk43NfZ)i9j&{q%Gci=WV$4&!|xa*r+ zJ~h6WV4BRCa4xX-n2V{M=sU^KtYs#f-m}er@?GymHctX~tg4i`(vio0v%{>P1nqEs z)pg;+wLOX1=aqOojGAg&=PbV$*gf%1?Yzqh>*s#BRXhLK6s!4UiIYo z^!HWVldQEG@A;hiBKG5VysYoWV-r#Z&CfqwU2&jl#`VvRv)RHLzg@5Re70>@`n^9+ zT!+~7KA!GzzY)Kql{rT1?IorAr7xz}{EOdzc;^cDS5l{QHrUIwh?)x=z0_#pESKTx zdqbrvK3(QVZ1wZMtEYee@Qs;y>#pYx(LJ)q6jGn%u}osyx~z6J^Yw%II+qMygzhs7 z?da9#oS5)1{3ibi)#WQ?>w_;lSl;)WW_N~9d)>MJtreD?%%8ZD8@|3=c>c7(&WArN zQolS%`~F>S!>JALuXcVY68>AX`s0h%;`>K7bhO64FTY+_`s1FUe1o%=&kYHQ6+8ip zl$lxfH<(ynsjq*2_7vZy1xpsb>h!;6YA2tN+frvH_`rS|hw>z$**C%)lA{7@vp?Ng zQ{$A!!xz2TP1$$0vB3V976zK1RkQq$O%rTlYn5!Dz!C1?!nUqrx>WK~*GZ}0F5Y|7 z+wn~MQkzicf&Xh0BmCdpwF&eLZ2qu?|B-5rxqiv2Yvv2`Ef-$69edCJsgAndqPT}Y zew=*w`*d>KMlGwB*mwVS9iAKVXyMZI+C{&=mH6EkbXmg}yY+)lSjL7=_1o`${3kPQ zHmk(HfP;4TpKmxGuY1R|;oxoUxlbPUMKP*>{#N`&>B$s(ez|F_^Y*e^RJM;hf#ePj7TN8DGc#f8p!41G57>Z|=@pW8-!9P`0sv-e!czwZJ&(Vnx%XbI8L9$CrFoBnK>tqm(U%QytzUJo}B#gQliem^E=;uF&6te z>0jHao6Nh5ZkMl5mGaOyeqryTSH&iRF4~4Q{z*UN+Gh9%xPO-OjkuNkq(5*`-r=n0 z$G+W?4V3)%IO?InBukSGI+n|0s^{KfnS1${_uJER4trK9z2%>0p)7lPMp4Z6P)SP- zm4hZGY1huXsjqwFX;r$D%WQS|bLY0hA!aJS;+p(d+Rh1&=ChxqAAi4@egBQ}iyt?J zIo+K1IsB9K(uW-%0xr&3eB{urBCYf*lb`>Od-y-JNj#coioC+EkEgn}bW~*6PZ4T} zz8?Ma+Vtx}GnSk%j5cYU{b-wC?W-sj9w{H$wmZLT&xP*w*Ztmiv^GZlqix3C&=lpudw<=jpHS{L>dVgfGf)@|cj|bK;|V;o*bq&G~0KH}mJaZgjD=$(CQl)U;Zh zc`bXxHtng#UsIcEvUMG%ZcO`Jn=P@wc+s|RYXk(QdCM7;=@dAopI)yKu^{t&i{7n+ zbJhy{JG>|CSX-LGrsHjgr$6wS&35>O+otLbuDhCUO*tMl<#_ed(~kwidv(e^J{^=O z7muiaxW&KQk@L^N&yJ^EWTkC*m)4{u{dPFKX6iSW>Gkh?_J7#

a)C59@)6yE|+X zvl}%)47lT;m0zw7eD{7@BdnAd1d>b z`Q=|Yc#>}%n0#RUcCNW$Ke=R=pSM{r9rxQL=8nhg#I~TBUCR$W=j~j@UbT9DrSFwb zjc={~Gc$+Wp8vPbpGj!tv9lHJy!^JaTjV)wKi&Qowa2&6&^-5q)GX#nZx$u2v3*~k zVk!Ts+GW1?lzHz%;+9yoY^{FpKJWi_y{B*H>7O|E{OGOBP|Xh?PWSD*@T6sZhqC(G zn#M=r9NgPAc#B{iy)930X#nar*jy zflX`3v<(khqS9i1@c*5V6FL7;*Xg-^Jl69I9@MOl{K0-UuXD|@_g%N-=C?ikzEif{ zaT2@oIk%VzU!~m_{&~Ut=x%mGq^tMK)w$=J6c;>ucQpFK*%k3JrdrcZNQE7}`dh(t zp=n6b?Ec_?>BbxN`!DI9_;OP2)2~ohgR0Foisf$m^7(i+8o#bTsAzIz?c%m+x&Nbs zdUyR%H9qL5{l7Au?|I4U?-RGJ=}z@rmF=@Tdsc0xMtunDomn@Ee{-LE7OQ^d+3u5} zQXdyTDB%)0u=+~magVTrU8YxTtxlaS%{J$ny?f`M?h~udrPa)uuf8mL<@(K&Ute4y z5TV0UIj_^lz%9yR=cO};1sCuNV=gsw4U>GL-ISx~j*@8Rc~Ys{Ng zh<(qfdXXo7t}FJxyR4b_r&|?=e0)nT@*R4;P^WIW^pPb(kCesg7C-&@T=lryk$u+Q zQIA_WmHRbJcys!;P1cIt)U<7CyVXC3L$=P``mee*Zf{$xZt$&QU*4m|mziRI*S7rP z{byOYbLFp#hEuh*JGFe?e+~1laaz1jw6#tB6Vs+s87+olQ(fZI*k$~Hvt+=G(leF`Z7;aT6Ce`%-Qf6XMt`pD7uPHEthxo^MMZ2E7_ z$;rv4vMKNEayz|R&H3w%#3Cz_>i6EOUd|j)5S!Pyd`WV6;{nOiS6_8QI8LYijGZi( z^?mREmD5EO7TSC=ylDUNT>T&OFYQU(Sx0+1Wb0p8v*e1;{`M)&x%s5oepZ$^i7f;=_i*D-IM7`&Wvs!ia=}rG0Hd%wY?YB>Uzp_o# z=6k;E`GY@$;y3Sb^lK6oi`P}zwn4Od{>I=-bB?k|p3seaUZ(AoBv{#T>+j;+OE&_J z)~u8ByfE>+gLTx|UoCqh=T3Ief8H1tahL5`_|{sPlW#jWU(j6qBFJ5Q&H1%$`oICR6`+ouL(eF>7u8K}`e0U6UWQc7Y=|MoJ4jq9$6-uGSb+O*cPw{hOG!)3dKx$@rcPQGV1!|J5@?YH5JXV%SV znUi?$dS{|LOT()J<)?g(Iqi6z-MB?dZswjXbpGMno_&vdz4O1Vb-aDm zV9BygZL=hNem;&bnPT;nrSzH3n)B1%?cDP(?$yQDPZawb0>7{S%j&tYbK3lvFOwFn ze|Y*sW&I_klfR!lnDKnQ?38?`32*GUpS`?rJ9Bj>8_%qeyfvZuv2yz?+8& z&r@`zWbN6-3G0_|ah^GP`{oxZy={KG{PuqlReSl}UHtg5J7(6CF9hB*X4x{yE_;o_ znX~e1=DYWqwmgjbWEgK!UHIc%j z>b%wMCe1o)LD5T6+ zdlXdS+l&sct-tn;ySm=Y-RRHDRwso6&Hu9XJHMzGz57?NeE)|N7cUh?r(N9>cgkS% zrj4~BYt%CK=0#@qnpATLMO<0K7qaHgpT^WBN;wvjm5+vB(yrUJd(z9wx)oD@Z=TOT z>31kcA|9J{>Y}u91NZXrBe-rwg?Z15E#;ZH;zFD{X z?xP=zx0$+i@$KH=6|~`d(agQIO%m*d&PfJ0{#$1zO3c((U$vj(b-ezr|B4J7PUgK$ z&gsiPZd5bH_s8B#X`e5rRg_&0ocHPR4Fi!+@vmYROv;MAXOP+6)LAm?Tl;RQ0}FIE zINtWn+3~Q;g=!Ph+p1AN;od z+Hd1myVUQuyY%e+=J#{f+t}}FqGdlA7&tdL&NAfjVy$0jS+;KW^`)o(@0X1G|NsB{ zd2f^cR=3vd|NqkZZ~XhO+52Wo{;H3A=eyuO#~-#Q3x8~wkth85VCww`^AG+u4F1@; z@ARXb)yXy7_kPLkK6fF+%cAylLH6`3+Z5P4b!`g&IQ%*<*K#{w`1Mr5EmO1Rg~>lX zRuF%jrDV^3>9-7lkHSJdIqt3~)7CNmxncT^)Dn&P*YvJF*}6YFsA{9=UcILJ4UW2v zJ6|vwYyY?Se)jk6H72ib7VP={mwoALhJaj=`}$|6>9PFv6&90Wk*@o3`bWFezWv8J z`OdyGKi^m0qW3@f2>ba-m4a=nq}J44S-4u~rR0?DyY|;iF$>+sl=bo5qtD6A_nqwj z{uX-aFum^Af<5jQz0=%|NUKM#D`-%&c(QeA6HlAQ)rWW9ZaVJP=W#VTpzPV@SsH#8 zt3PgdHrqMk7O%t>zll+t@|)`~Ip2_vQMTQ^Y>RH?enXonk2n8QdL|O;u(EJrQ@rUD zh5K#YdqeBDb1Egr99$Km{%g1ABZc4QpA09g4Ac`++m(|1qm$#J%K!B`=5ORasCF`3 zma}!>sJ?$)VfF3P;taxux_>^GoZWKz>h8GJ!OOF6yr282W~=V6J<~(IkKA`$y?>c} zTjclCPyb0N@3!jo&Rn-P=?|ZW_U`g`e@?Py{7QB+@lE>>aZx$jmGueh((S&_;@Sd^ zJ~e#!_)fI-ciHdt(z(a=FYRitI^womv~I=td^jS2hXQu{nt^8T^8Oan$?n4e%iOZ>y)32a@PBNf0nb_{}-D#erRa* z*|+xB_Sbi|zrLbb{UYq$@w(M9OLu*(d)*Uu`uBsWf0Q_m1uO|Tv;XN1RqMUCYx@^G z{gD>!eYSaZMSH#bik1snEBNP|c+C9u+g&&O*4NNqU#8!Ujg1Y~^?ZHxwP=uj>i>Ts zeh;Ez_KQsPn4r}zXY~DoWSPK1=6QeahPoAPfBKO%{_3QLklsBjo=a922?Vd-nJhmc z#)bKXAM?%`HUbr8|FYCNO?RpNTmCw%<7w}NDdC?t9G|v}O+xtf{j*Dq_iJCz(rG?> zx6f(O zhL!rye#}h0AYi@l*IR?i#A&}LR-G(wD=v;-w72SD-1;919}gXyt+t`PAfe#f9kb)T z?*vMCEY2JKUnkwy)^x>AYHR`&P+=dt{rCW&O-Pj2yY{U6IRq+B20diD3CL8MgL zCEtlJ<*I^}%m078>}%cgKxZ2cgZj2p!9fyB*-wA! z2`K*(yS(mUC8z$Z#;tMu$GQy<96BEET4RGKi6!i{YP-p ztxta=_DB8OTl73w<@=r+lSJRYWO6cmHo5EeAMNYy)8cREix+IV7-Sf*>ePwoqzNTI z&h?}#e1GvPIq~paW5xW!b3o@1i_@8^NPzCW0C zRi$Xg-lISMSz0azRx$1`Wn%)IBvSz+~|`JIdQ zExrA=^`dwGtyfLk`r?oKTKFP^R|GK{I1`=9s9O}E`U@BeQSl9x}~V-dbQ$iO1=-yV?? zCTYc-Bbo8n<5#Tu8}T>dZ`C^gxP7W+KW3``xOH2feyx1$V*m5~FCm`Hr2Pf1X6Xy(|5lf;)KBf_WP1K#>aJ4v zKmUGQIG9yip)!sC(Pg{MXQRGLi$0h&?fbs!uc!8Wd${|1_rI+-RCMQD>i)1(-&^w8 z()crrECPgY8uljsE&Xlx)NGGQUWQ&=|36NCDcO4a2koZiTJL4w3TfJk?T=b-7oBzg zk8$kP9q;$ot^3nb{%`%yuPrjxQ_g((Tlu4J>)O{Pd0#v! zkKV27H_!I>rSJ>M@-f@a$7?FT|M2y2N3ZH3H(`TMGH#!??dLVs|9mk^<5Q2#syuda zy^EKtzTSPcE`0U%rqhexP5Jr$=ZD8{9{t;+ysT%9kzEA4g@cs*`LGib%9ZQc>wm<{ ziN;S_-QxFcHTORG`zs7S_x2uq_utTO;r<_4O=>~$%WnM<%4;~8^GB>w)%W!7eW4F` zTe(hcf6yH~annza-+#<>@7GUt{m){PBELk-)J4HX!w|}_nYWCO7EAHNZ zeb4L4y<*otr?(s1^Zc9<^5*-2_+NVF&n_?jeEpn3@SMK?yzjn#k~~|_vRnJ5)P%Ys zjVrqs%*xsATcKk2zWu?j_qQ?%%=W*we|T7-{lwhpZ6*6Z_|B=UzLfkWvaaLa>S_Dc zmhNA5s&cn=|b_6hb10){r@ey-}A+; z-4pizYWf%O=Z(FanY|ql2ZM^X; zS3V0?-`D^CXtl1^{)2H_vpx4FU5`+>_lwQv~__KdWgx#In|M~LXPv$9# zj{hsHnJatz>gsE|qjo=cJ=OXC|A+aL*2ZnDUOg#q>+86lC)PuqTVP^pe=jV`XYD|GxL#^&av6|5jhE+8eWX>ASTCoslP=vnPAlPk#RNPhpLc{b3gI$J0#TpKo2z zyjXYj0;O3!UG4lwjb6Om_P=#i(aYcOzr-DiKKt_5p{1+ip8veQ{+iyjzx8z+>-O7+ z+sD1%TGaV^y<>G^g+t=I^Xv< z*@t_tAyeC{)~gHqOjhat-DWLR!*@Hp|2wNuRkljZp6~|?RLrK#FPIqYF36_v*>C>e z+8t_J%*zb?Gf%Ic{OHiMH3AB~hp$#O2c7!#qe4Y#hxSjsCjQ_EURDlUvF^SH%MPg= zSLlhl9NWLv%%tphb@s>29cIFF3+L7EEI$wuuyBG<)0$~V`fpo2zEWFPq%?Kj3DdX# zf9-m|B{E=Xan#k30G6`-{1IFIkG*3XUu(i+BCE)1Gyg`Q5#HotujKt^ImB zbN^pG`j@w#|B>9e_p;ufo_y$Y%X&4d!0jQ+uNz*9eTuc!)0a$peO+_MPOW)c*X2*G zo?5k=rD=U_9XI#f>U&4kcYp1v>bN##ukI~<&!}Hn#T|-aM-saZRX4O*P@>O`=fcbCnQYgjkSly z!u8kAJYWA>ee3B*S`%YKCjI(;>7n`BnoB=lx;sh4OuL)kUbEvYZ;aw2)|j(GFRS(c zaTd&65IS#@n$C?cU(a9v(mZ?m!J6EU`+l!XIOK7x{=-zq(qGJWQD3i2EG>@unzb(W zi~p;;!nd}+Ze3fP{4ZJY=k>#}!h4e)F04}cf6#8~x9*ew<=t~`d{0eHmA@AIPjcSP z?3XKA|FcA%l*rTiqrN0b)soTQsnveodc(pO{XbT-bXYB$Uzhbq;O6I(*1wL%e(7GV z|9|xf>+QN#Uw?gnb#DJb#n8I%EHgJoa7gTDNeN>)_@4iKyfo|4Wz{YA+{X?(@3U|W zk!-%awXN`B!)qS@lnagjAKrNIDum(g!G>GBLOOfSCax@y(XD^Xd~?OV9d%WIMe0f` zP15xLSULm-hQBkGe6#u3@f#|Sn`KMmxEDJuzolanzq9@ET5gYPqTlb{*1A4q`zhA#I`ftI{?5nS?ukN1p`lisGdD@vp9D&>IBAISXJpFISrL~(E zq+Dv-=)_p^{>BUCuqLMEdKTd_I*b1|?OwF_-eL{*cnL|>_QS62(VGuioT&1j{QsFs z<%AFC#ZT96y!-R@lHixADd%tUK_4RsQw^Cs)Re(zwZ|6t0_Klis*U4QlJV@g#<`JB)S?Uy@WI#fJ3{Dp5fZ+hVR3(^Al z`hWCyUD|Rq=rarB|Ib&ZwZ{M6Gym(=>w@Ql!k_LuVSoMZ_0GF?6VpF9-%ySZU$|u7i{7e;tk65xzKEeb)b9R<3cc?oa4*@(}%d@+FDxv`uam>Yk5tUmF802uXKNeFWUHXeQw>x zx^?mM^`FH|)}PW>#Qi#8x+d#gWx@Mv*6d%tlY7-)!BdZ3U(AZx612R|XI}56%D-|Z zX7m5;tP|2VUAq5BkVI8=L0)*=uJ@;(sO`~muil*XCffA>hm#hY15aL?^-m@IZk5o- z$^RbD`+qKU&z1ixgLm9u)Dr3N2;*vPRctf}(G%lhb#g4!Xi-z;>~KmFUboqQ-p-l( z^Xqg2k7pK6clEgT_DSy5qt^Rwm;dEE{8nAxP04$I{<~+Kr+e2r{<m)mWt+x04 zCvz1&ejj=n)b-KKZ1%r@Ev0pG0){L0xtw2P@$mYI>lK&Y`ukse{8b{{UScbQ)2H_B z_x*RWMz_U>RUB)pdwuh}dbR3ZIT@jy*6I~Pk$w`6+FnYnVY5tmUfeQdXVekt&=S(( zb$Xa^A;3$CtHo(y!3rNIVUCt2ZNFbDKUdu=|M7R%mX*(+tzVlKdNFEYmF(Z%*f|?N zy!k5^|7E>e-RtiYb365F^q*SAeOmCnTKdxGDHq>0e7ds#PU)i!e|cQ~{x2@mo%&X1 zPTWQY}r*0XtGOgt=OdIR@+qQ5N&PTlIP8eY?6_pG&4cS{|Ej^fGekRX;XA8Ag=9y+aNZeY|C>Bth{WMy&;STI9N zlAHC=LWdO{8sc0nhm;IC&L|0U%vins_T#JX_uHH6J;^O!v+~%>m{iplDJNfE`t<2z zyx#O_)g7~nUrpQl`6qAGE7_NKPfTB0|8((pUbzLUZiRk((_sAbgNy&ci?ccX-S?|r znN~j8#p%=4FIAhT%kL0faw$HBBVbvLQr=5hSz}?7*7|}BUxkE8M<=pf5n=i=WeT5y z1!$+Nhma7fle58&2on+RR;NZog%Ay8ftDuS;`K7|x7VKialI%zSnYFc+@~L_&VF5c z!dY^UgoNg&b30LcH3K~r& z4WWt_3M;z4y2voQD5^7~y=qM1PC(6Ux^e|yXfSQPKtCOQ4hmVpfXOCB6_CB@h zztv~o_q{4qDh@vTzUtBLna`HGA1L@xwZ-zsxpk#iUi@nfd3-;-xWPJe-NrxD{#|rh zy0Ez85BHbf=D(l*u#dAl5Htv}~n{TR13midN7 zRqgr3m);kN%jpBkyv& z%cmLP{i+K$rp5);>+vOCSz`TWPZ6uk3KrQX77uOmUVd-0SQEMM2@_YxwZ=&GK$8ct zL7pv31DZIP8yFZ^Upvk+;^|_oUsxFK{i!Yc{q_BEf3vr~`uh57{`GZFcdEy|jau~k zZS~6NukUuRijV7f61QsoRimFzCBF!}c-w2TJ*~={ShZH%T7CYn?NQrbzk9tkZ|VCh z3*J_F$Ngyi_-o^>@2`&6USBQyPi9xtv4G3_r~jK`%29o&inlM~h|2P7)8-!*n|0Fh zSl_-$M)CUp9?o2z%yO%KYt0v@rTVWQ^d$xMOffaC6TBo>>lD}!wc7YN$K97TqQ?*1 zFF2hvvOz* z=)SApvx}v@{PSPlqkhFPTkhUI8`^xTN%Ca#zdx!h7hctzDti8q+vohsS5IGtJxw<4 zI{P}bs8}~F{`IP@_1BE%|9Q~TqrPtMIj4B(u=gi?Z`sDK^{rpGH#~OD?x^=qC+lWC zpDrK%N8O=VT8(Y_^)&83yL-G`lRj?G4$q#xIJAD9*4CQvU&oKvhyIyVbyV=x|3wFH zsqAOnX)GfY!gp`K@U6Wz=ECu3FL}O>QQH{0TD@^2;K`%$~?&e4R06{-d)| z@9rP^bIFl!Ua_2L>vWldJDvX}eI~WcPu=z{Vi)K7*T?uhW`5lIP_5v8UZPsv{S8_t z7F1;HQU1D)bNQFT=dE|+I86ed$o|mY>|$2zxMDfCl*|fK_M@T|%5xI#Pk(*4oqOA~ z5N3soU3P!I^{#uj^rfr*6TSCWgyVC+uD{T=<^NNi58tFLALO1$Y;by2y`OLWrPe>9 zMLo_hO)T9f{M#x1&s1xo@P~&I9?ffSY$*^47Fg{X{m=LL`GmvVsa0pMpO5{wrjFCh zwB*nlftxo~Lm$SPEj23CoS9%_@yF!8{U@(z%k@XgE-M_ow>M(<>UB!W7p>2)k@_#I zti!6Mb${0WGLe}-RCH>N7Buc$z0sQ0ET;0o{F3jkKMc0~xKn=p+@dbm>8-EKthAnN zT+()9$&O=_FK+$+waJ^Myk6qh!JnUsAFuvtzi;#X_tG}if0HI%5A_$WU;QmfGoW74 zKX&@nn>PQC37m;AY+>K~Y5&404)qRwI~y6Q%GdvRbn0}K=0v-4(+}=xyZ>#htM1oi zD`@CrI`#Ex)%Wa6Mc?l@*4Bsp5e$ygKWL~qQLTL{+kV5amg^@R4li!GoA#h*nLXd4 zKYSC+e`~-0;$a_F*JA%#wi6 zzW$)Np54xR{qnfH)gPSPYd`&;yjT0r9p|YJZ5j6-)^$;4=h|EswC&&PgPhVy)w1*U znW_Crsr&Qw3+I(7^#|Qd?lfMCnEJ<_P3U3fZl?5jz1N>^l$ig%@_f~;*WZr^{GamJ zXwTE`v*C~GrS?Y5`d(@K|3v!!hjH(xYd_oaf7;ZawXd)KS1+%hW*&Zh_8f&>J8eYe zwce#I*vr;)zW$)3;hEd)f8#%U?)~AFoGM|vY2)|8OVJ*8V=v0z_tn_6Ra3Iq^-W~) z^O7A)pZu}@ zZT1Gw30Lp@5fr?;`l7A<>wu5nWm{|$#is66{~fs5r#|hxq}=P>@RgN)GSO;p+?%hTq{Q}4n3?fE zUcA1|zxc4lek135FRS;nnD5uFdU5yH*UDFOUDPVJf3CctQyXO;d-m1W*Y{V?`~G_U z)n5_)EFsfA?st+{!N9=U<~Yl!XBBJhg}!x8HUIy$>@UxH&T9YpYt8@vNB;i*w*Kn< z|M6e-D>c^C-_QC~So)$r+;(k6+V)S5?s9T1XKm(pPhBSK9oHmh1fg zC2zm2@8Tr6Ab-itq2@y3_u4lXX+;oO8VESIes_-WA#U>-($o*H?W!-Tn35 zq$;JqF}sxZ)}LCl_Ui7i@CiN1hVx$?b<`~iH<>cE;$`fs>-!^izgib{aer*_zvW*0 zuCFeAcYXE2mW_&H-uDcSAI{z5_v1{yeN$g!Ps)dtPKUwCcd3N`np!>q-)_K3a_SR8eZWZ&p>$`WyO?r2IRsPoGxmD}7sI}IuzMg%l zYGa-LEY2v8n;)F--uMZ+zIb7-Vq_|1fL3=BHifbD5`A6*M@-WnvnHKaV zqkHFay;!XXdlr|N({et4{$}>M)PI;5yZ3hH{Hy_aNn+1^Fz9)KU%+M!@U({@ojAPO8lbCqE4?o{J-gK>Z#;3 zNp^YHtCc7A&pVlS*mdvHpUNh7PQEXq{|LI@-BI^9nSXWvzL+xq{a;J-HiWMBe0BVY z=H*@QpDumS;xo@|ncbRwJ6X?MRFD#u)m*`*C8Ro|k!yvrPHB~L{=|=){Ui@D{DiihvMeuq45&yJh%j6oni{~%>-hHK^p7mbMrrrr# z@3)@Xdt`l_{qryPgxBnF-`^RtK5c)J?DwZyp8s7>{So|LChzL8uk(J}`|`j?+!y!N z@3s_g)2t6me%|_P$B&)YFYuhQt)Cw1eJ}q1dKTMz;jbRW*Oq68KfHN!M9JhflvulwTe`9xRvU-ZqoC9BVPGJf0l(?fOkM-%k~_xDr$_6SDZTYr7|?=Mjk z_KE(lbMi{oT;K95@1m+heR-Asj(gVYS>{C^yfOP`+1&dty!q!Xwmf*#d7n2+imSnq zjI!|SH#uH>Z8@^f=Gr_l6+?}mzArXTTw1R-JMKIzU#}a|e zh}y^>S?iQnPyF^bHa50OFMh`4^wqMuzyHQeufNiHw^}pd+Ft2ZM?O9Mwy)~PPp>)g zZ9iVSJ~ip+{`y4s-P)?D|5u-UB3Rz#E}XQ#^RN7puhmanr^Z&DTEF{Osp_0xhiw)ke_~iPu#osFum)mr>%<*pM5^wZR@YU>bu|W zk63pt^wq)3*Vl)y4;3yi+w0wVv3k>|Xa0NtsQ9QRdVb^B@NbQs%#Q!NdYb149RL5I zZ;e6^@4r9a?W)(;_nkU6Df-Rz>AT;5ZCac9BfG?EURv;ijc0^q{)gF3)6}gN@qQY2 zH1lnB_`kjC(_RSHe{j8EcsXXjtHJSyR!r44+Y|b{v+^4Nj%Q_*Pf?mLN5G& zu%)B?-W~bXqNT6dpReDbIO(X?zP-P`##VjLo?7+&_5JmG!x!z%{%Drw5V+?ttNh2n zx!T<&vH!B!dNL#9TXrUgEil@9A03z7~I-8Z9*YKg;wD z=EWbJa+l_vkJn9Zjn>?7WFbSExshL`zuendUbi;f<=_9wN8F<9c9*{n z<9O?%p!54fBFj9ce}3!{%Rn*hnyFI1 zYFm1F>4QnL|Nc7qL*>b2%er-c%px;aTI=6kw>#>`e8KM#>o0V@&Dt9m8F*vTlP^I_ zWAA5&7eD^HIQvn?zWp3085meN9cLN$ykafQ@N)l}{Ot9S()!i%Yl9!}@BVTBpVi*f zxOJMpcWpiW_1(qTpqB@ElRk65pYSv}e7)zVLeUpfV^#->#<#t>n^U;))kf~SzgL$h z@17L(<-^)rTdTHAzxlB)N_BnsqT2e|QTroQ_7q)SX6?C8AS^WW^`&!LooyD^W`A8- z74`9J(cY|9{ntVJc5PjMZTGwEw-Il%;_c#PO@sBXw45xutfTk*YxmZ!b>8cv)?M3| zy6S^#q>F~k>e*k{&3YRfv^q>bdHak0aP6yo*R-Z)ZF+zD;gL`KS;}=HdIt@dR7 ztXJ1|N5;L5eYEdt-|Lx0ldlDH#a%sqQ0{U_|D!EanFQ2eF0L zVV(Nle(YI(!EBM8?9N8^OUvq6ukZVw{$O(Pir-&fe=1K~|FwJ9*LQK(W3Q~bdv@1R zeRD1Sx+PM3#3xURTD7$O(f0pc(>^u^x?C0I>`|Tm$2hSk=Ih7VRYet@F^PBk|pnukBiOeMxQBk$HM|TZ)A){5Z(i^yA-~thp?|?H5h7m%eG`{D5bN zqv0-wUb7$sUR{D0hWDf|)FfwRtb-+VH9C;wBr*lQ4`c=gpE;e4-eesx{X zU(6^Et7`kq)%wrB{`&5%1<}tVJzw2lDHU`3V{i+r=$j21=h}H#m2ztYuQj+|JnG&4 zVBWzQ&OBMUCim{o+wW!3zTUR(Y2crY6K`xX(vbe(dfG?Q_T-=G_2s3nj>HSAUcEkH zDzi1?hDl$-%X*$xyf@JmHasll2ml(@!op`kz`;n!o$clGIh7^V7esTw_u5Z{NqJ{a?J+$9-u3 zI%VJ1$)D7JeaeiAtNw4d^ox(ye{RtWjyoJDNWToWYMb80lv@8|m5aoNIbH`&7$p5# z%Db=Tk4Tw$dKuH<%~zfWO36<@sF>+1_vT5wR?H9onpeWNUU{2ke@l9}Y~^~lcRylf zuXnC@smST+ESdhqQ0K;y?dps6r>|Rj|A+4L#j!W@n8S*Wc0))znpq4Su1h!mbU#rRQ9>~^vs`k{DKji#Mk-r3TsZs zENEt}Ql8(W&$nPo(wiso(&_S1CtquQ2@CzUF?{v?MVqpt{wC~ntku|kw|4RSFMe<4 zUW)tj#N>~V=v~&t?{|KFUHAL)tLra%_S@FXnjPOH9n8P+V4Pl3RLtiX-_TcGS6_>1 zzP_lnZk_k^i>2mkR=a8!25zD57tLUmizwvc5J0{RJ&bN_E_PN$jwY=mYac79Tk^hW z=HO$L6X{qX%Ej6AFkwN0jyg9hQ)0k^7%gM24yVL`1tCi1Yj@qR%f4@X`?HUJxk0h2IU@D+{M|_pe8df=#qaoY=8^I7 zL*HXgHOdBVPW$uly`=T#t9KYw_Ie#mRJy=$FmtWJoC_Rlz$2|HqMW=gjtM)am`HH3 zIz4pM2+(ro>Hr1v-FxbN=5?#jpMHPacC}HV(dyer!`4R%FL*eA<#PYY%a@*7CwFnu zpV+O>4n+PqBy{_qo+0}sJ$>o-|MutWy!xlh_WGJd#@YQZ_VU+SKV6@2!g^bA$sY#S z+%0l*-|duY&VK6piI2I{bdgHA{M7r6v&FmLFee{hx$*9zii($oh6=`4wEQ@Q6$>V{ zs`h9o_%VHBXEYJ%=n|1?ZF=NzAjU|Po0aWRV26j6AZM#nVj$;SzhC!Dt^Vxz^5f*> znCE_$E00@h_iEbo&MwI*t*UXacpZK7t-5G+r|BQwe*y`Q@@n$~+uj`yQ1GAHP*VEv z-|U;KC+~GhzAkTkZqeSFr}}yu8}hgPRBk%RUmxZpDL&6UWb>C~#RD!aR)N9qtSzd$ z^s{>--l^U3KUOcg^7z5h{tU+J4;b`WMP~|qbeVg#Q7B6I z%nzw}_2oOR{V>g%^z`@5f8R})Z8*PC<^2EImo96BC>hxqu%_8K@u%OPe<#6uPXdQ% zK%!$PW62}6unw;hg}MaLxtSeaN?fd<6cG_1#?#7lIN(5xmIzlzQ)0l305ws8neUHz zzJGdNxBU4&74527XQpp`wq87j=f8Wnn(xVvAM>|g{H|>z{&B zOcoMYa#8tA5X<%48+@P>Sye>2Segwio9JG-Uh(&c&) z*5lJX8e{ISfBCJ9<#0{BJ*$B6wPQ*_wWk-$H2$3aH0zPmC;_!JLwDNt@L| z_B5$1a^z|}z`($~-Eo!)FBfa=LQCJ*_t)24_dU1w|C$Z{ueRAo)?chu`u_jh`v2Y^ zxt?b||My~D>HS6KMNz-j?XB8WRhb)8(31m<*De26GM-v{w<|ozzV@sC z*{gSd?fmui%Z9GGvF7s@`zHVMO|lH%r?=O)ZfE?;%j*08?Au$mGk$&eYW``z;`II| z{ol3df4Il~)e-AMSMMo5{4)CMslC-(D{Zqs&F_6xvul6nnzFr;=l&>$+f_Z=qr7|i zHLb53;yyPqoUSiic=~hHsl8c0E9VwXzIHNC`Rf0O{XG8s5ie(5);;?At~GP~&KFLR zH9|qPtL`#;?T#&1U0=TW_uK5P^`Tkn&%cKFR$Yy)dirz6lf(AM-)@qs-{3R7CyF_2 z^8cRcRlXaw#h&d~`ojO}=+D=WSN5;hwJNP&x8HoKb4UEORo8dFUw^e~ZAk1> z_Fv~ez4u?&lX)G-EA95*eg9+TxyARszIyz8{Z9VNzjnllAAfoG*Rj|Czb1Zt|LW?i z_t*X&`7x)YU=7#D3k>sF3{Rf8(%pZ*^^4bm9K#1e8?~kfB`)|g@7w#&g74$H-k+T= zb$!~;ja3itmwtU6?)z%1{_0q-tGkZ=@afv%bfcu``9Dr>wN%lH&2c|c-~K$cI9G1| ztlyKb#|Q3P_ebER_1B*U)7#yCy_ep?A{m>ysfTCw5$@&}&!wj}>~;wFz4)T-?~lQ$ zRTY|@U7L^JFE3ADzbMbnV*Z3JOXS1v7gcSUerfIfjuQf`a+jI;Jkr*d{#@+P zeZlx-w|(hF2Cj12v@$&WtMRHzsU9Gk@fwL+W&E+3glLQ{SnuB ze~tWFkM))j?;>mGGHGuux+>usXD#{NxAcipvclqjCl~dl?UmE|{Pmwf=hyQ0yZ=V5 z+aoaZF4NbDejB4pe{Oy*>t3}vD7}8xZ~Jy_!!FK?vEmojt^X41ovrt5OV<9lzoG~A z-T&?So4sTA>Zr+Af6w|Kwcqvd(Z8H;CEs`MkGsd$7Nk|;!N9DuZ@?q|G54Cd3DKscY(Zgy@P5p{_BsaeBb{{^ljFP*;|uVEcM&AUe!DH>vyNA zkEb1tuV3x?dU}25i{pKo$G@I_x4y~R{m-V#e|y_@Myy|UGV;~m_^+#E{;mA|`*q0Q z$5S7z%iGzavt97@-6l!K{Dar)*3|fC9gV$zZhhVQSIh6XtdH@%UtVmlBG|E9HL|W~ zLE#rS#lH@H9hA5B?&+fYYXg?9ewj7Vs(iTK}t{bJ+#+K8-4i=Te?+s_xW@yfN( z{l6ytnH2ozs|+jGmpFg5fPxxjhdXlzTdv;4PTictm z>L=ESmkA`YOqR0w{>L~tyWslii~qyw*3~U%f4+Bful;{EIVsN_2@k*iv+rMD_wL_+ zTW3L$)&Bpyp9{#_+_`7}r0V4Zl7vc@-O~iyJx1)p<6ny|4&$PGDQAEeCx#Z zyBQ69^=BVHv8?C2At`v4_99o-{fU}L+m>LHLa~HHTRXi4_z^z z`-SJd!06Isoe9zt>JPGfs_+UHTA*iOkhtKBzu2Zr>^*|fUd;a@&)Uavbnla|OG>;k zqh|m8HK7mJG@Bp(csGCf_1gS-yHfYx+FSLt`d(#i*1EM>@BXHm?4KORH_IErV%x$jep#PqH%1frluUmI*)wOl$ z)4kqi`J26a`M1gYtb5g_4?aJb{a@exw!{13%x6CPEDp@7>VJLu?zbY%dHer74>$Z4 zKkfHh;ibF3PM!Yu*XQfU*$);xudA&{yH{IvdV^EU|M=DWdGqGhz1;O#C_=XX>D~LO z#}BfUPpDg0r@ri3%-ajn&JIie%r@dX{4?i%-1@-PvEIQ^`Ysh2t*(pWm#eL}t)I*u zCBH@_{@T{&?a5Z&tKSI8UEM9e$o&0rH}QkDCYhIRtv%k@r~hlR{Pby`{#JGDm@EBs zdG)`qUFJO^-Qp5~dB3hkm+RO5TmNl|*Y~~mCpa9iDyUss*F8Ob{Z!HAagtj57Q2M? znC3j5EPqhG{PY)}{So$j-kr9^#cN;8@BaTfW{LK$f5GDV_v<^hHL3&(uG#-N_RD|M zlw*694@Rzz$?SN)`fuLki&goWy}i|zA+Izhf6sbcX>b1fi_iM=@tTv~?NMD?GU@kp zUGrye?f%y;+sov%ujYK*TGi99SACsw{pag9O<}xG`_q-?@x1+eXZkInqLtP6emvw@ zzxVvV(zVzAu0Hu-<1ABN zFV><$dn2)@u|@04|Nqsx{^zxL{r|9a6?JRM4;;S#|Eq4{{4IOGtusIJx4QoOuj{K` z@7neC{kw_TtJdrPYpD{f+sA#iY3r+9&vw5Gd%yl=)!vxbD?9#1fBkstn}7WOrSDdK z{boJ$_=&xHTH_ANeVhOFvVD`>2}`c(-mb^}7E?~fGXD{J!jpVhxK4Hb!Mf#=Zw=hH ze!G$8a<+sk4YM00D{kHFK(dyIxzn}iKQ+;*VtGi!yVuLC^<(kai81X#t zm$vM`9WC?se_xw8)zF#*qq$}ij z$?IJ^Uc_t-T_gVL^~=&&eeVa`w`9$%-*NNxrx(BcGPmY;ynm#m|Mewnd9BU79p&ch zJ*U^quAl2Q_jLW5K-;h>`_}#0;&W1p{p+f)tGCv1{>#2w^EdU^mDH6&{t>Is2kv0! zep2;ljrL3-VJD`0djtZ^*bW^2Fgr0iJ2|&BC#UuQ(r0N_g_z$w=vtkcW+H-e0l!7n-+?* zUSHQV%#W;}Z@+k=);U*=otLJ3W&Zk_{Z(SQ()HE0?(6c}FaO({H0%4-`17^@I!>Hk&Fr`}$Dee>Plf2FwhpMGBV z-!|@YoqyQ7jrXp93e1jQed*VqS*!lU-27^$A1-ej+&A<1>$8dC|5bhGUkk4KljqO% z(jawan;9=}cWOpn`me9)Pv4q;&MqylUn3~@q&dFQHsNnE>+_x8T&~rhoL7JK%6!4S zBKvP|oT)4PC0zSN+Lq}}pPug5m3b*%docL>K|@vJ_lMHoz28!?I=&>fY)9CW^Rqs9 zXnem>vuEc0jO9Fk6L#+}y)v)+{$j)FfB*fDn{+W|_21KTZ(Z?UoEX>VwKw(G-2GwU zbGkn-P0f0LFDmlgp6mBtuly_fCoE3=>(zG~%GN$TQu&&1$@f3=*4J&#uW8cz`uhHw z@Kr*urmtF9THh75KkVJ!=&$cvLQj8+Ff0Fkse1DDLt0;3OzW&eUi~$HF(HP5fmz3K zmT7MmYw<7f*!6W=>-PToo;^ML($%1Un^uSYjo#P!fBM(cTMHNVU)SFGwo3on>H^;@ zi?*>zJ9a%-A3kaJ>(#QS|A+s$)e*JRHuBZFl<)ulCQpjmU-NrC%k12*H`IHl+9#Tp zv3!2&Fj>C;|EsI7{;w+eo3#4<(a?Wp-f{1|<0e(@+V|wsgdf+p#FlML`TPHG_18?j zP2sElhqLV7y87#P{nuBsLtp1zo%`KUn~)=o^w=)?QmIL!0*EQ|HD=4{(k?x!TrVG zbwSx7r-QsNg`4YEU75GT^R;Mf^~-I0BUAscsTDk%ZesQ;VbzJu`G@ab)!8ho8qM3x zCg*CK@W4JUTDrVkJ$q`^mGCK(>R+>je&aa3skV;OdT*Wa?spdP+hUepnf|Cc!yiuvpE~U0sGS+$U$Fl5?Y~uTtG~WJo)xF( zA78gKev#K&v1OlxBHbL1M%nCE*I&;PG@+gzd-4}O%sjz36eV^;=>V0Q&?(FVjdDVZOL>#>)%KrN5vFqHQ<4*0B=Rb65>B7sYDba8G-|XY) zRnD0I>DWQ0zokv5bIR81oYP%D)%nyPp^ptA=UWwieUJZ`rm{r;)B5bF7k)A+`n|E= z%d@`E+p9mrnRCkY?5}UuRGfMD>fLqz^Uwaa*tzb$zp1u<#-__Zd#2fIKUtDwn?GS~ z(7QKBo~@aG>yO)tujWlkKF9UmUs+imw==xS&ck$SrLJ;HRXvP9JWZS9Y> zzfR9_{j&W3L2(-{|LUFZAXbDaJz%HOrS z{OaBBrW{Yc{on6&Zc^1$tvB=je~SGjQu0T28u#6M3=B-S9A}yFcCi*!iQixE`}%67 z-TwF2m#V-2wejxVU$@>yyx(aW*RTKo->$bYxsg9KZVL4%oZKgx9lySA>#hB-j&?=w zYstQ*^>lrB?7yv1@m5=ZYCI13e@*XN{G_Wa*Uz5zzrLI0PEB~$0qKi(EU)bkTfINF z{MXg0qnFEnZ9gr%>gcU6|33ZudfobK;aAg#{0ohuzWO`wWVf2y|LS$3{Uiv&7~^gVG>eW4Y8PT=#o%cKMS+B1LRee7#T6DMSYy6Y7 zv!hG+F1v3{`db=vx$27hqU%}DAMPvvniBL>)sy8-f6Bfi>C0A33~e`itG~o-^7&U^ zJN|VBRq2MVmS5=qOZCOP`ycHTDyMBvcq=pS+Wwf`f4<2jJ~rC3|Mlcq`xhAszk2oc z)t9ipN%J1r-q0*rL;zm)QsFX9eE;o9>d(UCY*^wOKNI>aV?pf^wX^$KBnu zK6sX2T2#F(=HHRs{|$|uKe$UtDy9k;)K#5Y7gcla*O8jg>_sU@o^9>^n)UV6-qgF5 zOX5y8{{0>lx$^z4PZJil#iZ3w`#tHmyO+GJTm6BYzrUW|e_}RixBI*Qwj~l(vHf*t zcg5?qMP*;T7XSUw(!^N7Kzl!CwUYmnUtQPew>^J+`n3OHyZ@HPRn3)N_J6%g;r=sf zhb6!LS*BNeJAPtkQEYO?o4U2asZ$J7{hz$ zT<=-j_u404{_l6`TDQ@n_0PS%zn=bj{rQ(g^Z%WB?zL6l6Shx|dzG|zkewE@#4@f(eGxfzP~y$J#HO;R!aS$kI$bk zUiS6pm*sMo3ypqlymfzHWk&bb;)`MX!?mxix_b51zw^o`pDkNw{q^+ecW>jn|Hjxy zb$qSsF@h-p^w(@or~s*3b9X4b@jqfA#fs=&Pqy z%FI7jX7oS3+c~Xuzi0fSd(W=!UEOSdE%f!pdF#XDw%$_a-888_d-g|`)3wGrd#-r~ z{ao|#`&;{-hEIrIStnDf|h0 zKmDThrnoEjHth}%ZK^R}x^PKVj$PG;WaA6I%m3`!{l~rd!k7Qv_g=I9`u_Fgt-V!8 zqgQ`$|Mg`b$EzZ~qQ4hpdiX^(X5LSf4+X~`}M&i zJb9tT{TEk%iOp93{)(d`Sp3u9bGseksr@4nk&V5K7JcR1JRgZtstYraOW z3H?6ZeDTqrJ->HH?GIZW*75mP$XfgBlaCz}KPGf#_WqAQSGMTd${$wmO}~CR_dkou z`8h%HHv78nulG&T+?Tggce1SYKi}2)=fdU6r#L3<-&a+)>racRP9N{@b^6DzWuGiQ zb#}(89~zf3zFBkpty8kLxTTk|>_u2mLcDxBU_Q#Mz2RcPJaIXROiZa@A;?y zyT>~$Sf7&ovm^Gtck8dG;omkN{rSqvn*YD8@4Eiqf0mSA?^c*|-aaeK=n-1f$GuD@vg^~$`igH^9TmH6!u{<$ehWZ&9@B0KNsv$Plrp0B(2 z-@bO*&yTnC%P0TZ>)V@}CVa8_YnPATU+_b>kYZ?~86O}bS=_ZMY<${^a*up?`nxQT{Qj!7IXrP(6WRjx*TnYZ=V&R<9G7yUZD|7z3MR|iXz zUab6Eoqm6PU5-U}@%-st@BUW3d$%`ca_2hrt?AkEYwOmno|?U8yH%6Mqwj?uWqUSn z=&!%u-@W_mAIlz-uRo43J(jEwt-n`&_upRC@A~WO*4M2R-uY+ROOIax@y2pr9{d#X z+`9gq{LBS=UyHAAoBD6+l3SCn%T{gK;x$$O+PYW&<92q&tryaoYgzmBZ;4KO-u*qZ zg}N`*hK6{3z5DBGY|-`Ap0Dq(i4*>}>2J~>^_nXiGPS;Do&HvxZGUz5 z?)ttf4?12(y-nR1_eVIrchkwwdFAF=0>=X~mh?w2Z{@e8KZ~m{HKQxxa%j{me;3X;&ioRQn|Bn`{dX$v>)=zq{r0`;-Q?}B&Pj%xS^cBm zbzSKCf{9wD5}QIY4(rALJ6O_YRbV>X*!yHz*sQpXH-?jZ`TPl_pPS5zUN$`v9@gHZE`Zw-BD6?mY(7)YLe_B0kXVo9c z`0@DlGNbtW7xWb0XR7dp>95|oG-}$#s_Cn)Om6yp_tUH}$xD28_UfO%H(d(z39LFA z$JNy||AW_$|4jTJHyv7LwVxw_qu{V!VfWL$EygxSR6X)LVu>F+meCyNtDS~d9hkvvkHu5{S&ox=$;{Oybiy8X`dxY&-`qd}rc6wwz(XY5zW$SJJj(v7kAGfiH!WJl@!*9PJCYZlcACfW*bx)%RwV!`x0 zQ#P+<9`{zaPm2D{;#Rx2EVwO<&1P8GX`-jK2-X1|Hi4*QR6y8`0hD0;X4 z+im#y|DlwuU6T9Ma{Fw&;65n*Hm0Gt%Lobv|4HYqm7#m z#Vu_Kx!m zyruNyOVm2oqWCbaYl}GF{{MB#gMoo1!*P~*?<>|qq5BI>f6djt#`im7>DMCPYwNpT zitc_JynfNnU(5Jc@87+8^**(&F0Y^bjoLB6BTlaF{Xt37eK&-!nZJJhwfJq!uejY| z9s94o`W<()h%Y?r>+KKcioMtGRQz*m_pKfC`;Tp{+5ea8^Q9=`f6M&B&)OdpnfO=# z>iV*^zx3<&t&jWbbMaosfqgr|SJ$oEy?*vJ{%d)U^uH!qU0=U+{Uk?kjdj(3Q*T{8 zTlzh2*ZU7s-+bTxp_hH~rDc^L@2{=f81`D~Z~WJc&M-~es<_a)m7U*z{aeGG-F~F9 zUNj_Wa{j(qqpE{%hS(Z-(Bo?w(@nz`S~A^ z5>&bL@TyvkVy3kA$89G00ur1R>&%1JynJ)kzvIUX^_zY_!W?6(w%2QYnQr>&_jC)# z$$!K?|4GSsAe;JskE5klh0(Hv>qmY{Xv^)DJ@p~M$6Bb>^76t}83Mm*>$jhXmD+!? z;BVV(Q^(^x_BG!wu6wu6?_$jCBNH+N;uvmglsl^Hzka`yuV&)1 z_#a9(88Z7~XUvGSNcr0Q_0*pDN!-ihV)nj&vir+=!Jt16U+=$qxoSgPk^ZUMfA5tA z{q%~NU>uhe?D=n-_1~mFty6vbU!A|%UVEYR;o|my$$z3}G(Fk>A&~2&VAbO952n4j zVdmB*zn^86M0I7HVh8^zZ9RX{W zf=4soKD>BbU|&VN{`VtHn!nS3C4UmT@Uh)JZ}mS;KA*Yj@ogFcFDk_NGZIRse>tYO zFIaNYqJ8oImfhG{E4KVoX!%3iIsbJ+r+?Z1sUm%kK8xi|1^?@>{< zkChiNS+RAMUVOal=!1?QUHNt2rhl}&!NYYriI>0bYexSK4ej&#b{lQfC;$HcC1&Yo zj=t{I*&prdmlf9^G<0X{U-o_ZAK#mQ?#90k`pG@xu6>h)a_#x{{+pIQ@eh^F<>n~t zm0tMPd~LbMmiOUxLbujUiZ^`z_THiLJGbAz+FSj#KD6Z8);Nw%fsGI2zL?yvpDiQ! z>(k52aXacbetA0nSo2}}jXPzN{;qG?Q*z>e&{IX_p56NUR~-$%q5SEY=!2g1o{iU^ zEcyG(ZS&shW#%sT|E;~Hec$VP=h>9qPqt*NJ}5t{=TGZ?^GaL$R~)4uHMXiRm6;%OP_S;AmG(TAkIT#4GwxY zb;hoVOUx?H_!#WkbK}o+#+xDWDt9*a$ArKBBDB$KqrAZ7d6#VGS}A^2i}SPJep#R; zLvpLc`*#U8wl(&47fQn4yv#hY!L#mPf`Oml6ye7y54G=lIjov9;ZHa_qfW>~12rRF zr?!LzAvz&~EpDst?tZm9x_r9kwM*;E&t47nE>&xLd+?K9`iqMBt{3F2s^))T*u(l_ zqIAm4FDG_$g?{oD<_-P)*Ob4pm|xkwPJ2IV-{DDn|L#zK7teimCTpDt^A+c^?LEJu za*mumm~SYeRmjvZPvj+o_7eXJzZcd^>Vs8ho!=(!&ig*$(RW`4-tf&QRT)d})vamk znrp48`#r8}OQ@KiaENa4x*u!OPvs~|`94p1wkfDuc-fDR%=MYg|0gVxyZ!yYaB2B( zYk7k!JoWZ(PaHh#^Z9D(xb*tzjXR9dH&R_z?#^vR@tlS_lHL5*niA>wPNYg zrS~%Gm^c2Iyk~K(dTFRY=LsPx3C1;3W=UmjKk@$Sz2@t7aW2=NXityb6EJ0STLI%$ zS5?sju{|dO9L%4IU(v`5+0k-yrR+z=UsAWU%vMWZ4N|yY*sf*RZ!-IF?qSIa;Wih; zrdJE?ZT#v|gCCc6{|L%F%J=BSbgpbC;q8S2oO%{QhIgiTrWtSjYb2idr^@;2&-TT) z^uKxf7Vy10c%b@t6Z@8^=X(@#W*%VFxiUpVl#A2pV8enL25KU#tc(s5LR5q~1)UTF z4g~0kaLinn`u^@Z>$N}c#XkKi()GNb%ilC}!PDb+XUSKU?f-w-_RHIHg%6uGR@}4o zme}aDw>2yER_~LoQr?g5^dGsX#-*R`Ao`^1;abb%ds%!<^JbqfE^IrU7sTssB(5zk z@mldgooIEw>XdykvhzN)osZyo{POL;d-5i2C)F-9tg+5=C|Vf3`;ye{*I~7{LS18y zW^r846XImBxWD*w_*>ntFVYEf`^8Mpd^?+1vvtqu%u*AJww~wKZrjV^@AEtVVcsKt z_1KYn^Pf&jh*JLi_5DV5{%3U(cjbNm-?$J}8un;fg6fgPSOrHNGk!(cV}BxUFIiRg zkpKI`huY%o**oUVdg7lm=RmQaU2rv}CeEW^BFMMmE+lgDNV;<{o+T8U0*QK)e za_qfRR!%<6P{61XGU0-XbZbXr!h#45BQB0Z3msNW$PzR@de!q+>2;m6Ue`RbUgcIV z$yzeaqFwE(%;f3+?!~7sJ@fzj?^@Y!u33NQ`&9qW*Qt#!IGgm^H~zZG2m6HUGMkhB zIlq+>2;cm^!E)OQv*r_*ioWIr-u+oW!)My&8zsvNTkn57oVq$~-9$a@a51$%%k&RD z&{(^&_TDNH@41^#t+J{V&9d2lA}&8c@5~7XYngkYr&B9@b8w~<&&D6*H0*U7xGe7UvEZ|CSS^2UlYtj1P z%c5D2L)ovMR4wA)E*rIS_v&qPviIy4YvXiM&g!nHzoo0tz#wSGbb!%3;cM+*h6fA` ztf`K(Ecm!s%l4Un(+&T$zW#sxzrFYWfARWm`v2d5fxVyCmS27S>-%f@t?#FAegFUc zzw6Jl-(Npj|8$c3TGiyQQ#UKD`&9K`y~FDzrJavj0zd|DE;vl(@i^E`g`kMt|M->FVR1!TYZEO^ko?(lGkPj`_1AZILQroTngukV?)_5H-yyO%>x9{XOm zH}(Hl-H&!>_I}x0e*V_ek0$&7sLv5#{k@89Z^G`^6T*)aMeY5l_4mHc_47gTUPq>> zCms(LUD@|z*6v-qXUF}m-hX|6#H#n#cK`MFeD(H3CzJQPuv@RVO=NOzR?ghh@<(rwd(J+rB7F{p0Tg<8<%=pA){SJ za^TAL2VFJy{otUqAK!w@ zKkL38G(4LAzT~uMoAPYyKcN@6lFb%s9A!T`VgApWe0h_3{y%dHuSo5?oBm_X|LxY= zlb=lTUOoTGtM#tkUsvuCDPdOYHA?Jdto@;*GN)idYGakA{i17z{zVfu&ba?kuHz7s z8CSM@+LKBD_9dBJzP`FH=-q$$yn8Bn-YS)5_OZRI|9rJFs{hp0-@*U%*B8H~uXe4w znDjAnZ_%&tQf<%IH*M{oeq17Fp;F?Nc2H;UoBy+HDnkV1WG5Z5^7?owIXtZVo9vnt zk5``!v)9~Fuyv|8?qZ)he{sPx;i3}XN`uwMjvfDFzWADlR%x^Iuix%ndjuxME*Fi| zda~u)#=5xpy;a{$mrj4SbbZ)6pRez)tc+V{SES3Nd&cy?c>11lyFETNH+LwA&l7RU zpS*iN=NY>quE;;${|)Av|4=e!ntc0H(TYF2rLW#snYY*W^n-5xBkDQ(%RS}x-MA!W zF85=WG~+5smz)**C$uKZ3rK&}yuoLF?|lCGO=~)Tf1NgG(%;m&POCk~ZG0G*{|Mdv zqx{t9!^!vYQyg#1oPS15F!MoetB1L&UC#FM2l@5aYE zY;3;kH#@y^-8$Xsr{IS_dY=2Ge9tF`+n;>@y6Ms;NtxX}Q*YGV%l~M)HvEr@^Fkvo zW`ndFKa}jAUwGnJ|DkU}skq`x_nX#V)z|+~&Ykk0Vd2JeNh?Bn8|C}k#Df_#WEPs- zkPu>jIenx1p;^qUJ#HL+mAroL)8ut&+TlU9F%#CrPqOCt@J97Mhlhe1x0%Q!*E)lo zpDJqKPQP>7S$DPm?)`V(cdySr&EN5Mi|P7@Qj#mon(y9!ef_u4_ggzX?)_I(3Aq0K z@qxy*PCwjIHs+l#`EtEe?{-e^+*$vEM7yt=D|0Wd??1Ktx8dh!-A}$={dMqAQN>1? zol)_EzHh7}>mRIH*lK({QQEn9j(@@>;i$*j2NjJvILq}9O3eIx`m5${bxZ)}o95xk^4o30;33bZ56vmyZAMutIyI>#ytn z$Itz?{??~&8><$Dzm8v9xAoWctgo+Q*9G~lzqR+))u`qCE2r6Qs9XPQ|Nqt3S4-Ob z{~vwQZ~OX~eQ_(Lzh1gpvgvKQ*0udnOS7teFVB88#pwMtf4whzzc^j9%nTB}zj|`j zm4EBSUrdN^N}e#=fx*&viFobmiAI{<1B9t7aVkU%zg@Q`TRh z{co$@e>L(idbNhz<<9G=wy}@JMEzHNnA)CJUvcVf+}^6UMH}y~68>9zYj4%zm8pV9 zel3r8z5b&AhT^$1Z%%&*-|O<~adRX8^P~g+pRN?zXwRK>_1wGXUy~f~9t-*+5V&OC z|EIf^{=c_Vs0)~*TEM0ue^pBT{JPI=Z1o>}HQA*clX~~ZEsW24JpJ675dWg6jWMS_ zO0NxHA3rThL0k37)D~-*d9VNUUzsCu*s1B){wIgG>)L%Sf5Rdhb#|rnuBY#JT3$W5 zH!JD?t6ism?bGfP$Su6rD4_YHqVd_j#!I}Gl5N*o6BpcY>InC1{OTwh(`EmxZ<*QB z(&yIoEj{u_e`FQ6NbUMt@MYHYYx)O8L)$g>TZ)=3O}<`qXJ5B?{hH7RlcwFCRHa(S zWwYNS;dbaF&6iJ8zdpGaA74LV)4#8WFT43qbV*d!oT=sdeD#f)Pd>8r&95=uqt}1( zN&i7{jYHK_g&)7R48Obiu&S{B>g(%6Hdk!&n`C0Q&`fh*@6Hb`57s0wnbgSHg)>@4>RSAT8pYZwrMOW#^WVf^XHl3`x{?K~6^XXFxb)07J9-c108?&!}`iq{4=9A@F z)WTcz0y;Mw-C;6cYNF~3;cMcCijR+3{V|zd{zKVb-pu9D-|X{-vw3Gc@>M)sqd!e5 z%vbqK{67Uj9{c{wH-9WNIx5`4a^dSU%d`D9K5YML#Pf_Y|Mv?X?>_wh@y@iL`CrO5 z{o9wlb$|HZs_83~l#V&vZ~l~K*c&MR{`=9d54w*{7N|RSZ%cgfd1>bPuWG0NUnJP8 z{^#ZqmVJ{xte@)W%=lD3+HiCHKW!tX89SY&6|Tr1lH~Q1;EcF7k$`#Blom z3f+ag=YG2hWawB{{%4&ny(+c6*rSu{)^**zEW!4wxBtcbk1JlkJ^Y7?+L1`B=_@{& zHmhrK?KC^X=jmyt8i|j$uFHM@!SkXVkJaw?)vLbj>-zfY@uQwiObo~5=N;5{ zy>fl>>3Bo$>DT=nrI+}-SK7wCpW5M2B(eGwn+ds#~9?`cCon=@;IHw?CDy?p*zT-}|dizhwQ{viMO6PS$Vs|9!x~z&6)$mL;DTYxyN%?Ok7AeO-5L$=#y)*R{S{ z*91NNwR`>l>$}vqzWSQHd)KaI)35E0p1W?{)g@c&uEh3zdzBY;m2+*0`q!k+g?D?( z?>f)){u+MK{Kc$aaepK1{C>at%3^z;FaDbTwbIHI7QTwFQLk1Bt$s11`)^hDq^w#m z&&Y*eU8nEb>fxlnd!1Tz*m2Ld7j2_9>aUMq8~VNcZq?QJ9ZE_?^|5zPXKZ>t@9EYR z-EWir%$0a~ShGB6{pnTz!=L}Hk6x?(>U+(8;k&=??2ns&jce=mS9kV)dSC3GY_hZB zl|aE6oBOXd*T!XCTlCI9Zau5SiH4~{x3VKArQT58ap3d=zW3{E@Bc{WkU9P9=ic=P z&CJd{Q%js|?j^lCHgxsetE;D<+Vr++`TD5NU3JSg={2fn6s1p=4rdi|c4UdrsDqW!Z4 zi$Wg>BY{>X;iEN4D-QHx?5Ls?>kb-QRh*P5E+{ZRbm>j&}GQI(#FXbJ|)*m4h3$3%H(XniM--+(z^M=3-~{=RXv$`JDFp z>$0jH9Y6SoMxnRZ3>02+Hr>_3WTp{C>@yPxf$jpc z>-(kuCt6AW-gi*3an_85p7ZNkziTvyGv8mh<^Q~$Y@FG@Rnm-qv&>eK_$&DO>g!8) zR_5Qk%eU+>Q?Kzp-tI&9?Jj{>P)EGScS_!|%x!u9EB< zo%}n#^ID%huJyssp_}#1vr^#|J5;Y8v5(p{eZx9|X(FAao0!zkaGQMF8FWVdr}yOJ z2g9HC-8``~JAUCTb-ts=7T=qA@`iGod*u7D1@nuAG@AKl-0piYWz`Y(`iC8}ZRa+< zb8~ap{CI`z55;wRey=o$aFJY`P;lxO+u8lH&vUCCb_w3}N#Fh9LzYj7-K`%kbGDrQ z2qHDt!CKC!x!kAJVv``~T7X2z)_r~lhHC$w)i{4TF~@(XiZ@1-Q8DqYXD+iUl7 zsq8#A>C~!#_LSv3oW7P-+pd26oPH>DOH^lF%J0p=?_z|)JvUX?yPmvN929&!n>|ft z-O~5<-{d|Q==bb8=9+ZbL}})oo8QmujceJk@ye0o=d?2Ki25aUCr%Oz-}rt~;7TSL z-j(7!dbA-7s` z_gAsW*Y%&h+xs(o_w855*T1-!74G!t^sC+9s-9f?=f6K(u|q=Ag{7qP?i0g#PwT_I zBfme`Y(D$ul;i!uyWLU1Fwx3SVR@e(d_Y>$_dO?EHHlR)2AqckSBi#kphl2hZ}Sb&mdf z7#Nrh9cNkgWwDmmZ`Au+bNYJut0#Mc_RL!!^rQO!|L_0*#(%&6>iyQDb&lM1%h+Fs zZVg)-?%#Xm`s>)${A-N$|3CT8VjEkus51PI^jW64w{(IFO*fjpnDt|7QoOod-PW)_ zNyd8dSO1qL-}yR8MX~w6%=){(y7EeKCEtK#9P)=#|T^C(hik z`^u^-?*Atr|M&50(%)sUKijRAzeoenKykmED%fGW-SHgwwzh-~U{;F(x>HE?y z|HvAzJ#+kz++8QK|C>^N#OluQw=ui-hpX(KT6K5*Q!;)`!G2Z`Z1emy_bZV|(W}%il>MVMc|f#dq&!x{LpxwKrkE=*?Z8wz|I#|F62x zwPU?&$-Js#GnfBAKE027W zvAV3|slOsmRfRu;jvM*BK74&=jc~=8r&1I9)}QrNVf>osPv<-C3WwYci}jvr|{|0n&JaWnOc-{cE(!zbLdnHTr|>#Df@YqLuKChhpC zzx%HE(w9Y(Px+x!&eeZJeL;=-d}hKrp)$NrzKHU0XWF8`GMTl80-N;mrT$(wJh zzW(m%DU;QETZU27uws(L2)>$roePPwgdOm)irx(QbwS}yne9rZ^#?OuWU-RtT zoAqaV{hE0j!kg@lmCX8M-q$=m=-&SQ*LC*)TJ*K*`Xf*4x{N-L0975#d8dU>VIlX&O0 zcky2f{KLX$+*h4 z>(3jfI!Jvu`@Z~m>+|v#le%|(aEe~fA6kDl^y=#A#lN(Jrq91qGx(x|e*Uk&DEmv?>Ts?1_sjkMUku)QR!uuRpi5ByqQ57j!OVX>Rt7UQ zol>>dEnOGERrb4reQT>@L)pa4RqkckN5uI>c(azN$9IQcS#)p09^+4Mjm?huZP8og z7b|>iQ|q?B)6&aVU3w8ebI;DKG?!;;$_9tB)w?V_&Q6$A#?Cxb(n|dCyn^D#et$L0 zx^ki&wPt7Tn$T3PQ_!O+zHZA>-gQ2ggE%)FNaYY{I=N)hm9FqzlQ>Tpya-5JzCyx5 z!_8~0bxXyWug`*RnA$a|EM2g=E%S$j<-E4J4XQi%C%8+lXbEwu+AZOKPE_>$t*^1M zn+0A!WeKvpbaU!7vllNGF3l||i_f%)w4H0cqp(+Ee)$7~D-Vv86`F3C!)SD+M@x*0 zv!gNbKtzU^m59?qL(ZFbuHE@l^=kXxIPX(i6qh|MIazw(&+*BV|I~Z`p6q+;pWW^H zl~>Bk3nv$*|NPt{*Y@&Pwf#)4;J340TNLWA`lqqjhghaGMo%4#zWW)~~I`qq|Gk$XBT=vLEv!;iJa~V7gTfXm1 zV+NgQzn8>TVKKh$1%qW5XVxkZyYEpGIA-H~B5 z5Sig2Cf&x=m~dc*ffx_#gw{X>9}Pi47LRA2`^w`gXFXF>nOU?{ZPnBzYK^ab>#voV zWFM|6SNVDWckP+H^pz59XMTIew{2Z+)`#ef4*h_u^+1<0AxK6$ISXz9PyVdH?9IU4p4U_MYp!@TFQrRO0d8DHf?$ zu3DFdEix7rf2zH4s&?tgiHatB?riv8UEim=+QR)iZ_qVai|dV^2`{rxW-k)C2y{2yK4hu#Jkr^6dT&=Cl4m%ko}cSAZKj}cu$GSg!m#`Qs@S)l_`WF4KkRm#ZqfbSJAdu|k(Kk&??s=P z*w(L`F+a9S{G_vZi44BZvmxlgrc-fxP3(%P`MSYU=<&La7m z{eH|-PTudmZe`nNYqB@=exGZ}S*17noGKdp*x_a!q-E=I*4qlLh{D+}?j7p;hOe)c!#6|C#%z-u;zOpBd}Z z&M-OmqExortNm;mQ>AAyd}3u!K6UZ#+qGHMJ;uz}7cKT&;lgYbu;q{b`ehgHEZzHh z_C%)!!^Sx}roy)?WV;_9`d_&4u6=x!o$$#QXPyVXd_5zbQDE`c=@kbYzNYPL_1}H0R3M#y z=U<`h7TILeT#vf{dRv}dDV=_o>F(cCj{18mFa3=3eEZvP&4exS^?x_<1mrL<*WOb& za{kC4{c;h%*Gn&6%CKLrze8HV`})?4r537gIT7u{B(iJ!^xE~Ump}g6`t17W z{{M4+Z>ZcK>ApdYt31BqR)f-q!q-n#(oQU^*j%6+2$A7MX!>*^ls ze|qCLQ?7OTk^_5vPl|qyulE(P{(hqJR$SY&8wx>%mwvgFc~*+I^cBzf`z@Z?@2^w-aqGk0mU^xgOFr<0WA_i$;N>so-hX{Xv3lL~{SmwWR&RZE`ft_Nc=`23 z7wz}Idr|ajd)!g|*VmhG{r;L$7I*T;)b#8Re@j2z7yh-$zxK}bTc0*@g~zS!`gpo- z&Ak7wt{?fc>3&G)_g}BSo<8Zi`ir~u>i1t?i|k)Le|`PiSly~$*=w)vk9!*%=cLo~ zYjx|3r@KQIc>n17nps=)rE1^rs9AOK*Oz?#=3jp;?rPW8RdcK2!|V1uXCp!`5YOt@_%%Xn%ytPTiz;f0N_ePWtj*tGNDe)AmY5bgt{;tm zeO1SIwfBR`yN>P;&gZ>$edVRS;^9x!tY(~keZAEF?DZq-Yc~D*@{d#b5%U54I<-qa zybSOCAM}OUhtJsaGwR`4zK~6YkHc*D$L#<0&*zo(^}?OuG57YGRb78Avp#0^LDk!L zrXQUsd_m@5+tOVhBQD+AWn^{%n_qXQy9caki{irQwK%1QRS}>haOkEuc=CD&W_XWlPPW1 z|EDr(#c63tXM4W%hf8K`Ww|A{`i1=sWtEDwU(44e^>lCx8EW}`{2085bIxaWDZ}}* zdu7V@@A1=)FY(*@diwu>hsh1ATh?${F8?p+*Bn!D{A-u%tEluZcDw#;F{;~Dus?kM zy(MYe{(n)mx_RRGSK0lagk?JR3AcpBIp0ta>fpJ*JD_Ll!TTqFA3wx&zsWLUf1r5P z;;&U-W4$MTssHH2F?qT1ulIkBZl2dNfA6kY@87Kv5}W%ueD$X4`d1$UeDA$~cXfYc zWc~X60^xPG!yxy?5hpEOO%Q zJ1*sO>aJ!KOX2!uwKrl1YbP;+Wqg|s^%?hS(3v0K9A{bef%l*M57(ak@9K^7zrJRF&3c&^zb5>(XkhKXb^m9@{j`rQ zzyEht;niQCl?t_HR=)0>EgiM1w8?f$0OW6M4t6490McPJfjakR9X7t_VSNz{vAKmZM?*|1|dCw`Ip}p~vecj%2 zQ>pd8Ht+lW{)2~8rR4SOm4C%{AJy02wL$f+_^<7+FBP9Xdu#63N9O&KW&iaPj)nbG zfAY3^$&d3~DU0mGR==FSerf*oB|G!4me%c>eB{}lr&&p>cw1Nhw2%88x6V_0b=>Ox zF-v#q?b(;F|N6n^Nj=4fEKA>9U3mSyeDD8)BMS3gALRSA!eIL=b%UNv!(uU}__)35 zYirl-42#+H>uRC){q?E;fAn(~yH!Nlh~HRv({`(F_AcGnAi_i-gF{|J3|oo@C1OVyv*_aC(M%9(dfztea9OVsIKKa@}0-(TBgop~v0vijG( zRjSo%yS7&D71REb#xXDZ|HE^Vt`qJ^IGot{`stUDrai~(|2y62_~JhKn%39#*8=4> z)>R$uD*pdenV-kAze#fc>Cee(_NPK0vtOv!FaOzn_Ii^MZ%+ygZ;yUny2!WXkMY7& zr_LPSV_+Xwys^sj_eUnZfb&Kj2m7a8uKGMRc-#JOUP_16n~WY`Gpk$kH#JDOJZxj# zUi~T84H($#iZ*zs+83t3-@Wg}{$JO3OJv8bSs(X)>pbuEQB(Hc z`ud`Wvwq$ClaDuby^UG4`_0$Cn_hpqbU)?Q`Pb}E?mc_-_5Gb`PEX(caf>Q`HhLV>PrkSsHM@G+?@7@&X6ALi{vR&*VypRt+nX8~7}zWw zXIb-|Vy$18d%{1wc5V6g|L+C=ZM_>c=|TJd|FQLxKmY$9dieF;`1`Nd@1MN%{mY2j z?0s8Tum4(UoBj3lSEKONcUmgk*TxDxnc*Jc`gnirx9*^_^?g5={(ADwb!yckt?6H{ z?As6~IO$DC_Cv{6OFlN&q&x_jfA#*f-SJ^sS6^S>ANRLGY+Q>QU_M7aqUlaOzr7cg@ z7uDASQD34ay_cGL(4hZD|KzQwcmA#Z+WmF)*6S?w6J|b`$obv;j_)<`e>+1q#`wg2 z_k1W=^jdY%O#N$%Y=fCUrYu*A`K#Zt;>RI}KaI>$U)Qr(m)(slt^G4~dS&USzn`wJ zuiL9WJ7cfj$`9^!72(hJ{uh-$^4aLV{<>FA=N$JJxbJ=X+xXs|IcaOAUh#FDTXg-Z z$KJS2f3tS}yS}sY-LBLBR~EkR;j0rr<5D4ImD2Zw?f8~?3nE;OE~?r8y7&l`ccXagUEhhm+p+6z)sDWp`p$LNbc@1| zl!PA*Z*o#@`zxIK^*yU#anP2FJWR{;_n1UF@g&Gsf4F0#SLc1PbN+EzTiY2kO7c#~ zpWJKxTR68_v}((-J*KOC0)xfB&ztZ1F8(Hau>4OWm39>;hn6V4prY%mubzB$|DNxU z!@>)fz4h&VB^{(0?9I5l^UpVfQ~QrOrHL-)&|3O_`g)l+Cwm(k7TDSQZCPzZmGuG z^9ey1E@u-5im;RrWInMn#RX|{~ z+K-vfCwG2Z8vE+kmezX@nm48@{+%vJz%P;cL|zxvP4of|LgU!{L- z9@od6(e}sMVloYvzRCZYc+~j(>xV7r*N--FSpP76dHTJ3RczAl>DP}<&K0!3_-o#d zbtyh=E^p<3JaYE)-2N}xMg7BO|NKA4o75-&`Okg$L*w?!eXl<*aoblsr_a7^^2$5P z;W7Jlt8Bye?X3S1KQ;fiYWUu=3H{uEtP_$Hl1yFpt#xLOyP@=O$5yGwJa!zdTqbUzS)>(_eH#NGA*r`tn?szkEZvhxYFo%ZHzmfDtJ%k2+IBM zVo@&7ve(w+rfnaWsX{iI3-_r)w-cjD}quk}$=!W6&nHe2O; z@#(#(*Ldx8?u3O&|6aW>a_i~uuf!IsJw3P2#4cp&l>eQ#w@)`+r}aH){`Z%1`(B82 z3D*B+J$jm7z5dYMvlqT?Z2qP2_3){ln>N>89!)t{X4~Dr;MdYyyHEFj4Xe6;Z&UR7 z=TGNf-tSaB=bql)e8c+dX?Op<&)X^x{GYGj@4we2QGce!znHW((BO|m;i1>%*KeD? zK51hYdTML5XZ=DwmR~!pUU%&J;BzgMrX-stuI z`l_XGWq(BNb>12FD{k^XudOTfHq868`^Nhi<0W6e+Pi-Z6W`Twfr~l1`GNX>?V=~& zzy6Nh?^1f-cJk-{U*E>x6PiAC|L^d>)%&h5+4_2?ZdUZ4Wsg0~e_2~kda{4%D^A_D z^?hmuPxfzJucj2oT3UYTtlA>igr#v>*75;v(f4;|yRCj#wQSdxsEyv`xAuNCfBGW- z!*u;Rf$d+vuB?rU4&NQsvGvNn^{Q)jN37WY`=7vRcdL(ql_FuF1F52R)@|*I{IT`L z(pUP2*7?tkYTB>goAh4#&iBVJ6lNY=B_{3DKU>o0nV)WAD(jV3=ii-QF>lpB?oE1D zGXuT1?)zG3De4w1nqj?l`s@3v^{y{_qhAmf`1LB8>EUfU6&DX0 z|8g^PliiV|o!w$8#ZtNQ`@7%L_V-L?zOE{_-!!do`=sW>x)~k3su_>m^MB1-+|WAz z_n+F?<#*r4C#kRInlH(|zHagEMaLMK?q1-OkNh6>@xS=r=(}0#y8nN=TanMQp21|z zqF1H=z7`x~u9u6DRl2upj#R~eos+KLxK2pkUHS3J9)X{0gzWRrA6)h3#QVhsYUVd? zIx}9o7%jKL<6-)C|2N84KVC~{F0OR{XGwamd5-R|;En8@S^56!x~IFouC&=B zEEo5Pb&8l%nCk1l%fdfCaLLzju>QyCoh=u_!L{qV>^TXMFA=J{4*Y!{z$F)^^zMU) zN8!;udk^M)fzxXs;B+8+uuGTcxi3+q78c5r^CZ_1iw%Jf41DE=AM83 zhnedZit{y`vHy5=TcVlGnl{6^Do)3Xrm{2DKfb^J?7yT;TU)-H*4dw0k1x0Xnta{A zOSdmgzh-|^vCY3bQeR$m{(jxBWBt}sXlDukEcto+ZkO&3@=~oo7<+ry#k+zB81xsH zt@Ermt$G@+=Vf16p3(b5>CC|m`|oe^zWg@!dtjCN{#QZI|13FOyJSbfrfsIK|M(x? z3Ea3?R%T(NQPdx?_eC35ym@di_rW(ZS~hL`@CN22F84kd(&2ay=~0|rP?~1UHTo%cX6hs zR;+z{=k)*T-6He$hR%C`tmMDzh9Cac8vF9~zrK2Y%S^WA@iO_jEfxvpC;uK4mz_NQ zp=1Ett&fd`JUj+hbx%m}A5A{MaiZ`f`}%{dZ4!}{vVu}Bd*-ss+32iafBN;Sw`c!x zEcP&0IT%-{{p06j{YB+gUylB*-{Ni`zCO&d>}QR_qh|fp|6e_tdN@jT{p}!~J*`t+ z?OQkNrHRauuD^bo-C@)J_{q=z?U{c45ogK0r#e-!IU85F7Nu5b&*Plhv*y70WmQI( zf_@z}UwkyK^HR0otkBJV^+StEd;zFu^$OW9P#wU9e|@nt_ItRXX6No{&%TuJo*w%8>#0qa@7{k?`XtrB zs3$U^M}(WLnK4-*B2Pq_mF=M8h5#2qPL3n1C+&Ry?&O_`#k2L6rW{!=%3EMp{x|!p z{@EAv9qq#&{alc0w(mt%cf<#Mzq)j@vi<>!#RuH#DVJA@-(<`6>dr1Kk)hS)Ax%Gw-(G< z9CK-HKvd7oq$_v3;#8xK$8>s!>Ly%V``*fCa(%Egc)+(uLrjRX*)i$B3@cz=1C_DZism!E3qcWhJJaG7=k9->^V%}fb1K*#BFwlXFvMyQE$@ir|KnBk%1_Ik~=>c7P{QQ1@0 zZ8SY@J^j@BfOv+gKgz%IFI`%>q*rgX`Ca`F8;ufG3k|E(}#o=|;x;$H=G?yE{nNvLs&mQJl{0Y&psw)`}_a%zTDsKGDli!NmE#I)0DtZOAl>XFLh|?_KVJ^mnA%4 z)DoGZqarQEa;R}Zh>nO*E7Kx}2?1In60J@ul%n z?T&`u&)@t{h1ErOT;25O+V$n_t}%KX6Q0iL+vFp2?%me?=c4*IPl}LZ-|xTY#EE4W z9&$&CbsK)4XX++byCqyHC^?gz(I8~PJ24^FR>s7D90L(sX>!+`;t-X9;`O{NzXRX#n zJId~^Z+g-VkZy1;12P*zJ|4H-TOU0mx?{&U3Z$f zY2281BuysA`BC`2+P|#P$)~5z(z?v$e(CzzhdEid#H(IKe{;FIJ6hwVi29D_ZLMeL zxi_uiteW$v&~I9#-t;LhL4Gc#pvk0}|3o!(?g|FoH$OIY>ahl$R>fNjD*C4|?lk=L zy=<28{#B<=uyP(XyJoVbmL*TipI0nXb)U)e83jJo*L9d3ibEnJjrXj-B&(r&@WUlO zrO*9}K|a4HMvEHQMWvNl_iuBmI#+bh$5@K*^tvGL8pr7!jAt}Xcw9T6mn9gbA{gU* z;>;}vFCUK!0%!9!G4?K&o>pusT%9@R&SBAXi~MOCEDSi{D`6|S$ z?VQmLKL!St6vtUM{9LTH3-#+Z)~yR&w?FRPy7-7CTi;*(9n`%4+WsK(*Wy{;Ti?Cz z|MuVYJ!iODe7NeW&Y%B(sr$aZ{wieo($!__%-H93E1V(}_3yOo zQqA{YHJ*6d*WWYC|F61j`>W@PJI}=5`K58lfAgn*<=b}bST%L-+S2RmReynHh8cW+C#@v_}ox7r*{*AnQ>s9pM?7ZvQ@vHT(o_~FHhWFcl zOTH%U%~HO)^y~gNQ+M=-Pkuj5>5;8y@2`0~b&l27uMMpGugE5BQxyL?*qiIu%~O8w zs%O_Z+f98`E4FGye{7j^;o}qcT~Doxo_{4g^?N*$ZuWY?{qg?Eym*Tn}{Bh6d z@qDrO*VbQ8>w2C)TIP36rj2_&%jG50-^^un)Ss{^IPUD}e}&nHJ)hKHoUz~J*d@0B z2YcJPBQh=qR~C0Z{;ztyJ)<;d=dQ<0g#|};i|GH@&Frg^($>?c`tUX2BPya@3UG)E+VEfea4td1zw){`&e4=dMP6oK^oizKxUT`>e;k#V@s0j!$r~kINGC`Qg7Ut?=vx>u^zb z>7MjMG3?9gI7_-eF0nrQH05{n|5K_N>c5o#?ftd??45mE)tiL#z6M=yJ@q1&r&lC1 zCf09R!EgPa1v5A9JISNTdsg;eMC{bRPycUalfE5MsP;O(kH0yg;nF(kZL{aL{t7KO zeWYD&Yk$IX)xQf@9)9rf<&E1TG^26mA*Sw`9j321R$jjGXKBfe$l7}0%80o$XZ#Tk zO1AQS_g;N%Rg=-ZV40oS*SPnG)?ZtFa`%ulWj{qLT> zJM@0dPMdx6H(KBA^M6Pm*C{_}o!YtQDpEyh3ptjP^} zEBd^D_xs&Z{iXFyMVWj4Cw{(uSZ{H{;k&nl*Sc9lfmQ1SN%hOudiEM@ay}lyMhM}|N7>y z|9btuU(@&Pj+f1Tzi;X5pWCLdpSEsO=#C5bWxwt3@DuYV*cNe|Y*O!azi%>eXuzhzoXT1Kx`|0aXEVS6ikz>B-)t#7!41pQK43n(X z*RTKkYx}F`*n9Y=*QdB%Qn5=vHNfI*LBZ!Km8pqo>emW8mDdc*QCAKU)MeV=EfFk#3TD} z>#wh=_v=erN>3jCZ~LG{{qI}-x~+BVMf~Ug+S>kWZ{_PRoxb|7w_Yvw&e(MQzv!2( z>VhBTP3Cn@nbsY@$o78P8_Bo7zQ2w?nf3ACrZ*~)9{cCTH_4T$tUmPjR?*+U&|3BX zmZFkzAO1hl`TVbHHb4LUNiV*>=zI5Z_n|tDw?7QIPnHUo$(-YJu(~QFm>&P<>n9Bb z!`_2ipS+s6_j_@CQStj<`>wC%Z^`<-N7=Q@pJ#&Alqamr*N^Fyy*}sq?U;u-_vDMI z=BsyCUs70CA2xa|ZsYDFNxHIqPrhZc znVo#S{$dR0{u4eqF85E`nz3!lH~Y9K@AZkwm;0Tx)xzWn6Mr`sjiAe{6cWZC^a+5A_f8{{$`l+O=+N(T`tS-?@hWNR7Pz zTJ&?>roUM}Q@gJHuU~C%o2>F}f8f(oQni(DeEKH;J0G92yl2vnsp~$v+?(|N{F~JH z?7LU5t-cyEYjxe)xK5FIzume&Uq8#duhTy~H1?;)rdj{vj_Ul|TdEztzHHNaw}QL> zj9SE4;U_n_jqYZ^WK?@%GC5AJr~>sUG*Y$~$5w=Q_0=J5EJ(f3Cc~J7WEn zl{J6w>xb70?Y$^_e|6aEsTIk-Qs$RJ0&h=PoZc3)`ijfZyNq{IwCmSKe|=lE*FXHv z>Fc+?M}2>N_131pSud|PZ+(Aleem^mLEHM`T+5!ixV8cY{*y%=l`b=%@)t7$PZuz@t!tzS66|rv7BfrXLlTHrBX*sNxec`?G6e%YKu7{RguS z7%cJs7VOU^ux#$nWr9`_CtF{9_5Y|Meez|w>)v12*LuF#x<6>roX)zK_+@(WdhwHl z|7IzjacE{$I`Z^J?DRt)U)7$`UdiLuKL3BJ#UTNhr>D>od!8& z-Jkr|iLL33%haF*{vuwU37^yAuNTWnPWu1R;z{@m*)9L7{%J|E_|9AZ^+i+FzisiK z3RxCUmc5mt*fC+r5Et3vUt?=G( zyj?j_n4SHPb+&U+8!LP8vE#0yrZXJeUEBY6y<~e7X*5m2=tJzGIeE%)3b#tkZd)DR zID4iaN8hZS>8q+9t`bS)jyU@M^(j@Ig2cU+e{Keaft8m<0tIjeGjqhR-frJ^Nso* zllS8LULQ0y+hFYLmAp4}@@&g$megY>?2EsfFFJZ_u4?QvNfuxJ*E){7KFiwv%f0r# zZfouL$+!RQxijVO)AO&6uG_yQZolE&`MWO6zxZmSWlft*ZjJT#Nj#$uZc}dq0|V1l$62=hS**2x6?5wL-nxEkqJ8-8 zw^@I)*2Ra{tqc9Pt!`)BYW=IP{;7J*dp-UC>Z{_P|LqLduhqITZ|kd_o)0EZmycar zwzg0X-3`BWwdmivy1n|XWh<{u z+plu}hu*uZ7w-hGP}w$LJbiin@h?B-URrJEulmpX0S3erXY+D@Kqo0YGh5G z{xV8P|Nr%s!C$XuovjwWdi{Ldx}Vuz^>O=y)K_gui+c5b>&ka~qnv8~Utir<8|z({ zB&+;C=A{0k|51Ol*0)&nJxRZNU0l5O{^~E^Pae^IER;qIs3{<%&+@AB>0ldi9;Hn_wYBzApp-kN*R_UGsNCL!Ce zUy6>`eAu)7^xeDpS082Hi}Afa#r5@PvE#`)fV4*UXRpy4t_V$-Pfbv&O7+Ug^))<45m#>)kmS z-(xCrL1OabV`fWtZ{YKv|NHhN;prED{fRBv9JsXhep`;eHCGt($$GD~8SQ^x{Xd=g z^|k)dcmKk8A3vUGZsM%Sy1rAEkFg-LY5qkCecL*l8UMrVor>1De@T0PSZn2S);vo=4Zz66H}*N?_Rvv>s!6;f8K~&Cts}d>{zq(gHzJ~xB1uC zh5pym{OI9T8@hXb=&G-WqGOZ{rH-UFFltGnu3anlX+4YEx%%LJQ{yY zi#mT~`H$(l_Qrib)Espwre>vH`l&>{Lk)3HzuNU#)tFl6FS+$f-TnMmQR!uTiyw3K z$Q!ri22NzY^3vri`wH!tiCy3FHZPd%c%u5b^tS9RDISjN+GBHSj}$*mdAqFBuuuQt zoj)bof%RvM_pB=vHrnEQae=U_HGAWt#e3iQshyrFFkrC?BH3?dtVnVdEd5bUD~Sr z$m!v#o(Hb|Kk(PSJ5cSdXZ!v|6}9){L8V$a>@5UyQe%~tbEd5KgC9R&Lxf;M}?0DM|&^$Y_n1J_C{@E z?~@;IPkQ6M?f9o9y6Fku&q?mz7XIV)9k1r!Q?BpP*N{)Vb^qk+^)BB}JZF@Za9)`3 zlJmZ`P4%TWb3%IaH)PM_x-nZ+N8yL?ZFWX2ksdD*VJ?lE@y?(3AN;ZZ)1%~F@1FdMvQzo~?|kmcL-%V; z-v6KZ>;9vPs-oCgJEht9AHU?9YoY1NC*!d1|FSK*lJ(nawj{3p)%WXM)BfheRnfMO z|98$1I>`1tK50q(?BkWoH_ixN|NhS1jWfDp%hd#5UvstI{~+I6tbTdC*4^BjNrg8n zp06+8&KJT`bS1>cWzU&MweM1oS$+(9Y!?{UFS*HP(Z1B$R{s_i_dx5&H`7%2Z&l23 zytg7|^5&NEn-wctCrxsaFS(N44X=pV)JK>*c)x zpYDLx+fP^~!Y#{maKeR%JTWegL(PE-Q=HPvqt46ReH8cHR`uOeeXUQcy;U|IvA@3R z$GL5P>yK}Lu6F9@(|VrQU(JIq?(g55!hiMuk8dv)&Mf*r@!zZku0?y5iyynyZ4eZ1 zUff~uFHbCE;o2Qnj{P+;Em<(@@Lu_yQ~qwB^Pfp1cKesCe_6qgQetDy=6v*)dp9$4 ztLW?AeKF5XE!Mf)27SCA`0L%Lo|2fw|NEoA$Uc9h`!wTEeM)ajQV(Z%8ue+t~ zZSmYM`{$ZIYt^~eHa?78Y<+GHTSoatyAQ|s7&SzCG{m@gK^r3?49rA1TN)DFP4hjwUw!Tve^NVV#Rk8ghLa?F0Cy)S2a;?8a+)eWzD8sBVLIGy85XZ7SaM|~zAkJ=g#_3u5Iu* zCH{K+-&_X?wvJsZ+r{^XSawfW{7|7De^zt>`@b*y_i-M}Qs(&Zp0;^l*16fk7`oD<4mP*f0(Yu(j1^F{x^7X7vJmu}r!*Z==#t@C$z znYUrv-c;}I>(T$0$Ndc|i`ShH^d&aFIQG}(MgODrN0#j@*S%>||Ecc(8SSG8mqS2&G*UJ<=bDhy7Rl!{Ua$au=WI*tqqt*8Po({2#7$_4L*~(^b#D)?DWw_k4Z$ zTJF`WcDP1G|BdMU`m#3Z{qyOiySJ{o8ymK^Y^&hAR{R=(QYvESp8ZcnqfX#K*qr2bsE&P{&a{ z@ACd))!yn)lT%(j{rYNi=le;k&Ux=sm0t2C-224^@8VnH61kGQIRm~vJwIV)_Wi_3 z7m7+h?%qA+`;VCqQhT;leTuvPDf{Zm>_uCvzU*tYTDNC;LPoK}wi=1*oaO&7D((K$ zRCZ?i3zzsc!MB8C|2fPwhM{Qvn^`%h)xf76v5e?H!B z`vC?9CQrv%cKxeZiwgJu4{Nz@TlM|b-K%TM!*9KQ6t-*s|NsB4{=U7x_TP4^saF3P zIfQCH6(>)>zuxuNuC+;TlP>GrH$U_4-_=XEHa|J}{(tsYzZ&IGy`ZXBd#e&-EA($~ zKmGCU-1-&WEqPw^k4+FdDBgGE;LqfQ2V1WHI{RzS^!<;rCSThh{(jfjm3rPRDSL1I z6+O1^+0M#WrS;eTudUl^`tx;|f8PGDtF{^i)_-*?@P0Y*(K^lStyN!(qE|)jzxt4S z_ui|M!%zR(U%zg#ZE4-sTUWjY*8YDRv-8=7sfQ;`zP3N=op;Rsxcw2UFKT(a*8Tpe zeQots-ON8*vo%)FzxK3BMd+92>Lu&EMX&4Ly8b$R*HNW#*1vZTmpop7#v|`quRVwG zC)&JK2dRn%oYg5Yp2W^*X zgsPs#Ir&azW0}@J_5bPcCu_sk*G$!MpK>g@`;*xByH$t9!m55gtSO7JiVaHnY4mE% zn!3G3?{}!+*?yZ-6y=vFz>$h!f@83Ogw?gIJbJof;+j~X-#;5Jc zamu^+_4k6ieerIm;yFYv&bU?*X87#wr@u0tuR4xQ39s7cAE9I`ynL7c)!SeHw3nY; z((`0{Sj$!KUq@B1-d|){KKb&WIY-=W)-0QC;%{4b`rn`HlP|xXd||qh-e1YPr(diS zdi_@LeSQ4twCdB@SL@fT*#Gs~3jgxBjp1wSHg$YaUVrh!e%I!m@%y&NSM7{nvbH>K zWBj^6bMYbv1_q`|$65CLr&vpmEX=E||6lw6|9#)PRqx7Gh3{Pdb;>(x)~nwyPOZ&; zwm)v#tC`-ldZ8I}!@9M$7VrCbvtZ?~qxN51Z@t>7eLa5l^w-m0U#(xwRMBUudtiIn ztbav1YvMmYp7z`}wq!%V%jh#=^8YrLIDJ1ErFy^i&i7XjuKCsc4YJw$G3#;dANN;R zFMX}ry*2CllhsS>Uww|P+8h7+uCUFkX**_C+$o>pUnpg=J#Jan)#O!IcYnINRQBKc znDtB7U)`5!8#C?o)!%=s@?-t`|E)>dzx4Iaye$3A6N}GW_!trs`^UBF>t+9x# zv&3Ir?^s{qthVaq-`M*v_UbP#FVEPk^Q!9oz8As&HZA&p`N*%QKa`i7#n`Ll*PSoV zH?#k`>TBh@<$Yh*pUphHlJk$>)~(?Wx@XOv^7Y#4YrFsavFkkcNI}K=^y$gh_U`_= zQZKsuOSsYhSBc_vzyGLT6nk@m!&E!g<=4?)+M%oMT~9?%x;SgE=!x$2W*qa3zkjco z^Ka{~t-nlNDl|^>Ym3Xf)*lqq^*OTJ_4@O5%lrRk?NLjv*Wr@j+q2{OzyER5{<~H* zdh4&Rdlz$k^5NohcPndlevbRL^+$(8<=huU;|8W$Y(7b0-{7K2~@6RUG{crNkxRx zZr}asdtJ-F&gp4~%RPU;Tenv9b@%$Hzj1rxx7J_N-uLutwal(}olpO5|MS&`t+W2$ z$-^gh-&?q8lGv&e-37P)z38|3TKZyr*U{>5-+EzFt^K7hT>b9;*>~~QyLIcoR@TP7 zTbVNPom16Dm+PIP)%Rw7QL8`UUhCIy{dN7d7^$*jj#2 zwrW$>{eJKf!|cOMifS4*A5Y%7VDE=(J3TAw%M}u$?zemo{w-6qewXpKV@FHHtJjAwdaAQ3 z>z#l2{_x%R*FRCtTz>z>`uw;(0uhQAkA1nIovP|^#&Ov(Q_b~H|HlYCFyW~_pLzRh z{k8aY>#qexU4QjoXV-e~r^nj7j>vOP3_2Y3yYcD&J(U(G+mmV&`Xt0T?EY)axwu|* z!}Au~{8v3S3O-fqo_OwmD9+Wjkufgf_F|7^`PcQ&&OiTM?Zxs;y}fboa$P5LtDV{~ z`NzN9lo$K!J?Dg< zx3>$v4y`@;!h6X_re(imGxpd2TeGREe)IOcHJ`Sh=ic!0^@oRx(|D%G$$G}WG2Uc< z{fqLCr+;h8PNfB$;F%+OUKK<_4F+Yjt z?8^Lidj#EHhL+W}9(DbHZBM|FC+WNFH}96d-Z|OFcxsode^~aude^FV)sqihmzT2d zdiwFN1@{+^75n~uIqTUL&iyyoZhwn!znECh=~+$k|2Vu3tx^8e@mcnM+}^NdtHoE( zKao}bbMxQ$XJ#;VYlE@St5pGtO zLk=4sn2B<6g4*yQ8m8Q=PK^c=T+{@W%~#bfIh&Mn_jSasb4F7Qj|g5Ukz31t(9YfQ zS8eD2{Ygh}Z8bBkj`Fq(Nv>-&)vi-Tu6eGM(GVZQw3|H&HpN74^WuGLE4Pw(Ft{dU{Ce|L=SdF)re-M)Ii zvd_UMPapOFu{2?4GhHJ5dfVB~jJswv0Xi9c7FBLChc7St^gMIzdp!nC#?o54r2swibj-zxuv> z_tvD8`2TimkJ-dM(9Tqq>kMdJ>%e^B{QhlOVpC5}YyP2WcrxnR z!=NN$E%3*nF)drXvzW0V^N`Ad3A6Sd@OT?2;h`RPU{0nf({Igp>)XwCKb~{=ciE%e z=AX9wr<;<-1_%Nvi3_>cj)Pt{okH^f8Y0Ke}4Zw;yb(a_5O@a zPZSq+FL?fG|J$q2X0aZ8$z6Y5{fpf6#S1?D=BSfZ34DxdA~V!fq@|b+Hf+c+6H{VsS?IVS zKu3hTrAe*g$5(U9SLx=Fl`BiSt|jGpE?O5MXTSc>x&9yWe>cCLy#BsR?xcx--mg2q z_M`3D&)@y?i_Nrj-D5v<`M;a9fO+qZ)bUE7 zNbhohh{#7&vdm_Uh*^b-#bJV%^by4-9;BU;aIMXsdwv=Xl-6DTS0!kV4^0(dgzFwL4uA6SBsOQzyufH)7SN7 zgT=#lU70jhA}}s#$%E*{OYSedT(ke#|4E1M{nP)u_o~cEhwt&ZldE@?ddBY-n;%@K zX{l(#>{u3Lf|Ja=6I;U{d(B4+4@0 zJr}F|5%>MfzjBA(^BasRPuZ?{T-LBIUBKMr`(499Gx)x8GILXQliMomr4rUn1wO}I zHZS4*{z9Qqrz2?wD_{4f86UUs&ksKP`B%`My*5JUcpVK}+I)pgM4B8|U-`MFg?++` z&mMCspDE{l+h74&m%B=ghqdWp;({3_(%dbF8WR?PPoi)-=x8uSLs03{x$19cPL`U! zzBO~zO0U_WuUzKv-~99cMb&rzU)>+7&3{k4`Q-1yb8AI@zdSzwRc+q=&xd{oForL# zj<2@qYEAjRs%zf#*ZYl*PyK)6&yBTn7V2Az-~T$*u{CGroZpF0&&B)PEDn#|dPRsK zZc27~$e(Y@oLlnWil4O7PyVhN74I2WU9|c(GnLa%ZQ2$=7=-^%hf#x7~_d+OkBo zx$@4GKMx)-FfcJW&T zUtOBK-}}|^?9d}RyTi1v{$3yUH|qV@LwojB*@tPr>)f^Kf4J(tr^gQhre_B|2OLU>+7#B#r}F2x54v~#$(>v8Mout zH~v4awR_s%h)D+>RVPOq1$V8R{59*q^tiou@7#Yl>GbK-rY$e8uZ~_{R{mDz!Mkji zcdxH5KmPCflCLjzzyBvBGZ1_{3}}j;LS4s&q;bpzvn&Oe~_>GUi{zc zBY%|FyJqHpzHxTj-dloGtM%%`f9;uG=cHD9_j_Do#fjFVzossW`@c}Y`^nwkVY|c1 zS|?7Z>)ol|WHYa8)pg18Ut9Kdi`V|j4tf{p_y6<-=Sx#hzWDVn^_`@?di`df-<$qF zGP-GaSOOtDjB2rholv(Vgvg`tR*oB>U-yvewq8Ve8cwz1mw9v(ER){*zO6 zy_d&H*(_rEbu`vqec@GIVdq7;b!&eejI25-p{7~4w|4a%ZS{3a8yFZE|2occn4kr^ zAJMo?f1G!md;~e?tUuGZqNEO`S<^q(O=86gYML?J9;)e z>(jd(b?c57#jXo~b#>Jo`PIU|zW(~Muf;1%y>(@~%V`(!f9k8($L$Z(zS{LQ!9Q;I z`)84!R^N|=o__VQ{MOOBt?K$)1xt?{z4bJ%<7w1U&|OER=la=;s;25))6Cy#vZyv@ z_3rhWA~z>rEcXqq+Tyxe@7n4!PWz|5{XKhk*O$o9>n6Ka?~YmWF=fi@s!Q+py*`;A z`r=*aw~evKqPAzumHW5;NJ;jrNmaVxQ)aFdESNv-_kOq2Tg>+-DIS@<>(izDpRXUU ztW}fxI<-nCuv~rh-K*cc-~El)RA2vZ(|Yy9dEetYHCDT?zqU7JXRXlg)k{mm<8&^5 zi$CAFzC$B-TYW;hK=x0y{U1fsQf4il;-1iEXI!~5_UJENw_hqj_cPY&uZjJ9<U`k#LH`f8@1$24B?|9iUob7}U24qv_}o;hJtb?@xk5kKSHuW9Qd_C@Vnw`Z^U zucP-k@1Au=x%XL|#2Q!5Q@g_d zR*P;izqbDB^4;GwW}FXTU|_FyoaHFs#adJ)cI#KT_I15yTVLHxn)FZo_5c4T|Ng&! zYE9YN|Je`!{o38}^<{0U>2<-+A~W)Xu3!0jtXyIFUr}wFy{o6cI&!b$|0Mr^+&MWs zKd0TgA9r!{<)39s_V1l?Z1ozuxWX@823H*4t4(RzXOi4t{Y!Q_Z~3Qy_cM;`tT@^C z(Q5MRUt3?e?Adzs{;HDG|6jO=Tr!>?ws)D%I{wss7vuI;f6cqTdQyAc`?{$TOp;=g%H+&5?LXGuNt=SyM9 zs^?Ed5zKIWi{)ok{fKYum7xo=SNpLb@`-t{TJ?;n)kUabH8nz7V*9TlUCH%_ZbwLf^8 zV0XT=yZD7dy6tgxXLi*tqY(F#zjQtQ)z#f~cE2sXf23}&-K&4Qe{oDMnowETeD|E; z+S1&)LO%Z|&Dr?v$W+HW)8<&57TsSZm{DhcBeUdF|LMOK&8$2tUO)J^V)dQUS?*__ zOgFtOd--3>9oHjOY%#At9Wi0wC3gPMln*^iT2vZl{rq{R_iam5sdKLWi3k6l+D~!U zb}IcfZQ93E6Q`@M6RNs?X64?F^+C-a)24R6`ns2EQKn&p-H+-uPk+6Z^W$45%lXRS z_!j=9>-Jp|&62c!{q>u`pFa{)1*+?hO_u+$(em5(`>Xc(ui7$2z&|l``olL{>YjE@ zRuN`jDxY%cQhj>0qp{j)o0LnrXW((B)vvOm1Nj%iN6qC&mDeoy&VZ{PjFEBa2c3UBa@D|cHe zR7$*dF|Mr)?b~#Fc3zv1?2M_4C6-)Ow%YvdU}b&n!JK20)!x@^ebIk(SKWm*|2Ll6 zv-<56%VeYZiL0yEA6|R@?&z_p+I@S|M<);sVkPhwsN=|8pE_|t$zGBwJicV zeDAh=me*ajTW)E1&pgS8O?{_#eqJImS@f9J*XjFq1&b=)XI#r~|4m@UyqEv2=Bq6K zAAQ%;KDO}1%e(LX&OVsNwL0Hc<6rRO$!p8o4Axc~>xlnwI9RcD<3xD1{{%QYv{pD}FblA^>rh6(CyFD;y&ig&>>#8|E+>hjBk#boS6fYU^x=S|579Ra>axC1+kQ}X)nneKiq$FpW&5IS9@os*n?KFKj{oiJ z&Hjw{>(|$H-~J^(S8+x*Q~ZwU|9)FPXR(WnR^R`!>6Eg>58llE(^~E9+4eQ1Y?F5A z{kc#5Y=o+APKs;y&Dxhiny!7n*H%P7Roq_l@7hhN)Bn}_uU^{p{`{)<*55h09MAqq z>Etb)TYEjyM)!|VK`R@BA;dLDEt9LJb9+;%hHl4F0nOb(+5X$d;N_U>`(MbE=>Fu{nOyeFYDC5 zHB-tb<==KaJZJa*@>id|e|?SHTP>1(`E<|w7@6Xy<-$!COb{*=}p`f8Kk4^7R;Ry{H=RkNVT6@0ubozq~8^tLv+;FV{@Z{>hW% zUHkKf>X&KXX0ES4pZ;!Q&+5H_)jw@zw%W#j>S@lLzO-P(h47tBh_q6Y? zN2{;Ludmx1ufND^t!%&FLe8kln*HiqjCUVkU|@aiILmQD7Hd&(veMsQJ8ylh+M2ib z>#wi3zP{QUxAxc8s4tUN*OtWpi4WTTf4%4FxVr&jHZ z+Z#M#O4}o?cTx?Kw)Nuw|Ht>4WUsw5=hNLCVODo?&PQuDzLHnd@10p$sK2!MLs%SV zUHykwY)e~z?q6KvcJ}_;S68DphJV&CTl?#KrOp5MS65%(A2xaEOTBNa{|2<3d>j8& z+46pEli<#kJ67H?=H2dn=xdn4m(T&uqN-+J9W?fSa-Yg)U$7OkKB zv*h33tL?$FHvD_}Kh*Q6r@PB9oBdJK-e2AQI$S>L*Zog<>-Ju~d-dL{Q#<;@Rp;$Z zeg8`@_Fo{g?!EpbgStKP>spgOKb+%wO}l!v{Gx;E!eQ&e{{Q3fw6WP=CT+gI-0tw)$(;&f2WK)zj}B*~;+m>B1=9d*><~k5&KLee;6e*7Dn$)APdrw9Xf> z+gtR19Vbt=RQBuYn=2H*?c4V0?WbS1;p<|(ZNt05Z@qey((-%a`d7C^yx7iskP+It z7?qO?{=MJer1HP&ZFYT2{`7hLwiR9)PvdxarE3*_ z&XP)9Fva}+zXIp;%{AwP4sUW2uzzw^|DfmN$|^R#&ElUqxaLH~9ya)S>B|&D`>FQ- z`BKu{l(tw+)ra+j<*4}^R zn73Zw?vK0DXJ6aBW|8&#eXlP5{#vbm|A(I7${_hozB_6=mNXncdg!sMX=m%EsSkb_ zHS9YgsB*vVca49@$@zC2bG2S=sw;~v>w2`Pz{`E-3DphBl|TM8fBZGU{F%(m9k$1W zt-K#~qP zqLrIXFYa((aA@NVql+IWTEFysxqL^$(FwnlIc9IoUh{9q-SbbreLpHQV~<+WyuY_Q z8o%tj_(kB(e7o;2_N`v>|JAeJFL}-y`}U-tgxpXT$ksJ`ymt;TtB3*67=SXWIbw%r^s{EQ)E}i;& z`kR)6-=rp0OKku2X zSQKSl`*q1J{U)dM^vOT{>K*HUX?xc%dU`MZ|8JKo zlRp2nc_NViD*WWfCDp&y#LoD=FYfor-~U6)f9iY>->%;BZRJws{Vnfa{i)@={R?tY`bj!9!4FJ86LviqKA%6xR3xE_UAb0 zura_ul)I%xsrc=_&2@VohDsjtS{WFu>8pC&d1~$1kL%Cdv){ij4p+LSADf4CNVUYNj>V8-j}cAjB}&%x`C z&Cl=sjfnUCs=K#A%JPx!yLXHl$2=y4xY_Ti`hIiASNV3^JFowiu8^pPEM*7Arw<8*i5|EbqHmx`QNC|t8=1y8`M#kub;K4)$KdtQu-wUsepMnr*_ zNL$lEhYb^aRD}7M4o$RhQBmf4T6pj0qt}yOsm$aINr((&)lSl%ub8*)gO$+|q5skwKi&D}uW`GBCp|6y z?Y?#AZ~cBJU3Yo6zxs^AkL%tZePI}W{4+~psfx?{SEhFMs}=hF*F4zUpto*PhkWl6 z!(YpufCiK&=%{eBG97GK5D_3I#o7!x@)k6*+^}JRpOQe!q327Jug|^r)Hmm~R#$elr{;f~Q^T&v!;+}!x>(!6bn*(@Gg}Ql-DSwIa_iydIo{pBFznxX561}; zyYKz9(Y-Ngq3%iTig=3!1{=cP>a4QS_|$j+v|3L`L|KgKNWq2xKQS)WwuKHGLNp|V zS&uAKwA8F?(C~RkJhEVubQ(gn@?qOJbs;d@Xe{VQ?tan z1@^zBz!PJe!XGw0#{2e;Mawzbt3#a}8g|5<+JU;Tyvu`}Px zJPn?!Zv1>I^!Cl=PnO1+zG609eADQcMM!P*cE?LGulDc%y!ZFJ*t*G1*7rC0G++Ot z|Mhjh-u>u}zmEU-<-axb$NUXTju;>C<$7=Bcu?}xVVNHx_xipu_dRSC(Nl`i{r2vl z{*|%^!6gN|Z0nd`!96m<}~;$S@F7Vr^wiQi#wH@;RUXTIh&))q&Z^7-qJ9SxI;Km8U7b!t&MEnAb` z=B2b$_{yW-g}f)y_7o{R6R zu{wFSrs!JX?9;Akn^Mndy($q-f4yJtg-vl&M)Y2<%X=E)Yv-FRD4Dszrel8Q=U>aG zzG(H6>IxD4{&iv&kEWyJbkAdX@8{brFTbbt^L6!!4YTu_bEf@!nJ%5)|7i6RDZYP( z2k*Vq<_ipwz0)^q|5Ve!6;E$Asw!Uo_Urso28Laec1Hd-lv%%Xe&ErU@r}Y3pS83R!ca^poESeP-Q#8`p?;tE@Q84chiDG9gDqScvIh!-(-N(1 z4+9psw2Aj-&p!9_^X_$ZyH{_y7vtl5YWfmCwt3fW9-3tMRYkvhEPwI05ce;u%NMMY z9mL*G_S)kWu>odA$GmcE1O&**l*z z&8s+E_jc*+6$--B{oXn?aG&7cxTSn&Ls-ndyK%4Q_n$Z<_@>f8{LhZ4@1mEFPgo{j zTy!Ahy4&xs$2RX=emHbu*`{aTAIfDYzx(*2xq*R!b&KOHCxI@|);&*;HRYey*7*DE z4ZmOdI`;0rO>e84Cm)*oM&N~p!jF6Br(eIalz;iUb?bM3&3dx>U3+CC07k z@~n5yTN55|pgp5khcD}BhGN*>`&x&8YW=;g-?4gq#QyNBd+u?hF8AxM`Tly{)k(#7 z>zYclW6w^1wWjFqlkfjoRC>29c$72G^qbJ)DesF`1m9h>H#RWL`pdrcHSVRww>I9` z7i+oqpwZ4}uYY}i-#Kx<`u#7$HCmUArk(F@xhFk-r)8?PBy&Tr=1K4O*Ir02y3-J`v1ZH1_(L;8 z0v78^n9V=VGdV_MdxcZ$G^cYrXEx28)O5O7sC@nWLo@Yu>R-&OS{twQ<=)!+X=*|n zs=40PM*Hu5{bBCBZ_nd*lq{Bue7TF$`~5nLtGcK6zp7cBcI)x~BNEd(((}%=sIRK} zKg;Cy&-!?lu+)aK*se`~e?EA(`t|D7?)6uHP2(3{GrwcU?AjEyB`0lf{Si2LBmL`- znBK(N%U7FQEjF}XlXjV$WdCDVgY(q|8y>0L^Y8zt6lovAtNV}ht?7v`xe}#6)C^gl zC2er2BEthe@Q*_wk}To zrM>E)WRqFZc_R+(DNP;AT=hSRZT+EVxnjD%H(zn|?l*CEzv{k~XIb2Cn`9x7vWLl= z=e7Qx?O!6ow&lqP^Bry6z5ehS=`XvQKJj(0|K#@mhHGNC^PTVEzwD#FFU^zs9CbD_ zu?YxSps<#oR;mOgBGWuFoMU+wOTtMPN14Enh*hI;L7={x&p z=l$9z|K99V4vCxoH*D3Hr&B^EJo)th^w)pu-z+V=zxn#dg&z`g)T>_JPggoNp?>AZ z$a<-z_wK)$bol+IqFJ%C>`p$ZyE{Ade%6;wMbq6RGIxK!eS7=%$tO>~SbOW#UeW6{ z%X@AY$G_j&z1Q_?PV&vYQBQt-`qA%nf6u)6U)IdK(K6{nt5-lSFf#}mZzGuJIwlx?Wf=!JCx`%j@#c+mCo z(xTX?hxcE{y*hr@zIDX}X~PR4c0qUjLi8W$X*qvU*0h9w+b3|67?R>2`2VQ@-naJ^zTEY0st~ov^>RS2p26`VV!r1IKdYitA*AE%Q49C`MiF+SC6}jo+R#;rq{xT)%dGIM){Cvd@xh$&UJ}>FYHw zmsvhvr?j{FYv#${{|tf!>p6_4y$iGOd8(UzGsVo=`8|`_UTNXl$Me3{*FO35w(67U z#uIbp4)U+^%l)-$$LyH>uCKqo{+ji*>rLAxqmp+Y!bN}n`}Eze!upMR_1nGGpDr%j z;1hdaSI7Kj&z=9fH&0F5_JaSL#OJ`KS@A1>vY#_3xN?5eN$pS3uCL!l@4vqEYx>%% zZL5D9?szF&+qU#?;+Ng~-|VS5HvRhZbx->*Kh=zI&Cj^?Mo8HHwfg+fRK2H7`wqE3 z-*jP_ZJf$Ai^2=7(Hp)mzZjFZxpVUMC6Dxevna)WGktx(_WS)r!>F4|`*Xg8-{d{} z{eD%|<-fDcGUhyHPrX#TPiu`icYydNVW(TiGPIl}+w+D0U!Q0DI`;dm$K|^pt55p< zM(|9|me^LS$-0)ddwo}{o9uoQnD_48-)X@Ds~UcW9TiyP|2V_?)-+DGyD!5ocKko# z8R)KVrqox*Y>yZ_qn_m`+Y5q;WA{xAM|XWx>neZLjDHotL9 z`+ND#tn8ba`}h6+qpxD5GJ8Lpon(HXnW+Ef%DSUBzIZL3R9e3&)Ba@juXRhezPfV! zuB_Mp)pu71&KjX)=jB)9 z>&3gmu3Hr}P0y-wuvuF%xfFaE`sled1;jB#HV zzb-Ior@^r=(}OOWW__yKXumF~+R$xv-Jidi;fqe+nDw9c@9M8FO~YqjU-duk-JeMI zV_xh5FD|TGefDJUdKT66-sybnFK$@9-u3m>s<=qos11F4#e=Hm>Yvc6aqwkf{%9s~ zoIN2W@ckFvWjkx5_sYb~T2zw0DRr_^rI*k5vRBjgCjG76yK9$P+4Ol!kNp1;KSzM| zl7%**-Vc7VHoA zd@=Qr-I3F}=fg!y;_pA%;>PrTr{(UyV%qw@{!ivry>KPp@O#PpN&iK1YM*l^Ys5dB z5Z04ttA2gaTk+*P=DS=k`lTXdeLmi2x5JX}e=qFWUZw6@Wp{du@v7@lb$_eg23@LC zJ{4neZl~S<*LPKR?|UmAcK_YCiCORO{fRh}eoOmE{tcDqXT)E4?-x^2uPiQ#y;~)I z^>yg0PgMu&_WtDMo3Z@#>aP`P#ntQkf?wZ_|M{vj`^nd-@2i{MT-*8C?CFBv-KX!JerNyF z_w#qm{#zPu9>03Zx+PyT*R2C><6)L`oaH>>6>I6yKXVtF-mm|D|L@=bzm9(W|Nr}~ z>#xQCypCU6fB*mQ@7M47umAr)eEpjJ*LFwk-IupCN;MZN0MoP0*F;CEcM>*UxNySrdNt*WPz~r~gwoegA8F!mgK*d*?^hw0M_i ztTC%PX7c=6@Ur-D&s%$AtG3rGeSdwg<*MngtuNyRQ~&?+pX~Mf{f_RiRlCDhg|2=U zwR(NjmoN&6XuEXD|F+W#3MQ;n*czq+|&o3oPkL&i{ zzrW~e?ETs!Uy_u|R_w2z@KUVDZtn??A19ywuT8l3_uES6gqG#{UL|j<_O$i9czHSh z-7Wi{x|ym!kF0-eZWJ4Q_22rdDxZEV3`*;1WQua}%sLeu7<)d=`sUrgi%RBfs`~P$ z?Xebed+${{X)8LUmCgYJ^xK1EXI~^@xR<%`~GE?Ec;>>6s3IjMc@6s zQKwG-y&Jzabl!fx&ChQs^ZtoH;{W19)YtWz`Fh&>lXOG+-}#F_b`w7TOvR>X-}6fv zSLE!?&$^`CTd%rMJ$swwYt`%1S808J{&c-%NJU@$-=zEZQ^c1&`{0v3wf9Aucl5WJ z>lXc7_xr8r?zdTc#h2{AzCVoX$rt7IXQHhB7VW(n1X*P?87r;2_OU0fvUfgdy6@qQ z|1sq?e^j~mg;!*JyrNUsUmE@E{<_cadusG;g>~L>-?^@Tb4kM4cMn>te07=`zU$9o zKEL(t&EWP096CA=gFAok_&D`t&gVyazl+V-CVKeNyZ&2M{&z0^m1jL{m=}99=tl$F zMdk@P|L3KgnrnVx&dccz0%yWB@7{c8vGePnM|GEe-cU5Ys4l>@+k4ZysL7X4KKyzy zu9|u61JU)-cp9|v!`K0{3Bg5K$!%RL{lzq?-_vHL~alxAPmIUFzMuy4BF+<0GsfA!Q4TNghT zI{rf{>;3EB=j1%C(|qxzJ|+2Gt#D<>6~)_+(jT6--G1Uu zzyF4~KNDEC#1-3oPF|J!XMysaUl+n-9S<;SiA?Ykk!EE&Xt3dpftXP5p@Rh*0<=^( zTb)2_UeyG(tgqay{+)mO^SqN$Gb8mw&pyrbT&cH1W_Re%9rq2NJwF`ZU;gacZ8LMZ z+WV_)b2{a9wtby)WnB;Nf=)Y*S0D9CziDrLw`ty<+YSGfg;cBFa zYrm5N9p&P=S#Juz=)J74>zCutRrTK|R2~y7TEBnwxt=78^*_v}vOYaAse^GrbmQ40 zrKX#-bjs!4|GIy@-uU}o_y6bSzgzqD=InU)&2qnAO;Y(`a`oT+RaNPSGG!mkcxh^; z(qvRid>QCKcY=axH1jUEdhLwk;2=`5K(A_xm<-eIil!|=-TrhFSAL4ES@3Yp zf9vA7k25x}irW+yQ>^<&X8rDog16p0K6TvkPv1L%>pyJoo}YW?@B6gkwa0_fTCYZ) zRBM!2Z?Jm->x|9?lB>4wn0RvCCxgO=vlWW}`nfDvWpZ9^{$qpAaBUrhMRy-R;Xila z0i&A8gf>v59^SAaM8i~=)#;$aiU2nitrn+;g7aj`!{2|sJ@;#A`I={2cCAX~GYxxD zUaxX@yZwd2v-cgAFJC7+U3R%ZvR+g_}*21 zF`M`;b&W6elkOF$B5=ifZOVpA0H@ zuzZ#H>GtN$ddAQ%R+lTzU)j6pU{(Ik`d@dyy!I`$`&pjw^Jl|4eu?-;%%^1-1zL@p zcW+=(ys}fvwNh?3BWONgLXIdG$DxM_2VxAQxFtc!G(^LdTaf9Yzz&ym%j?E9zbp2Y zA9__?JXK|3mqmAh#eRu<72?IQ8S;shwU)0%1#pzj4Sq~iJz`5 zXRzDq!1I6qe$F4c@l_{G0|M*VF8F%ijyf1OF|{pe(!J|De>V4SW^Q0q5t*VPD$L4s z7<7rF6t^JL!9s%=9TBNcmO}+AT(ktGkDj~xyZV*i=k8V2`C6~E(u{+{3Kx8@_pa49 zyZb5Q&Q|sb zuSHGtOXZed`xaj~Rd{szi#YcCcDK5xF#Wi#GVLZyZ`G>oYP}PA?+;)13Si&lz@*Fb z@#4N;CMzzPezn%%`n4_N_PM9~Ty;O)`XHKlZ6?R{soI5c6WJLU*bh3+auMWWEn2?* z!oIgrYpdRGJ@sP!zW;H5tEWGG8&UuN|NE`UTMPeR-Tk*(a{8XRdf}zDYs0C*m>kuOhZze)8eh^?2d+>vO&b_MHAdd!1Ul-*cBZ^{?+cpT4dC^{ZXW zXWw*9tuX7b((+%g_w9?`;4QD@P{U!``0viU{PpQCdh9rgnr=>XOA-0MTDB>B%flt- zjFTIF|BGPT^Xc>dwB7HncDm~R(iSbhbv6Fh*Qzh~))|HUcb@MbY5Vuh;eytazpp=N zFUwzl-&Lvl)zpn528}Xvw`-txi<7bO!76qt1 zyt9A9eb4GoOEXON$9-pY^BXItuycKzw=Svp!ttUGEZQKr$g3-my7@H{mm2e zq*6b=U2^%AA9>4`IsNwV$#RZeu(Culjs17Ug!~7c-0!8|?{8aFE|-zgJ?s6Z2~YPj z%V&M^@?QT_fAW{`qqSG#*H^Czl=vI``p%QcNwefnekuKtb*|;&S&bvtTO#)Nf8#tM z^l_G=9zkQ+btG@r`^Ye+u@4nZsnfL#Uq|9lkfZ=Mz8pNK1KEQhsV~-U+UXN_{#sEV|?KCE#7<2;xm4J zp1%97{66ok>67nl&8yl}``2P2Px9h}9|T_?`20&_`pMt1j~AIQ{C)IK5gY)xJK%lE6f?(=+AI2h!X;kohC z|1y`Fc+FLY;g!PE&P{wd$Nb?AInh72t{-kc{jt97gWA$FX=|i*Jztx(%Js^T53j#- z)rQCk?7Cr@vD)bM$Hm7QZ`W3&$rn|xZ2jc? zPpaX2(SdJsHh1VLtgx2P{-oZi@b29_ujYhxFs8jD34D6z2ZVy&_PNnXRL^ zt%G~U#NsGn*Za#Cbnf`^@V2FxfMnr-NZ{qw< zvI?im>f5qg&nZ>E__MES-_)BtJi#ukn|}CZRC~@)J^AD7L!)G!EjC{!-+N#5IJ`N> z`+c=Q;MUd!>oW?=dzJ|sd#}(F|DUKS@^epw!6!CmDM7i7Vr{c*CQXsQBXRG6c_kL59YWfIjW2Al%Dy!`oX%H`_n6`-4|%qb}j5R ziEx~eupvBs-oDFQZ@zx4ANt;Y^6x)imbdssKmGG?_mP}*(;w$^+xGQ^es1Z0@#@3W zw|6GWcQ5;)CiH#r-c#`tLT09_KMmdSmdWApnw>5up8sW%TEACL>(0M}=X>OjyeQw< zX(rE-u5A82x#ynRk(!pJi;q4_KK7?JI%ek@G0Cuxf!>pMR>$nxmG|WQ^GC0Ky#MIL z&iY)?rb5)tW6uroFQ@Dr~e%XVLSu=8$ZZG(IcgdbhhlR~6x3UyYe}DVIgjrjjKW00> zr({R<{qmZH)l2iYF(5q8~wQT_9ll{~xe9PRRAzWZ(7`6)_O#-mR&> zs@PfF-@88=!c|$<--Q};<_nxkQeRuWr(p&pq>;3*CD0%;V z@yt4zH+$=sADQx9Z|ViJrT^l??Kb_t|8?@a-RI-aI`p%eechX>;T^gA{C{y)qxARn zlX^`nwy%8{;B3f#FLs~di+DZnZ+`FdCzr1Y65qY!W=oRPnGce)( zKI!*)Z(NeX-*Hs$mbA?;f6jUD=lA;Xm*1m*UEh26_t`R`&hPW)t&My4J$$nD`iOng zmwa*Gy-shzB-c0dnmE^WirU98FtDz0oaHi6i?#fc&?TO43+0b$pL{)K^7mWc+ikt4 zXW#$)|Ne`O)nC6~j7eKpkpAn+)Vj_~A-g91QvSN?bjYH`&UMz$>|2Ze$p6S&JNf(n z<~84SH1acY{@>cDd!?m3X{vhp+0%Eqe($^?n&#H5Ed3VuT{rLZMb+@Hg|Es>9d~UyUjJE5ky~3+{-UZeE z|F>R!RefkveB9*g`&6U9zPr9BeD|Dxf3(j1U0m_x)y1dhqrY@tu`KP1`=j|YZGY!< z@6#WDf4y4ab7h^)_5Yl1?df;wf3duBub;lZrItU|bK2Gtul?ELXVyzz+JE+>-geu% z?&%p5Lpw~lLeDI={i=NRZsM%(|9pS#xt{uS|D@mVwz9N5J0`jQScJ2gfpkVx$ijtp znR{DUL)ogYaAe$WIHRD(He<>`$>QjZSAR?v<}5coxb<$Jr0xA*`BzI%hV{LA8@F`r zwIy5YuI-MNUHvnC-Nfy$H@VJKepc+$`$EraqW00a?sAFfuL5E1->-@LsD69+@=wP9 zLbl8~TaM1Y|6t#w;7O7h5!bIDmN5TmzxT!5d;U|Um+Lk!j&9a43Wx|u&swo%L2J$H zq6vA&>%2Wu*7h98nWdor@_K96qwxDS8|Ef_-u~(DKW^T3 zt$PW3MT%_C?6Xdrb&v5p&%Av<7r*-XSWT-+XN}!&mPc>*em8EN72j$4?%Yqi>UDeO z#r)jHdu3{!A; z&B@UB|LJ6&r0jnJg2z{;ZVEeX_&xZ?LdBjK#wy9;)yrJ2<#Xmo{L+sX)OezHWzFBX zs(-rwlKrDE`k((1al;4eT&g8VOe|k%;US1~UmEDys(gvp%eP6$Qhk;NF^ZQLc z%i4c9Y}XY1Y5Vb$`12-5z3<&SdP7eBy6(C6X888{wb}c>SGpECojoKXd4#oL($lwU z`7{1?#s58d@&41^Y4vOWpY^Eey!3lfZT;ku*JonhJ6{nzw@5TyPUz>ZHD*`YW<9K8 zQMu_>S(nA-lc?{$_SYs8%lCojKluDESo!tbA4%bB{g?i1*vX>0^}4U+#?w9G9|AAx zeA+zq@ay}lS)W+Ht-ZMa^Ys(cZm!MVy8eSxRQ~&ddEfGPy4c(HKVJQ(S)y>$Kwq_=!K)C%uf%`c(D(n!o+OSnZeDtJhEZa;;_SkH#6Vw$1n{?0x-{ z?3Z`1H{E-aeRkQDc_$JdHO+rr_5N+*2bx_?iM_kVCLzw_0l_}Z#H_nwMgd+}>hNXY^h7pMJ;4X^&P5xN_7@O)Uu zo&W2?85mef9A~)-da;&j3E#cG`rp=D|6c76*YDhA{=X}3^7Xyp{8!Ik*T25xYts6! z)rYLER{ixZTeEK3(od7c>ipKn{T1KUJhM}zdB@koQf4|cUM6O06td6%^;gzNb!W^$E>ulM@u~FLHACS|6FUE`P;=#DwFK@og$eQAMVv z{Ig$wxKp~iv#R&>^3Wxjg86MJ`g-iep=M{2rConW{n;S(Puptmf89kt>(b+Ms z>DI6Oz4PRA*MgrU*3z>lR?mu$ zbd8#`uiWLH z8mfA-^wrb2>1ULta73ND{pj_ZCY!x>YZTqr-mBktT+Sr^>sb-`8%EEL`1>_@8o0>c zXx2QLA=CUq_T<<1(<=Xdi#VA2_uJF6YZscHQ?`87Kb< zo^?3++g!1$dbeobGQBvpz5KP;J8t$)E6sVo!S&fZ*{%f@zk?3HH|KSB*7;T*)_%7B z&-3_x-aDsX%)RyX*7xi{vCiB3H*D3MbYf45&!NI+b2@gLJPSMU{g!y7{px?Glw!XB zP}=*>&Tq}X`l2PezrWP%ww(U)(a*2H-~C&9>*K%QKPP6ZpZ|4A>C6AbVC8wvF6@bX z3rm%Q7t2W4U;Tc+(x&#;*29|wt!kSLpPurxTEG9o8GF}LS{MCe*O$M_zWZJE>V$xg z{p+2o{zm-#73j>d%vEo$)xLuYJKr44(^7x_)5lG4(;q?3c{cHjEcEoBU*2d^HtBba z^>_Eg4{qtTeXm}fau{|_hm+OFBZBvxeJezouKww}J9D$=i(zwftg^foi~`tsWEahdk({GaZAv-IMe zeRpkbU)E3G^;x|9*3&coul}o^zZ<_d%hBJ@-#y{aB=0}sZ+rgE+`XDdAS1~Cf5tp( zKHoVjkH>2@so$UDJt?|=zsu>e-*ObJ`@&gY-=?%#?jr{7l9OnHb`nC#lSuWEY9pYtzeRsMBPHGW&9^|Z3?h4ata zdHb4je!u^G{e|c3x^=%&tB+JqnD%}3_oq+4*G6jG)wH+&Uwb!7=wWfdnjfpeK0iC6 z^|fnKe6;DWJxx=8IIBHToxQ&-W%9r1{R|8oxsJ13CuXr06~>B}D%#fV4fWl6J#1t6 z`q0?V(*IveKi=neO^>rmd)K4gyLYb+R=KJ6p?z<@`}zleu4{dNFe&hP!UUVW@2>9C ztrmT}-9me*QQ6)Qbw|C!Z+~j*Uw=LQY?NSlxMqFz`S{-7*M9UQzMo*1w*0l0jnL`T zofq_XOW*d;H#nE!K0i9t>TG>{lkTzT+buftOaDLoZBdZ6t0bf=zLsn5Z4dcgq4dCK z{6Dqq!tcNOZytWMB)#Uy_p&wX<3LxpCH?3={a0H5T$)|1-s}8Z&js|7!&dL#_H1JL z+r_V+tao|OVZF7q>i45M;h(GC1lq56c{4rg>&;g&+^PSA?v|fCUEZbsXn)*_@@F%P zi(bt0`ksAnmgVIY&4!=UKbGr!{MP^T_s@5;_R8*$n-Csz>gwzGwK2W-v-Muj`nhNS znH-DL7X8_$-`vao_GgvokH6vP!}?0rOr}kV?|6cBJ z#q?WZW%uoXsQ#&z3hPdDY%Oo@x#5&@+2Y}J=FTkFiuoZge;til8*chhcA*-p@OS-; z2}G4qWNj(Bee&)7>vbG|T0#W_!krg+|MKzZ z`#tqtNKo9TTFcNI^~rl1PJ1~wd|l5fo9pg={p8184~wO?X&oqEZX?;q{j=ccj~6zq zr~aPZQU0mCY~$s9%MuOLMKX`=KeyZE$HzVf4mruWUBzR~` zn%(F45Nk8<=DmkTE_U{ZPnOt4xLpYI>VKTf#r$}m{ZISS#iz{`1((R%Y~hz#YRBy_ z@%@lksYwpU>HH)^yP28^2X9S#Cv)=26VV0FmKXmQa4$Z%{;b|rd#?@g*PBix^xZzz z@nKzU)GH&ix_GUu57%raFUTtC*{t%vt?%I$TU)01jqDX)PyYV0RCz(*^C$DpE%oA2 zO-`G!<`J8znUfj!+5JyGDK9?~yChZOXl-p-hSQ`;IWDkd^c1?sHkVEc=~z8k6`g=YNs4e_wgO?61nZqwQ;Kzue7x zap3jSy!ER-3SDe^yzidJ)mM_!xm#mbpEQ45RX^ur?Yf?r_g}j971$Bj6ADmIK^Y@Au)MWdd_w8NK6a}6Xv4nWbDK#ENCYJyHhU@+2 znmRjVYbgJY{l8;&+ty1@yHd|}Fs?RO{m;QyOBb>I7Z239ytHQhPy4FcNjHpQ;w}Ez zP33HnNw~J)!omi}zbP}F{k;Dt^1jNdnfXWi+2yGjpEK8_8~4Q4>+jy>u=uRQC&M@? z{l7oo*sfT-zP`Zy@xJ#L=6$-_xw3lq)Abs?_upqt`hSs+ck4%%I%z9Dfd@}#e{i^T zZvV1%oUHq9CD*6R>R9 zwr*$l9X$QFBI1sLq4-*bgE7YOfA+;zsEf(`X{y@Ca=ZQFpZ%6oO@bVrIZWJtZsTR~ zck9IB#qXw^S#(}8_HEFTsCviXm$zBW>$j}`5z~>mif86i9pS&}r6=qC+g1uVlpf=i zv3L3U>-<%Z^ZqPe*I#)@is*~BJ@J(}Q~o1l16S!Y6JeXfb_=fF`Txgvr|s!^`*xpv zc}sB4DYv9kAzrSzO?mLHM_2R#u59qd6j_UrUzV4^Oet8yle>X$T3e}vr zV@rS4s9uSg`BwcmU&u|)l56{AJl058^L~-cm^w%KV9^I5Puc66rv$0_T>Sr3>8$?Y z+R!sw7Js+D;giXn|K!$&2`4H&Ypx}(@4Q|VQo8xSPB~X(^{2?To7bOym|v5hCcOLV zHK8fCq0^qU-Q2%w<+j?lS01s+SFV5iv&er>lPUM}FIEX1sqE_vRCLAvw>@TVQ*z;B zcv--6E&0&q|C7I09X6cSzxkiyf}Cl|?|+`!Kh+`2zdl^#`uDxsyiq-_5x){2>d)jp zvAZ_+iR!ICF;{+he+bcw-EDOEQr_N3t^MonIw^7AI5oNT{kr1jeYWerW}WJ2_HGv(3ww)CPaVTt$2R=Z-eD_89{d!R=vNQ^knt5-HIE$@^e3Tn&;p9 z^3&X3{k!Y5?n~4EZB9S_dd<{p{w1&0{5#7M+^7Cuz(>AbPT|cAkJBGtU%4La*;#wi z;rUm)NB8#5yZrm=jK1GL)%;JG-oJFEbIsJtg4g~%5v~5t5`5%XW%k!({ol{mFXD{; z^xyV)^M9Y##aAS?-|bsy|MHXgdV`IzQ4H3^?uY9(1sZ|5uw&r#)KUaL1IFzO$!qpLUcr=S)G~^ z7DVWYa!h&l^Yz=WpFh6Jw^^U-7pUeF7Pse}TD$1||K`U^(>v=ef{VEhJXC(WsRmX2%TYgA9;JZQ$xi(|EpQW=QR#myNPZs zlwEi=>C%=p5l@;pPS15b0Gjl@AjaL=%;<0ev_puqr7>W^6&=utYzG4^T>3>#=l#il zeD&FXIjwhFPX&DoR{fou*LHNpzvS<}hCd#@nB2YN-wKNdZ~i;}(2dVH(73PoVTI(y zf=^!#9S(o!6BfVPW2?;ala&>|SN0!gUO%60)~2i#9LvAIie$cU_gKKp*M0ICQhfEV znrjd5+`|8D^=ZF5>4_Kj`ui};#P#b%h=?D3VsPJpz2uqB_k}Bt>hU>FXzre9*TlXo z)Bb1S`B%I7uLUf)w%z+d?A5S_YggD9KdpTEX#(H6_ve;$oSNk_sp2E+-KBby4r=%F zX||>)Yk%%rPyq6=jtXaMOJl==4A8WGM`MyjfQ~5WsKg|VIZEyZ`?tP-|9)L|!!CVY zpQ*it)59UwpZ@X}R{M z&iS#oq#su;zB6^tq9>1U&S0tAe{-^R#fK?JKA!t(IzM?*`oiok&5GQkK0RlYa!m#4oj5PD0xe|w7+ZP{}WGYGQ znwfFM(9>A9S~q_m4`h}O%?~tY-fu@K5le&|G_ub#H@7?CJ*FWU1_f9eVw`fXb+N^2I|6W;F zS+}q4kZ9E5oUflQw=6u@k$wNI%6X}VnavL$-+uV^hgJelj@O@`Q43u5-9Iwv%+7@= zuOl~Zy0^;KC@eqZ0OTqL5pI@4hYLL-45UO@TN)Eqr0A${buuk<+%P4{ef6ndf4837 zUKx|`dd*|{)`01S7SgjHFSgMC*gog>_mJ;*pWcrC(=9(|)1Qe48ou8-KYM0g(f${5 z0{3qQ-U8>B|RBKtQix=-VOnN20W0k)0;+ob+rVr}7WPNQ9HkFq3xK*xS;-#&~6P3K& zsL@x|^oZV>I}Y|raqnKfde|PkkJ+x{){4)KN;CiNoszLzr}gR7RQvP_uMKImkvDkN0 zLyLSv9uz*ge6D=2^{##Eck!1$I5PR>zlBbluRYW1Ju&YdpStf~(o=GwT)V}sc4AprZcNj^;}uJ88Ldo-e6p|BfBCkx zes!yJ9J2OF&O5Sh!@Momt6mCE`O5iWoyO_{0sY&BANH|LSz@bQDR21W-~+LFHe6!C zELz?<5e|nn&($Bah)LK0I{tIQn%HlF^-#_02P+yD2Ux?a4fIVLRiSo24{ z#q4?2&V?s^cs-!7kR)OX9Js zr@j{D_`Q35-2RxO=eB;*p1-#K*TIYX*Vcc%|2lTH{Qvv4D@DS#W>1;w{aX8U_@{4= zYhqKiRAPdf>;xa~w#YpvbNBU+rzdK;UM_j!y6%6>>Qb+%I#cVm#%&c`we4fA@Qx2X zVivQuG@tK3S$Fib~CZFKgr8a<08!xAiX9ZRcHe>bt&n>Ay;idmH_=?nF*a z=bMY)_Ap%)$#1r@33vIc{ozvi^}r(G)BoP-{jBuzJ?6ExG<8Z-qnE**nU88rmL+u9 z`|Rr$nOc7S+*;kUuP;?S?Th|Ar^n)u{mXg#%;ta5ZZ(?yp0V}wsR-9q%oT~9X6O4) z@y-6!w9B>sKq}k2OU$PoXNIoy)GUwlmznOPkQc^TAl`9rzj)s=p@tuql2ul`)BTwq zw=3>{)ZgsgQ{(6Dt@>zu-75J0<3rCb=|#2hy;_~y%wcg{ev0A8$AYnni~HW+$Sa&$ z{(0R`d-2b&Iu_525yYIfK%@=-`-FI@=+J5dZ{1C2dklztxu%cAZbU~f6;D&0J z?ScizntxB1cY5^rP5Z8b#&?00w(=i3la(LIC(M&87P@R^{b~6j(Z-!=tJ=5oOv%+s zSoicx-0Ice!#};hzUZd_tAvW=l;`R+ z{eCmkvi*jmL2%;cyNhSXw_ksJJ@t)F_1&BKc~gEW-zw6w5$w36mOA<1$qxw*lVcUr zc!jRL)KNZ^SUAQ1MU46-Q@;I^4}DkJab$_!QS&J(WeHaIMStYZx;mxS^HBf0DBGWh z_DsKft&8`&-I8NrIG#W$A?XMx6Czl3l}L(Jowx8dBExTYf?5W(Vzc6 z|C0S#Pp1xiKeXcFb;^RKP|GVJBecwaBEB)Wdy2b3Yyo`w0mn?oiR*9=6IWG~l+jZ@F;6~APUl^6Tu#=eVC4lWRR`{Q@;e+}qvpW>w@kBIcZ%lY{iaC zM+=%PXkWH-*B<7?NA<2*Ar~L?eXrSB%{f){bGm!?X1Q;dU)A5Q68ic5k?ke!fTy~r z;^(v!z2Xbyp0|%9Dyi+<*S`x(d8WQ8vn%~(zbb0FzWUeVyG75}hwTpg8$UbT`pqNN zMKbamzO=Ni_%UJA25*~f(#DCC4DPchZt7vGUG}Qr3+Md4e~164n=QC5Bz17ZisIiz zHuLJA+)}IQS3mSObCH*N`BE(bw}j)_PqL3MpQGhF5VZEM(Mz3<5@vBSyqW0gkn>_wcSjhC{@#pI=c_vED%lP`IdhgdOp||u) zCf~194!qVg`I^AL_vtV5+uw2=F0@G4#bWd;HJW$b837-Ixwjg-jvP4D8P3|H@g!F0 zPW_kGIfbPREA!r6(=u(eWcK6XTi~Jl)?kzSh6jiBuOD}5w!XL0@XdzLg z;k}&nN#E(s(={ICAR6qr|`P?L%LgkT|TsI^D|Y>iuW%BUiug(xqE)Td(?iX{S&WEs7^31 zIOVV~?~{v5)&E-G;5q}9w{zKr6S@Ui*zX>%Z~1+j-{r|V7WSWA{!czd-Ddg4)Vyoj z-8XKVY6X9_E;Q`k^ZRdp$;98Ye$W3V*`K=4;(g2B>+>S2fAdP1OncIDbFY8kC4G-| z|J%)F1;jUOiR0Yzx&Fo$t{v8io2NUdepiqBALHb&*Kz5VpB-aaVcFW1mvebbSseGb zn|%>pKc_tN;X;w_$64;>|JU!Abo|Xy;{W|d))CX>HAVmbgm85~?iIh27k($v;AZOC znv^wHRBFoicfPGYqZA#t`~AKz&!${{9;lkwSEH+(yq@{cIlE>F9y=TRHQ_uLPT$bt z^=NH8elR5E!^c@WZL(hUm>zfHUlx$2=9FJ+cV)NwvAULuKsf6Pn+UPm4bCLmv;TmQT()hV@#yv_VW7LjXK>i z^RD>+6Ev0;co5&=RwlSmruo8!jurn`{Ve>b(6MCl$BJzp*Y`4~{)>q=)SC2*C4I-O zzy3$KIH#R`?|=R4szY|a{zukrt*nXU?V7HBQRwyL!=){;9P2`LUIAga!V6oN~McJm$#xeQ(v;!r$dCmAg8Q)C4``cKc=X&Hj?w$zS>Z ze%4-ov%LDr?|ZAS?|XM+_Wc*(Irh(4Qu=@IJo4mYNoW4vL*l>0{;w}DEwAfrnb!4- z$?k8^Hz#Ae3g4th_K|sii?&UFe|4s>_nm(X3>;4#XSq#W#adqdf7;GnRgbJ*y?gij z-THlhFKpAgt}(A}YWBWs`=kD5?Tz@cKl|zXuSs#I5BqHPth}~kt&r~Rq#r_i``67q z?jHaB&J@4zeeeE??GFE&weQFL>-?RQol^GduA9$w-1~n4%a0?Q_{#6C?A?;#%f8ut z&s5eY+w87;e*JY)@z2B956ogew)`(B;<7&`cj2L`oxlBw#&kKkxF;$P#ow>BvDW3E zH2tEDdG@!@+GSGpMSfy`avnF$nAmJC`1tIixrIQMqmp0K^5r(|zG%Dbjn z6{MQ$8=70UHg50PUz$gXl(V->2fnY}*Yi~TPuE|kMw}|{7cF*f~{k+A#;okqx6>D#`zrXnF<_a2Gx2L&Ae+49dJDJGzZ{aM*IF+8CHA$HiNDnLmuiU6^)+!V*X>`2zt;L% z^ls`673KNnSv4YQ&(7*Ty0)^XZPuslJ1gJJoSG`Nr*cii3Tv(_>L%&;FSs9^BE6V1 zinY3(e{p5KhtHLHyubPdjb^26O21=wWSiQI(}G`h{=VXt;F-Mm^w;++rmxz&-o1*S z>or64dy#OB*Z;3cd!xS|uA8A^i^_~+r`3GN7M}a{{^IVIynk6=v$nom^ljow&XQ`qqXEGiEqgcnoojj~tFu)} zM(0OT*O4dYpN_xgS=6~+vu3{SE6#mlkNP>Td7OWDSSj+}KKX4MxBZLrUwl%qz_CTD zY$}v;e|?SmuJzsZ!>KK^K7GF}@%??+ zbE{`7h5oB(U-J97GaxOXWyi$-oN~@bIoKLw#r7U_WIOIAciya8{o8L1g9WJ`*KK>e zK7CHetNs7Mdr?e4(YLQWr~lT@$yDC0eez-L3WX^9Gv5Z2fhY@WJw#nylR`o8DmGYbEI&Tqb`C+vReZ}r!e ztir#3Z~b2We{o5dH-GM#gW{~2k6IZX@c4&ctvht5NuNWZHjnLUsqJ~08_~=S^Gn#j zt6%Ir-PROvGOfggeTLbi`p^f5qS8;!SfH`C_w+^8>0c|~m`K_j|Le#x>2T`(#KXze z@lPMNUaZUcYWm9RzSrIA;Ie&LpW`F#zOA{Md^IBfj-_3$#V?+U-3A*DR6n^H_tm!Q zd)F8LT}OXy{Jb*g-Tqy#=c{j9GfiAD(Xr3Xj>985-OJc-e|b}+s^7vBi#P4!J9N6> zrvGDQvpx2@>|Z{&`E<=>iu#k_QFb)#ca8Sbd&*W@YUi`G={_5~`;Y;7`4Zmd`yeld8 z-%J({3CTV4%wNo&6wp5V_`3IpCdJLa*q!oim8!vwIXmMIZw^|0;xDU~yU_o~;W{yo zE$u=xPh2>1BWGz+!|VRnTQZmVxo`1%^`*HkLzk&%o|>er>wq~^Z&TeeD`qX zMG>YoPaiK$|Hfgn#{Ky5wXFUQ*WR0ozcJJ53gVjSzV`V4w$>%4Yuh6J@&5f*r1pn# zriJz4nKG+?@L#Ld(}{75*>zSmeEFx^*-IDiU$H3eO8m;4mc1|P)xSP}I4Mx;R84y7 z%N1Qgk8*PNNC-|e=Kb_JO>K(Z-pIQ9cOPxEXERC)oGdx{*7U>bk~aA-Ce@rV75Z&k z=kjaU{9C)jCmgsh;Bo5D&24P&{alP^wVgG5mD;}T$?j4;vqu71^<{c?7Duz+K3QNU zkWnHnJ;Oli>O;Ro#ePPu*IV{0|JQBd**;NuXZzjv)8-cnJ^Q>N({_Gqmq?OT+hd)* zbqtqyFFRZ*R&c8MFT62q??qenI^Psmv-e#Gj6xnGo)m08W% zBYv49efb>uu{M`B_y!6SR_=7z4^65`^fBOGQ>gl@;uWRpE|K4&?)9AvMyN~Yp zaev>veO-#Pim$;Xf1`@9{abXrcfVJZwBP=9?&Le)ufGuE=>NTEMpoLnE&pEqURkor z<4xK7S|`m3OZK095f-I?*Kex*>$@*68Jzf;_;pvi&5y$ur2b0OX3yF8kApg#|Fa7boOL;T7)7M{a+dZjETmRJgc%Lp=>+OgBDSrCV{&%f( z&YZr9YZ-F+Z*HQ%*PMimy~MPKh_wefGU7WxD2bf3L4QJgqBh``44- zU+wc`lG%CvWW?@w@tdmuUtPGNeDd@EU)E&k}rE|Eu!`A z^K-}ujJx0|*3yjly1jAh%gaqyi@uI~b$_L>{@<@wU;WirPk;3^Z{_5#uXgRbruXV! z)YexgANp;1EovzFc)j;$f4iC$XBRiDEUKQcQ>*ge@kvtig!WD4nb-YFWbeO?A$z}n z)V_XdXIRw1TN{5}*|#=s@3|taX*I?1>aYE8^w@U4X4!v#CXiL_=kJEeuUkiGe+sX$^$*}(aFQ0eeWbxi9NAm9oUP!LFcl_bLXh-XbkMsAx zx>$Sn?w6hGUd@UQ+ZS5&Zqx7A7R&6+*4~f5K3%DA-QlFXuUmEh`F#C&^5)kHMTM8e zwZ}NJF1~t~AG53eTAwC%+T_pYIW}_gv}c*Y)RIzqI-M-G57{R(*Zh-dO$Q z&Rs8;?mj*F`p)px*RR%b|0+m)BRG@UE8sw^+L3?VSF@^*{QerZI!x>OdX6hEZLfY; zySRh@R}|Ca_m^9|YIelD+w|V2ms8TJb9$$9+)UYb*Y#(ujhIxa8>zbb?!T=%ySBR= z_Pe0{R(*E#v?-_5WSds9_%Aiz(6nUF<35ha6Q+}|f0^qj#&Rk4rIq$K88gQRf4+og znq1W`{iY^obmq(|4KLBQPM1%2zyGogT@w_wE2*yj+t!PvLThr*@<`9t6)ioy@t33W zp0wGnuHFxK-%v39+0p;1Q@rlV&aXEoAND@<>EEOK2_{EwZ_av=*?aTw7n4U@c>cTj z>g~KSf3~qu#Wt_ZfA_g=*d?x6rn6jSg8$+h3;*9&zyHhlWjNQhEf(nycXS=de%Z4B z%bEF0Uhd50HTs%4NhWD^)UsB+o$5`Q)gMD_i;M2p>ulFwsJCbSA+^me)oacq$u%EI zJlQOv_@=(+YVr-1r$>J9ZQI2hQWd0{a_HG!!;jsf%<}qwV$FT0{5xNwttPdrbpKM% zfb-SgB!#4Yzy5f3#@#m-;k(ygcGg=M=lV1Gi0$q*H@?n3(IMjG&L)&uY?Rf2t`+{!D)$_Svl}zT>=NTEeP326?mI2+zszTHYId_vYkt+YhVU zn`(MC#&*#s`ET4`W+>c}(DNw zu6z8~qO0$q;JdrMM;{i-rKu%H#>@E4{JK4(@L1HoABW@4`JP~o%?A4pTk^^zyb&;X#`pN|ob^SQ=IY)5ZMm>z^^2k`m$@tp zasqALH-C5|P-!=Rxz_vBZyrVd*_BkyvZP4R;f3L~)lsi+aS3mp9PiR8D1VxlKl9zW zWaqa1>eH_k`qdrcUAF$_N3Y&ZOYchxzxknO!(qezH|xRf;~JkGlJ)+4{gTQFKg}vuKUOp0>EAhFYrNJy|IN#7;Yi6zJYT-wA-=!CASkwFUr)%Dyy;iNBJW9?C)smMA z!Y54c-Ft}nQ$=m@=Npj#pf;Z#mjAM=UjF3G}Mx`yU&~V)wgDTyHJI^_t%?+ADKQ?gg@Mq_{ZS) zIn9Nav?7167UaD7R^;VjpY4n1Ykj||@OGwtVw}O%urHpABZ{Z~yK9gTwpChh_d}OA zHI0e~Kjen%&-^{bTgP^~>f#Xh3Ga8E*!BDM8y)TP^X)AQeiZPGHJz!S^ISa&df?)_k6>C^~s-~g=W6A{o>oL7kab1pu79h z{e6dyMJJTc?w_|TD5L&V>nWFQzxjOAi=WyBIUX_I`z?qsc)od)>>-sp&%U#BKlEqc zS$%)e{*AW6zqX%#RQvsjx@6|U91A1!;&mzKSN@-VZvFa$Jjw^A)F-Ro``UI|O42j; zbxYrK`;%|$=j%TSF|pUb^ycaNd0i$|-#ET|9oy1%*yVh4UE(X>>`%3Vfx%5x?|Pf7 zbMG9{^Y5SU-Yxe2?xh>byYFg$<6?F8oc-}X*SeBlMrW&Ieut;>U8}EOpL^%X`MC)P z{NDY_s?h(lXCCMFZQWw+i@B!$dt`s!x!}q)M;AG^ee>h?oDhG$On}GG`2LSiX9_MJ z?@Z#)U$*@3k+0jQbap0tmfh|1-+uq|g24HE^Q(V1-F>~y7vBa zCzGz^gNy$D6nkHNe3ns5Nm{a^@}%n~Hq*Dhj@xSIKR>v7(eKw);qUZc3G8rsA6zT` z=Way#{TmUU_SNTXQVrI0{!jX=AFldKzo2jF^!wGTMNhxIUo?-?dfMF6U!-sD-tcGE zylZ+FrhQqqb=~W$zp7Izj2BJQGyk&I``6c`?VnS=u3NR$DtP->0R!*pyI(Odu;x3? za-aALbg@p+_m8&!@Bja|_x}IbzjZsy|7%?juNPYG|NC$D{{Qd)|KxuEkTv;hf|vj7 zXRA`Hp1nT#@c$>BO*Q|os@aR+Uk6gJQWCs^_I)(B5uHL4!S3G`g*}7>h zqVv7J_@DoOZQZ)-tFHE4Tl%&B+U~bPa{lt0PqN(dfBxqQ@9`)($5-acFaP=Xt~PXQq(Td&+4AF%3l*>i%(e=&At}#c8`jB|KHR_ z+jp7mcc1#BF)*oHP<&&Zy8|`?|ic>+9~V9{-+;BPyLQt8M^c7-KwZ*yhlSg zdEJCIRExJKEUG9z$Xl?f(EIfEL!S+vIZmA|EGO7Fvr272_OI`M1m;w(|2^fAmEA@D zf3^Gk(xi?(`SPE8y1YS;ShL?(^u6qKwd$Qe4@xfk+O58O zzvGEYp?eQ*F*-gJIQr|Kf4ZKJp2y<_3&M}j+8zEku_d%7SX-(8;qMFDEfQ+;oc(w0 zVt%ajQQPRvSX!}!g+6bPA9LM z^Y5U*m2@7qnDj&?2kU=N_w0Xj-LU+9xbC}8Rr~+ePL|!1$=cng@iBFK^}PP_r|V+= zJmPixp;S>Kd1KzeCGTt9pS(F=t)72B<752#&~-tDq9?=em)0%}1s$_h`~80H6hAAM z)wSQaYn&%eG`M`&r%deWBEI!4KW^{4YGo(zrQ&+J%>IZMKdj#@&ok4nv{SNwzhUBb z%>}Dc|9-!DWXIo)N7jBzK{dBo~hX3DhDm+kNz`m)g?e3$! z*0oa}UfKG8)4g}zTidIwudWSUe0^$JYq^!s-EV=?`|1K&+Jzo$y`U7~_weHUeZNhY zTcqEz+xE2eq(XvmTe9c8A8GUMtpCRCZDV`xxOm=ywllT!4qIRU)*Cz9Ww!2gULCu0 zP2ZU3R^(Of+k9qa{f%O){1snz?SH@T_Zz`(JL3(NzOS{;9=F;4I_`SyLdoLq_wAl- zzOu1*SN+YEi64d1e%sc`6n=ca?^DIUOWM3sT0Q*s9S*!ABRlinlz+cjT6Yv4-)8-$ zaE7qr6$`Gr7uJbC`7pU=bI*IXN8XhY%l^K(Q+B*~>%XwiKPKfJ<#?*Ax$=I7pYr_u zrcv)U{ykZ{(Er|gGhu_EDoxKliu-=El<%zOU38`_+xq*tFZbsN-F-h(*+--Fx#7I` z>TSO!#L3Qo|9AsiWFTJFhU$bl9-nh?MZ$rON ze!0G89{>M@j&sM}jLv!9 zo3ADxPX4=R=Eq(8^taYu-|=U@T=d&t`|e(z{FXCIWOJ3y>d>Y|SHAhn|NH&?;`%hv zY5&XX(>0@vCae|yyZX0#^H$sNK%2i4l8q)+FD|VY4zu_*mGjibNd2$tuEpLlP+t7% z$A9T?%jMFk^|dFj#$MU`(qGqiO_W?*oY4EsuI|OW(7#pMiAveof7hP=TAnp`q1Rv4 z>1WH<2CYA}C+KU;*Nh7}HNsP_e7^U)l>6Ml-aAv~Y5uBGT=%JR>le{E6O_~9a*lpF zcYnEDCa;kGoz%lYn`%5Sf8LaO@a~>d-2QHXm}iswJxjFkNU0dy?(kY%g)**B3#GU zt>rxRVr$m!)mw{Y)=d?9_UQVOubFawI(NiBuSwec#O=>%`wagtHz#a#eEnB9 zPCfHGYzKZdH9xqWvFC}%;+eMHe-f7j$s|2?JN-p)`6{j!d0l{l3%9D=z+PhoH1+CSQ}~reH;V$Kaexx)!%hd~DWz`?uF=TV`m& z@mXIJ-4?Ad<}i``=fB#@*O7O7xcG{5?{}|UsUyPwdv#HuqlS4;)?sH6voOKTvqe3I zulM{Zzk03ZYSs60|6fN1S6Sw(|Cv8mG(bf{^yS~Yk1otLB0U^gZd{UIHW=vleiOTQ zH}%w?c%zrg>MuLRPdfQ@$=oKV)P)b3R>?KCEPgJ@T&Jp|Dz_s#&}p5hmxfsXYi$*Q zFFf1MF7NDaOB9e_emms%{vAo5Cz(A~ab3&(L-%ptv<(v@`z`FAxSq6~`=|7<`0RaI zQvLM-*FCz#jF+ydd$dyP_095{iyZIY@7KR_B%LKKMf_=GfX2Jq+x4>Nn3UU?_SOHH z_wClCBK<2VOOBrWGwV@++wbG2{?Frmlagg7{o5yU>+F5+^S)dZND~WtypA_<{?EG3 zr>@Bx{4!o8*(d3gm_)ymv|RA$m*45~qw+Gv_UAY=W@W8;zGbJq&(=NjOmj6t&R@K> zW7eJNXX~OP|F7t^%4Ay+STrm9Qpm!H*YDqoy}9zNxyt_TlTda?``!**6{B*Ea6?oxUymq_D+5qutYQf$7|MY1UdbU-|a!&2c`4x*MZ5NY#Y!x== zhwGv6eLpo;KH2(GJ#KZ}nlNwo?xUp~zjHT)Se^R#$>q!BZOn@v`E z9?-j{LL)woN3qlJXKE>*{-ma?UEJ?KUvC#)`M)*y&dg9}jZHmFyd6J}^~`WO@^8kk z@Y=igy)zGbUYe@s5@r${^gJ=hAXTNe`9S>^_Rryy7JTJl^pUyi%mC3m23)H zw$yh1`S=CW0_S&`^@aY?TlDnnt4C_;=f6I={AXGD+rRu1cFvbOGS56x^6Y(s1eGv5 z73&o%o`wV*s(pQ={p90YbJU7;q#ilXUs2Yl&fliHzCCSe>{qb|6Z{?@T6uiIjnnU? zdam~HG^fmXDzg8}*L1Ueoq`F+uT09=boHlA_lfK3Z`S>LvmtycZ(QSix!X(FfHqe{pj?(pY8+eX7a-rOlE$=V*V;yg1h<-+!G8 z{kW&@`(d}b(3?{~?%vuLyIXJ1)9{|}pK9_$U&)x>c;)%<$UMGn26nt_1ByKOw*E+s zoXzz2dXt?|3*VkkC*S_%=*g31JH&eD(D5s?xD5%)W6)4yDzfu{mvQl z@g@75bMg0U^CqM$JGUd(>)$TRq#v0#ZVRv4?=^Sp=B$mbtY7u-zkbNSz55rvFMV?CUGm$iCM^l|PU|}_ z4_b5^AD@|EJ7K1W@1)vX`=|MOuC@Qe??sws{GELK#r5*quc>?Q$LqgX_bYx<+M;Po zYhA*57nRq}IsMEi_my|ej_m38i>`0_oAO|H+0UXwyS5g-5u1AJ^nB@6n}61%ZK{Zk zuiRx`xA)igZF|qHsVsfr*;QG+HjIDu%YvUa{Y^#5JO4ZBpF8!__Wz^YqPOv1Uu{v= z`@q1!Vd*%_W0DqY>5KTfy}!QxdcAtASGUtfK7_4T^0*L&Ap-Ph+{EG0ARw6bqWoV|WX!{Tbm!sa(G8Qqy` zAJ^>QGm9;^-tGGCV(vEo=Lze1N}8GG-tK#zcd&i>bl#(>4`%PUt#x%v`YsI;L>qC|*n!oG1 zSzpiV|GmJ}Y|X|0pC^8BnX_SDY0UgZ-IXM^%thCj{EXItZc9Onjp`j@A~U@|E=01Fe_MEJ}fuNK74)Yznw)TdsP1DhE2Zm z(tLer_L}+oU$YlQo&FO1zc9&2%8-vecVV{vl%Iz;s+H6CY<>Il;o)SbQt602LVusx?YAx2FlU*kW}Cm_Bdyi54vN0FlWMlw z8eH6Yv1b48w$ptkwUxaR+pP7hr=GOS5xal-)$i8MVxbnR_yf`FdrV5RuZR^M$f-Dz zvuU-%%Yed5@u`3O1CE_Lo?^=5`fqywwh1qnF5m3Zv(>x${N$r+Z*wG=2=|;)HJjHl z^QTqym%2~?{3h+KTDf|be|X5BYyNCS;hO6!?Y7K2Y&6UA6jNv_XXN+)>P@>IteBG~ zbh*I)t(x3*?Z5AGUfAEx^l0?=<(szuUSLP z@AcJ`SKNQuGcBn0d7Xfg!DEw96ZwBKt{0PwyfzBPTer5%h7KlW3}XBw}ti`KV3Jvq%Ic~0R( z=7cwu+f!dpNN96C^8eVksk|4`jeDex=UmBqG%2l>w{Y{TtFjt3l?G2Y{Bc$ocyv2{|zCLmtbkh^t_bpN0{lWi&FV7*htjq!qRk2TTQWMWN7?_9s-B5p{^|*>!^1=Hu_f9@m zy)4`IOXZ9A$I8!ZN;3YKu6_Aq|C`U<%`eXA*r8jcu==mp<0O~6 zCH?)DcDp2M_Ti5A>R)-M>$3YyT7M*HV*Evq`AwhmKD_!TzGeS$t|=O`N?CV2sf$A8_PI;Z>3O14>b(fNPl)fZo${L_Dl;q)WYlaANC{;t)p+*f^d^2x^?mbN9U zK2-?6{&{ZOscCio??1Tm9^(1^PiE277A8OD)boKs^#%9sB9cAk|9KjJc%wtP_0lSH-`)Fl)(@NRHKu{-(ZIe}!ZBe3{cv-cLBSPrs){$hLQn>Z}J(=bpOt z`p!~$;aNBG(+m$6Zcg7b-}Y&jzJ94^{|cj^YTNs#6pVkF7bh%xzh6f{!fJlp&D`C^ z>)KmNIxnUEHcMGJ@3{KjI?<;}fopH3*KWIQIMXJ4xvc%C6$QU$-khu%dZ9`p!8vnI zdRt?Mv*zAvq4shM{cn?d9xJrfiTGG_Jic=*-6gy8=M29W6R${pWP0-J`eD`BD@yB(^uI?bQ2?pI?>U zP&s^ZChO)6cY|XC3vb+P`jz=V@b%;6^WCr5cJpn2d}+PIj>-_b z-y3eSsMgvpDx0>^dW)HikXhpwb=MZ%!zL6DkZ{LFn|NrMZmEKU-(YM%l^`FG#Tp!O@8jc3Zcb}f_UT%Lw)zZHD z_LS@Mf2Zl*{i^6w*Ro~8`PzSbbL|!7QW8@Bo=Cd5&;P7<&HTFPCtv>a$!@v#_{d`Y z=KSE5k&|!gZN7eEv1Z?V+j0}mV=SWFi;M3lUy?Knc*`eoOS!tf>F!(k8`Jiy9MiJ> z{r)t|?EBj|yQ1fOls$dQaq^bE7wtYo>i+m^(f?HM-TJ*#7azUf^W&~l?fS4)QHv|? zdd22WaDBS8eA3M`@ypMrUr5}3ODw=S_V%3baz>N6f4SC&m;dX&yT01{{^}2&>$ZIQ zX)W%wTm9z#>1riUpM3iN{qg4V`i1&x#=rJ#zZ(}?*x^S%FH3w5B zYr<>e!=T&v- z^7~BvHFEbCea+hY9U%gKb5}Nwmn`mZPTNmnbO8@8Qw-bxuLf> z#WF1-h2FF;-!3+<+F7^b=;c#M+wCOgCF}mXAD<`bn)&rdg1?jPzpkT`um9h* z_04?wzyJ33{j6J4svW+%+|>L1_1$rArSvwRn{srI@ti;N%9r|E>}9qQtu8{Z$w0yT9J|XR21r-hX@VPrfMo z{nypI7e(t=|G)oScT<)9ivi|#{9Vnz9{+b=EwfD}{M*j^YeGA(3dVo_w)fqd(0@Ct=e+qZb?%YTh8ycVxTSMG zJKp~5xx!%I_a4E7_8oQmGqXfy)G;o}6RIs16gU%nOWx^TyQ(NlqmH7>BX{9~yboLM z&cAYTPU3L})7Y^lp2-6dd)XezyrRm2y{jvmk zpZUcPvpF?&)+M;CC`_faIy8yce4uJSMu+ZHLLi(<4mUSN<1=`v`w*U zO|&>&^=G#JrOy{8+D55G7V91{xN!e_^|WS#gi4#=N1T?q_A@NbN%E4kkFcj2x7 zMo6`%FOUE1Te(j_R9L%&If)u0N!2TesjSvz?dMgwrZMueb|$-ryBx+;RE0 zf3e%NgVxQT971;$PVSrEyE5Qy^Sv5Nb;FsV_KP*)BfMJzQ7^%%{=qZ{0-^VGmTv@b4I*on)XuU)WP!%tFM?XE(xkh_A(5p z;;6Ujc5In{>C>~N$GzKAnKat64=65kyJ7icHKSdbe9xQG@(Z@5pK53C)`)77{hFnk zr#0oo{^iZfHCMcv{33cMQ$c`>LSkIq-It+HckE(|Uf#2%t;lZM<(TIaRaeyhZGFY{>dH&eYW3L6`)WN?vz_D*Om~XU zm{OwmT%vVyo7`z*j@@yJ) zTjAxkjn^@^dBS7HfByGBB<6mepRkZmbaQlCGFxqG;8e{i-}Z@3`DOShdDcVC8)ipk z><;v&cH6~2_UsQ()_ylP{K1?X=3*N1nRA5aIT4}R8qF)~8b7?#t%-bkEB#s1j@;=^r&BXNek`#1)Op~_(&YBTyS7Vy)oRb~ zm@!L3t!KuL=1I$c{oeaww#|>jF==J?S8U}k*vx;-C#5(o`nh1-m1Fbga2QD3`|Zel zaQ-XNhsz>l|3xo2Q}gzB{CfBOZjJ}uulX}$-Cpsfn}j`e_Qr(lI{AB7mdVuHi7O8$ zK4w~fgE>9w`s1}%|IA!wFk$EXduuzJ^cidKUFbga&;Cl)J-sLFK|$YM57(lTDvXz$X(edpuP zE?&0xoN8Zg*pGb1lh3nP9?`#$CmSwYck%T@yZd(R->YI$R``4P+y6HVOIpRp!+)fu zBFRTroGnrDQ|;;>6|0YNpUpTfoKYYbW-32#(u2G87t8NY|F6V*UoY|86y=R`?N_L{ z^s=9vw^uF5yS`m%`|m$FN9WEq=5~)hSfJXPdMWeY(iU<41zEPjzgkWv9G1*)Zs$UZ{*Z-flVE>Lvjc*gY z7#Pc^9}WyyY1$RiyZPkT-_4BTmk*_hEIU2*)mizg5&BYGAsgG~vA%cftqs=~XYY#- zxz3{Y@Yw1+gO@K4-!pKzCUjv@%9rqc`QEMTr9bP;pFOS2Ykk<>R^J4-vkf0V6qL$7 zPS;uQaQ1NTyl=ASk}L8P!lzH&b@0@U<^R*QdN}7gZvItrn60Ntdfs`dCO!#}5QP)X4W z;*(09vDda%O7-#v`~GgWk`zN*?~cX&OMLF}|0{3tF|Cpkyi$|TV*lI1K=!%!{*?#i zR||bMQ-3_iKzg0glvC6HxSL9d1wGxj{m{m{D}}{(-(F)g*ZxPrf|I^pzROh7mVS9v zqtorh5+>W!w0!9_^YFEy*VlW#zRp=!{>!{D>reUQ^GgqhAJ)0@V2Y!xk$Zi5%kM@; zZk_&^hnjXz-a3j2TjXEf_gwwD`lb9GaVOd{E+|h-``B8+_v+NYhdVN-Eq?vs?bm-> zH&5fdv{wGcJC#1ZGgDiq_}LwGWsy=oF#qn!OLyy4wAH)5CQQhSSsHK9Q&XT;-uHOj zZl(;Q^1ugq1T=-)Qgk?>@X%;>GhLrQtUw8r_o&wm;EWxmZ`` zl;W$jr2(_`uc&{xcW=iE!Ku&pmcLhgkPt=DqU2B(=ap?fvPRsrR$)t-d}zJoL@_O+309x|KH+cC*d%s%h!$e!4AF zRekf#A1Pa`*DsnC7aVsm?Nyl3<9n00g{J(z|MhR>iT%7PpPzjDy>wdq^bgbf`M>Y` zTJ9(=Kkv8b%_)1|`%hYb#sAR_<=uZHjzr~ew_UtFep#~h&uZt$w^#38%h%WXp7lRY zbs_)r$)#ETRY%TW_fz`4_cwRR{mk`=M>QX3y4_oE8>X=T>%{8|OHbcAHA`vVPg{Pm ze~W&<4~c6lH&tI)`)g~}^{u|5-|I8x{<6LQbMLp6zb&ml>iygOR%Yeh>oxvXCb7S= z?>zrj8~^g4_#OrZE+fZTo|Cdz%g_ElQu4yzH~iMqQtg^or}kf6ecd){|Ld!-*DcMo z+-EC#@4so4dhOqJv5%(SUngsHYUgyVa@pXkUccjQ&p#3l-JjT$I5Tv)Z2F_$DwkEZ z$8}x4Z?j)t@Z+cWRZrizO}S%wIM;kLhugLKl`l-)YOeDx;`y(>cj_|Jwq>Q-QY~Au zf7*oamQFaZ_^|bM_Nkrv{dT(!nQ=~UlUZM%*cWsBEWu z=*&a+n4_2PSKb!2B>eO{-M%I<;fhqzP`&u9tF@<({P`zl@3$uQd(5iuuZ~Q*x_VB_ zybafCKU^;PckTbHlP}BvckTX}xH`0ahuXZuIoo8){N_|I%d7w zdU$JQ@ae1PBzL|4#^aQ}df!UhtiRd&en-7qx&E5g?rF>8Pkx>LxVF7hqBeg{)6)a* zx8z2)P5i%Ot-{hfPoyn8Wv_0j?%z^-YJbPPi&4?LCtY24Ez~sYZ_5r9HRerEd+g3m zyQj0L;MJ{f`@czxSj5&AmH)PX%l%^Rq;>08UyYd`;vT=c^z!Lk!PfIlH9J4Adpx!G zM5Ow~rT>^EQ;Lfj7khfvRoF~Arn*%8(hj{D8{?*$l;u7Xd#cW8a4OE~;^`ktZudM^ z{>A0<;gU2rcj+Y=#jvQR46~Vh-ge2)%M$Z`zwWrP?qbpN=W)~C&+?eMX5P$5O>T3( z#UHNd_p&_bEB0%(+qY|**+g@fIrF5q-O$wHusdCKRh&P{aHgN$2WQ^A*=x_QKeXho zq@RJS!j@@s*>5KMiS7S0p)Z;z*hl5I+-DR}Yu{-XJY4Z^(~Fm0QuS# zN;w%DBQ8kw33DjaB%T%D*R5p1&m^9q9mURAlb=vA>ykp+4wr*B8oJ-c`tnRu51JJ4 zQTpZSgr@w%Xn6>)j8h-tBXAV}J6wyzbrGi8^=YajU&pzw0EAw4FhX=o8m3 zkJtP8-~D5+aH6VyLF=|$kL6+UZXXX+$!<;9RZ%u+;Q^C~n}WALgddYyq1^mVOY{1* zH){kZ?2#9e@_DYOE&k+3V8%}S`hXynC-)p5u84|1FmcEG&qY7ZWt1u!3q;<$pmoD6 zX}RO6qkLgn-alq$>R&u@<>WdGLf56^skA9PUn z;r)+0zC4KR4`z;^Zq_X-e0}}$KXb0?7GKx9H&OfDw%=R=jSa0yWlvT-eO5istl~$m zSgNXv`Wsz_!1W)@_s{RU$M(9eCY=3*gt!Lp%*hD{WiRfVbNivs3i(a)%JDzf7m2Yx z;o(?z!PUw9-pBY4PP&udoqY5AnV=rug1BFkPikL@$>_A66t8}5&)klEpY1o7XB|?D znEd@n8}}h`(YP%>nbY6$&&=i%m$H0Q`2PGiE?(Ka>z!8xE>vf+3zI!3trzq9NlpLG z-%;yhR_y<`HEwHo+}gPAyCJ7M^sMLK`=JwI%*ns^jZwgh)}0|m`-7V*C(cTH_hjK7 zfvO98r!^FGHs8w-JDk$r;lJX?grX|(6&*9?DnI_TeZxwpcYjMm`4|6x#S*<r({7u z|FcWf+EGwt8})5bK4C$%ECGg

  • n|w{dDR073Ful7b=k|&N9)~arf=>7fvsJ<+3;Ncu7Ujl?|tqXU$w;`hM;y^W6#Gc(|7o z3%+;}+ut{}ec>8|uESfxxvGx5zP4sHe!Cam-G_CD?(~}>&u1suPS}6A_^zqJ&Q*u6O{Eu)lX-+JCpi7NZzTc=+yz5Y<^y88RDKG&`(@Bck=Qi=O~@Ndfz?X$b`5XO zj5?_F^`m~lltXVGX-}WL?dJXsjoWy*m>d$XIQZrL{P=fvdBT%_d7G*^IAeYYInpLQeT_n|fV^l`B4&DIWafa3G3v?;ZQSXQ!tl) zj$qjIZQo?n<;=7H6nreSXk1^ixL$AhvDj0|g1Pf7KJ8rh=Jd(e`;-h4i=PQ8G`!>f zG2L%o{K*eLv%EQ`e`Q(Z{+F@fv2Jbe&Lj?joxb*Lx1Edseu}TQ-x6ECY{Sghd+$#^ ztzG|mdi~l;i*+GkOM&?w0zMk^#-r?E1{lCs? zkA8T1Q;kHGYFqiIib;Rlw%gtM`{B|2Pt~XEZyKnsd-!jEN8j(uyNq~luH>wKUXs6G zT<*)i8qZI1U&5~C-hK1hb=gU6apBUs7t!C=yv;uQsq*L7`gD=S9rnK!lNYO>zNMS{ z>!Doe^u10Q&XP8^|LbbIq8_e(x_keHNkQ@JPjo%pZGYBf=~EV?w2y@zUD5N_M%q64 z_5Y)MAya?y|F5PuSDsjM<$nX{x+yI&F3H1;6D}n9i1KhYJxtg!K}$qvV$(v0&eoc` zto8eh%VKr+aYm{NT?&1XrEm25N59kG|L^VmzvgGjC&q;zf11}@If-G0{OPCnRa)Ns zj#~e^Z_NbB4(m7jR_VVxtIb(kb8B|`kw$5q9|xB{_Q^6Sv2eWcW_k5L^UNoYZ@nux zYH{}OHP(I254SnV{fXu7Xxn^K@Y??M&As8wHlZii=RG|qWMf>jsQ7F8s_%>c=0E3t z%6@*yNlQ)%kyqmPm;CR!r@!O<1NG1~D*fE?%A$>n@LjCosRv8_thn>H_g0js&e&X)@%-!iKl{)B zi<3@$Jy9a}opyBD-}Ixu4qqtbE#By*+s=Lev|f2^Yi=WJm2Kd)^0McT|Lp#Gd;XpO%zF-%2=Pu(ygN&=^|!i#^9r3t<~i%H2*1_hdiv_s z{zYX9M^5yHY?aYjzvS$eh6ZCLr=|Ikt-NO2YHC6)M^vk{itkI#so1qQDO>DurG(d&&V{>P`x);1c>UOZ{hF)t(lPcS_1*bW z?p1gGP0la){L5f<>(&6p>3c1Hgv|WI|5*Oe=XvY5SooP-UtJsh-}JG~m%4d;@)y5- z*|))>Qe*FRt+Y9e(~lTGQdq}-`Ay3C1@rYZ9F}cxa6h3c@%QBazTLm=KkRG&k@3}d z(pq7K0IpSEJeosYe(zFBZki=0HGk8t`j;0g>#Sb>e}1Ir*}+H;SHI~?wQb*) zdmr2NwcvC@0qE{KHxcPJrb7)I5^_p}SzQ_nR;1{-a!fsIx37HP_PDyz>1uDaEY>bf zv|1_4SN!?^)r@tyKiLnwx1Fvh)+0T|_=3IWXPqq}uU74-FZ%LAJ#I5k=)B&d_zRY&QkK|e|9e$^!E1YCEbk*B z>5}7SS97NPv;27LW60;eSC_t@s)wjj4xb}i zE{X0s)UmDh#?4;yNxhW^)ITw>U*$|&W7D2nrB=MaX0wKP<_m)h-tQcGK9}}}w?2P= zfxl#u_f?kkjq6t)mu)=2s30<-$C;ax>9E6w1P?JT)~43v9Ue;VJ#+FauYHcaa#e3> ztm(7B7# zUD@#><_T)`(Ki{hs!z|AF@ITCU0f3wC?j@b-wWop->LcczwUhBboarBW7Y0|njiMR z`@8Q=W$o{;m)|UUvqQ3(LsE)mg+tlN_+|OeDo^Mh(b{+P%lYWfht|g)dAQG4?7Mh^ z_k$loc3vMA_Z=P+;g(`L>bN05OGKKBoDsa+{2Uq)?S8iK>{_pX|f8y77YP_;P&%Ud7-~0CtN9HZLy}O0u^S<2;->Nh3 zrmb>XsdZb`v~s3TVlicE;^FmMWjr!NlcW+B6=K+S@=lu1APhNHnWA^S5BYr zbt2_`!#ieU)@GX&(N3GQhNTfLOCrmk+CN^`?RdI0_;#;pajyrXUiqn{gL@} zz0~TO?{Qm}eJc55rTr-O)-A7>2Mr1s7}&Ns&hirKVy$0T80#H-eLZO3sOg%R?S8xd z@Bjb*-~Zo1_v1fjo%Fj?HNb`ai?; z=DvW>Th_JzQY*A^axDB7|Iy-QbY^YQ{xJKvUFKg`EwYb{{kE=dxqoTV((o_pS1i1= z*B=kC+bUSwyOOuMt)R2*)E=oNhwJ|`d0XFzS*KL^ragXX)vwx!lvO>Qvf5MqukHUE zmAyY|b<~foRe$B9o_zcIskZvTv>VU&+t^Nin_uR_`+lzck}cjRZ(YwW-dZ18fA#$2 zhZ;LW`*~8k;^RZ#SM5|^bya`YmN`DtZGW!?t-0CXWRw@mA7oqpOW!w4Gwb&pr!DUn zrC$3h72khqZ?DCn^n$p+x>?`;27dTk{bm%i9qGAZ=#lTWr)TXy}LF~?xjlYjRM?tIwA*y2&H?>41m z{okO^I|Wz8?R#$3cW#^E{|_R7De81kBFO}tb zbN0B3|KFeG`8!@PB`unZ09eq{m{dxhr1I$e`mdR@&8H> z_D>uN{~X!x%-;WSp)SL*hswT*GY=g4zw^Wwnf?C{{l4%tSlV&TX>Wer(vzQSP6vzs zuk-u;rFq%}qoW7ETTaoF3=H^ZGH1gSQ_c@zo7$MGPQ@Ir3l&yaxQlg>*G|XJ?dE%a z+Pz%QJxScn@3+?(t7mb}|5@*v&0M&}!07${;7^++JkCElbba~L#E3;d>aTyCes5LZ zQO^BZGfqCbcKyRef9_As)e++V-gFiE|IuzsztvcAvF-B7r#0zn6BqtBFQ5MRifHn_ zzsqgyw|I+m<5ew;s7_VvY_yhYU~-~MH1;<7e=`ll!E z-I4P=R#WCb{dKBDBTaQ$i#3l_Ma7RfS1k6qtN&MTUtQ+^DBSwmy1rj`!}$-jMV~C% zSbJD2jeoE4^%Vs*AFfY2w`hv3{r4NoR>r@+X?y#tV0GAB@rPL~-|sBbe!OJgGUlJJ z-#T);aow8s-?jR7+)u6R>#r|aDZlH_tquNm)qC$o+dQ6<^O4cQ|EJCB7=h{2Z(iTF z>BB3%r+*?VukbS8T6lbgo&Ob!#92As_QrvCCZrU6@`!mSt?_1lyp`0I{!7ZQCboR4 z-1dmSZTl3J#KhSdGBI~~{@zV=OD(bbsAhPYTmODR_tfl1JDc7#vFo!t|7n`%aQ(T| z{uUnL`UgD+zs$0G_)$cA>5af{0=go4=_Hheq-@PxdIeK<|twd%^{$&=Qg>on+anbF{qBj~T-qX0(o<(yzwy*dI`(4L8)cbg=kK0;mwdI?_x}vH^r`bF z^Onm$;+^7G^x5FfTCW(>Yh0pD2Iu~QN=iPTX z{q=QyJ6D)S=|_p0_=Gj0_ki+P6c`4_6NsXqUS@eTQm|18UD zjb}VPUNgTsYt=D-+rOfH<#*q0e;@vjUlqCEn|kl+{i!uo>-M^SinrCiU#`DU&of&zZ12;zwq^To zZCn5K>)ML-7nzgRz143EJN~Wv`m}z%(EnL~zd7yw{e8vczR0f{|JUuGe_{K(kj?dr z$}d^wbG%*hZ;8p5{dE`5ai0%a89!Ok_Wz&IR22pW)~}AUye6$;tzVe`>#O+x*VjO& zO-6O?+S>ee^`p?ASO13gubX~<{i?NDanrRg%+$*dxBYu@jo#<$xB7T4E)VPbnms-H z&Zn5|WlQ%*towWQ{ngba?kOMY)$h!`pY_x3dw7g_h4S9_lMmMZHuwAzvv_~#qNQ)E zx8B}b%$c`8KWtfx{buP;X()L;Frtz46tnM|2lb?R6dcUZ;^XvCl|E9k>{#x%;PTrws_hKX5ch8k#tzL3&*&WS=t{t!c z3NkV-v#_w@z7$_4{L?gJ{pIYhtF~r+E&5xPAFjXj^x`s?z|Lc}8~oda1Y>vFpDX;= z(pVJZU1s36D=YiO6@fDor<&OqRs7yRjqAf|Pi=?O;lDVRX_WRDB~B4ee0Flt4M&ya zf9(^sr6>LTF>lgN>sRZY)|-o~eoGE8+LU@ZSS0FjS#0#D^YMIkf4iRQB_veZAK$4s zWz%2gA58+S+fL1X>Z!idzA4IVkxEwkn}KRsXdk(I~W(RoW%f0h5@!oA1kW9{PJcg&s?y|q4}x%{-Cj#;llvWkZH&kT#q z8y<7sXzfpGxhNIF_P)|qWdECESw8*=+a|rQSu>wg@4ad>6)Fn-u^Pbv&W8ibnkubCm>hIs8}|0`rW(7XLN4=p}1JO zXB$t_^XD<^1)uIweX;xLr0aWprFSoR+P?1g0*&NOekdaqjMD*y7s){2vRj($4$ zg*`|kp`D+{^!PJjBVciz9aHS@2&I$8GK`R}`5Tl6wW z^77^O4cn*oR4$H}dB@USoVBrTXUK}Zk)ro&7hSJ?^CIBI_4Oy5=Y(Hhyx`_Bf$XA% zyW7G(|64G#jWPWAk!N-s^-1sdJvF@1{Z{wY?+ZVTp6_+4$f%2cx>K+9_gk)aCwZ2; zS9Kq<-urQzBVV)1uiQhjPOJVd)K^otH$Shwx-ZJGTYYN$l+!a5u7tOop4cV*IH@p% z<0%KTg6GqFP5%y;&viZD$9>*dsAl@pCaZmiYri?gsxUAARFNNfUi_Gw*DI%c{_MV2 zJo@#UazxV8a7y0Vq?XOp+`JSD3ana{f>g6|7PW$$~yvNZh zU^XFBkI$j}rdOExrp*!s0d{7cm-ANGHH=CA&J^7&IMOdJ<> zym>wMY5nb~a(gVc%db(qbM1w|vQnd!?>F=mxU8`-Fy1)J{cPR5ui{QhzK_0LNPi=x z!lRgY%xIEaa^T-dP8^T^w&tCg@y%_+d-0z(wn3@qo}WHs=QFMQ>g%g@QJcKqt>1UZ zZIbjcw^UuB>fNvJC(Nr{z2o@l;$DMW^KN~=k*{K5-?peGDfV08n%b3HZNE7NZ+ttk z;bTi|OZw-Z1v~xwe|idM?7gL5CTc2aJm0kP;h*^0Pu;rbXPJKs+H3ynRGk0CgrC*H z%gc_Ru{KNjnf-oS=+gV&Vy^7_ee%`Uhec1nZ=e3*&8y3&vTwZ+6kMtK*8T7IU;1Th z?AaMeec+-7j5f*fBJ2) zM$0Q%iRs46i~AE7y*qZw@bnD*`?cQF^Q+$&@4f!n=gZ{htLlzS<_R{_*mmjsYTG*N z>HEt|*5C6s{9gP0?p>}aVSB~3e%c28p7iG4d)Hl((`&!0ertVyJ?i7u^>+U3SAYL4 zK3)1cZnaqa|8@2AKIfgBGx`2?y_f%E{L4%EJ~-yco5I}TU)m`Ze#qK^w%QSb-tIs*1x{K`(@Ssu*ui-ujyZ%{$g&RY0av`mTzoV zDK6Qb@%H}fRa?8ap4ywLd)?@2{Hx=q>(+#?TXn|S`_hsoS?L?kS>}A%JoWLR)fayk zISCuj@z`H}wtBu!&i-oo+EZ51Yr^k~w@kTPJJEK(dCbb3pcoy#nrt(#gF`3w%_g0>HpuOY~#I-XWcb_z0$Pj>-6mStn1H}j!*yf zC2Cq}x9q1i+ToMttxcMJo&W0j>#Kghjo-KQaM_C%?@woK*LQV`ayhaU@uqz14*Ie& z<;(w{?@KFflinunkNcbY>uc4{xD7My_kVpdW9AIAl=HUfCcOSPRE|5j&5QoZV%YQO zlmz#+1&7rqzm+%Dk?^_v=jqPy3re?iV}q)We%X8f)!qB6_lM0ofBYfeoZp=x8M|4E zn;6dSFsk{#`~8ORv!>^lm`pz$9L8<(t4V10{-~)Hn}p}O{kHwjA;)xK+Zyvb_kY)Rz{mscF|da;~oWXXcu#^6P5p^~55h!x#0ltIkJi{^mW;V(Zq~Td_Oi`TLEJ z?!9;U8S&)bSAF%y*MFNH+0C?1-aN7Jj?J~asonE-Tyd>9+n@3G_FN|QlTUVS6W>r7 z5_fFnv`HJEe0^8BHtuMR>)E?+;y2IY=8atcrv2`dAN2?8%=j$2y3Vh*Tr~aU)7Rgm zc3sl_aYJ)KOv>*I_gC%_o@6y+x=h51*c&OY*;Lt=Uf$ru|K;DEdquOSfAaLyuWj;u zH(UKo{>3B9_q^|0l)hp0i>W2^Hq@>Cb^eyJc<+ z7rteM()NalN9L#heip@CK52}vJh^$Uak#!o5T(&yY+?_A7M^XSK)HFY7T3mohM67Syq zCf6KrsZe91aguSB&^_Z{#uksvp1k-hUw-q5>yJzMo0s_JyfF$>xH3PxVAI*gS~Clk zVx1a~?B6CmYVTj)JU7Q?M@YeT-KLBFRfPxjbLM5I?8sT1F#mP`(&H9Myf5W0Zk_yO z>k((S^w**BH~C$D?XgfdoE~tx@c-Jv-NKs}DYCcCdw+`Ynn}ijgyVM~Z5Ha|GjC1z z^E{fBTDW`u9q;zGy&t?+nTSX_p8Wbp>9<&3z`^a?O6B%PNZbn6*f_b=+vNX3!x%q* zp-(P8iueAQs0ehI-&f!D)2eQb5RYdOw|iTa^ug~vo%RvRT70fDCSfwhyBNRf=yYxW z%bn)GZs%dQq84VQ*4~&){|*W!&a9h%df#N%6a$xCGoIUJXzK*O`zj;w&q~EU(J$xW zZs#EDpSK@Q{%Lob<5;V5tNz4*+j=db%+LC!Z~Po|F=Gk0;i2w=+@90dA8QycRy`yv zyIFRzv)`BRS=}=8_1B-Xi}~bu`~39NOQvdU%aA@<6#G!t`-Y;(*YDM(m!3swx@M@& znqZ)NXI|)}g;|YV@0YyZaIwY5zkl-CtXnDx5ADuuaOYj~zrOb5ilUjt@iQ|;f@Zb7 zXN~_`vn6fWbM=q2-^tiUJL`UK>GpL0S7b9mW<%?u+Rv?z7b%^utFK$v^t$`6<^Cq8 zt;Vy9W0Q5>x0mbsy1Zq3_c*1$!|0>p%uhd0URw30S~mLg<~M!2${zI0c$D_(>H9SQ z+0Q=ozu5Xq-?Oy(Q}v=+?q^R#0`?{RPw3yaG3C$oLl+wNpTD%czQ2EICO0SlQrot! z_p3z=?whl-#?AiwtL^u$e=XActM2_53Kgu2%AXs*dD1@VcenbrYSzyZbAF!p({|gQ zrU(7y_K*Hu{dtUQ_cKe)9?{IaH@PQEQr5{Vu`K+(cjEqURg;7NPkLXz|7hO(^7@53 ze+1uGJ=&Xgv&Gr#`7d$No7=1QUET3@o&EYv96Vb~_61zj&z#?`JALw_@F4HaaS*@L@e(HDkt0^-tpQ?0~eY#ZctH76cJaaT{_Nq17 z_Xa&n_MFuA<)Z3MgWd1XY^r&3`gP7`_dj!|{n`8Jy?e!7-Qw#P67OBU{h)vMj`K15R26qEt-YZnDs}vHVSUZ2Kf7f* zgucyvUoZHVGt=-PKga*@w+YWze0`O*w&?TZ>J|Tj1e;~!%R)93E7|P5vNBNQdid!Z ziV}Q#{+{D}-*|8L;+>Zn7}%aW&hno0ina7bY&t&=m@(cC{qn(MM_K-kb#M0G{d2YM z=;G^_uGSXHe%pF(>#vRKCim8sN2RC*3!nPEVU2fZ%)SF^yNVavzx}E9KeRYhL-B|? z%UbbJtNj_5c*W-Xh~{g2i<>HaUwx%{7I)r{%2~%&$<6-d_x^2MVEi#JtH_Iiwpsq+ z9n<~gTFw0V|HfvmnZ3V0#D6@b?bi}RawTtz0|kfwa7nw-xC?9ZCA6!Utj)M^jx_7^VNTXf_>Ffn#+X@_2S)A@9$b3A2s>f zuK3P*zn@1~>DR9>ty}x)r&Xbfm~V^O^^j9#EvNS#lvvC^?Ptu9@46>nybSw0>BYXf ztFOD??^?b7(e)McYo=Y;ly~y2v>lga&YC;3C+wG85NP_;MegNVBl@+np3A4L~^ z{kD8>_>%n{>m&VU%9uYr_b=B_((a4mDzWq$yK|p<6l`_V7VdG|GV%GF%ewYAUrFp~ zuDffn>~7`PS3(;6*G<^hEK2?efHq5+g6jrr+@mtBcMMep&`MQhczp5 z@%Cpg7kMxIAHB&*PSASA@oK{}f7@DizkdAt!qeSFUnaMH?UPoky%wgvb)uO}%u$Kg z;fwa(NfcTl7QVau-hYh*!VyOC(z8@XnQn1{B#q46W;Uk4|@ z`~P0OzpO-i(ah?y_aA%;PX<_+``$0rFH8A){yXOzo-N02mj6hL66+B0in;%5(j_aa zaGs}|;=sd*cR35UZv1{eZtwcvPv3v>TBr8s?w#4KXPp%sbU(B<|Nb?T$MR0`!Iyt@ zHKf_raP3h}-Zo`I()!b{zeqh%dUYdpX-4ALN3Qq2*Pm0Y$n~##|2{L%JOBUpyBF(> zKc-%Ae$aA_Pd2ZU{f~d`qA%Mv)~$WFHe7en*CS~X799Rd7X_Z%QK>WkjgyDMvnDG$ zo`W(ajSao?Ou~L`IC6`#r2OjoX9BK|Ip;svqV%qaulKA4?<@7l!;4jtdg zYg|9(sPd*>F5b8Q(Ov)S{z%sBMo#Nuy0H8*#Ohu_?(y=T7PueL4z#h-Om&DK~i z+PVMugx6aP#H_2&|9zj-Ir-$DdGDjYC0abaws}LJTJrNI_5JhCZ1Hp4yfoyLciC6F z)4OH%pM0pa`|a%;uT(DHZvAF8C{|NqAMUei)v{y+Qf z!pMq!@6LEPC9bmTUfS(fd1~K5Ii_UR4E-zh=cM)C?eCc~g?I1zqU-Y?G;z$(je6An zp}+R0&5QNtI9y}z#a`Kc?EdGtKaZ^UZohD;owet3wwP;Gz529A^|cdR@A)XdKl1zI z?~I+-t`^^!sO|sLrsw;Yti0&A_iO)E{ixMDUO#8kZ=*IQ12fI%ymx>2e{pW>tLt8T ziv7d_o9*>)L}uH+>wHsLe){zN>GzA`YrnN_jW2o|ySA`qpI(dg^yKyWiL>(eznUcc zE%v^D+v(j;e;lfJzVqL<=s zW#sC-m^GDe&K~tM)vA>f`u(~Z@i^NM(d!dLr>oq1{a&Kk;&fNcn-7ht+xcq!Or{)vy=v+|W1B@1#&=l! z(|&t4ef_6d@~~#+T|LpWyWM|2%xf%Kyy@QlbL&o-Mowkk=Nt2W-W*MX;l&Q<$;HU9qPAlUP1B3x}OO{oh*bVRr%kEr;Sp4++ldey4a zUfr{&M;z6>w(6ey+lu}d|6PCVZ;*;>-2b4qET?Mvp=W+|uTvJUT=wc>>Mir$+Pzbm zQ(7N7db(@oP5bwF%K3d67Vm%N>#MV$s`=z>%K!S-jtd;FyxF(^-uLflU7f~#c>n6_ zJwk$B8Nxgzo954!bPN74CwZHJn8bqjyM7x^4^O}I_}U!tcKiCdJsYS0-R`Pd_S*E? zl)aNP&tHsC%Afe`!kT-Me8;b;o|kOP6RD|;)tc8keeIm`{UMFD*0%ZjS}$fx@9=8+ z_;s(WL)5g*?52WU|NTCm`}#e~^Pey~qk%||Mwn1*J7dC*038i(PL_jC8xnLhxLJ>= zc)gze<;v4i&+xFG$J%_Q{;Jm}{_2%nUO8>S&#U(KH}ve#aG#={j{Z!bRm>lErF z9OO2;_wCd2j=EdSx#=YupBHLcIlPo+ZeY|CnXpWPyOn8Sib9A6XsZ69qrn3obAgUS ziayi6$8TR5(sDIq=CPMutFCkEJNsnk-zuJG{YUs)yyvg=d-qSBB7Z<#@6px#`bllG zSAU$UxG(R(BD>$OJoou-%w7ApGE=R0lF+J#U$a;9ZSPUN^h#Yl`~B7Xk)oxQFCNGE zsoq_mHh25^I~knXYqZ!msP3-&+2%GYb=&g<_Cf|Wu72I=$`fjH#}}c zd;MnZJ!Ly-MmO^XPu-14>)xaCPxyb|eU|I; zVn5PiEYIx9-u;Vh@|O#fY%;bTj+mWcZgB1BUc-}h%QrGk`T09!tN-?oyFRT^eUOx4 z68H60`N7(siDyiw$W|UT*W$=tmB?}N<^IJXsVfquU48#$Z>{~$Zv`sb*CeX&*+)P7 zS@>ba(qm4((jT-Lt{5sgrg$(guH9PB`7`OWzWWXrmRBZSCd=;~Ui#y&?V3F~`kYMv z=U;FTw&<9@X6Cs=lP#T>s_50Nof>whM6jLD;Q^zLNDnBc`yM)M63`J*W<7K?U`L3O zYvw+?(pmdL_Ds?eT9r51^oV}rA786EJ32qj?%5yTTB~? zEuT7l)zRa}>v=yd4GoOn5phN3xyiztjSU*-x7T|fl@&Gxwaay1PH|J~K* z6qp%)^^{uo>PeGdP0cLZlCypHZId4%`)m~+Fsg)1IOD-B$aL7rAi$@co8^#7)wH|$ zJ!>mo>jlLZ%=9Ps%dXw2|I+;7(}k+c*Um>xRu@hFcK*~a+y5Dp{y1;{ zC9he})U^KJ>Z?9W^F9Bs`Eu+3ywA4-JAS1<`JGeOf5_9~rBG3&55tms`u{Gd?T^T@ zaNtc3@BF%8qHIgaqsv;qgj*hb-qI}4ZMV>B<|@k==ZCG2uJCH9N{Ov|JG1YU#<7>L zxK01<;rq42kGGod#2&{AuJWG-_eEd46FzmVyh+x-Rxk7Q*00|}9CqveJjI|8pvda} z^5yk^jDa8bJ-xSNpGa zfAyGpb#>XU-BIhqcTc{iy>(aT-=DU>zK7q?Ff)oRcp~mEUKF44X)y^l}#-RHZ%l_$p8#|a)X7kD4{KK=XlbFQDIQfn6-y|sJy z&w^#kcfXh|dwj3ee&U7v(wyro3FeII2o(_HR;)@ zgPigP5#Lj0YJ)b>z7K!$<-g^08~Im%XLrxr_P)OEkbA$(UrCj9-qWvd`hQE^W_cwWEeG;C2 zt>>A$;Wo?pDZldX*M94~`)%*lU*GF>pVZybkIV7x7x=CG;nT%Cs^zKap>vt$aM`gk z-mh8C_atZjth;i1Os2)Zxqbgl^!JVh-tXVV-|B7q@BO~A`Rb*||E1T* zy^9T9v;Tis&#_Yn6BUhy~5RrAzRD^&EdI6TY4rK}t=HDc zy$^3PEWX&j<9?-WZ2cylqcy+QyC!c5TOT(mv&#MBrudR~9a&7}laHEVE&3d#d`nZeW+*dJzvwH%y!yISYud*|*{l;;gS1IskSs?Fw_jhUr zLbA^T&!3gs-#cB@eWG3e^!mjI&$X7XG_NaB-BP#SY4>&Y)<+e6%T8pK`t$k6X7k0~ zdd(6q-~T_!d#>_YA*+vy5uhz0#kCV3_TGJc< z@jRt|wovAF}X!DDhji=6~vjd%v$tf5~?Agx#+n z=L`62-#^vN#I)la%VPG|_6Vh|@GrhH@dS$TODwSzS#PY+ZaPMcU6dTN^0 zZ&|yw8kx-tqTdHvzCK)ZwN`X`k@Y=)?*DuHc3nN#7~5uNX}&>mjpe4o$=f0}z3Vuh zq{hP*R{V29%+9}mX3hKSRXD5edZ3#5Y?aT3uOq9jpM3s%p-$|-wdId8w*H#9H)mc_ z>CPs>2O-{z)<)jawr07$z1o_4dg|V{C*SXx_iO$Bu$(P5%a#Y#E-WnLNc{5KdgnX8 zv)?$52JYTBXU=^6fA`mx=LdI-pIo#1{T9WkFW#G9iQmb|-+kt9$*Yr|Uw)*ln%mjm zqmc7OWS-nl)#Nvq>eXM>r|!<*w0%|WJO_h+9lmNy+)}<>p8h6&R`2)w_IBshYCeB{ zaqs=UZ%_XC?VoHn|3zN7=(OKI!}t7+TkrC*=>N&*@3$Gf`zxSUUVf=wOT9d<|0>tY zpsi1LZ0afh{7vu0r)NL*UAVC~eAn`Mn`S+_zRR}oL)_X^>z=<6IO~1qlii1|YgT*sxqRa+kIpVK7!yZ&qO zr}x!U&!`(q*8jGxS#j+*=OKag*r@a8Jl}r_yZZI$7ylP?1+@Q0t;}AUe_iYA>aQzx zvqCG)cUJv)`hVB=qLP)TzOLLG)3wu9ij%c|@zuLAcUDi?^Zj*e)YX`In|i)heNC#5 zSbeqY>#DCwe|@;VWO_c;yWF{7HA`Ln>x%gzx%Y$JzP~T2v0n7=Uy$+sM?0_QthBzq z@B4p~<`-LC7r%PGvh!Vj)Q{G#zg2tXLmzIOzN`21MgLFr)&IK`{WF(|JpbGFa^0$) z%ME+;0;~6($;;pHOr|JmV$lkVhJW9F=16}2z5DCx37hNpKk7XBpDH2F&{y zAA0@C?yp}u9!%;sJv04S_{uwO-}@8}h<{o2FHg4f^W^8>-)~Vp_TTp6_g;Sg-@D)E z#jj*>cX8rbz5XC)?hCv9wtg9T@xQkId0IcmA*HtJ`>Q*Gw)bDR@A@)tZ)C>Iin$KX z--L}NZm8r*Wq#gwC+DWa-|Byfd-f-_#J!u?_xo+>?%!{|9uL&0`(L;G?;oqV?cuA6 z-ij?0wc}`sG`4vferw0x|EDG5m+nj2rJr^0^}qj4H)7wUO#FV&U%uvo(S%UD^M~)3 z>)sN+`s&H_U$%9vCDW?@*cQj98*NmUa{QmzG{xl&ZwuSSX{@p0VU{bpzpn^C`SUNg zzEwki@6lTR^2OiNCC>LuDc0am)lAG;ReqC|cfr*K-()m88ovH`=$2ZX^!HnG3tt@n z$(O&I0%i6j>oHbu*l{B?wMgRQwY}&2j_tOUWfD$)(^Fi&LwWBznNPR0>$_w%QyK2B{r_o<@l zor%%)C%?b#J8j8cJ7=fo^8arvqj|POiSXpi{CBr+r>)ukHHPKytCnpF{=WRe^pn4L zJu@q_ep5I@Os<>H(rw1sw|kHNo$!9&>0Pm#)UABisqe1)S^InU{<~jmzu)XM$2{drao|-+$qgv_`BWmrc4=;NpzOaesf-?EUop zmeAWT7n*k8+57g}_kI6kmi(W-w)$@T-0H8__oqq9*1pgCez|kbUU|p=_fzivKKbGQ z_xrWo%4HLLm)Cy3pMAGu-}}0HeYbdf{o3#Md9Mp!oL*Jo*Zt3Dp0DxG-xv4Se!si( z|AkM>%cmdt{Qce?zu9r~_mqEa+AF;JT2S`#->ct7Z%y73e0TMZSpV2-FXqO~{r;%B zz9nMWmIVw9tVxcue1*MO%a1J7y%_s z_C2rdudim^U4QQS?)UqzFIijVJ?YBJ$5LO_=T>dXkLnGxm^4}E$E>^Ji^}8PMof-; z8!_!S%Z|5J)8p^F-*>`y`Mve2XSZfwJ$hqN)sn9-x-zZy$8Gt^^ZxF4`%0Vo-<@C7 z)?d@RIw@*v)YnJ*R~KJ1+8BEBiKxf_cYP)AUy2?H->1KN^40TKcgO9Ho2KSBb=CD< z*SQ}3ULUdj{n{&wEX+BXJ$z;_-dc4u!fETTjd%WSTJ-&Lt)&FaY-iRhM)t4AOY10h+{@_q$gdN`%Hbd?`CNqBNKiwyM zcIzdz8sFn?ZQN>gEeBQC`-A4%ufD%J>DSXpHP!oP4Od9>SZtLSPmG>Dk7G_}o$%Hb zFOTm^d&Kvg?*q%}3jRkt20!ap%{?2o^hd|*psOO=4uwo#E#FiTSUdG!V)xshC+#2Q z?S4~d@ulJA)pOd%T`t*sU#}I6+qGx*XAT?tV=tb2O?Tq#tv#Rj*sZVXr!(W!GWo;% zBtEoU-%-e6&;I*q#fA31`4{8%h($IP&S^hCMz2>zw7K={$b)Jw)Ep+!9o-2lnyrS_mEw&b9Kk`>-(d+;y7OP)alFW%)VQA zt^eWbP0O!`DD%kwC|K7YxvFITpChMR+mD`R5jV-*^i$>hM=O7A@juVK0x~%`hUk_8~NM*?>YoE3L>_>l)PM$5j%8b9~^7`?UqqGEpjm7^zr%bIhY`Fk(Q z{{70hdw%P8#&2*^d2;FX$0Wri6E4Q>X1ROc)Ld#}zHN3;)mQya*T03eYs+7+)@jz1 zP~hJjW|!icMi;o8c{`=YkLjOP0+`D7-M7}RuzpnFI zx5iDzyXpI7c~hlX6!Hr$Gl{>drX5_kO2b~a{*j##dGE8os5Qu3$&?Nq((`6~RY z;l1DgymjBlMc-R1wmhh%?0e4PHEc}VBmV7QwE1h|`!696wtxNp_0CW3wo=NYP3+>otFy!VweOc1KYbtmy>L65>Fd~kj*{y8#it5? z_x^nSw4sj9s?&@9?VtYZ^PSvz^SfP&)xSS=*OqKyy~^AF=hny4mDamH6uqMroL)*{n_#U@$bK<`C>Vpem#`hcTMkF=#kT}?}uCI zP0cO(Y5Qc?k!rWoXa5%MU|?X;b)4lpIg7RQh4v9H^Sh`*7EGT*9Fa+3@@#m z>aBO8dVhJw&zhjGxiwQ`V`Ee9tqwo(UEMamMSGh6`9J&q&pG{c<@$*1`sbf@OZxTa zM(@ji8Mn9S)z6IZ_!6(9;nl0}FSfT5jul$}{l!%$yfhZ%dah& zn)>edD#f^shb#{{$$nqzd%gPNx&1rLPv75PK3Q$*=YU1oUEwcwYR%%%mr=fAk#Zs6 z?U~(gH>ZWa-}JsbWzUPhwSUaxzW+bFFK>(b`l&^;roE1ObZz&8?dt8%e>|#7WInLo zqnnxgSr7C1e*&9iqAgNI51qev|HZnK&%f+@FnJMY#KX0Ii%!`4fB2g^f8F;uo$33Y zpDW$=`fvT$?f0WgZT59*e*bf5b@cW9B@_Iv{#p9;?fSUyEXU=2vM) zl`yy1XbUJuZQAolto+FO-|-rM zo~bJQY3ft3e!4oGo0EfgdVApd$M^o%bx-~E^s~-7m$;+4?^C|>zb?3_98`W`>8=0Y zvbXwVpC;|^6)oF%h{q3`~F$#C*Qq(b-ecbyEUQH zzQ5nQt4XHL{f2O*trYhc)!*U!Bdn$_-FyC9vGwC6o2QvYzu#Yc{gZ>a)k&NA*Ejz* zQEPwmB(QPg-ug+uSxRp1-&tn=YtN)_-xuz8`nCV9n0}~LdClykMHWxq7-z~}fBfm= z)AxI&zwiFNJF=2RFXq+VPiuD8zKXr|E$~wO)QG>lP3yL(?Ty-7H2I>$^Q*kObieF> z{?UH*r%z`~v+w!yyomE)RGIltRKn+>rqMIcoRjMnixn>G2XW7kVwveT_1fzqt=|S8 zgx@{76%l>>i>axXPm9*0OR=YGT9a>id;G0DnBE)2@}auKT58>v#|fM*1q>Ns1<(C@ zR_@&_7Rq>OnNZ`+!qs0N1YXx|+Bczj-)f&5tM&YR_wQ^BIa8OL~ItRlK*+>{jq|t}=K~B(v$ErK{cwR?ADPd{UJ^u_>MS(LCkW{X^HvKFrwl zZc%){vhI!!Ddse_W5!A9509&J9$-|O`A<1rEi_y;)%1IQ!U-%S;WlhYua%a=_!y(Zw|LRtonjgCN z^J4AO^w>GI_WJH&WyhD-PY%5I-FJ=Z;{Ke)nNvPl@6!ekOI_V+D*VS07^di>w|j2Epn z@GRM{>=bAHy+7#i)yvK=&R;R^x_*O|$ujQr#V4H4=AJnCU~YTnosb7*j#VGJ3-6lo zCF$Ou{q^mP*R4NwCD}htzH)i?nHv4_%ze8zE%KgLJc;#4U(3y2Spj2mP}>*UijtT zi}u}FjF$I88|zMO%m@uvyX?nMz$hm&K}&?Im6a)BK}3NHS1ZdwSAzvwVq6^_KOYr` zzFx9l+;3;hs;O+DuJIEW9*eb!?=Jsv_1ZahwHIx-k6iMbVLye_T>Z<}6WsqdJbiB$ z{J1Fg@#5S{>h=+JD7NH}QV9eRKUZzvDS6daMHLp3mLPrmLQD(^b@FM$fnHjt$E_bq)sd zzfogU6cVpnSGRxt%m1Bc79=oDSoZOGpLs~t-!%t*EwR}vcfL`$Z#~;Gqq4Yl6W1>E zxVz88S!mha0Cv!R(>EeqtzC_Y3O6*|xLTQx1#-%5oj&{H(f4tt=azZThe~&$K)f?xVuD)dbTmD?d{VS*3 zf4O9Q^ZZ)EW#`xbBkkZCp0CrgSVEllrLW)Iaw6@&#w-!G4_hbogtOPTDHY$pa_8Cd ze|5nhu3T_WV`@;6mGWD>QnokuRNn0g5xPmsZf=_>ver~=t=qGY$=}Sfx9?A24`9}d z>)iN+`37?&hw!%KaQpp-WnR8rLKbY#}+8+?p9 zB2zSsgn5|`I&4^=VYI`B_m%LvN1cMb;Cbi{4Oeb9rlx`g5dmUCyiN}t zb_A>fO;9$1W~m_VUd>YyH<vkKcDUy;|}|WPjuDQ~M8Wf3^Gg!|&Ps>7v)>#m!ZnA8;Y$m95(8L(eTa zUK{a{~Y)D*XhrCffECjEH1v&Dm^~ed14M*==!=%eD-|%qisKxztvEXGjDwLyRJ)~ zlQCvRS<^RPmU~+`R>$ArV_;xwbDZTT+{Id&5nH#;GduRK_w;S+d|$`@+Zz6BNp;lU z?3LO3uId)v7ru2p>+#>ULAB{`v%gl_M(i@pioN=0|Fu7YFC()>$nUVS~T z{q_2y4R>#Cynk-%;s3>Vznd<)`mr=y zqx)}G^wRayfA9K~vhMlSHuKW#Yu3Noi&wp@kDIh^cemjd9`m`St8bR5J9Ue!vJv>M zdh(>2@VCCJXT$H8zP==U`v29KsK|r~cl=d%*39qz@n@nXyGqHmnH=h;7A^OF_SpK# z=TF`z->1YcT*~dZ<;4Gkw{P^%I?wF>Pyf#MTlO)3_TFD#w(gzpHLcTT@4lxd9%MhZ zklR?Fb3=Lgyy|0!jQ&6;(*_S@@P@s%ITbp@ZO{Ple( zxk%9VDx2N+-I5Ou7l-}WE(coS#_XFP#HGG#-|rtY_j0~&dC9u2bz_pz#Z7GD3+xWx zbX9!$uG7szK=Iy_xb-{abNcdUc~yR3m)ER2{Oh>*VS=V^w zou;Fl^n3N&D|hG~v%UN6I=a>5!r+4$yf-uqooRrlC!mpW^=SC;kT z6VVgemo0Wh3C^7UXM*Bp!GCY3PD$N8`QqR2mPx$R`W-B8Oj`L-a9Z2tX)XI#s&2Yj zotgGtw`0<$IbJ_rl^wsjzHF`j@3lXle0>@GU`5dMjjsYUPVeB9Utm5z|K8zAVvC%A z@G+M^owxbOnFopcxS!3-3j48cqM*SAmu`lY|6=B8-(R-x*YBJA-pnp<<2L`Xj{oV5 zgS(b~`X&E2uU}LDWXz`8|6jGgp7amvelCCiiOgFCJ)MbLzs5N6}Ac&^q5p%ezLgf^Ul5HWp!OPeNzLx(#k70-Y?7!WfRM}{_Es}-`6kl zso&)NJiGk;M_F~z{Y4KZ*=xzIGt}6zboWa+{p;bSk~8*%pLVkT;y3wFb$-v;wqM(4 zR@UGDoBA_x-EN2OiA&zU?D}}H<;Pq7wUsSAY8ATqRhw^I&wiEp`EF{km7&vONnwWTpP?kIm`g?BKiKz5La~R{T@!F`Vr72;t9r*P2<;v*macU-2Io`DYiamkYdOpML#q(dqv; zkDRMXIo9;2a{KBR^F!DD%(}4LqBDNay!pRt=GpG7TQ_C%fy1o(;(r$<%>1sq{N&xA z^-YiNJ$-Qh)mq7K9OnBC%+5@@Yd!6C&$lS&b>X2~zi_U-wjz0_lYU+2P21Z1k4_Qq z(>ZQL_dirpHo9kZ*DHSiVVf1sp06yH2V~T%FRF5_tbhOekI18vN0!r19sP0dzQ@kA z_-DFvzjZw?l-pQ-qtSog`|WqL@7`^a zJH_tCD{u467uw(U{&Rk_`hNGbL&^rHa-N>`);mAdWTN4oz0W3b^u^R~jq}{}WY3@f zX{tH@S~eGL+k5oIQdTDU=zCmWbnia-^?&#Kudn6=ZfKpW_x`>5)1Mk=vOb*5J%9b= z&Hrk1ZJR9rAJ+f+_t)_vHKU_RE&@cM9JW>kX_w;ZZ4n?4xzj zW_L-8{wMpzEBcyygneVg^A zYEu8doW%3*PtU$|sdiuX{;OiEbj7v5T4)JBeX=J>c>pS|}dtuoubB3Xo!it|IIq!j->3LqF>4-A(_~xu#qmZhC*_MS>8#)NC1&04b?RR&@2&}db>yd| zP1T=y@_#*wzY6T!oV&{|s=m-7KkZwMx!|-G`CZQ@6zOZNmwg_v{{Pg7f8HVc*My%g zcit@&*Jn57PyT)Hb^r9^*VnCc)zc4uF|Wg5QU9F!yIs$}sE6Mz4G-^9|N82s&-B-^ zCnGi&F5ANvvpK*`zx;iP+Z&Je+{cIiHUI8>y5Fh#-PZkaldiA6dQ$bqx8C-&u2U}m zcNTn`B$8{@D-k=v@T2&0OWoRqKkM!^$p)X#lW_=r^6`kx^a+!%<#{S)y}$VWYVe62 z>wdXb)y*hK>{WS_eeCz=l)Jyx%WwR={WLLS?L7NKuFPe%%#Y#?j_gcc5oVjq`Xke> zZ;My`zJrA_yB+5wHBOk~f4{bE#UXZqiXRgmu79}lNwvA}45QXP>azm8GiU(wP(w?(aWZ_tyQmdQ1QOSJ%h3 zZr@lB@rpSwDPQq`XR?pkuaX^hFWETR*)PA?pXAkfzxsW66X%=U`M3Ggc3o?76FPrE z;oNgsmN$2-HdWv5aOf*+a&oxu`?J!ccoJj3)4}}p0iTl2$X_o0eQJ4zciqMih0-Kb zec_-V-bU}Q*GFVH-@Pv9()iy;b)(#h28YlYj$djK(xL=!9XC9j* zVfp0qFCVeV76)sd&DWXFt3NsW(zJJ%U+xl9dtd#lOE7C?Qm2UFr;3*qiq>r&Hv8t6 zNwr_vy>DNK?Dy@(GtYPU{5I!}shX}4@y!0sEc53oQ|1Whi0wP4dcRWs*U^CJxV>t! z|2EbOEDc=LC28gHvd!R-M*j-o`AqB?|DSLEtI+?Dlhc0xg12oS)V{HN?mIPk-~TI3 z%+6^R@qe4@o=O<8u}M!B%v!}_skfuJOT2pT_dJbr|K5E2{6DVkitGEmZ#}w|{7)5|i-dG3Wf+Z(V=e*W?%T zn0Y;yu-a|hWB7MW*(|fC9A2{X=J$yGmf;`QzQ0)Yw`h<5?w7J!?^P>p z@-MyO;Ohxy2{CI_$SRq}GG|)VV~zhCGmbhQ?-Spmx|_de@74a?Eo*+8Tou21d+E1* zLIOu$MBnS{3BP~Q<x} zAdceDj}za2OSZk=a{OM*9QU*D<60i>_TS7={d3p;Nl&(_widJ9J13U1NA5dc)eZTl zEYX~&zfS)C>y2SA^Sa9?r|odbW_TR4#Ouqf!pJ=tM^5(DHz{o}yx>r`^PbsXhrT;K zm!Ew9y|Lx)*|W3ri&mW5|Ip%W{SKq+x6~r-Y(GWt-2G`|{_&|n!;T%lUD(Wi=PvSf zYfH#}zR=e9R?{hkKk>V7G=yv{GLKTy4SH_zU&e5;UirR*x>{@(cmGuve9JxS?620& z3-i{@`npnmmD=v7kIG(3xV_(N6P0lF4)rk^X~hd z>%*4yXFu5;sz&J=>{s@7~W7|9*9?`0v;1yVhOMXw#hIpZhLPrv6NeQ|TJ^ z!x3#y@809|toR(eMK+`EP;IvAK1=0EPd-8xzGfW$;$P^K zMHQuP^*VDtoct92%U8V-3s+vRQZHPwe801GZguXT?($Ec zel0Bcz3`UZ+xWY2f0NeE|5~+Bv`(Xcva5vAn&S7TUA%AY{8||@=Xz`w#}bpQ`}>nv zc7Cs8U|{>@ILlx76l>{`za{(akJhz*|GQfL|Nr|(b>g!{tIpId-yg9*V%_WO`@`4A zEV&!~wYYiy>;G|oQ{(m)*S)^G-h11w(;xRc^vv0kKUF+##y;!!dLiBK#Lc3V3%B|2 z`0ZgF?Cso9AHRD3`rr@W*4EZ-z4f&y_TSd9<(?K(->3g_d!I33&S&oW3%WC!8f}jB zIUdj17n|E!sLG(Ey;t>LUDRp*c=_+&WO}E^y?#04w4dvS^DR98pILr+75+1!Iq2{S z`#pD7yo-DN(zg2l?Ah;}D*vBc{yD_W+3wr+Io*#pJ^Ii!r(-7L%-+-dcecO&a?EA2 z$-WMklb>#%{MPzi>-QX8mhNRqYSVaYcCMKoaK}fgu-eMHyldh)?U&;B6!FiNxpiudhD&Ue@xx^GCGvt}vf@(feNQ-1$%GZ&c@l?pNF`ZgKuo zYASX)d_VbMYnZ3H<}!ixPN_P7*M(?i1pV3i>iLOvvG?^JO*Bk-tXuH>)|TnJZhsf@ z3w;{1Wcv&6`#Rh9-p{&JX#3MQX71TdWgKVE9$mS7;otjZSEk?bGc$9Fi^yuYvb8{O znNXtkx!CMapLlCG{;a>?C$vL8IdLHy!ORw(7&ay6L;a*Z#`Xe-KiIANP+-?PRJRoQdV{jNi=jDC*jtnzl*(?{^6*e4Avo zTy;udhnJ?gj{IC(o?UD|wyUP6D?B~@IdNum*6l}E`%IcYIrQ0`xCExY~Am zrN`%+FUz$bt?*X9C4Tj^{(4`%d%41Dfr3?+<}&fh%;Zx&lD=TFEYFAd)6I|70(LT` z)rZ{+a^#a0=KZtr*TX4S|L;E}r}}Gk{h1^4KX(>B50rEI7qsuw|49!81&(pV-aqGY zulLvs!(|QX*ER=7IfSn7wZ4>`R3n|$>v`~T<;t3`f8_*T%1nE|Y@7P_K-)V8i_eHB zJ>9(^U*Vv<-pb-bOZ$JX))QR3`A==;vfTfUQ7YRcxa)#A>=(7E7t8;zy}Af=J>aXo z)zhM1FZtObGO@?W_{RNPEO%Ab))u|GU;9|^R*X6KqZJydUR58~pI8`KxHQq}a@36L zEIGYh_P5nG{gv$tbSu1BG3$%*#y0{}AD>WPzd7IU;G9km+j}P}d#=7#Ybl$x!>de3 zWx|>*vKa}-W4sg2zVdzfm#^~m+ja@=8*^JL3~%jgo;9VA@vG+K%m3U@e*Lc}_$9t> zh5!EGP97}#aU$g09@&2;PPqRPO{GRI|FZm6cWYJ2r`5Xo zyZA3nQvYAO`pxT2vEqBXZl2n|nNO{LcJ-~uN zPQJE!@2w7PzL>U)|1a!aF1w*-Y0HM-+SKRk7b_`FvVW=y}yp$y8c@0 z(ci3fKWn`1Ec={s_EVkoG+`zt^EaD&JWO_YpRls3QuT@S%l~5i_~E9ux9PXPB(Idx z|IaZY&?agAuG9+)D=zEJ{V#f@K$qo)W&Tgw-CIs&&pMt}r7$Vsv-@vl{j){)!``cx zpMJC=`jzodz1ZlDxAz|r`S8NIXP?{mAB%Q;c=7jsM7qSy_UqTbU!QPv!I||R<}Grb zCg*YK{weM^Y^LrvuC%NTolt0d|LsR9o+o`?%zA-7!OMStQ4X)M-f7Ff?ECSS;~#~d zeV3Oo`Bb&t>1*9y!ItmB_iJ`nb8@LZZ+-bkJ?~@5tP=`Gm6`WfeL6pXzrL>4<&&>p z$4;r9ZdJA=FED!FO$m`w78UL+9#y><@DbD4m;NUegCk0^{OwWlP_^Jyrs6fL-A%!Sj;z$_0^x=DIHXM zsr_P4Q-0Z76UC=HJ)hTX|KfD~_Mds4`(K9iygHEEWG+dEeW*Gwu8ax7bVF-gi&l6>9x{(&qokyVruReixYk^#B6{%Vft{{*zy^mcQ`sU4MSo zx^;WAJof)y{cdmZ{k7$`r)#gCEc)N^Q++>svd@3r8!DR*yfrOfp`mIJm@c<`}?x3cl&(BbH(rBuJ=!V zPLE$HdSdqM>wB-Rx^mX#)Bm{daqpBqRn+bCy>9M3W#4k|!`3@a$KCVp6U?f3wkLIQ zQAOPCa4Vs&(iZDi#wlIX-nG@rds0$C{j3Xf>pJgDm?*xpH(q^e%HMA-+w0fIuAZk| zz4}Pho@rZk*Ozgz6sMcHo?0rtQ@MHZOMTb!M?FuMPv)Gar)F_Ve|CVmP3g`0fA4qg zeX=&TO7n8||2bV|8BrPk!**0n>#y-xQ8{bo&)3_<_l3*f^`DV(dtt4hcZ~lL&#Zf% zE^1;fZoiv6ZiVe||7XW}BD?ZX;mP>$hZFPO$4~A4b8L^*vL}D;AB@>j@JelFocjH$ zxx$j2{YOf8E}x$J`_+}!Y4Nq|)^xn<^=p~A{BCN1yY2T=IxffB8jmg97VVkxIlef# zp<(|=SL1fq^LJlgS6{z>>#IK|J~G`qe>^?EVPcNVm)+NA%-*}JwXj=tdAm^Kz1EtP z<8o_H+D_?PrQ_R<&K@1{RG=Au)0(EjB6-%Zlz7K=aZ zcs|{vX1RrD?Vf+V&S9(LkDZr`I}>xUYP-3yyxZP;dpCyfYk3-9T6*ovNxiwBn{0b_ z?!0BaKho`U?DxsHu7CS2P$}1Aztd|&eXz7u8}m;uk<0_;7iBeB&GXkS|8wYR@(nfP_8_C@ zic|Iq%-*;AdB+#i50kDh{~fU|F3Rhd>8cy&mYt1XynD7tT7T==`}ZFf9kTO3nlSBE zu!Q-kpYdPwR95MIy#IfN;+~rR=Gv|6CkRT^|7*BE*~~|trQYh{&Vt~oeR*5I=<_#uJZx^pJ(uJJRzzN@IX=7o`L+Ge%{M-M%In;fw&H)pvHAb1{y8@D zs5k!-`(9YQarUlzb5>8j`gP5Gk=@_V-&l0x-kX09W_^FS@>7>y==&)?Z(65r?F=a^ z;!>A(TGfuuzeHVji4~exM%>~l$|`7#uRi;}{Ka0+diPh? zmVbS!(S6U~H9mNG?f26+ZSE(v-us<*`hR4r?SJQ@PWA17y;puQ?2F}ZQd+p&_wY{6 z$3m;V+S}XpKYzd9e){+7koShN-`!5E&3b*|-&EQdbXE2HJw^?YDH@`jtSpBdH6jW`!259| zK)bJ6k0=|gu6p|RukHHsGj}?pW*Ubq=2QC>KjHbN52fG!e6-J>8|9o|^z*avgW{7{ zi(ghf{Z+{QgKbUUt4SM+-`+5C{BCa}Z|&l*za!s2qf2n9TvXqs_BVnKRa2WEZRNXj zKQ{j0tvBzbzi8T=?Rnkjaxi_<{^_;qOBJ{{-5gfN-fFrZe*WHcm8G|?Jbt*xfA3AF z_cbMLi?^InQaZGK){3czH>Lg(5LGD4;rS!p8MdaSVQ>7S4}0!)7&17C2HAILmuoJ%-H1BH zP^K^`^}A<9rvabHd5e4NDyn|WdRw}(mFHOPojEBJ`)c=vY~$KDUOFhS2N`BaUm|2mW7ERkMXp!GCFPuNE0z; zb$KW>y?x!ke=X16tKK`k>!@LIpt|#cg3phwYvOHwyqxdHdyD_gU*SuCF38O7pX{4e zuYdZ&0V$UkN4n#_&M`Ns`MFwUa;b;fodVW%#pl1=IdQqLnojr; zx;OH5@e&T}cEm|7ztfnP-NpWUPuZ2r4|v0mh6imq@ksD!?P=n;zZe-}=$i|NrNTW9v^a{ryIgrXLCrEGo8v7|cRKx9jrBJ;D#ty}-T zzQgJHQSUfYc|qy!It85_`)*9GKW)VBzIg(p_oWFND|arpRK4P7vcMbcrzt8XGOR2Q z9X13QhzN7C9CTdaqan)0+SVAz8F`-f^W#2f#LY~RBmRl>)`{p5w@-}i4t zr0Eo{bbjs2Vs_Ec_)Rl3=IJmb>4Yg=;xfu` zGGfy8_Tqi`Y?6oJ9+nByizhu{`~PnBnvlN_3+p%7R<2dGT>gnsb=fq@cHKk58jtjX zy1qonFlvc(XasSyvN9$dh%gW_YGrc_1npE6?C_bsH@#mr)c4eD-Vjd98D6h+lONYU z+RN|u!~Xw{&-Tk@Y98*$TPPoMdc7=fdx>av-}T6ke%5~l=lXxYeZOMO>aulyzu(@v z=Tsu-ZFcOPbH(4dt)lr7J9xT-Gx}6C^OTeXH@zyf{bjAQ@$Twq+qvEo#NIzB-bL&j=I4DrucGGpy7+5`hrZ?Zyh>fsdeSTW%8n(1 zM^;!cYKU}b7;&|>G!=l(dFNtvI_M}7&?d^o>7||@8NFZDziv-<)t*^-H%l!a7n{5a zDlz-m*SzETEA{l$nCy3N{wwpxzdieLiCp*6hj+HDznj0|YkBRz*1a76qjv9)S(Rk}AvCuB>-69M>;J#{|L<>o*xT%{ zb^HJS%@VjVk43r4Z)>f^l%D5Jg?5}rrXAe<-+b}rx211=b@uI5T|IwI{JQlQ7yZq% z<0!g&XZb#X^WKUNQrn*{|8z{I`ft)((cfYAaqFTFTCeUn{``=np6~10h~Ah#VcNf9 zy+baYd{xr%C9UfG3;%HS*B6%Ad)G8`^1bDKnOn6sX@A)9q|K&PN8|SDy}G|$Wv-(QPX?FnBK6kByLLg_>D-Me0`S$~#a-1SlIX;PSJ?ZUlx7H!#){$)+wnwqJv z#TA9Bw*J~^9dB^>`|s;r&sPhjDZleQyZ35F+>%ErPvYLjn%>$K_u=za*#sf8Y6I)?L}x!hPklcSh(h?^3i^Ny@g{dYfKi{G|Y2CFy&HA-x3+j%E%-0z3$LGru~ovNRrj(ddw-7o{OR9DUj_!| zdycaLrf9JiT~%LbtG{k{#JVSQlU6O;UVeA=^wQU_md(DlYxu{XWe{XG*=8Z(tMD??EN(X0bF?-D6@i%`>)T_hc z`xif7Uw<=UZe_E1a^l}8^^3pO?M$xC@^+r_wrbU?iTB=Rt-CNSe6`T8U3-)MW|?1D zH#sR$Hek;F@V8m{al1BE{lBrUdeXK1Vb;^HYi&&yJ^fm|-c|76k5gqYqC9J}_C`fk zN7wC*+w?b1IrP;bxvQ)GIx3t!VyVCGpy+8AKIXdrme;?ur@jwQa#8X5P}^zfeM_1( z>foL2<>u={@9m9^&j^XM{rB(ty>ka!=FYwHw`noA(Y=og7haP1%=PG+nj*{Z#gcX9 zn-}!7Ofc7#eX5-K~1la-}o$PyLtUf)#KugRj<~)^YP2xZy5Wv&805m zvy0Ono&7HFey`VE9p5qKmUFV*`+eKrZk{aoZ~uBj`O*t%vZqx~m+cEyPUc?yE>3wR z=enIHtA$sXKT!GCW0m-A;osw@&z`JjnU}BiiMu5JOj_e@@#+6Qoo+q<_1&~ro9=zR zCA>EC`jt=YykpPbx#gXL=fgg> zv)4T|EPl>%?|Ag;D<5NpR$qI)zFz0L=-0KOw|*J4#$SKbU46JKHS3pLP<>kbyO-}@ zuKsh#xPJMunbzVReO?Rhxt%k*D3KfZ>9gU!|0k>eG6}t1l{`sfN>}Xnv-*psYDMWc zUB9<)&ALtdwNJhe_g#4DtX)Y*@u;zSFK;a;aFwq zwbM(&RZe|>9s2#pMn2v*JN-_5ewtOOBVk;9r|MIO=B0a4b|=5yk1c%>E&Xlx?ukFb zpMQ^xTYJXvuF9(H%lE^>W3&DKpKN@x_rv_(UrlCA_nKZgeg6Be>-=9S{=dcX`uX3i z_ve?~pM3YtrmwGGpH^R6@krly{`1-&)9cp0j99wgWy;*eJAduo>Lnk1clFL!Go$xk z*R)=9|7DGm_kR6p|L;05Ft9K=&I%OqVy*S?{VN$ATi^f3n*aaf|I@Dicl`fb%c4ZyitpnH-^=}T_owUTFXpP-%rBVKM}Fi?W%u0p+M)j2t_ekV{#MI=I$j^{b@%(Lz4x<@t$)$_>ZWMt+th;> zZT`ev)j!Rm*0M)>HB;^FBM)sK{_X4Z-XCTkHUDb=((&r{^f~y|J?fNnQn%;Z;|Hu1Qnv`9V zda7{qDd%zTdB2Tkd0moy604Mc_g207(R%%bwMqLYFZocUxBvP2xYg5YXScJeO!*Ki z`Txx(j@upEcK!M4QYvKs_M}iydX)L(2m8OIZWgwgH+`RH$Be!`UoS~#{@(Djoc)xn z_OuJGC+6HQe^LGCp7hVk`hPFx>8*>|D?KOa!`rWa{hQYFnRK~vGEZw&Zeeeo<|O`b z^6@{qjh{8V?+3lgFtWPwD)B?Z^v5OFb%dYnPg|q%JLr!@P+v#)a%qRfCt~a$Ir7_y z_1vC)bo$9Rp^Baj%^|fNr|Yt$x@J9DpY&_uuZ{ORC#U?hTb~iJX5Y7%=c_lZe=EMJ zqRz9j`o@lonLe?>$2{gwnRWJC)kK#MzovxQH0@+PWi9@9-sRZ0*VXT@+Qhpl$mMU7 zR>bd!m4D~F|M+qH+~|2SqIvh!b_*|%F*L&{ENo22>aC*S`% zy8OJ;sllbo?X5L$-OT-^<@yOd(f3Y2>)ZG02$%N~uiBO230=QmZCxg|XP5oeQyzOorc|NAI6&%(*5}oxjelqORLC*EiFpz~PiW)5R|1)~+?h#FoTneZ zXlM8D*ZRY5nu}Yv%J2NY^6uIB>$f=ZY%vjFciD3!^joarpItUCTmGC$eZP@4+jOqm zr|NxA)(6_|SpUxbOWW6&33DzlypgwGz3q0&oxP!@dj9PH_bq?EYwhklb(N1S-~WHK z^nAH=UCVs+XZ!zJpQ_&W)BbGr_mx#!!p&Y!{{8>2^yly8Z{z;{>${j4dh+?@zjd3I z?fd&@-k-TUx3AdHa9Mxa#h?tb|E>+4^Clm5ou+Mab^z5dC!|A!<`7-s(aTfM(yeaw=t>Di&JTbcgt z5&cwEq1ZIhYR!J$AELcC+r1B7I$i4D_rCA+T1VBV_g}qw)%iE8`~UaX(~q9iw-Pz` zUHJR$PseL(zqRaY)$NO0`ZBNR`>msQKWk5|y?gcZ^@YBxzm?vv{noZOURYOX&Blne zQ9D)FRq9XQyZ>pqqIvDrg)>91W<;yp&9c9~zH8R+z4?;@?|Ezw{(d4hW_xAn{?}J# zTq@U#lWu$Ze&73jyT87lymN2d->T?GSNr1CHaOpTfBt^$_xroP#ztL<7Y@I*`D^9o zihr*_`!Sq%pM3DM!YIbl>}Eqe;YS^RKTrSJt*ozHgtS z@a;eEbTQT#^`xVJTXl0TO+P5Iq|5te@s&rGoBZEidj75^l>gNGODpEZ3oftg{`S+! zGk*2D&X@1v{A2d&wt8N-(%W}VIW=eci91~r*X!GQxJ>BT*Yx#C)c>P@Ui<1sGWm4# z>#q72yZ`5@_w)Lf*b1Gw`Rb2FW`6Xt`@3(4?hK!=zUcph7N2QN)!#$zUbBz=8Jq7@ zwe$Dulds=TDK&~){od_f{C~s3+q_9@PcGcED=uFA{)9Q|RaVmXS6^4!e`mg5wTOsC zg8AKbzu!EMKW6;(W$Cw7MMdd?xu^bCuY6XR_GOOi`Mch6O*$QuPX1nJ`g-Zb>!ISW z_rG|^`X%CFi@$ZryzGbT_Y15xKee-bY3;(dpTpOd3r~M~ZoApGADXVVi}ziRviO=_ z7ap_4DUobnYzJ8t1uV2qU&i9S0nzH4lY0&{jC6OsAB7&?=je!OUS`yrB zObZ<~1++wjS`IY^E(lR7UuRTYx8?3<!ahv0?s+HL>km(P9`A9;JL z*xu^%c{(<;4`f*_i~lLKUo88(QRchjYqr%G*BL!{WG?IbCHU#I>WFofb9a8>dVF@$ zN^!HlZ(apA^Bqq=vuEC}dfAt*x&Jg5+?abr;3>Pk{QQU)ah8SKGA*CaV~)FbL#uz? zk3+f}Cz`!q^v-4(LrB`8V|jr}7gbHximVL!Ryp;Y&YOmt0R@a&B2zS!q*@L&CMu+8 zh;y+VYBpGKMMH?QmC4azMSzx|)heSsmG^$U`@AYD@5q{|Mn9SgWGv+iA76d<{onq= zYpUfRrrUAb`fu6K_+$C+lKk%>_VJHi#_eDIGxf#4XGUi2&-GqTR@eT%{iNEKwyM`L zes5LJuble#Vx!WLiY4>Py?^u1Iv!&l@HuLS#-Db5gE_r-QgRimrfhvQohQD{Z_)Z! z+pAtPufBbn_g&os7SrVkP4g2?^j2;c5cBWgn-J(`0P?_u77=0CNP%W+Q&ZrADH_Hc zbETfz{&;TnaqY*TsL86&ob@|Btz_bxAKA(C-CyzE&-J2A|BKK>|9SuTT~>U$skqbp z@TuD7df)Jr|8BS2+drn5nc8ofn$o%N!H2@PE9#gI+&Xil*5s(~opT*o~?nnEZRRyYN zE4#OA83h?T^eP&xnm&W)&G}6S9x!TzOh^%tW@S3uupvQ1O^CCpB~UR$seFFrx;yuZ z-@fjvST5?(>3hmgDCEN1KYYIbp5MLi@}yesqwW7+q2Bw7O~s!6(a(Nxqj3GgN?Etn z;Ty`sH*3#nKD2~!P2rwI&Rp3EN8}ePpU?c_!#(Sn3eSo|vU}fe4OqG>y~F%&;gy4W z-D`7p$9^n0ICcKo6^E_`?sLz3cSTHZzwKA`UqARBc3m;osSN#c&T*c@ES^NkAeZby zN!Is;th;q@cHHF3&5dhq zgMJ!)^T|9MG3~I=B&+RV_cv%qW%PV0lP~Mr_V)eNu7jc1`VVe+&t0u2{d6YlCe^ih z5n{#{iWo$=Z4PdmK3CId(xh1jT+VqnF+S8Q3Q3e$ez_$0&w>eBHx8?QJ~m_0!n4O* zB=((td-qhZxoK%rRm|i$1>&5mH$J$#hrN=!GpzLHjNYZO_D8ckk&gUPyZ^5f2Pek7Zdo13+Xx0Tc`?I-u z_AkFR=pPO#-<~z?s!f!2L4DSojIHeJR9z>n40)=u*mRA^UB@8Ln4<|6j0Pc7G*m(3 zPX?edLT*;@Sha^L7fVY^!h$PKX`4Lnms?&ppA(oG>w7X(IDESH!n4Qs_#Ia$(2HMl zYtPYxcLls3^}e~gIQ~y%eBQ@-X5u1$J5Tw34-7t?%p&!hD{+on$?w%Af7gAE>Z~cd zu=}@IpR66Tn8a~Khus^0$Is*ND`hdw$gwTYebS&2m8TNgqGYc8Ux}yIZrTz>k+k5j94 zM7mh(J^aF7$Nt+}x3z3z*sJ$fCs*w+x^Ew@wl)0L-p}5z@0+hWu6KRc>T97VKUMUf zjG6sQIz=46+J!CbBX(Xh5A%=z`hNEQ&iPl*PmQUKSvB?PpZ)hIBq;v3+r?sG z*mS-4*Qet}zqja~e0Q|Grnltk>q|LzL%%+k*f+6!|Ne_|KC`|EzWVxZ^~#9X$A!-w z*I8*ATqwWV^W>-hmA2PjzgijjWv|(5@mI&QBc6XPuG<@P>b{`hYpuIgewAj6oV<&F zU47j7C12^_CyBD^jD#2emtA_dqwh())mQHr*Eds|@^`5p%f3JDkjs&{CyNC9^saMS z?o--u&ilputM^*23w{(el(F#VRJ$I%Yu`)J*x0Lg1@9mI99ncS=I{0QPa-SpcGjMZ zFWYHfzgvB6b{a>~uX_Kx-zVQqc^8-W?oZubmeiW*fAyZLd+ja1Z+rIC-JgGNIBdUb zUh`j|{cG%>NvDOiO|O1B^8Nm@f~NoSze?tY*EQMJhhL7hy|1$_|NZp&`+mPyZ{K}q zs`q`HeS6DQ_BmZWd)MmUuiq?eg-j z^)znnyRCKW!q>X|+HvdE--5r{J8iR9uT6UYY}fVO5!Tbsr5@ZD|2|T0{TJ0$|HJo( zPY&(g`g-e+JF`~XUemrJ(9>)Be)40N|54Y!R(;W3vNiwu?#Sd_nYKymQdfOlw|i^d z)0NeC z_|*S>O@#lo%r03TE z`TFyOztZxKuNwPRu1$M6!S`UT*F5XGo#AU;Qr^t^TfXi0zdr}HWbVJdx_)xi*^S0A z6U963`Ruyibz#v_tuMN&L2~&%Mz1<&t<&}~4SJPkGo~Bis6jB$l-c+3=KkNU+jQLgn{Oi5HJ=#~kEAIKa z$IIud{Z&lPm5pC?de;)CdrHE~pL~k*y=A-p>h4vZa<5gcuRN~5e75`j(pA>;_9_WI z4dnW}-qSNOHud+_D_>2Q>Xq-TtpB(E_PbZp|3~cEcYVh^z4gNsb$Q7LsUwCR_5bsOPi3uK=Baz!=nSu& z{hXVgElDD3YD%SVYx2#XzpSvjqZ=zLu{(Kl-}zd>k|0&@KfG$%J}7# z>+7of^QO(8t8(3`@6DPXt_k(uPNg)@$v3q1U<`cqc(T*t1a?M=ng611x`k|7qP2Wp z``+3o4E%=wJs9}E#Fa4ccmAKiATRx2fx-0e_dm4?3=9ml`yC(unZUrn@LT`w|KmLj z3=Fd2Z||Bvz58B-0R;XFKQQ?g0b()ydwc1B2?K-NopLajfq}V!fq}O zw=5$CBys%IyXD7Ey<5S+7(ao5eQ5&&BLfJ5q#&3PLcvKiz5`qWD}$MVfk7fOKcyHX zE6~u;z|P?R|382Ie+C9Nu9Bj}Yz78~n38O;TNvSPfjNPFW&lVZ3P!UZ)v*i=AonmC zFfcHJ7!0h-a#J!B85kH8a#O%|Fu?5q`*&tQ=n=5#iWw<6MPL>K14CJ6N-B&ErCDQh zGxO3I7#Kv#a=`{6J3}P}A}5nllnPP<(gTv`DJ{wY#Tp|61FLLtNm3321H)_xJz`(%FAi%)D%zWZw(mO^5){Jv(r$LSs zO3u$I%1h2kW?%$E1_p`Zk`i#dL7ldYfq{Dm0|V2V;*#PjBwLp;FodZyFmNj{FtF}n zU|@a3z`%Ntfq})Gfq{jcfq{7&0|T2Y0|Rq00|ToN0|T=f0|V1b1_tJ81_ov?1_tJA z1_q{W3=GUZ3=B+F3=B-i85o%K7#Ntv7#Ns9YU>#o7~e23Fur17VEoF!!1$Vhf$AKz`(eZfq`)X0|Vn^1_s6+1_s7k3=E8i7#LVD zFfg$1U|?XYVqoB0$-uy+!@$5*!@$6^nSp_`hk=1Jh=GBJg@J+hAOiz`2?K+G2Lpp3 z8v}ztA_D_YG6MsTAp--q69WSeF9QR=5d#BPH3I`LKLZ2T0|o{zJ_ZKvYz7975(Wky zGX@6EXABH%y$lR2KNuL;yBHYQ4>B-ter8}`KhD6wb((>J>kkJGmybKJ?yBQc*vKbhd z9T*sx4Hy`h?lCYhuVY|fJoo=jmJ9|4CJ?=qfr05J0|Qe%0|S#U0|Qen0|Rpp0|OHq0|OHy0|R?K)NasOG+Yb} zOfwl6m}MCl*pD(Wu&!ZXV5wkW;3#BZ;F!t4!0{47vlc_bg~OJCfjg3cfy<16fo&H9 z1M5!)26jD&e^`z&FtC8c(ij+63>g^Mx)>PP&NDEu{bOKYna;q#}xr z3=B-5G*!yLz@SlFQk)D;t4z>z%mSsg7#J8}VmpdUlJlYJK;kPH7>b`UFm!)mU|7Pz zz_2}qf#Jkb28Qc57#N-`Wng#{&%p2_o`I3Yk%3YEFau*?D+6Oo9RuT;WeiNr2N;-a zf*6?6N*S2CWEhyXEMs7Lp2one%*enTyM}>z-gE}$S1AlEYFrE~c9{$;KK~h5l5aAw z)HyS-%$UuAmf{L9@#VI8PAiuD<3xpJ7$q1OQ$H{; zx@5C9G&C>?cCa-xG%$ivHd+bJ$jZO~O8hLKT!B=AgTxuWGcYjRW?*1=$iTqxl!1Za zB?AM)3kC*;4-5z`$?~YW^Njg~`Ccu$h5@VJiaz1Bic) zfq`KQ0|Uc0s5pq;&cML1pMik^M4x3~U^vLYz_5#ffnhmR?M4O$hJ6eS3?Q>X;vjQC z_5Lvi28PoN3=AOlVFm_<6ATOtm!KHrhOZ0^48Ir{7``zufWz|-0|NsHgZu;X2gvUr z3<{@v3=9nSp<(llfq~&a0|O(d^#jt&0cHPXU;u?R$ShDSGm0>PB9DRLFQ`Swz`!U7 zm1kvOVB}(8U<4`p%)kJSo9_@lNH4<=1_m$&#mQR+28PEB3=D4=7#JQwp};r$aDk03XK-}%d7mI@101IyH-Y>G3M)`L0@(#}D=065nsK0b z2Bkkx`T&W6+zhez=00Iy*r>?B zaA-LL!=*k3hWmXC44*?782-IrV3cBIVDvi9z*wijz<6{a1LL1g1}4)O2BzjL2BvjM z3``H#F)+*KGB8K5Ffh;bWng}?k%2{VCIgFQIs;4WF$R_j2?mx)Sqv=8(-~NH)G@GJ z$YEf4&cndUevE-NIEI0>WDf&t-%JM9b&?FM2VOI>AHq@5x<_zSy#6s9m2P|rM`fFy!O)AcO$b!J`6X6V zvE>y6Rc?I^MlUP3zO35z3O}mZ@dk$8)$Bl^+MVxlp}Jik!0285?)Qyg)U@XV8p0<2 zsc8=gHShh56}9gF3PEiLzQIxZ!SBeZS0NAOvA|ocZ5z2CA~{)c@9#2sT8j z^X&hwbN{=~|L+8&_6z^pFZ}N!5%pa7-*e%A|9LPv`@iqZ|K5`b)N|s0&x!wi$N%>p zLq~nbLE><-|HS|P6aV`l2qJate;+o~bNoLVLQ~d%{67?d&4idd@g!7m;>rII)PL$f z2%Y@j4?_@fI0-QXWH`iCC;s;y2O+2jNCs?|ctObvxeSMB6{qZU0cW z^&K45Zg~qw)mvXzZ+%_0CK!K_cE8> z&RB9Ieerd8Wt6k*4!knTU49o*8Rf0Gho?S*)JK|(4ftd$hFN|L;bsjCxOkDkJ*WM}5bjRS$AG2B~ErRSc93 zE=%E+Q9rDng2=+_uc1;OO}PTCk0xLFKj9*%c|gVb2un=_shhxIh*=-?!K!AgwGU<` zg|$AKa0*|21gVUAj{bu%pycttJ;(k)s-&J{f4YwRLPn4(36b4Un}v{E2`6#YL=bU$ z)<-o+l@X*?8aee5sA9s=Kf+oc6|8vxsfi#Y*7^wEQ7T^l7^$O#=o}$;l(1GzkjkiH z%PZuX2*M_*KEhKaVb({G-VlU@bd-?U5D`c#sA(Q^1l>ye+Whkv&n`2#_n$3f+M_v!!Lr$L<~c>f6AIf5`C{iCMSupSe* z4+ZH-wVnj^kI=eEooD|;x=0=8KwTvA`bXX8K{Z<6x&J+9K%F>HR}NeyL3&0#C;oTC zJ4Z+V_aY%sIgi}^f%J)xNn}|hHd^PX=Q#Fm6v%ue)B0h`AXQTTaZrB&)DeJnj?n5O zxH6DAu$~UF{UeCI5M9V@hzL>_2}}P7saotm`5#m!p8}I-LH(mi7qRz`CSCnM@yh@H zs~|Mt%KzSr|9j8>?>PtRTana1g1841EZ~X;&VclfAS9%2g0jKNkn1CO=LOOjAgN;N zKLP5d!TU!OPC>g}a5LcyXzg?Cf6vi>Cp#T__;bdGYD-_BWfD-Vpoougani?63Gx`y08 z!qr8}S#byJcpaoZ%3pmSt&0TV(zHG*-~19*CB1-DLvRvaAK~aCq4bX+V|S2V5@y{5 zu9Du>?Rp2PcaTX)?Sm`=VZ+lQt||#@l15JaZ3n)=t0X-2QP+w89Vh;Go%-K#5`-W% z3#1x?lf=|V5RGl8Ktp*QXQ1^FCF6CF`l#m&sIG+ckKi>Cgh5h$1TSYmCC@QL{ROHN zU{w+o>m$$*)^X@~UGH(|xEZ`Af-pcL zPz)mVlJInt&}uG-8$gv2co-034m@GN8SrYT>+l~0!qy=|tA;=W5lB5Fw9JS#!?y1K zid+q0)Lc`;2vRp;tB*?8KQ3AKsA$cDqBW0-);xj?))lUPh;6X$0c5ZaUK!=BxPx3DLB{Lg z^-;=#D`@o*gbN?I19gs8+=JIgkXk8k)jfDkgvNl>L=Y0KJ}O%G2)SDXXT$qPr5m3k zBci)VFDkZx5WGHuFefLbe|aUqpav! z^%0znmUNr(F>VH>N@wIy#5iSCIU@Ao&Dc^28>Sr?*dO4L24pM=cxPm z|L$Xi5u}y@DL($c`veHVOP*d>O$4eMj)7)FAd2B6L>vnV>zaYeG$cbH6%Dj}22Tk= zIzAIm{GWK@fB!Kg1nE^l>JeCd1W^nkkAbJakW9wn28b-IR_Z+tn$G}L?Jf8Wjjeb@hYU->`b5~z#R zcao_3s1IBXK`R2J>Jm~J^_>J&M%`et=M<11q1ge|B)f~K9flL>I!W`mjq*@6g z2vLbfBKM>~QYZiSo%jba3OOb8BK3(7)eyW&0@p{73J94*>k~n899lMnaM7|VW?qJf zz%x690k3`FlSB{^cx8mOK7v#_Xe6i(+V%#Stls*%Zrhu>ZEtF~zD7fk+6P|mRBd^M zRue&_(E3NP%IJCJ#^;dPB4iRabM&-i-4n=oSMl11g{$vF5RPi-0i^CISaAmpLHb0H ziY9yM?W`rYvXow~G6T6XLSy5pn;?}DTJ2N0^(C4JxF&+Ec>q;LRomX6)kJVEa%BYXAJy-C+py~$ z8iLnB$P7qoL?a<77MVm#fK*A4Y6zKxbd(@nr1rC*DhV=P*Lfb_k_Jdms^i@McJNpqWW=ux zRzG%uU5KcLj)Mm4x{m$tI*J8hucW|Q(CQ;lAqFYc!6dvIf-`W{0?6f9AFQ_pSBzO7 zq0~f>T4`|9N1$%Vng0{dfqFd?FGK62?yK1Aqy8KJd$0YUe3hE@5vbBR4XueFY{;k` zgoM<3pi1Nf`l=O36o6t7TsJ|~U?nl@BhWMuY#9u^Hw0(EE2GXszu^dLy#uLz;3Ook z!AVFqgp-hLi)*+8t!D(ObdX8p3J8sj+$Tb-gCJbwItb2&)HrC>5S$CCZXi7%NM!_3 z4-o-ZLoZ?E^9t}tUHSUQ$n{a>#;3@Y5zlwFi(&3bgzHTp#5xzYRf< z$|!Hyt?Z?@VU^LX%*EGJ=U+~lcPVMk1$YI7#z5{OK`J9yee@s?wBqhT{_2OQ2(LW-m2~r<*T>Rg05lh7csgFQisx#1;EXaDl zu9N@a2&F#4Q5kg|#nEGe)JI4psB;9V6u^~H-?9Hamd3%4N>CB-G*)0U@n@=yE%7`8?q`s7E&O_pqszE^g8jSEk8^MG3 zgL+RORscc<=pZCy?h=`VtoDT0N1$0eu#cc23<)135>h2WV-u_rA_5_?=mM{l!>oNE z)excrI{2#t86En8wcddU!ZR9#0m*%E5|TF|H4dDO%s|fL@Qe>(AXhz*dIur`VWW|d z+6O|y>m6i9_14!AE~Ij)hINZ-w!Es@@~U?0YY3wTwyX|bE5X&ndr7d$2sD3GvFTaG zrf1;p&(qTNPs=wxFW(62B2^-h-meX5o8b# zJfj5dA3^FPG!ouRLSnqB-TAg21HmgJGzKI!qLIky7HdNbo&_KbNbYDm@C{NE!AZz; z5~MzAJ@ON>_PgcCzxLz*+fRZJq&{jn1*($Zl@WpgUJ8!fKO$Hk!K);!^%0~p0;xF* zo_+$)WI<}Wu9N>ePk>Oz@&8@NK;0t9$Q#z45mrG+cL~&)I)*Tz3p6+bZgPWb5Xf?f z?xX*^k3wrDC;1ywW<^^o!qG9m|ILr8Ev0_k22czrbC`2PvVp!E@`Rss*%A=iw( zCqT=#k?W)GtN**N{_nW*zw;^xL29M$>;JoN{OVoXx|XVMy`+G zRT88of{>Us5n6o&awNEO1g}A`R575~L|G1pzB&#ZJ5UqAD-(K-{)1FN5E5PubsqW+ zuZG$W{6rw!6GQM^24O(59-Kt3cd%AT@Jx-I*&&q;YNCowFCYv^7YWiif{0XZep#{U1-MoM^^ef%BajrVGJ;e?rE4EU zYNC=gk4ix`(Iap-1XLM8dnn-g2v%Q!)^g`9yOqEEHl!;AuZ9Yi-O5{X1BAe97(kWL zvKxgO2n`(}RrEb)NxEMZr5nkje;B6LlT?-+2Vf=p3jHI`Y2@jxa{>p!E?%99$)J z9sdtOpy4}6j|nmlg;XD5_LaJif^3192B{OFBt$Qa1QpJx{VYhu1R}wuB8YMP|HNbe zCmj6`sxpp)NJupS;vNH4A`_5Ec;~412(+$5b{kw4!stDM)UkrEPKPb10BuA%37ty< ztpfzFxad9ef6~SOJ(vIYUH#vMiaM|S@4NyM?6~&7;~J=Q)N}oR-xW}N^+D#mVD%1K zHH6HCjG;k9AaxKj*>wuEUZwLiDAhsw=G|xicY`;?^_~TFsUY=H-*JouZ?N@mkm?Ci z*TC6u21FcE`#{)`3J9_~4o(uB5`tDt;93b%@1T+J3JAi0WJb)&2wwYOGKj2?(5fd$ zH3T8yRSy~iUi%<3Ak_+T6$2N62!iVx=%5^23R$pX(+dbHN1d`S-S85Eiq}0aUjMXc z-IJ2F&x+SPEn52wgw{O)_lzDDz}7H82IU|t79f*E`QT~@(ldh8M-Z}L*{%GgH;a~m zQ1S9xB`a}K)O8(DMC7tKDA)V(91WUf_QJ6bga=hCx2!nM=^UQfX*CI|z$ zdV*9*xT+zzI9h!KseLfX0j!T2_I!fmg~q*~A+=KT{;!Z)sRh%lK zc_oN~cJS(j4x|A(mx{I)PD?AF(KDP5H_Tu={x$LjQVKe$^Vm1fzagBpe?bJ zPW_((Mv$FQkX7G(Xa7&W1R90w1y?=DRT88=>bwG~j5@FX?*XH}E1)&}`0FF&Q9E4q z5oETg8@wGGQU~>%1Fa;7kdS?6@cAR;`l##hA4tuDNn%PtYaj5kgw8|15nS;41W0{E zxH3BU9bO;7t073egGoXLE+BajxjuqaK=2K%5H2zaua6)-AP9+E`#>rkxCn%cOd?l9 z$W;%V4e92<$K2qx3?>6o@4%~@icK#-LvY|e5kwFo0wK#cg6f^(^-qh}KLrg*ZFmW) ziPk-b)IJ4k9u=*AQndO>;hLu)1nwLatRYYz<%6e+kh@5E;K90*RrktQ->X=2zhuSj z%()jb=A4i2KjdAz!8&(=MPi?lcZH~B5|>^$yaIwVvX|b5uyetkBUl&dUJhts1C+$p zMS_mk;p!g|>?^@*A4~?M8iJFQ*GE`;NtIh)L#B`*B&==%9qd5-cwNJ;5AgH}XCS9u zI2)4YA?Y8D#H^1X6%*RphStM>(dr{eAE)UwXtoH_KSHjRh^dd7!6!Vlf$JkkFRAVP z|F-l0TQ5LS8+alKbZ)~r&|wacnWOGgpxGiwHPn3q+KcHr{=XAa1;DEnI0LyR!o`Lw zh8S`TVJxVZge-dmG+qZ8wS(3>NV1UrQOD8$9dL7S)JG5#kb5ybN5Nx*kp2;v>^}yY z6@rWt!b!w@5M+i3Y1*jg7|3mCX7(T{099Cr|MwjM84It%AS>&pp8XF&Q_lRK21bxt zX~L=hlg@xnW`Ng6klqk-mDF<;w1%PU#{bUip!Icq*Fn=vUFV?D1nOXdmnAwrz7z z*#f7u31*@7%1$|oR`Fu`A%aS-Y~of-JUR?4Dv)&!a1v4zK}h8K2(wm#I1)}msv%?& zxdK9CLq_c&B(70AbyF^b#_M`vqjTNY zKx1@}TB-LMs6Ohv{=e(S{|Pt$_ul~3N1($uPJt%%dQSd_Ab5S$1>QRVseK@9lIkN! zWz+#aBnmQK2kL4a``>jKG2Me!(ID4BXl#@U2x+k#NF22eqGEm2xc4)p0)mtEyFWru z1Na0mv}y>xkQ<)OaaBghH4daELKdmn_6E5h1n&yLYae*k17SdVK$s-F#(^`cQ0pVm z=-hh98o4K+vAA_lO4ozN;tIi)5vWfD9+xXz^E7|eV+hJy`6y@kgPaxjz>_kdeWsB5 zsASCpWCW_A);=s*_o#Hkqw-BpO4r=WS$s8p=9%QFC!)J{dsnS?&Yx=>->c+ZA!MDw zZyd|49>^f(z`$?Gz^TW;q|U&k%D|w&z#z%MAPKLDAPh)N1R+VPk05I;kn1A|8?7dS za3M7jgoM{e5C(EJ1ZP94Av6+RD?x_r;3T9Pf^Z>hI0+uLqe*=Psg=OX@W`o;;8_4t zA7LG`gOA!ls-)I~-_3Ho0*Z)rfA6VCa9aQ!7fd}P!u7egZ zbYA=4c@4zqxbeROTpz*vM`$yBkR}9(JPBH*38{}DBxphnatsK#(**A#K{g!qodX>S zh1oyqIR{$j*?khU^0WKc{|==32(8jV=7K63w0Z}L3ty-JV!)1eM6QV-Y{-&1(5WU! zCqsfdN8tVuq$`A#g!GT_)JOH;1L)A|BX~6gXAn~#q18mll@Xo_2vQS4`a$rT2vYAL ztxo{;j39!g>z_jUN0s1?5`1*76kHz_uXzfY6#{jRHasm@|0I9iQE?s;v zcjnQwN&BK2R=MR&vyN`k^DEVGOp`K>5Yq7Al(%P=FkukXW#Cd`V3A>9mSkWMWnkcE zVBlk5;AUXpVqoB4U|@w*LvRvO6G2F-)JKqB9;6zAlaQJSPC{xTG!kAJp)rtaC9G8v zr1lx2^%10E!c{B5Ya$2(QXhea>|nD%>SlSAk=)~ ze=~SV1EfBJuWEqIA3^FP2nnf`poW0ENG<1}^-=qI&`A!ErQ@w9|F<9e-v+KeKz)!S z|2q!-Z#xX_l7I%vAa%`QXw3rY)PSldq}~w0`l#bDtd4^WyMgNz=qhn=C4yY?L)p0N zBlw&VgaIjgK@||{EE1#!K_>f;{s$rOs2#laflnks81TxdA3Ved(GKZnK@Px~a`yl9 z%OEuK8mNAmedGW1E1;UF_vHWn(@1L(kNxjE1*)y4UH(7o+W+~t|1Z4vf5F}Vvv2>O zb?g81+n_3G^6meVZ~vco3)Be$ZG8nV@a_WFN+9wUs6Ohy0UEOFzYIFZspl+cSw4K_ zN#_aB2p*_=OG`Uup?!ciZgb(HW_MxY7^dGYqaUy!N? zd2|k~8iJp@1sS7j+yA3=-*-IK5Tr6{#@#uBbdfN7L-1+{!oXD-)$MqPRtG_PLufS- zL=ax@z!}K34>B84??5Ue$jBRQoQzY3Aj&Gvi2EhVrcEN zg4Iv*);G9D%TrtysGfeg}43__L+JcbM$x(uwU z3`|N4jI!WPkT3&-Ft{dyR7S}45qs{^+h{0n*_}M_q2G`t4R8{?(C%LL^4qZGcF;qF zAUz{Y5>ms`zdnMD)WOeUg&iw|b}Bfumjpfm56Xs&-9d)zAahAzHp+M%eBiDQyeSFZ zErK&3i4%=PYgs|KkWo8G1%yUo)<=*LJoGNofuHS%ezzV5A>>|C^KsD11n6oy@Y?U@ zv;UjV{%<%1LQSVYy`-k&;9k=4|IOeoRXezbY6UNNfK)~h5>h3BYPU20Th9D%Jqzv` zo&Dc=2GpDeZG8gof$BU8?%+VGq(jj9sO>PUR)W+=U=rHX0rz@9RTV_zk^dcskq}rV zq=rIP-+2_=&w`L(6S^QhBSQ62H>~1=)IMk=s4_yWhCquMkg5_$g@Q~%`c@!ikfr0` z-AvF00LP$bto0rTEvo|^j&$;W-)T@kXv#%Uy)*sl|C!hS&%FG9>UmHN+qmz4*~b4_ zOaCX%`X4j(f8>P!5&i!oC;m^K{XcKz|H>`@8}|KgJ_uUOHtEv;c{l$ry8C~@!~csO z|DXHl|IB;;r-ISsyZnkiEZ9(npcQxB z;Bz^^{UgvqP{?x7j>CT-Te*=*%*qH$9fVvZA?+uE*F+Eoqyi#XAHgdi2m_K0;Unje zoCv9d;A9=rDcs0e8@Ucbn=nGIgCKPc8VRX!;DQh?G6}7HHodOg^tx>0tI`cGOE$eK z-3(=yZ+Ka;@m2YTm*pE?macnVvi4c&y65E^UKX!^Tm(LUyA*sVciB47>D*=O9u=>B zShVIr@#^~}YaW!Xc~rjkamBhv__@ney5 zU=THB5HMii)L>vyW?+7no#YV1OW21_l-+#LmEguRcOPr2*1c z%2`3b`UuyeJLLWm(%Imk72uHW5~ODYua6)Mv|0(73#o?SB)ATGgIphhrjWM1s@e7m zxiZ4Vh9^I~&wuiN_LKj!9{- z(7I{jz5f&M{qMOALcO>D_ud9=iR}Wf_wKp?I-;TH-2c9_p!%pAsaAsQUxSR)LFyf} z`Un))Cqd^W!dG{Ls+@?n*&~WP`dVE3Fxf32SwmXpR#q2 zsy9Bb-Tbm{%ZuubPs_HxDBkq6VD;Uc48pbyBDM@-_6%aS3=&{c*or~O9Ml71QDb0IVPKGBV31~DkO3o61_luZ z20;b}0dO@0sf-}wavb2%ISvK}P6kGpIF}@!lqOo|2*QQ*kI?EPNJlAq`EBGb5~Qz$ z**SuUfI3HO?-#5EO@!eZtSemm5N)Im&P5xkgK&whk2bw7M?w{w-#`)i^+|}%QPnn3 zHH1_Xq3oQ6Ew_X2OM;|22nm-$J=g(svl1jRqml5W4qC&r`%41`YTWY`iuQkRL`6*p zzd=y*A<$7mpjrvsKZ2b84XKY>j{R>t0UEV~^lsY0og?J>sOc1F$gb%)h=h#YffnYS z_}>a!@(tpG`$`}aPXBL%q4qN%Hl);ooVoz%mvtZc-*FJsE$RSwj@l3YLt}vEZVvwk z&Ep*U-*xDJ`w;|!^n+0PN07eKk^k+7p~~7IjHCbC!3bpOQIHf=?-6jn30%`bN9rI` zHsJBPo}-|7CCI7dVZ$0upaO^uJ=u|AICD6KDJn>iq9g{?97+ zi*53M!|)%9UVo)szDwBu7qc5!9e<_>)a_;}NL;ky0{EzMXpR@db$@>2_ z+x|E00S$_Fodg|5*LVH@59$|65Ois!mYH>iB=iarA9R5C@zRtbMu;{(_^9gTLAj{Om+R z(DmEMJ6^%;=6&BGsAd0;mi@n)_x*%$oA-PL)kol3sd>*=1OlJMjoc1I&Xve)L%l7+B;PIMf)VEjiQ!MT}!*oy+w?Iv^DgCJC8B!mN@Y^%0R(5@^V7 z%{{dG2vRY@Cy(F^cr^rPKq??K60JVM*+oLCjLJ8?hE_(K-@toD@D33s1JXr8BjJ?} zgaJ=)pz&eY1%Hs_icBIWW;h!;wPRLB4ZFWUE2BMM>-T~Xa+L(BjG7L9gV#qbhkmsj z`UUPH{cb+|xB2kj#>0PGj{R?hb&)XZqsEianyBgc|HfmW&J?I~1g?)DyJg|^5vYrF z@_!2qLFS&iPJm{sA#3M4j)Ce3P-O(JkJ=CZZ$J1C4MFOrj)RE$2r`rhsg>FefyeqF zbr2E>s*E7j&=JtMA4oB{GJ;e|ATIK--cjh}6t(IjZS{~4D+-IiHb|IfY#VoW*<8hfnR{6A~yr|4;K{5tR3RoyhqJ)@I+Q$FmBi2FGK z=TqXg_rz`Q@tMEkGk?Wn{Ef%>8=ujCe&hdw#{VU({!2Lim-qN@koe!F^nduI|7iv!DK-`Q-nM z$NwkY|3Bd_s6Ohw`M)20WYC05|0i7fKM{5v5~MN$RWY!`Q6ZHMa%I$c9CS<~Xg&x$ zpw|iRB|$VIkJ>@jTXvp<)<>XD7I-WdQYFDxFA!fJb%HAzQtKmdHT1K2-%m&;f>%bZ z`@SP8qdi|C-6D8>gq$VO*pO@s&(t+L-hr;egItQY{R5;9!c`wZCW#;4Le`g z?R;Im?M2zlu+>JbH#BH*e=2uuosdq6^9QUF{N@xy8*9`Hg1UIqpT5@2AJ zWaUxd5z~`Vvy-t1)^<-a3oi6ZZw)S*liai>p>0=u>rU|Ts<+@I+QNpcWp|Jfc-1$k ze?(G!l)nag9yp{Xf|Kx?2$=z|jF1_~RT5@>1QCG@)*+G5+ms;Vb#M}1AHipi;0)x7 z39T~1-K!XM3Bk|PC{#?17DG= zq~=54TMmORNP<^Mt%pIi5@?g`anP9w%_l);2Q{4m%^cO80xfK4I`h9CywI-kBxnk$ z@ffH|f=nmD$;MOv8&Caj0(Y_?f-T2EBYVxqK=Vx<;JyiD$O}4ibmV{Aq5rK=^sn_0 z0<|3c+j0<8p|l+WscAb1Dy!R${BJt~s>nc<3uG}iQcVQuD?#g{!=PRgRPiy;!ijc> zdaxVdYC6E3BhVTN$T|nugb}D;1{Q&i5W?0}bb=?IdX9m*MSbwSKSw}o6($}79l`*q zgL;obFY<$ILM)LpRwqF{>K01JO7s+{a<%Lh9sAdE1UicZuZ-{y%0Xn&xvD>m!u=Ai!sF!zYH|j4JT* z07!iVxqApkLiQhRhQcu76&z?ipm4Q2yGdd22xX zRg2ewj-G>IkDU%MwHm~)_n_(Hzr0kRQv2W`VO0|0`UrpBgjok6i$Lll zh)BiuHx=7KgL{w^i(Dn)s)-=t^}9b}Rzt`l@X83n098o`zBac$_QK^ftJRB*Un8k{(sVOP*(`l8-mqG6Hb86Ntg-l3Qax>x^1EL@c)`E|MQps zOq%*Ju;qbE$vyM*i`o(AR05Anxt|tyIU#I)MbPpRugN1`lSdrJpLxu`^O*eSHT^GO z1nL9{8~+zE`7dhrU)`@fjMe_>;ghy=Jw(hmP`oB7|n^?%fq z|0&BrRZ`(WP<>Q;>3{tdP<_;K^Z$f9|EE0oKl{=Dc~Ac@cmb-67QF%u<;{KZf6jA| z_>BAir{4KL>E{0l*FmWN3aCEnz5qIX7j(=N>@p)rEdyH80PfmA4(#nX_OIjEzqTWP zAl&xjptTLq+6r7Zb({jN@rK;<*l`wWX4?tSNx$u=^%10(1h17Kht6SEF_2+8G;w%+ z1gU_~NVNK>dEa+Pl|*EHRJZE`q*j7gMm4)XKoDAegw{W*+xDgbe788HN`e#%5E5P! z!5Gl-I&g;wdR=(sX3)k}NIil^Lh7U9wXY!cQR%vu#cQ6HtbJCv{(1gd(7qw~3Wti# zZ>qMwtKRxLXW5;MMK>ZQo$zno=2*JIEMum2Y@du*1-of7t9B@ZtUGAS534={vns6b zBL$x5kp}mAM8WllAh?eM8C!$T^{_E8ure^PgDW5|aFxUjo)Y2!uT0dopj%se*ZD z3Kv|?nSCyG+UfKeXCc)PoFrHuWi7jvwd@vr)DFUc)I^vhXclSpT~L*@8hUyIybi)- zAor4RO)9}VO33vQ=8<>cDhXO2LC#`@_m7Y+@`J9pL#~yu_LY$PO~~x3ZJ=rhQW-%= zNJpt^C;Bz$klF{6Bv>Co>Ly5K1R)#2^%1;Ef(+b&dPzw2QS;%yZQxNj(5eRTRwKv> zLXcfZkh-bi6sVU}fAW9b2@tA>^_Ut?{0AXeebfl<7D1|^rlbE`A?V2eRxoNg44P+e z0e5zq5B_gC23?!YR;dgqaur z&$Kk_|LBo>iY;7|K~OM&j&{QhX45u{|lJ@7c>J^I$~!370myO>;30e z`NS;qibwgEnC^cuqyM6Y|3ytf9VKap|C)aP&9nZySN-?u`5!*>fBcsJxrhE2T>f8t z?SKCD{{=VxSKs_!b>n~YjsNX;Kx1^%pZuTm;{Uu?pzhM*SN|8i_`l%U{{@f#&%XD6 z=I#H}Zv3Bk?SDV)Mx^eupxOj<Y< zNJwP_?;pXrAQ7bg5u^?RWy4(`AlVaA2f@keU7%Uu+MVwqO-*P9-|?Yh+q=B)eSQP10xtQGcdr} za1kc38pw(`4se~t52}C|gh7V$XfR4xi|Gc)*`*o;lv^cq`4-O&t6!7YwLQB3K+?41 zsdLVy&%2N^?^4RVO9^v8D1FxXl<8+6D0$}DPZsKXs7TUUP{|i_CNt*FBsN<<)$!+7bvznn76?{&JxSZy@nSTs?cNo~OGKjqu)c7x~52}v@!Bvu|)qh#1|5~yCt#bd{H~#nN`tQHwf6T`J zNk{+ZU;3YW^?%;A{}o`=bRASB_1px_8BKi%8o`_Y3^a(h^!5KGufPi)9{-ph%-GFS&5nXlOPu6*m;if!*ewG#MpPf!K1?G=2*0;Dnm_k%!VaggP6 zRhynwZ+>33=|$ofc<^V_pk?Ywf?ss0wHK03TccEqQ2`eXzd(iZ3Bdah{J0iP`3y? ze*`)G2DB6byytY{S;LCm{6FLD|A~kHw`}`gw)B6>TK|M}{`2Yn7dHAYW%^&j_`itJe<6eag8CpNVEA9a z5F{mP@?XOAzmz$M6gB`=E+PgXDM6$Ef`*_TlaMiZu+H*7kKTU{#d{3gCm1;T8Q6Lm z#g2+;-x4?YFJ%leP{ar{peJMdUp?@@dE$TT(*KUt|E;_JJ5T!Wy7GV2zWD_2(*y_TYZGJ8bThFgRs%6A!IIe z1rhkDF3kQBa&CpQAvqe}{zNVikk2B6jI=J!hs%#zd2dMg@md ze*F+OB^L$}GX`!gaJ?e}9(IE)E#L!JI`9EE$iNz8*o_6&(_sZOIKTsO5H>^#v=0cj zEeKNkaDyj|Aj=Yj7#O4&_zZ+}{B#4W{EB8L^=!+Vaj0C(mL^A?`ZU2qPxZFc^} z)VUW^=3Gdebvbd?xq1#2FlRY?#od|nBuRwA=L!a8sV7oYCj5EhXAQfkn5x71JL@Y=>SqS1gSodM%6%-1$Y>(9kOH&Z+!%@1yVz` z9|o_%gBS=N!2=m}1hinU8(cX;4TO#ObsPfqZXj2IL-wxrodhkto_Y~f4NblHfBNPB z(=Pp=couZnY4z6sS@Sm$_+VbCY=6|ns|NT#bs-%Rg|KqRx&$;|R_X?;!s<`~W^5XxB z3;!F>{ck!CUchkW|BM^|=imRoeDHtX-T$+0f~uq$SN?<0CD8cfq;sGHBKl8( zE{uTOiw6$9e~^7cklF-7Lh2(3yW{Y`j>G?u&k1P*-+|I{G-G2BZQ)CedmiI2TeA)$jTYVvtxLAwW#E z=fz8ICQLl$U9(Xyeu{{F0fR~?gSZQWpdACh1p|j61EU%PgA#bZj~uuMBn}>ngU|kO zfma|v>KaJ>0El!>xgfNgkXO5K9O zsrzado~l`Pp?u-#vLzP^=AFu(cP4w@ne=%V(&nB|nR5YNA0^JXm^kBN!i+NsGtR_M zKOGP54Z-Um$Vgqnv{P|YPGl~=2}huA(c+s~OQ4s@W`ny(kd6|Zgv=kIk?_h0je#_y z1UeWTUMs;F@X83n0C$ugKq@9k1%yO`?j3u+oeMn6KCtJWXM-`tg|1V+qU&`pef(59Gk+c3UZ3F7Ih?#(TMxy$l zMF{*l|G72(^J)DT)cr4L@L$pbv}{}4^1p=ke-7#Y3_@QSMSpN=f_g@L#{YSZ{&SoD z7qR;pzFsE(V5K3=F0WtX2$sWkSl`LQ1zJH6L=QeiPCMEdm!d|1ahAU)}$| zcG7>N%>Rak|4mE(>o@(kpY-2s+kelK|J|;DP{^hKVORdAUHPAS`G3~M|Cty5mz@1y za1z`UISM)*eFFFt)*079=MF7@@_*T55M$mAP$v;GN;&oX|0(A{X!6p+MM>#Sf764{$b!0Uqvy)I<=n=kT9y@FjYkhkmz%QQN^^ za0FS>0O=*c7tx{393ijehOm)qCFBL(jeEX=>LB>Cy4_zOxeT8B;9VgI1KyH^R6`K5 zax0dhlFBV_D>lD@Zgt!Y8h(LH4ZvoC-juF`T`IH@bie}SR3Fgc2^*hQY<}9X`)$kK z_pSRr)NOxWx#>~C;)@a8JH1Ng8Ai7%xa4x^g!0O}GVmCIjynJizDa=RfJDF(H<0Dl zkUk4!#8?19!OUqx{tmA(auf{t@>22wpM48IZcEaKqEW4NoDJ5rl+PNyya@G8_11e(BVKMdLr+kG4~*3@_E|AaH3F(}9Zag)x0 zdO)??|K~6NA3ODfSHlz2?DL8tM}%Cra$D|TG1|zWzK==kAfwu4PQBZFdLIQ0K8qUu z6F2@ZZuno?6f`0yWBy;p^uN5te?`mxDmMQWZ2!xHDY_Rkjg&} zm0w(He|WV2i<^P!BT@7J!fO8+g#I&dzF`pl#;pxnQzvNlpWo!afYpCV=l{x%|G6~& zGxBd{V3^LpV9dZ^!oZNkz*)$^x|f0H2m}8Ec9pMOdjEyZ|4TUhSMm9;8uMQ#`M*}y zf8E^w%4Pozn*XaU`|o)0zw^caE|>lXT>znkbN`dUl~K;+|K;E+so~iF>VyBA_Wx}? z^uPJg*S4cS`cD6ydg=fCJO7tF{J-qM|3!EHFTDAG{&i5rGy~jGg4ajg$NzR6`vbYz z0bVP09r*>VeZWTmLC&LtEO-D_J>d0ta0YTustf7C8tWK7Wb`A*OR26%l`v*R74GJwdFo+2iE#?4amg6U8Mx}WCOSoz`{nkeG%w2N-CQ>NV9nA?m5a_7 zF1}W{_*&M2D_IM!q|LjO241KDUaoKjQXj?7231DUv(CrQ1R+R$6hHlJ+_WZ9UK&mmP(>6VwcP|4;OkO~MF2~Ub7)kpBu4q>3w zN050{$QS}r`>k%zH#n-_`yB~^cF5wdk6^u|KP`v;G#~ukeDHU}A@Fn(5tsa4j8&5UaI8+QI5Oz&$L`8gKBZ z9pvcIiDyB_kWM)VIty;<+5c0{fUd^tJ@UVC>;Jq(|HE4TJLZ4W341T)a#z&y6|dnV zHno?`%Fmb;-?OQFVpsXgqxqjl<3F$Fe;(ccf=2%Z^g$CtLWci^3_-(XlHhGYpha%x z|K)7|tJ;FGESo5_D6z5l{m|9MsZb1A=MlYh!CdzW4IE|2mLDZ~GgW*{V` z^`AxLCj-wzHt{zCx}X^)F;h@&C1v(s&g#FC)qf6!*KC3{3=G+f?DC*xE?t##fyZ`6k04?X3diMW>Q~&!;{Odjbw+DPtIJ`1~oM;NU@dUCu z4!M5>seK?M%9@8mzuRDQLy!@L6qbc+nlC-huRqAS6Tt zGGYKBk^4uGx(T^Hssr!%11)I)XTZ9hpCEY?(t5_nKn_ojv z$tKX*K4lwUm23i`;*BpMTZBNngjPQ+Ui-Lc^Rv=zFB^8gtlab{sc)Bc`eX&?OeR%t zMqy*np`)OacBH_aAo%#30C*q{GK0ebp4Nd^E0C%OJ}U&NRv_bY5Gi=A1gW;*Bqw;z z2(oci8npJCS&MB$PboDrDLye#XVbnddn2DI z1tGH*-GCr?t%Oz|!7HYWCATs_>l~oZ*=ND@xS-f|Nb-oCtmakH{$>!l#lUxkf%hx}|9J+{SNv+f zMZi^(tnPmfiN6edk6FcD^J@JUHv!dpJlddokX7+No7_tVp6g7!6$}j7EL=(qY;p_? zI$-3_z!1W~(9bHjfm`Fhgehoytg7pO^^pJC5&v~!{_Dm4S5En_k@;V;=fCy3|3>FQ z^^x_(|5g|NyIug*N1mttyB_-=e)zxt?*B;}z9(<|QL*iN@y1V(J8`PEKCj#Px_QsL zmc8%04}F<_?*GE8p!#Up{r}7EgZfHyu7H*|OgQzw2i!S=R7Q}R2tq;z)*v+tq&|Yw zP>`AkLL%2k$m$_|AINM``+=Vw2Y>`))y&DZ}_xsRf+B8wM=D@c4y!+U|>*#orC}xkrRWRL@Erf zjQGG4I-u(qVD%9@QdbC4A3>@i$btk28!|lvmxV~NgJyy_#JPo)xTTCll&ytygA{EN zb$p8L;#z~t7bdrF%$a(me8Ktrwv@>Y+5o8cAcIs(J zWrR#-EV=<7yTh!TAl)UT;XBYWJO~$aF&^YR2XK`Ht&i|@kswn@pd*IBo0Y(I5OgjH z%qFKk!a8IJ=_Nr(cm;&UDBB8M3J$55;MGw1w%3q42tq>kBUJ2sR{>rK3gJS?>Rs=V z5u^@+kX5@sz!7AZU{gTaZu_czqpciQWGH zEeD_oGM5BdBi9PPgaT6UKxTy?b3Kr?bC8+{LUtVf54uR=IOyhu38(%~1mD>`^Wy&n z*Z$AF{D1oC|C5jZ@7VsoXyO0Z&j0Sk@AX4pOWI#$)wsr>c%DJ(HiP&L2C>JC@^9Fc ze{!k)5!C)KX$&26Q?U3iXZByl7=*+P|BLGX7t#MOYVcnGjD(H_)|D-2xg7+6;_ur6ib zSivB4mQVG8q|tvxtN+q^;Q1rL2MmG_*yUgFtAF5E{>m=>mRD1uKzUy{_BPQ*AM$|5dL31 z^1oKhfAO0C)(iftp8KzL`M=(^|GHN}Nc-}CqYM9aPyaVP@?U$`f3sEJ)n~tmn)4uI z)w|*~&r8<6C|UEMeEs9{O;4(~y{Oy%vSH`z_I)2F9RE4>-2b`P{x7@tfAQ`Av#x-; zNWI7ZLI&&F4}OQ#L=dtKT!}#HBk0Z`@K9Sb^1cHkHl#j+=z^$+?{9^#>qe`L@br%$ zyI&zS5rk|8pI-;5i6E;_A@i;93gPziZv}v~KB@tceH0tCkyuHc1*qf{r9*Q3bDy1KmRh9)*J(ae#aa1EgET z1+Iozz)P|rn}gsar0Rjp9I=7dFhJJSLC#EooMkEs9_5o|;L~MLauc?SQwSd8-~|ExVJqD}%NSbpgY0jnOxtAe~q&b(MbrWPY-K@(A(=WtNKN~;&EU1eFnm>XrW{97D zI%e9bm}#e?r=CPZ$Tbm!jlVvEbeF&#B}7dGsgE!h<7Ih)kSQeikR7Blf{7UrwUvd zVb(`gJKqsgA3^d3bj=Tr@j7^YRKNE-besY8TsyQb5~MzYEVqN#M@>im)kDx>@PT#U zJNuf?g6=bFJomr(0thvo|KD={Kj@0{lc4#d2H1FA{lWi@2T%~GdIDEK@G1#jAE8x7 zpepI$|2F7WS=b zLfgLoWo!P&PyOIhaYrlekht3x4)di9Ix84d7BEPzXOQ2@pm2^!^)jc%O9A~4!iN7O z%|W|LB~1P+TK`vb_^)F7U)ct{Ai?CnnC^ceP4J|T+J7GP|9qPNxwJr4lDPSQIlKQ- zR{xom|1*e#5WCuc1^fRdLH`Z?|7$schU^q<{wv!4SG4&rY4TrK9kft^f%6gr!!ZVi zMGOoJ85kBYuqmGsbn+D%9P!76i;A!=yC-vK2H12xSyyso_;qRcq&vVfF2sB%C2zuo= z;uvl4`Zq|Q1ya92Y7K07B0&;xj_RI?3K8C8K#W-i_EvS`yAFnU$I`E}vu*CpHD)$IDzxc__m!S4;bKGbb{ zQ@r|N`mFPQjoYl!XDRtN@tLMDCvXH4_*{As%i79Fo# zeW7;U^}6*ps@C4FT6?=>?fsIqpox;~<)BhMW67=D#W!;n-N;&SEekd%2dR?a^%0~) zlsFr-VgXL3%(;{@=MrRfcjAnT@zc&j`a!W%PsdC-6*mopVyB*rg;hgQQ%*oo#H3>o zl(yg+8p>F3Eo;$@3@}PtbR&K7O=JWawS%m|gI7av23i*hTnCX>A3@fK!)qm^fjdzD z2)qFAQ6X&94$eTEOCqU0!mO1bbr7UJf{>67JMx?n=zt)MJ`uczf-@i;BKY44?3#d`yC<+$#IZ&VEz8@4X`D4$ZNkL^$}!KQsd#j2z2OQ z%Tdq;NgZeYx1Rgoe&K%$7&V;-RY}bk|2LioZASuaN;>wx7JNWZ;}OuZfu=*KRT8*9 zf>uKZLH#C3-PCsQf6IPQ{nE7mfAc;NyJg>h5Q21?AcKD38meR8|L%kTdk%vx?}psb zFyZ9?sptMrKLa{Ju4DWEisk>)CjR%W_+c3JUfSg*yY6)cg;NZ|XBqfUGVoks5WdYI z`J7SlJ-6n6eqB&EN6hfQq!DPGO~T~Av^i)&f}-_*RT~gewgJ@+k|v-P+L8vKDoIf5 zKcD)4E|nh~O5b?Z|B4&_m$UeimC|oB!2r{#U*6 zU+u^#MLc2kIFe_}0GvYby-39ry;R7a(;+>%Q;sm2XW4el#Mj z`GmKW;3`3t2)F`*tm1}T>;$|G0?<%*vt=jqyr9RsH zvU>B&+O4naw!Nv@@(MKIx#d;Owl}pq-#737T7TecXlY6!w$2iH3+;7K3k z<9ay3%i*}e%k9J&SkxE-AqZUHLo1{CH!|m4hoFpkS2E^ZNt}Bz zaqh+BxffIBT}qmBArZ8t=wj;J%b;p#_QkY0mr`e6OrCi@X~wyv8RrtFor#@#I%?91 zm`TT?CmxNNbS!SliMXl9u0i&qNN24YmjhTEndeWiDNrxgP9z?5*(!jG+pt&kS z^-&gdItf&t!s{al1G!d$tpF#se?&}uRImoUN+P~Kf_Ic43`k{!Iimz0uftUz!D}33 z2BbrTECQ)?h^dd@6;t)@&)}@|t$O!YY*iAd?MG^T1lkyTtSf$2RgyDA5tC5=EmBPlab&q&q&zq;LjWh>AEIaxDMWh7zzU(x_{K98`@e{Plk%(5RD zB;T?rd=u3EFKZ6cBCPYDf&U`|^J50Vk75S@wO#)k`~KJS_^Y85ou^F!nPrO<-W@1=U9!>lvjE@@W5&wfQe@`5#mt37utN*vY`q z!@$r1-e*+9z!1d1;LpIO%fM;C#HYftzu=uq{Hb8E)}f24Y`$|VC8KPd)2)X(4fuJ^35;n zz}F}=?fKNa_jCLHugLXL>wakO19F=KWVHaKnt+U$VUm#A1VTcp5-1y~_5oEx5bl2H zIjFR#k03px>YX2}c6@-MbgIYL)h!X>c83Usp1A{gLgF3h$q{6@eStbYR z5J`bIuA-e62OpZ_0(WqbD<*F6KpZD{oi;?A6)eKfz$V2hq9LyBr0*PM9a|?)%|BYU;AG?KE46EH*Q~i!vHpJ1%A46sZ)PpIku>j0+M*lT%b*BS8D%cH zmbUn6+TyFp^RK3WyG2 z09iqoI{PA|K1!T^Hg3wP=!wT8CmxTPaw>l6NeGIYaw2-tkr;3_1gVT7Cmo5Jco0$_ zg-EK*YmA&c#oPkV29iqu7dw74MYylTe>2p5w0v6Ao#2-dXw0ucdMM$ldoq~1X$tM`6~quTvH z;i!K9&xQlP;HdH7Z^#;krbB-q8 z<^T2P|2Limb&)^^*`4^`aP)sYxJqgSZ}w?{^_5x<{BH(VNsaqqdteWN&RYN-iEsqE zlnyk|cNlbhbnp596V8DSrI~&SbScA}i~pyb{NK6#f6c1@S(E+;RlT)|f2ZVng-`1O zgXDe&?qgtdlR@+`qvSU(rT>EJ|3$Sy$H9pi{TDU{AzlN}2|WBp|M?96^XY^3nM#|2 z7HNy?{}Z2wTd>vQ|T?`;RZL3&1z`U2F!*#qh;HShi03PF3mw!*F!YTWa^anJW=7;4@3 z6I$Krv8AQQ5UZokB zl^9r67)DI1=PbFAx#U{rl4}`@KnOBDlsq3)F{RGCo-+4Z z%G_(obFL=OxtcWl3S_D%an3n7g6wjIkjXPaC(*(CM@cizB~Ck^F!fyA3f>tcF9Ri&Z)N}xJd=O|U0%RZ1QSd$8C;#`I0bR8- z>B@fyntTy-!9>mWze#f+yH%glPS`B!yp&OYB7^J%2FV@w|FTy9#f<;+fc6CaVBomUz;uy;a}NW@ZU*jSY_d0a zHUG<5{g*cS&nWSWf%6520?29|m;aEJb!zsYDoND|RH<>v{AXZ(#=yKDJO|Xuz}yP0 zk661v^%2`b2C1Xm5Udvi? zBWKyI{B@5@x4eYZN09yzsOo|BTp;yP^WM)OL89v;&_SlKSsq9o)C{^Z4|;1Jyc&Wt zAax%^CDcZ6eFQmu4l*kQAt5ypgsj>99j!j9-S@e6A9xEnc-I~%e}M<RY_Yv z)NK7wx#fM;RuH#g_m}4VU)v7-XgTn$6?_qG{m%Ers~^TrJ?&YuQ73+ih;tc(ULu2H z5QBgt1Cu@jgC=;3kR}6@CU{Cn5p-q(s|slQDvK0&RtU0p6|$}Xvd>f)TqOyBkCKDj z=>Q=iw*^5uN1)T;Kz%3}8?&9mYi?8P_zM8Z6Y8Dt}FS(Wl=0XJF;u#CBWGuLn zy5JHB&A$xULzFV_N)l*j?sD>+t0?@F|qWD>7;0SX5Pr{5d$+OR=&OV_Eh%?6*gUE!0C#LYYn>LN`(8aL@M zxIO~82jVI$B)G|BV;^*PjC&*8n=Y;W%imUBeO3 za=XT(;EvUC&<%YZC;xYz`rm!_fA9JKz2`uv=feN~%l{`_1fi)HKevdX<>k^jM~0XlU@*bsChgQ)p` z8QcGI_MnvmGUlLhHbEWG)=>_%|D5U|#Gwm5<-qvAfCXsDft2NcL5=^65`P#Z{_t!3 zSFro9@B81-=f9dGXb4Wp26Uo=jO~AUv;X{R|5?QzFtA@_U|j}YfiRVUVHyMLN@j^Y z94f!$to|#SgU90dzVNDoCZ2ShK|L&GoBvAopxRB|`aie&e+JRppfwDvGZ+}BFt9d* z5OWs;TQ37!I|F+M1N%}2iESL}pksw3&Hu9~{b%5Pz`$^sk@qN@&?yGd>kLwNSf!t_ zNIhkf*w4VahJneIfx()AK?1Y|iK&`TWv-C!KM7;d4HOb)AiwB%{WtXeujBDw!wocG zsOt7#)$PBc$A2Zy{|r_CwO9R@y!l_|&VQM^Af$Tdzv`X;a$qEX`@aI1A$8-w&Xxb% zoB!~8^)VQhDq5#1+onr93&53N^nlbxA*~)tbtDlr^cv-vSBc#BG z4#dHBl|n}6Q0gOatppKAQw-U}jV1`=euv+h2dReY4}6B8+I=69_kBYykAOG^vKj$G zA~))4_I#@a4;|F*|5*h_)qB5I@BLb_?{nFnj}^N=l<#_9x$|QM=xFW_727}ubXV{E z*tqL+^PaCA`+syF`aR|7p9u$lbnbXvv-)1qtTPb}n_N;S>H1ZR8%DD$dNYXGF|g=> zmmkQ2NjdOjkSwSN#3I4KBmzQ=0t|2jx~dye^+1l5gI}TwS?vbd-zpF7@~|j_7~EP6 z?3xU``V4YT0>d9+~Z<&u@xtJdAGUjLwQ&Ar?ew=$RB zOkaFGb;-@tB{$O--A-S0J9F{%j78VdAZP*T*aXOo3}{IJX!r|M+JhHtgO1uxnRf}& z8%mi6+Q*s*8jb@knFFnwgY<@GUx9XuU`x26_0i0;ap1#$5@()G22CNIPndBga^lg* z{=<>|hogEAMfDyE>Dm|AzT2&Oi)+Y+W@JI3P4BOL3bfx)<^J)39YXLsf<8l zci8$zplz_5UO?_lyWuM)3LwGHq176Es6wwd+$Acct`m35L zyZBvKGU!ZTknLv>Y-Zq`!XPx4L1H(f!eJiuo1zAfBn>~ynf;J41@!_z(>f;q#ZCW9 zfKN7%wFlKl5|*H4Y@!CBLwi80;57bosDX-iM*06t>i;=(|8pDv7q|bf;SSm~#31~g zf%PQ=|7St1|7tG(jRXIi2mROg`mbsaLUQ(?Q{aR^OAG!nu%2gNILg2iplp@4vA> zs6GNMv2*#a;tH*gRDJ%lxBWNX1geZ=@BWttS4p_)BfV??r4Rp?3Z2QIkt1aiCvTZ7 zW}PK&lPzVRC*xQk=Tl{tGAXoUZ|#PskkbY*s}e|!0I7GH_kM2L^Qi?1VfI`g>XAuU zO$3>K0*&lJL?E?K?Y@t2RJZpN2!RI;kWHxH^%=5AyM7mFSpt0I0W#TI4bBc#`+t`2 z`Bt&(Q~A!1CA;1i?|fIb^IgU64>fzf)bIb^eDHV6p+ArnfSreax9yz61@ z`)qQTDEK#VS>`aPhBF8_F)$l2Fz7NcYJ-s`1G6>*n;ICYg2v5Q6d9Nm7?{Ku7)8OA zkuaz-ViIIv5(F_omv=*!D+q#?x-kedu*$LsYjet&39C7XXuC@q_^8-KtJ=n>J01yaTQeQfX_iqoOKyAMhC8p;%8h0p_!ob7UE}~jh}fIau{9Qv{O-& zjs&;R{Z{+Z)rRp?WW4I7 zJ?qp$I!%(M+vO~@&0l1ZHPZ45X0*v&ZhQlAsewkm0+6HII<1AvhZ{WCtO!)<^Kp5qv!Z ze9j21e+20nA)inOU$g+NjKHn0%AFq}2()wu+)IMz7c>T>4niZ5XO19jls95G4O6+;9keTb(lfqEVJ|zE~VE(n*SsX{tIaR7uE&M1PSYd?gkPt_|K#L zpGy;T=9{PqXqS+p)qizcP^BPm{$JV{bO3^&F6jOR5rhAdhW|x%|MP2nXOn-&F872- z`K5p^sN14w|DR3eKLg8s28LS%Xe&e--Ed%1)r+J8ifBf`0$yqW`lj{I7QSzw})YlDP9<;?94$yC5WW z=RXMD{V#dvzu3M1Lbv`a-TW_h_P=ueHU{-PVciH3y-0DBXeskJ8Ou01>jW9cJPqF} zv*c;1la99U`PzNpXXE~#^?ScJ?*0Kn;8`I^y#T3qApN8Ey`S5`ouigLpCIB85|XeX z>Kk@{fpll!Tv$y9t&c$5y`LNRey#^~i$GOU&E5|+dq36egA^Haml zPjx##f!1m709`j$zw0w-vI*SEsy_@Gj;=lUw|f8Y>I1)O4}q?y>^%8@;;H}rC;s;z z{N1tZOUsgq9yRjxOSovh$g!l@n0AnwG#ZUpWGX@I7ISkxITdn!XO$Zsm9KkHw*Fz!>bp71 z?&L1No4xclX#Lot8=&ohscZqNk90i#YdM2fAQ*Uo z)@;jI{AX4C&%kq=f#D3OGGdqos*IRB7+BgE*gHY>5#Is^iATaF|5R;3N2jvL{^!^D zuVe$Nk3buaY(b-TvS$BzmH#tve`a7^$H2IXfqe=C+hhjzi3}W*K(|G3^fPd_Fz~c9 z2<+g~x+G%$U(WHrnjOd&tWrN1gg)>n{F2ZCEyLi^2Mspzn}F7+vnl>(7GJ}_G@XGn zhFvU^QSzv$?oC01|5BEq%1GAXzpBT7E#LpDuK$&tK_ioj4*xZrL5R=$zkbetiLL+j zPJ=2VDR5~!=6v@`lxo_M|gb%X~==Lje}27fOL*(w!W(aFSh{gW;*nv{qRrFhP4Zz z>oX>t2km8OJqhZnx19oYE0Hf3Nn0M$pVx(uwk3~ZVp#G%2!p#iFl*pxv>XRygJut|aHBWRTb**7E#>ezzn zBQXXBDFzND22m4XO%DynG`sMc;G#Ksz1yqio@!ljwQcR4?oAInwmjVys6K)pumGu#QeoR#Q)XXD0k2y~oP9BA&ZX$7 zr^EV=1a}_{>N*hAbs)I=U_i${pO#&ojoV$TH#$|UvnyF;nZML5XMsW5EThy}2FWuF zQ)WVtaoTK?v;`JfOKgi*I#;fDFI%Y<*&}3{#319zDB;E`MP1cbx+r)YE?Cf5D31&eiASeO57OcQWwgF|cPcurx5RHZt&R;7~Xs zr1M?Y9CRWax6XeKFk;pD&!YXGMdv@O?tfOD|7_r;2=Mz-d9*>N#t9mLPIZ&B0Zo@E zS%E4e&|(IQ|I%h4B&GkKTj4*0&;z8gfh=m->GOp4SZQ z&$v}U9T{ymP$xpo4zx5*&-uTq73iD=qLuR z$qd|6Kzcc+f~q9;i40ub41E0zA{T}9-$>d0mvj2B;RM5{~2Z8G6+6m5WmbIf16YFr>Nn7VN=j?tWtLWrS1PKI{jC2294S& zIe^A6W$peexqvRJU`qRMKKsA?MbL&L@jL&;@BA0N@n8HFbkt7b_J4`npq`P~z5jxD z|MT7YFMi{{=(YbcMLQVeQw3DKgw%Y5G(1GKyu@@p#r3>|4Z@|&<3z18rR?)`B6`af zUGLiet7-eE#@#;}c73nk^Q(T(ue#me>vlue&Vj3vZ;;+lF-V0m@VkQQAZ9ZLb}I%B z69!IG1}-BI;?`#1)@I<;03mi2P&EYU6LBjsa4Imc%P=sBgOBkM2h}?a(x6!+7G(xb z69yq$MwMVui*)_aPM55Cq4nFcr=BTYe4~Etqxy}n8#lkL-}1I*s|7F~l>J&-+~pi34O!>(XhbUS0=tt>Et zR6y`<5r{GWM(VulkU9u*nq1nvD{0^k5$KEs6vm~Dd6yt{Q~KP?5C*&hl|J`!(u{Md z;Kd1%{fGTpc6d~8aI0MBT)Eb%VvS4r8mH1#wgpQqvKLq(A&bm;CaJT`(`K5d%`{D( zVUjf6Flo9;>MY~b`Bpj0oJv=@R;+c&ou%Me#xCc`AYjbEug1Wy#wwt~!mrH4Db2tp z!N@Gez#xoV4Z+!%^$}7>3B7-WwLZ#Nas$>yf^JZPFKmD_ApIju5>hciNJxDIUiA%H zUk9m@FiA*#ghs*#?vUq_Ae9l?L{iD-7e!!Hf^oDRq&|X^kSeJXeDXZxlm<9SVSNOz zn6NS+%Ot9Ie}>dYkSYnYK7w?VAY|44pVhE|yL#~QZ>(h9fnQYz{!}0QTYC&tCDokx zUv=z%*-;RxJo3Nt2xzdb?JTIZnZ5q2Tk{2}@Ffho4GfYw4E(tad^rq!1q_144E){f zGBYF$uBh5Rlr;V)X7HOw7gQxd>LV6ya5bd&pG^-`8F3l@7cu`YX8E7b;6H=>e+H3X z3^M;kO+bx*H5br^Q+adHvN|Pe(5#J&Ip|0qUeJ<&w+sw>!S@8s1>erFh=FYtgXn!h zy&sZh|Ah_ybBljt;(N=l0qPcMx&Bvm0QF5Y9RBM#{#P*l&m;ezf$bRs!y52*p4KQgf1XW+QSDES&P^TlWU zpT`hXAMxt^7tsF0t@fQ)_ocA$dvW9cQf8oCw9>Z!A@z~09e7=bJ*cLZvi>h?_g~89 zzpVX##n%5JJ3w2F)NlTmxC^R}AaxLA774-@y9cV0_-;Y#BcaRx#d9|^$R_hBxbZ7{ z@~gV?tGaS4*>Ne_va5LWX$SEdCGr|23fUF=R<7$k^moe1|1JA|w;lM~eBfWx{=bm= zsA>1N<~`pb^%1D*+4Z?$=cmS9pPP1mY6i1GgK6M;q<-gn5CYdnHQQd*?|cj5?gZ6J zkfqd+`UpJq_py2J$A�>UO;YS4MB^c73P^S5;j{e)S*!+kgD;q!a(9p8nr=fJ{qnm4zV?Uxiiw+HWp1iQoG=M+nNVGo1S%Vebc-BUE7wI z%^ROpuYF#*`f16^M}^Do=Pv;vXuY%KZZ-_%F1-yA%w2jLvOfrU4KwH<>%})ArxRo> zyaht@Z)Aa|ia>P`a-V44bx3^#sf<#pZGA*rZP4HA@Cn zMywhPY_cH4DhZl^1FeULEHC7N*Fne(to0GBuXHVA(Ty}Hf-U3#FXw<)OvrsD;`>L) z6%(W~!aVE_vnGOwpw&bWE=l!K3HTaW$fzAw64G4)tpmq#V=UzUVt5w`(oq6&x4o;{ z@d3hxuJGCM0c)jGvFj5qR0$f&`&td|96_oiNEfMU&$o)b-yx_9ywDE5z6i3)sA}IY zI06l)@Ba-Y5B#bI_n|-o?0bGR9sFB)@PFmO|7C~%mmd6Ia};zDQs-&V+PRK{{}X0D zQwUtkpxMeGRscdGg$%+448m~?Lb0IE5pO<&OBMNEztZC$3_PJ8K5eOcQOOdBnIw&5aQUw zAby%l=fAe^e~X0wIwAkH?Lb#fun0e6V7kh{xSxUV46F1LA&rkbhM@Y0Pyat3_ykuz zUC?Zruraizmb3;{I&#*a1?qA(pu_8wZ9#*Fau%RNBpBTOi^u=xU-w_{?0?x?|K(u) zAV}>aeg{-NLFSUg?)?|N`(NnBe}QZNC9nVIJ^f!ab{T_Y5|4rduc9Njq63eTBaebP zlb9~ExH+$)qkv|Bh;9h4ZoImCWpu;->3e?9I{kmb!T$|AzSnL0TC?M4J*!{K%2{rF})36=1yB2)t!JGP>ZyR>J11*Hu{iSio z$EIEHYPUiUKxo?WrgiuGzJuSU9RD-r_@BwgeosC0wSUk1?v0NdmffhCdnUYMxkYG) zs#7ViN)Us%D+8YyXlxC1Eu9hryFG()G>3f`uS-9J^AraA2@G~U3>Iw+X3Y%dEez)M z3}$tVh9wMIxeOX9po&SvlY!rbf!BtC(*o4T;WT03He}$?1MMLK83-Q(6Gh8zR%mfXr+3R)uvUdfDD z&kWi7nZ4+C7Nkl7lWFsBAR|y^H1B!}_LW-;3Aq}AvvJi&xH?L?YaZsVc?hY7;3WR~2(60*pG$%>Fl!}vm4wWIA8iLY z>9=(2+tRIX%eK9PFiN++h1XM{x(QY>RbZ=!YfD zzo3;7c%HZVz%NLhRlDa$?VcZH`~H>f`&YdGfAw+DAwo^3|7WfFU{SS?$8{2edI^JA z5rc3(gGe3-3FR~JW-;(ZGO!0RFsHJ}H*y%<6105Grw7^`Bw-Ai7U9wQ&mi}OLF^Tq z!WS;B|2*J3z6JFEi|T>uBYC_3Vy56@c^K|7FzjIvy(wh@T195+3#v^F9R8cQfVu;6 zHvgry|1%2yXJ9$bz_1Hk8MT5dqe=#bW(MXC2F|_QDmNs}Ks$Z-6#lb{gQ^T|@aPuk zR3E$lT8{tKE&lV${b%4g&%khifpI#hK4R@*;GDp~GZj>u@b@$D^)YZwV-TLtrtn44 z8dM)CTmR=*{m&r>x=TXE8gzV|toeUg)Boc7{{u{y;MxpAg0mQeW-|!Q08N?k zOkfb4${>1@SL>dH)qkUq|K{=kH3I%Cn*3*%`p>{~pMl{Ls6OI8$slrLVF5&}LrV(Ek<<|K<1nH#q-a`UVJz z-UZb=5K`hUqCSF%2;YR(M@$F)Gq_J+5Q>J>M;vmt+=>poiWZEbIt)Ta?9$fUD&BnR ze(dUz%*vq*D#_9g^}coc+O~h6apC{Olm8p{{;S&dy>iF5s-53!c7Cnd`L$uk*ZS>W z>UVs`S|2s;degM~ZS$@-&AZ;z?RW?76Md-J_6mCZ&enI3`lxyL`>sPjCZ7C1>BOH2 z$9}c%dS9~qPGrwPtK3Bzp&g=5rM$+e3`!vkVy+Av<_yec3{0jB%!UkXCZKb8xQsvx zr1=aO1U(tm;i6C91oF%uAQ8s9F4pI|ANYG$i)}kBPi*G_k>L7Jc#==`^kO4YK zw+Kw8%)JH~uY=b|SqrXZ%)0^~uS=bMK5_c#xQWLidk*^4Z*eSHX;-q!zH~K&v@cz0 zQ?$%7XCY`fE@!@V{v!LrC6?LqOfqI0rOkv?M#d>KjZ@te`+}uTd5c{OmpRw0bFNuun$W_a>IvTM%LT56 zSQ!`?!3b6xF)+aEBVB8ODam4BBZB&=dW?jONBN+1Sok;6?01F|UzdCx4Q z0>VndYa(O@yjFrUAhC>0Li$Gp>m%eLyMpyk;G=e!wGw<@3BrK%n~=xxAeB+c)>m)@ z5rI@pkZP!8%bSudZy;>QI35}a=`NwwN|^OgC1`-|GlZ<(jov>h-}4Q$t`*ils@VG- zQomH}|5*Xvf>E{qC!{{A*!K&LAZjWP{Hi?gt78Ap$^*Yj5Bx3J|EJ;r=yagTr~mgJ z{a-lurA5jHZv6%Zp=1WWXa>&R_->Uj`;`2JU4-dMCLI z|Fi4;XV>}9XABxTV~~Bvz`uxrVG;wwBvzSqJZis$b^mkf{^!#B&!-3K2*_Ce7cuWdiCu@T!1LKo!#eujc$;6+G#qY71K4reOG=RqQVV z$6nA02`m#qE8;kM8Myip^^xF22KJQ02tH%rImp1ZpFwyX6bY?m;5)z|ev(D;m#EQyS)2drZXjRCTm2VT`OhNw zi-GG31M^u1?%hm6hZw}pv&vp$QTZzXK2%)L5W2Qd%p9~LUCR2ul=XiR3lM_bwWMhN zU&-RXq7`UYFN4E>HlP0t-TxIg{1?0QU-~*|R!IEDe}S7IBz)(;@SXo+cmIpt`7d-A zge30#7r*^q^d^YZy!oGD%})lSdp#ps`G5M+|DF5(H*Nn}v-Nw;*6&r@zgF+~ zRvaD5SK0k zw;t%IY5{!)2;$dd;MZgjuxC*XVYBVvcbmxIF$;t|XEC@t--* zV$dvNP|RWwiDKaNXJB?=;PPhR_FxbTW>AXaG0s-x^R>7?j>f znh3&x#5g9Iv-&~qnup-z@c?W82;NsBxV!;U4WW^kwNmkxmk?9}Tl|1rA3_ncx6qkV67o8A(`zDkFG(BxMe|zfa!uzr5*xX%o;nv1~s7wF>^rulsMX|G)U<|B_ch zOB6(}{pY^{LV~yc3*P!Kd<(rk62AUl`u=~_tN$4$-eXWsU=R)BQ?%ugw`P?wW0N)K zmNH@!)nyUY=a4k!lCuMi<0*LaDf@9Mhq1^7GKdB+FxfINIB@9Zxs`0MUHxv-@&A)g z{BPU;zi#`_@-3e#wtlYM`MF}pC+HfUoo_0(zpB~xs&2=d>a8!Ux4x|3`M!DY=eGSn zIu8EoI{2r3-_N?u@A8-2&zX5Sy5o>l`Yg?`PGQSz7OfZtVFw0QBk+N5s-TnKm~k~R?=kP(q!P)0wFFH25wabK3hnAB;?$^|!?*52#g@v?8< zo8G;zJ9ocn+wrn~+l!jbPb=0vDqVfQ2t4ct>F;E%yqB~3LDtGUkopKjt^nPt1Ch;J zaSvV@LDtAYCW+80qpZbuKvP2CB?_RiI9R0vsdvCvEr9m7!cT95h=Z;LpMNz8Tp7XV ziBiG$5JBprv^kgJrkskIbRx3qJXQ5g891zM} zV4ODBBz+EKskU+2Y=h((7TF8DYc>Wo@9?Tw?@+YVI(LB;xZ`7!JrB|evPhq0m@wHa zb%tB%ijd}QL5*9yD%Us_F0s#!n={`!XQ6H0V#mCN?j_5;8aKPut~ZNs2KSF3 zw+dpZj2IXgQs-WTAb4ejJaz}^86lI9g$*eSu3;j0#e{q+IA$*iULQdi7~Q2?kc~^= zgIMpPEw_WzN|3QxRKSCqn6%+E(J9vGB)=R?QQNpZ`;G=nv&Jmo1*F+Eoa%BYX z6IEd>2#2ithSW-!H4#h%w6d-YTn!=DMDQvJUMrRE`39+yK;)hu72wWM<^JDQ2mVwY z_)~M>Pwm0K5O(GMzf}kRmG1jraqxfo>W})xI~gA$r8f6&bf+)t(TK|5HL9YO2a;$S2RBS-&B1F~yGw^<3;Ml>yw26UpItX!3V&I+xS_mgN2{cm2IhBFu9Jl&QdAt9* zKA^6dzSn;h@IiL`YM}ERg!Do65ueU~Zq5JPYX4c}f3nEFVU&8wDD#L#{wcfC3to+P z+*+U|8IWF{x-GZ{(f-dN@t=wJ1q1sd2A;D_0v8!XPO?axWt6_lsr-~v0~8uUhM;jv z8LR)GK{?C+a+aWNu%Mfjz_%=kS%3!a#4P>`8vkdo{mJ5qB{4WSb!gv0Q-2E?j<-gS3|7M5&Go-Es^*f|o zI20UMWNcWZ&6%al_#{oZL=D(PbvY#r*rm+4WG$gc&V^Ueom0V&TPc7+!k2-=mVw=# zMLF6$dPdEHyK|2JUvT<==kD(<+dfrpdRw{aZQb@yjk`bxUe)gfAHKTx^Q04hrk?&k z`S`ECLti_$J}y~sF0gE|reC$NQ4Gi#0_F^Cx}a5R3{tSeb9lkm&xtUw$uO`hGq5Ux zmKJboFz{$HaH)gthvHNL9XZRT#K5f#VsI-lKo-AoD>3leGpdF&*tT+eP6Sm$zVjHo zW-&NTVzBRLueJblVQN1Y33|2aG_whhW_zc+Utj@|Fhf^}tEU0tPS_wBrVHx&pLT0v)pe>J)&s z7r_=Sq|OB$b_!W6mo(#CWdBjGhHdV(8(pi{+m)=aE?jC`yxcr@p;g{O`{HFT6{{Ue zmYZkKGtZu9nKj=kdjY7U1l}}cm@-o@eu_oryuhaI5uN+|8aBI>t+dTwV3RlBK5xEV z?mS4}$1HV*QQ{P{)EVw&D??kh2RChXDp_uwyTCGgo+WJ5&L($}ZQf#s{6%i1Dnkt%N*$hgLDchv^{o5n2}sQYArKk~==2 zRZNij2+~V}kg%EvGGs?leN+akm>{!ARR{i*?*3k~>qq5&(8{{(E#K9%H!^7EGl<4A zNF{+PBcVtJp+FE4^kv|3V-WHMA^rdckx&Ma2nNYm2I*c=pLqm47^hq_$Po6#}o#E zIgGORrOkip`~5cy{cqy`U(fTuq6O$+RRQ(?{A&OCwEqj}{^!;H&!hEUKpWgk()iA& z{!LKllZgITaibry7XKtIKsGAagH{R3nSc&}Wf%X?!2Oef?I8o}0|vhHEJByrr7!R) zUFT7IA*}meKp)gC5;p_wGXhmcHlWH#-U>t_*GFO&|AkEci<>vFkzE4Ws@{zmonp!apF~U=Ti;lQ4VBR4C7D=XOInL;B^HZB_SH5 z=+luk_2Q%>|0f*!*L4_#dXE0@J@&ur1PDzy@vnXV*NSzIvKCzNYuaj%GF`|nhe6UE zeCC@v_!2r9@QzVg@X|F&@D?612GGHwVhoJp44~8elo;5+$A_}3ftFsgt1z%Dfe>U3 zHk6bHt;FV0X5dq0;CF=7N1X2cpkX?<=?tDT8T{t4MXeIc+ND>2-hIZ4$YozrHvTWz z`M>fI=q&n{L;qX%{BPL)zj^z=hAqFVHhnB#|E_fX)6(@%i`G3VTnoBhym;+n2r6Cs zxNO~%vUN{N!IczfVrUiU@aL>mk1|#~098gv^%1kY5^n&nz%<>Mj3Og^B1{Ptah(n=Tx@RGIxPh-a@PF1=irX9_yTiW|{L0 zQ)cQXOts9K52=p=nzp)CtadD3;!w2MxoELd;UcHPB~FD)EHh@CBu|6%k0AAtOW8{M z!X>u(i)`{1Lh39~WmLG#xoD|-*-G!GE$($2K>Z_4U+{Px1Nd%1to0G-wpoaeASA?( zY4fh6gS`y)ICKgLQY8^nA0c;lFA3f=f~>Er01wlF`b1y`w3oE!N9peGn5cN?*V@DXJI;W1 z6}27y?^J)7$Elq`DV;$zmq9U;K{}a1B!)pKhCw)%K{1~}uZBUZmO;CYLAQa)xQ#)l zmBFx!N$U*1`AsIh|11We;#yGezqH8@4#lTTLYWKNHffP0c28L(`;e1x{ zRjlHxSY-BdDIH~&*~1{TiGgz-XdkH1T@?$^ayU7g|1wsf%1G7vzpU|pPQ^D2%;&&s z>be-1%0Y;^f`P4?fvpmXSSuK~b_nU+k+B5L8Oa#@XA=9*CJWkit7r!r5|p<99rz%u z_Md_GHE8~bc?SdU5(d7dpshyyvq2=^EC$xi3{ocqbwD+mMdW|0=>JB3|214dC*6ta z{O8vMApvbrH6*C}Ur_fypAPttJzdZ#6B7EM1$c61pkr4-%d>4jXWvN~{1;UG&n*0( zf%ho`=R*eG3k-Z08HBH}N#5dAcqX9!R!|ePE<)7gzoZ3dbWYmlzmzQq$=HDIK!@Cf zE^i4Mx8$?>&u#vn!R0?+)PIJq|3XXt^X~gEa`ZpnDG=hh@SpF}f9`YtdCr3v0$2Wv zT=_2w9>Ej4{$KFge<5&pN$~oAt=s>l&-|CI-oqdh%fPS6DPzVcW6daI%_3#RB4x!X zWx_0?$0BOTB5KGWYRD*N#4e`8DWS(9Z7!(dAgSvssp~7C8Nj3J!zS;`BI^yBSK@a9 z9ZSydD(_Sk-?XcD^Sfz#zPGM=l-7MXu70aa>TK(vP7V7q5%p*mac>4z69xta1_oa6 zWj&B9dmwklaWXJ)!tPdpT_?xL4pPa)4?cuPjDcAiwBnjo7Q7u*hJhW-V3%j$kO%dQ zxKtQ;)EIa)KuE})K`olWvW~&Ng~73hDP+D}!EVd$Ye7rjCT{tYf9QYdiT{O8GLd=X!m}niZxcbiy+7{XQ2gn zwKil;f_?EapN7rB?YrEn)>`K;vIbWlw)vorjBVaxi_Ce(;QoQ^z$6x)exM4TpvMtNszfD+7E zyG1v<ybjbCQu;zaTv4^1Bg&6x77&;l4E1(EoAF)<| zY9*lq(&kSUY(e#rviW~*`TxvPpu1lsOhKn`OPheMLlRa4ZyMq|462g&mNM`!V-TDR zMZ)tL#E**T-I2EfEk7^_{BIHpI=NKM33Nx2s1E3KBLQ%qNKgk<8A0kJAp_8T77|9F z%1G7(G$bf*{$I@EKffWkTO|9Rf&VW9>t_b``wScp7z8dei(F@yy3MEfNJ#U&q`@~y zL(pztNej?vtJ2_uUg7l-yh?)9N8INBh3)?f#r@Z<{Lef8zxc}koV)(>?f=hp41_q& z{O3ITpYzOruCt)}i0{&Wp-caTFM|g0M6UklyZm3^Dri}q@Xh}!SN}82eac{42b#MV z1RbFUsgGEtty!h5xui|mC5%|bj6n5~hyjDBA)BZ+o2WLcgbA0drI4DNgpQAxewesH zxPWFjr&0i@J`(d}5buVqnk%uR~A+lNt=5 zn}7tsoge{l1;h?sUIiJnVPjxm2bWGv;9F2xLDd#BM6oafqa+A1OEa*@fICex;EG8O zQ6I4>Ft8{vFv~OWIx%P`Ft~K`hs{>2IP5m#apLBGg@^x_90#GoBmZ*`{4dz|KY#E4 zqCNi$cKt8i@xOT6zsjwDOE&x{1Eb1KU&=RpD&6oJYkdT{a}BbV8?uHf54?&SUKwSr zc#ys9J_s$p3#xiB>LW>;RNseQpxtLz0P zsk4ldXISSf3~b&O*}X5Yc^ha7sbr~r@e=#uC3Z!N?F$##7cPPf%~@p4b*oq%)UrLa zeV1qTI;WE5kX7OiCCfpmc!g8xDzA#Q0d<>wTekVNZg(k}F6EWWBBBP4aYk^Y!`3mt zXh@$3jf7N2Y4bpfH^2+pk^4uWej=Xw2o#1$s~upQlAw$3AW@LM7<4UE){+|#l)V&m zQB?NQn>ov%lSi=n=vMY}$n~gj?j8Wpy?#=tx2UDUY2fnS-265Ha#zbb&ep%fI~>gURvaiQt{^3#hYJ4`bQ;O zphq{9K{g_Rk8yyk{f2jokQr6GK7qC=Ve2S?7k-0A?J9SDtK9XieE0Wq*ljNr-~|kj z9#i4oZ-slmJzxP}Ip6~g4zE|w|Rde89_lf^gF8-f#;(x)C$IjWS1uQBU1Vb5E zJQ#R_8N^~4q!Jk9QWylH8ARe4oaQiuu4i!H!r;7t!DKOm#&ibF9tO2e2Bl60wO$7O z`HYq;Idq@$8od%U0FAXz`$I{z@-WrSAmT1bILNXOM`YCaLcj_Xmbkb zGB7DIfaVLhz-dwt#OB+%+~%@e$|Ox6xG;KnNVpF!XV1H)zpmKFx)X4n{AJp)@k z1A93Lv9vI7buoxNkp?dzPp(Awy`k>8I ze7gU6bwG$m`@fJ5=xz%U?f;@W|0Q+)i>UwSSN+c_{hwLn3j@y^2F^PS9Jd+xE-(mQ zW)Qi{D*KE}>5GWYKVe-^MJ8?XU)BuNlTxyPc8DbG{)^i}Paub^^yahv&tv(YA>cn( z=6{B%|5=v&XIS%}aovB0&Hov;|7Y0ypJCsBhQt4vkNsyp^`GVRe~xqiInVzWJpW(l z!hf;L|HZHT=erIUd#25w6RK?4>Eb50oxE?G-XX)7)nYaS_c zZb>s%2_t4P113>@MiD(`F?AMk4OS5?W+4qGQ4LNReL-a_No^-t10NZEZ(bD#1`z`W z0RskJT?RpO1}*~z$cdv2dZ4Zlvpxfp9s`R$1DhcOvleJs0;qE&54(H;(o14z0M$Y8 zT8RbRU}gi=L<~IOxg;Ub*d4P31G5Y>n>+)PG!v^F1FHfEu_=HWIjkz6i6=%i24)Qg zZc_$nR|e|>#e}Jrtw%!_K8)Y=Bk$nQLye=!MbPpYoF$? zeVV)WdCr<=d262Ku6~-o>S^A}CwVKLf)QvfckZ(Lxy$b7Eq?%Epa^??=eJ5r9t+a(8GG+d)l=-(Z7F>fYqRRj;oXc5sBX7x# z{G~SwmfgyjcPVkoN&lwpE~RU1^OicmkbUkFhrFeBIg24D`8bxW4C~w%)wkcLVWVT| zGH@-k4Ahq?T54YmTIp?^Hd`-chEc|xfTpdH-TMMsw!2rab1q%xSiHokWT`{ZVz3ra zC&(&qp+(Momx|Q^E!%_JcX`)uaw=QtP_o>qY^8nK3J@w?=}@-HqjH09-B#DuZQk8` zJnNUsh175>*?h9$BwBrxw&)tTN+PyC z%3OLAg3ziYJoOQLUI|hYK}bk130@z;t0YqEqvCbXk?W&E@bTcFGle$2fK*0hn-FI^ zK>A9hTV6uyBg~Ex@?NB}ZSRn)AqX2Xg@kp~4%S`zQnBj`q&_O!^{s5zw@UCP*)rHV zhqB#Y^LKyA+x0nP+xwhdpGyz@tvUI>^X&hY6aNd>y|Js_$?Mp_AQ8^M?#{sL3z`X% zier#XVUSJ&RYpP)45A4pnOlq}yb>uq%@%c>!Eqgf-dqqen#y1_mBC;-gV9U|vt=wc zYdCbD@|(O7Hv`p241AYBCnvC%GB6b|aI1mJ4bY&90t2Tk1BVO)yCfrz94ntDGmkoW z)<7KG5@rCGJR%H?t5mF?$k>7guf!okSSFz3Od0qOgH~v>wlc7_F>o}35N8tuXA=Wg z6{tR9ZDkOh%A@*O)$YH#3#fmj;RqU?<5mJyN&H&>Mf5=%odh)h^D6)2R{X;y|CmAK z27~w^2FW8V3dcBAPxENp7dL(-Z}DH<;lH-qe;xP#TAu&aTtNruNg0Cp`Q$V-x#<)fsTM+f6l;l7qsC>_$q_M zEjGF5{OTV>bpA;j|CccXT|p~v0kT5L03#yWYFZ>s|_@C?Yf8J~Vr7rzv z*!Z6zY&ip8Jm{!uUTX$ET}Dw84rx%8#3^mXBWKGaWzHjI&MsxbDq+MbVZV}DgbYHg;Ybh>LW-^l)2#*V4)C$gTWnvj+%|8i zQ^9fvFzHmV+^Jx>eeM!?eH7VqFsg6AZ{sGHij@u}plZmzcp0cWQvzCdXO=lnFJ*># z?n3{@EfHON16sCw*KcwuTkcrA#JO~tbLldNlBHm@+^%S;b^apP$~D35yTZHo`8RKK zuU_X;vD%|%y>rzX=c+YM6>D6o)_GKJ@UGeH(z4a7YqxjfO5NmcDMN2i3^Fi-mwZEN zAp{9IwH1?uR7|k?2-;DCaKWn$FDK7OL68&9QORrIx(RfKL+X<2sY|ZIJ4a~!qpT%2 zvSD2$NM)3{^d`JM%2<92jvzfFIGM8oI-rNyKfmKK>dkh(@gLjVdz_-ex zb&-nKJuhDO9Mnc0|{t=uFS_cm9A7R!<$YXb9+dg1L6+1qa zZTnEN?OoZ94`n+(K*sJMB)CfYQoi$Z#m=u4JHM7f(5^3~Fa%Lnx$|T7?$2#U{!Kjp zf6AHv6&v3AHyn`k>SB~fumnv#FxWGIZdlL;i?nK( zACb2DFKzW-)a<{Q`F|P1|EyB~8TgJdFzy7MRth>7u8o1Kje)NYgt%%L*ee-WRx!vO zf$XBy_W!Tq0Xl+P)bKx#>VIw(&=o~|TF~=axwZatY5wO@{l_BzgF*fSlj0{fRnTe* zF#}NT02%AkaR*gNS`PnJtp3Xy{1?{(?b{X61#Ok$*9R?x(5wYxQ5o@;{^Pe>SK849Wkw>i;t={m--UKgXv3 ztn2?XuLYxZ|C!hQXIlTCb<=;2ZU4D;{Ab(spMCd#mIMD;5B_I5@}K3{f1cC-15T?-_y@F$h?LD+LJ#J~akjb!HJgHc3-f2@`fH6E0bEP8l-}X;ThqQ#MIs zR!Kc(aUE7sb!K5zCLU=9RuKkf0R|?15Q9^KiC2+BSe;i&PfXcdLElx$&|O%`f`L~V z-1iY>U{Pe?R0GvPpqo;`{UbIb24;N*Mjg zf7PG-UwPtx=@C#Jly~5N{{H`k2Vv;Y|3YwmRCwTj@&5mXdqLxMMO*)tZ2Sj8>;II1 z(AVO1Um&wcMQdL`>ZAO%PYc&PEm-{o(lg3k{WNFQll+y>z&#^S{|Hhaxc0YGH=zwm>Oho$9yATAahSHYYNdcpzC3g}R-%eV1H)-MBltp(E=ih{!w2(3X zR?gykIg9V)EWT5)2!tw^-^rhQDSOJ{*tV?!rHfp%raLBd83t4-yBEki=g4|hn zwll12Z*coA-^MNO)$81=*SS@#b*ownVS84s_paIK(X`FCeYbDx2Ah)k%C0F4LWbbJ zFdL#Cg0V1bBDe@>nhZ8%2dRnRBwA&ZJpVFkO$54qBV_^3`Y2<`Eyze6gamE)g4IVE zu!HZg)<Hx2gAPf)*-mQk*KO(z6DgY1bJR?xI`zN% z^#7_IpS(H_iTZXiD8?~xdo!@PfjUHDVGLs7AS4mVAQ=Th(lHF;aSVbH4BSzwK@07V z|M$A|U;oB`gWgZXL(tL+SSyCL-vkKun-t^c4! zYYd(2!sbkTMxZm0StJ-Zg+PcyoPk3eR2eY|GO(z#2%8B=#tTTs^GW52D3tI^goE#) ztCLXQB4!F&X(ns}t&f-`{xk3%1=UBKlNmTCGw^mY@O3c=HZlk_FmTs0@HH@q?iMt> zuIl{XBTeFY4@?rT zz?IQG2L5Z%`iT1igWx45;VUfSx47jVaw)&$*LWwY|6ke^bR?auC1{_loGs`~R2f@P zWh83#U&022l&wMgc4aI@RtY0ENn=n=1ZFUc>oAFFGYhLS2q-adN`O1C+zgC-pgVn7ML_o~ zuuFqFygUjF{HnY%Mv@wKQkwSSYIdNS4m1oa3SK*>%)q1z8n5Fp0UcP%q7UM-sDq9^ zV3G%4#~=(kF@^y$G{+6@6S081%xvH}1tHM71ra@N1uH%^HyM))+x^~s&zpC>YT5m|cF)I(?QcrytTH=1$A7f?sC2_ONPSei=3UX+7e#AdKt}2cRzHE)M>(sW z%bl%kF0_y_dc0KJrQicx40`vP)fjI~k0UmfT5Nawlc+ z{glP`Qy1S&TLL;wD0kWYg5{607u`*tcRhaE`RLx`{*Alsa~J8yPErr3mvkzWcF5&5 zif2>xXOQ<~kaK5{bLX|nHcpu6QnVaWAGsEYk$my z!vQVZ-Ky3)m92!=N02JUp=7yb!6NITC4LQ?BfIy7cJA?O+Ui!d+O={OGIFU{?OM6U zscfZt^}2wT?cv?~!n^nRHEs2%S&v*FdDN_j)JJX&TYTDf1$Aw8tzBUeRmq{{$;2ni zz%0bTzyaLhsWgULvpdpQ}5;MEYEku>iTT9t&YKDq+ArVu<{cO6VZN9{6D z>mx|51R+tbt5|j;bJ-0@=O}Z2RJ`>q z8iMqdN_KuO+4;G6I|!BU_)@n0bLrOil{-JT?Elqw^8bvp|J(O`&6s@FFlHK?Rw@I( z7Xzmo1CJ+zkUwY#x_AhKXb^)~AcJ@igLnuCNk%Zp$1zAJGYCX57}R*K{w@waDVpy* zXgG-J@PDSm|M@rkXPEn6uJSvB|5*l;4GhNX8H{!?nD1rK`N?JkI*FQH^FIUM31h9l^D3ptbj5_TbEvo~q_tyBx zrSzIz?lz0`4F<8348q44gbp!?9%T?c$sm53LHr7{%uPBM3Tu zlu4O^QwvlvF+--7j6hRMjJlwcd6+c7Gl{a`zLhZO3L$<~0Zms)BR@&|1f8H_r@~p$ zU7Hgo9Zs8ZGIzzT@=Z@__Pwv#^R|B1o2sqPE4RI<-1eek`b@+fC1=%~89rT6kc=hQvOS^6Mj83?5=y`Q@De#)X-3G=QcEx3`g2(<7x zYuTNG7555O+$&jeJ8#j|oY`lRCml}e+!0!{+A(daVOXoSSE;N`x~y5OkaiHisw<0> z6}yBHldvx6N+woWP#b_#gF`REGQQKTc$rJ#66eBYuEi^yidQ)nt+X##YMZy%Bz+F3 zJ}O%o-n}no!r|cdU66{yrFyk<)hegT)ehw=A;_k5xlQSE&)N-o zwPuZ5)oS}nsea=)R{|bI1Uq%&s1|B7FM~NG>sFH!3fk6PX8iI@9sgH1Uh(O&UNF9Vm zf-57?cpYS637+~W9ekk@bkq)9AEht5nX&v9uKEbMe?(z@gloJGtv<@%02;D`)JG5! zGMWcxgQ_Hq`lxvGYqUxVPi0iP^*wTBgxpao-tnn;$ET7VU&?lUtK0v(_1OQub0E~R z|5w87Yo=L?nJo$#1icxUY#7*`7zDf-gaa5v0zuV~Xb_AH0ByRL34FBt zBrX1{yZyKD{jX;IUqIz8lh}P0<^RG)pc}e*v_R_+_|*TiOWkGQKF7eZ5QGFLG6+p# z5T48+F@s5Audv}cDYO5oj-WGzKvj}6=!7?M-T$2Opvs6r_%nmZT?WBh3?kPUM6NMP zJY$o6&8_%fKn--{f`I0K0UZz$(E%NHCk?&?P|5&Q8A<4YuGGFo+*#l0MHYbAeUvHn++{VeM}c2EQeYL7Q8p z%|Kg#nz&$tp)B{8f4En{F<1+If2A}hd^ z(JIiW9mmH192>z>i~lps{>RYwk)iG;L*@~NfcXsCRSc}b3=9t73W5vN?gU|B25tpV zeFQq{Uz34PgMmi@)Y*Z|@o`Esfa(`n25xx<9%&F_7X)i(2KQN*z$6E#++Y%7WD{fL zkYeGM1!-Xt0atna3@m&MjNIVjg#%3Tg6kR?&|n?29s`>x2(cJ2up2XQ>VuIc1G6%y zCSnw4;8Eh0F|r8EE$&&-zxCSmz4s>Uf6>41dH=yreFr|a9{AL_=UwIQHznJiS8RS- zy5UjjrYGfFo|SHSS-$Oc@s3ZWyS^0e{RXLziVprOK<*rYN9qa={wp~64^$a}>!Y0A z{|mSO%ijvBj0!+28U7Tl{ROIwFzO>nWt6w(3A{eaS^fxuvQ|98tdAg5MOom=2(-ch zTpwjGxu3n{e%g|IDU0qTEVz}l=uZ05yP);qs~+X9ew@GhVfx~m$+Itp_aF7H-(;P? z$S84=QecyebCIxBCZ}F3gK7waqzi+f4TFd^1HT#Q9CB7w2GFuq$i6KBP-l}#kzF_3 zBCZ2oAAveb#jBi(S2+|e104;Ny}%@WjuW^(ik)~Qq+_>7&3aI^QUj@C*1A-$h0Oan zRIYZYTJ1^) zN6-qZVm)LEDX?j0IHbc>y~HMSl5tp}vRk5*rafqml1+-8Pl-oVQ%2PiUi)A&@YF{{ z_K!e!<{};a4X=!_)<=-D*5TuIpekt@sDA{hk08Ay$ao#nVBKxx`UtX;0dnRlq<;iq zLwZS&6?fTd9zyy@Icpywcab3dBa|v>{qw^0&+|7tg><06<8@CVB&Y%cuegKkOM#^4qE>Rtv&)zExjw*f`W>+yeo#)L9k>eFf2v{ws#*9|{xgdG=MuliE^?bo z^caWm5mupHO#Is!`3|!P9%B%>&m{JcLHHvG3I1Xh`_C-?pHucfhdk(zI)1hPBHEz! z58?(OH;5bk7c&CgCo5n8Iw4lX@ISBCe@?aUEQ+5PB(ir>`Oo0His)fze8Ey|Wxlt_F3BSl9h$TmN5l!+(+W|JB$2ms|2* zYt{$h=9|(vTiC+qbGbA!=;tuV1TgSAGB9d_)}Db*ssvR+NEkd0#-#w7@!?Yj^_4g! z7}!KX^#}_elw=lQU=s$7uX0L&qmK(boC2A8W`y;X*g-uYCSE2sK^9JN7EW^7t~i`RAgXOgx~T@XQe}1FN*Mkwr=9pY>B!fb&2P%qJ}X`KvV6nq@(r&mH@&Ue{H|*AyUHzZE4P47Ew0-2sdV@E zvOPbF_Wvr{|10;v|NKM$a}R=0-l6|_hhTl8L;v#*g19*c{^#ubpS9tZ#l506pgm2XjZ4dJWzN5xJoN-<4O`7Br@T2fiG5oB zH8M8IA|}y7x;`ArPMne!jDiNN0=kSmnhYFrpfOC)y$syo88^swEqL6rGB619>IT`y zHn|opbtzl|9;{pGR=&=?V!eCCT9;DLjYXC@3tcK!hjr~um~uR`#?ONxWbxzf5ovPP5R;}@C+8W(|C~nfxsJ?@cfjCH|<5~l%lH97+!e^Zzqjr#r z3DW0sty}{#8KS0qjcdhPuj)BXh#A!Zt) z;GAv{T#i`{!A0Qp5$32JW_<)7zC#LbYf5x6D-owkZr8I^2#SGf6Y;pVr+Ti+IKc>}453bwv21WzQDY z5kyeX9V`{hz~jvz8tp&%ro*lOPIvwbT>j5;{6FV~{{q+l%U%T`*17)~LJl%$uV+wS z&7iV~L3J^M(rO0v4Gc=}SoMF48h~mLZk2273YQd(W^qb2fp*h_R#I^>F!C@ks4(-H z3n(5{x46Tn2~sI-3OX)ZH{icX_isc@HsgUt{3C!NB*1QREY=#D6YXP-Vm^3tAw@sqkMw z9ai?|kz+-gN|1q?^Wyn3t5V3>7eky}v zIfGy%1G_&1XqAWo1A`)H>K}5-0lch2Fa$wccsS%hgCx943?L)}nk3>BX5bKFU=v_q z1}}bN;s*`vvx`CNBhZZi@ZF1$EkY2I6;vxR@G>w7Fmj2o@k_Gsi7;>pFt7@N`cMpP z;PnZRc>zvv2TGcOK^DBEK?yXz#2^FiJqds}BEg*qnN1L8;M0-zt&8c~-@NN{*TG-) zJ3g1JeNnXf8Kgd{-txX`>${rmAF6kMuHXN?a{up|L;s5Q|0+H7H}4>LDc#}!`G-NZ zPd<2z4!*uF_uzj>eU!QTfBw$@nVWy*Z~c?K@kidq-}#&VlyCb}yy-{L#`k$^U+1lR zmbdO1q%z80{Wu3SeD@48d6cv2N$%>W@G1#1UY8A?FiKx~H*@*@jAi#SmxDGUWi7o2 z8{&IZu=+{v^1G??uZB%H=G(s4u4IK~{ABsS79p2%HlqXvjW8zl5C$1f25~zEVJpyS zu{_$Ko0u5n89)IAnaJY<*V-(wz=E%rW(0?!D34B{ZETZU@iMpKW$vY`yerqam4lYI zc~x)ls9Xm+%DZrxYvr1tww+1SPC}M6c-L?8Y}f$m`_!#-uUqF*yWXXCJ)~ChuHO{g zzALnIPe{jZ-^MMF%E+T;J-j}0ty}}ClAOv`Lb^h(m1~gakDz_1inXp4Ydxzr`qgjq zYupypx+}DOS3uKN_lD(W`O^%ds+E27^!;)zBg$PfdVMj*;ep?Kxm|hodehpfxx8-Hc_7Clceor|5 zzvuA3`VH^G+K;LQ^f9O;G4OgZusSiY+cWUEFbH@u2>UPyc`^ulF^G7A5TpWv)IN|t z5wZwGT*8lm*^(h_oy)HOlGp#s-1^US;y?3A&{AQ(>;KiR{pa2LpCkD_gZW_wt*s22 zTNqRpFeuJtke$zYM>i_w)KuA>cznIp493y0oPjT# zTlJlY-ggdl(5f~eR2kWVhQdVj{|l=9=aBu+B>IYh{|STW5eAXN zEaJPFM0c_Xf)E4$Ar{dS4AL*zRX_8og9Zb|4E~E5{O1NEF+I@IZfV{BvihKd9i)x^ z%Nv4_v@xj9C8i6ir-Ze?@u<9Km%q;_c@xw#l03^Gd744|B%|m>Cb3J*QujFIAM+{y z7u5nC+8}1|U)1=&q&a8^PSWnbm;(q&Is6y51udl$xBf3;`CrWXKdTK0F@*o;D)`Sf z^S|Vp|AO283vK_exb?sM=Kosj{wpo}uf5>ERR0(8x_j(-=XfL5GC0p*&?{$lWm+jKvN`W0#M&YNq8Q4WYEo~kp20ldw0cA!$1qNPuc0oB-ei?3~AvIlN$al@4O$~7HpAJ0AXqkH$)=FM-**S;!W`>Jf) z_o`jLi}(L8J@~(9KL`~b04?V(I0W8pbOgMb?l6d52ww7CbPyz4eBgiaf&T@&|L5)e zm%HO{?vB6tTmNOQ|B=7xPws|adF#LDt^Z!Q=|ld8clqm|=dOJU8?1YlvFri7GJ@1Z znadw$E&=s@AS)2kmfTBUdOv6Rqx@A*E7v`*So^GM?UTwikE@p7%Aa#SrF&Ok#d5cd zNtV&AhJh6-F4=MxQ36^%Jj(VQ^46RZ226sQ9Q=xm98!#|0u0QMdWRL9gCIRqc;Mh+ z@CoYryCyc-pfyMq}{)oWdA*16S!2IU}KA#i=P z(xqahOWAVI>NSp~D_kqrc-C%!m;mV?fmBwm@~mFtRkPN+cAa&u{B9ojpk3tY zq`Xy-!{Hy0R3AZBhC>#IBUecvHn{gxu}IH z&;?1bOOfC+N}wfnJ3fFG%N_XIbofWx!Cw`d-vqQDk_(-{pq0VE>&?LAz`*9h!0!W^ z9uo8fjm`;pGVpsa2)Kigpc{j*2ZIP0LFyercLqUs6cJ%hkcfaAi)ys;%#T4QK=qOI z&Hrp?{_|Y=&vO-YUbOuA{|pnKGuWSIFxtnUx069_BZJaB2Km_x(sLPP<}t`#=G1w_ ztqQ6#1av_4k(kzhRm=af>I)eda$q^fno~T1PvfJo?l*2N&?+?v%l~Tb|BZwH8w7yr zBL&OUX#tc*$$=QI$<)rLEm65pPe`&}65_bQ^tUy(g z1bE1f$N9fv#D9gh|Ay25%WwKGzUn{2;{ObjelRq>V<>sRkanISa2JE~3I?O;463aR zQdJB>=?rYq3=AHi#UTuu;C-f&;9WWF;Q9g<-uP&w`ba>TL0FAVSd~*ugHJ-0PePST zM3I?S3Oe$}E(Thq&nXNV!~;zU!Ou^?G#xUp1X-=k#=ywU!YRqYE6X7u%g!&u!Y&H# z89^3Buz_Zym?c?xl{p19S$I^yJCfjYTX2&Z!My-(22M>)>u5fWXa-gb29+p_*vUC_ zuC;If*s+T=r+ke#U`(JY4f928t?mlZcgeB;CCC1k9{XQ%6jT|3cETP7 zE&oQVk8*eX&D;4eebe`>4L=LE{m)(p>LQhH`&Yc>Pucdbg_}MUZhV;w-UbU=@4fzI z-kN6x>t5undXl;PVaBot=}YftEPIf*@=4*E=lN?vs9@cT+|^G(+*MDr=HH0uJ?7i6 z-8yfneoVh|K(mZ<5wA%yn??|WqBnz_D`*->z=(m*fPq5=*8LF!FJZk{`9vxpJ*X&H8|r?XeS& zBuqIT+OyBEbvvX!@@m-PS-%kW1New~CeSm8(3eS9(-~Dl*5? zWo~6lTuT?bl`U~ATjE~1%DZ-5^6ZOngjq2q&AF5`=MubU1ZP0%BdjER;10q-u9d*s zkw8^a8t7Q{YZ;562v#@U%vy2_S|5RHqV(lA(w9Tm*FowdWD>T*0Z|`8Y9i3^>?+Vv zhwwTGGM5A)A+t!Bm-j(dgCmdNLD-Of6Ea!22DBw1f9=x(@ZN0@2_CzH*GCXWG3vk_ za(x6jbG3NeYXkz1(Uop}U$Ns;?XGXl2mW*&`rmWpfAjV)2~#dwrY+(!F9t0_U~>d@ zXSf^~M7=;r*o{HhjX}^6goGRzK*)(f2!;e5L2LnM&@DECE)4umP==rj0|>c+?#JW~ zl1o`FviHBj$^W9a|MP>-nGwA9pZD^AfouO|R{dv)yU3uoi$QA}gZe54rKJoC(-~x^ zFi1~jkeb3EvzkkH53lZT0fT?6TA)?moZ8^!cJlW@=PEFSF)&0iFwNwU+sdHI(4tM{MR_`i?^sH&1M0u37qX#eL|`^ql& zmP7g;o8%p4u?tM1=NUyWGl^YclDGvzQcv0CUvaDa6VV0jSCTRYU1%!@UP>ox2O6W3 zcm6N!3PQ3@pz2A+=D)1%e@UDFiZ1_^3;r9{{WqQXU!~!bY|e4ch|O%ylNj`B8Pv)c zq_P-g4Rj0@PT_S zkRdoUlObGYaLvfYz{JbMF3Q3w&c-dt&LhRiFU8C$09TMmt*6T1sB2) zy_h7kfVie#OiN^Xzno1DgMb5rq>qAkT}=PQ_FaGKPW-Pq_CM$N|BS=`vyT2RI{Ux) zGzuy_`M==A|GZ=WvycDJIPyQ^00?Cr{GYKOI-^v5=zrPX|K&UX*X;XWyYGL=*1v@t zzGbiakhthUa#m>bG9SY{z=g+mv zooAmn-ywg2OYsty@-=Ri(49(<+6`h6yfVVdfYe725?&dB4o#YSF%bzNS4@y<2#ti* zN{Caz!B;7PkFEt@I0fz;-NITQK?dvKH4&s5f^(s>Mc@um?((~k$_TVcZ560Wg499C zB<1x{(b{K4Ye5HzKyMVnrMM(4J_DcSy}Z0EbO zo$tU00e!CB`@QAJzmB8-E4F@2nsw8@e4Ds;2ZKr?Xkv)dm4VL_v{YNdk3q}_)cp}~ z1y?IBpvp+VfkD6yOMN8Z0IG)goe-p;3#e8CiMW7{{L(ClS@cq5?|+%2|3z+rsv+(R z|M{o5~}BV31nLr1XJb|1YQB ze^xEf;of`({{=N)GKgIhQf-se=@r*KqGWNNTN`v=xv0^91zXTfd8$tTHC;eQ^vIcl zE@0)?01ePd7=q546W0fw(!&ovGL8>?To1n{=wu#2-T%V+|Hbq`2y&8-pyq!*wg0?I z;E_794-EV-8TihF7C#8?1=UC5`Z;PPL_4O$;b+5eZZ`7dGdpWo;|kN$rK z=T{8o_ZZApFsRRDkgH@6D`en}Vc?Ep;EG~k3u9pQ2iFRK!t%%iG^R0nNI`_g}~!%hzq8XN=ZnQo)J9I!~yCJG4L?32(j^qa|_C_@k_AqiZO7C zf!x3>1X@SIAr9Ks1Ugz1$qWP&bW0AqsI5y-m2dWJm!wJ3Zj}sLu?&XAx(SQ?X525@ z^S}Dq|N2}1E3f=7Isd=x-2aLTAXIklf9cu(C1?H@p8TJG98@1=9|u)Tg(v?Pfl=Op z|M45YW^ekQyX9x$=AUUR-o?&)=F@xCrTL_N-4WZmJua<>!>67NnRLpsWUWs60=vpB z(bFy#u6kOr@lDm{cctrImacl5v+zz#|1p>Hb>>+MHKHepyH>H8WP+}6QV3#@^JI{8 zVHCG#5VU0A)nx#!g;Qi;Rsi=0MZm#Ao-H{ zBcG;i-i@IBLEiP7d>TMqAjQc@W zK7bb5RjzWYS`8Vmb1GR1IZ(*6dYxaxX7Ab!UN!5TOIO$zF0sp-W0yC_I(w!~&Mey; zXqDt%w%o09oofYX&#ZU-Cf~*_puMzp8(eEp`bTJ0QsT_>keUdMB%?lpRZmwzcN&7v zUxn94X$!7k_Kz|kGe^ro{UgX29i#$+R7ROAZfC8$13}P{I&ggiuZ)PPkKh#(gn`*V zf{4H`s(^QqK!bJbpFl?HAXQR6xJrW5LC7TL+6H*71g@msmTmh`v+GOa{-2$P{`MXJ z-@X4&(Xz)OEeF)XCNb-zF^Kvz@VSA;;>3L!B>Wh}y+BC9jX}(XLEH(1#2greY(NvB zBGwGTwqRt(AZ*7VWQ#z8_MobY-xfp)*nvn;W#j}hna718|CA+oD<$7)(A7Cy7yfgc z{m*j|w5*O{%YTOOmkeen81y$Vs4Ztuo6DdylR<6@gY-lOsZIv5RtBMJ2Hr9T=`9Ru z``NU9bLjl#(E1O$0!iy1hvr{y&2L;9U%9nG7sB%B{^!>DuW0sP*9mkw9B2Z_0(1z2 zsK$RmmH&bopi|%kb^r5ggSt_mo6U7V$7u6t{o~R4$EW@ugw#N_59Br?0oDHks{i@G zh+pMDoAgg6@w*HH*BJOtG4P&b5ID>rc!WXZJd^AVE-lajagejVK}WWlgRUi#Gx;xJ z0@^&Q=m1_}uKSHc={2kD3pTlzq6YsJZ9tbhaOr?fMrD=$#wzig6;>aC7I=$aVGzH~ zEOn1r<~5h{CvG)RH%in9bc2S3)qioz|6;cPr5yjuI{lZj|1atIU(W5nwA+6v*Z<-U z|AlQp3+?y~|MTlzWRTd+Adt$y6wbhI145jZ44jq>?A8ptj*Nm1pi_7FY{8@z1D_QG zuO$PwIfH-&1E(Pavm9t*1+-d#j7BvlXrTh91dD(?tB?v4zak62G7G;lE1v=zzamIb zP=Q@ofmKkBkxz+LNDZ_Q7qm1&lz~Bnkwb=!SCLObk6%)sLs*r8U6KLRyk`TixWhlk z!3gR!F>*6=ig5BvafvAMh$(Z5X>o{Za)@ZL3u`j)DSi>!>|68wvP}7C~XdfqzUPS326Fp zD!FsWSThJ4G4N?JaH}zK%7T4_c5*Dlb7aICL?aO-yR4e2w{^zEkj7QsONk7WfvP0Ph@3~wddMUZcqPLo7x1hQq%v}^+u#OXYv*3Q#-j$h;?ARHtxv3k zaFjUnJbINh`(onki)i&x(ws}kog*|hT3-pmMXrw^RT6T21e-s?K3)f}k06y1q#A;g zRH=_39jILJQU}N|9c0BFq)LLww-2bzB=jXiTk9^w>szgj>FfL$_2w~v$WDxLT5D#P!_h%3TcZkG%7(_il zNX!LPABj4EYaAQUbv8nlpejiS%oel(?Nfk|0@ffk&^pK#)WC++O1uuBjupGTka4#9 zlK&k3qDB zL9&B^tC&HcQaR`~pY1gU;U~=MKe%;4T_i5;|6JPt`E`E@=>6u=0aY>FI{*3fLG_V_ zE$FCHevSW}a{t++|FcW|=a2?1w-(X_RYv^U|GB}44_qJdfwuf0>LXro1tg&UUkHo@ z)Ip2lAoUT0=xqkRE1>#_?-aB?62HW+^;E**zpCqh6-UtB6G~?PmCgSvTY`|B6=?KN z!W>#3F-kvS;J?Knc!x#)r<5tETH)3I&!hC0LG%N&*i#nqXRH#pStV{WNnT@>xydT? zfJgZ`ui94;-9KW6|D{a-OIv`>`H`{vuk89?C-}c{)PEJv|I!ZsrJeuFx=V zX8(Eg{xeAYXW&`Sz_5UUF^qvBn1RijfytbK*_eUdoPo!IfzOEndJ3Ek1FsFJGU7G| z9S6r{#=vLBz+u3^A`d!&7H6LVf2qX@>W{$cBW3|5Rsj`eJ|zZDX$DSdMqU|aemQnw z1yGeFq{b$!&cvqz+StOt2W{B1NI;h7aR{q&il}jmDKPU%fR-^Z5ZT&iV_@KBU=?EF zlV%oB;S|&2lQIyLH5ODf6;w237m@*Y95CH4#Kf&8Z5S?Z8ZBy>q~cMi9W}!ueW^zN z7WuT*j4`YEGItpE-HqM#zx3+=+PhFxclUqoo&WW>L8$g7sA9_4^(S)mTaRhi^lJ8~ zqUo2|;W+g9%KYC9e^<5I@52g!5J>7-6)H%VtPOyM=n zlyE4KcP*E+FXS>vVo(WVkoRMda0fO2_$?W@^ch&SKx^Tc6u~P7#lTZ$Jm9fS4sd;q z=_wkCh)QbMX-71Amd%HZeR3hS#lrd(3+l3$gN|r`*F+FT=8C(Zi-VE+N03!^$n_MQ4Vh7bbci5z5Hbn6RBJVK!9(_% zhmgt$UOmAXkX_q_YahcAWG)F(JwZr#l?1PqAY*ip^Hdvmfv#_E-}$+G<>T1S111U6 zMXd4|r2QGVEkKCh5>(3wT0=P|9RH`XV~&z=p<;c4m3u0@;}p=|DrqpGqiqWaJa`{ za*4rUEraSJ29=o%3R4+mdKo0U7{uxsWZM`x6B%?{wKo5EIQgF?;}?U~83xUt44OX$ zwEhX{{O1BA9vx5>!=Vitp%OR!FK!CD%#K0g4+Hl{2F~{kLLV8VesU=O;!^&@qw!5Z z=a-P~KOxn}C)s zNZ9?CwEr*R^j|&jzgqZz#i0K(p8tg%K)ob+m;Vy3AS7n}pHb}}tI9?OuIUWiE(~n; z4BQ%^eI?v#pleTARYCi4cr6%sEf~0+L5Rx>bi@<4KLc+d1AiEUP$Yvy3aAr&5RO@0|Y5k+H3Wg`J8O$H`j3=eQJFpBakI*4m}2paec8wK)M=7>6#Duzz6 z$XRV!cgCdhxJ2$j@tgz39asGpe+XUiA#ll??A^c1j{VQx@il7s6R!#9Y?}`n7j4kZ zS!IyD%rIxURpBa!@(tF-AmmWK!LeeaL**8S$}P@yJN-J3WG%Uox%j$w$mnWhZWVI}RCZc4;e484Fek69zsF(7j8n(%_mJas(e_tP;cdbQNId5>u71Pj$-d z_pDyyUAqoaA3@eLK!)a!>m$#GO+L+AgW7ibG=NstfyoWt^&sR?vlcQI=Tx@brDCOP zgXr&03%8HC~l# z+)7t?R;>1^UgurA!Mk>YU+o4^Wz?|QuL0C$gA62k)@(#Z@zc-35oX;KKl3~sL24yP zcL}*NLSrM>O^I_aB3De1TPh$Va{ma<&RBFUZShrjl?1Jrz$boV_L3moCCI&3kQxUv zUWcbX%2@%vRv2=*FqkBxK7ydK)qiuV{$y4KjotC+fDSd~(*4h+3pxl$Ko4}W9h>5R2JycPg5Md0 zJ~D`YVUh+R7KL}5s-Jn(fADJjQf8pdO(3h) zA1MX>SFQbTG2y@I%`1O`Ge{sX$Mfn#BK4PLFEqv z*CPh@DWLnFIGq?coj{cl==4!VP<_Ov2A*ZmXW%kr;BjFR@nH}TXP1j(SIy+o$l=m1 z;?XVP(kWn&O9IzNO5jxi#20{&2_1IOrc?$I23AQ%9tB1|B}P6a&@mcJg5Vvg;A`@D zq}j!_*~PV4MDW)tGeo0S$Q2(q>Ab64ag;x0qd@X%vCNHfdFzylHYwz; zlFMG9k_AHASu4y+H#<~sv#Z=3k zIiG>wnnBQvLE3>))C9C4h+7MEc{+;%=(;5aad2QlcK*UQ1R*|#us{-!Za zP9;l0tE?M9%NU@=K6thWQXfGM^YL!l65Itn!^g80)OCUn$3YeH6D_U%wzreO| zk$v$J*UD8MHEW$qm)Ydax6GbnojcDucb1rmcS9Iw-JVb0C-juaCUzHX~O?o;4ferk#nKb_Q#egw|t%_MV_Urt^uj z&nL~f07nomgiM-qA$jh_S0pX zKD)G8%5J61YN4Rc3#SPKk2wRk83U&Ys4U_(WZ*Gk;4=Y{5H_z7103<2f$AL|GX@Yc zVc;==O7Vc%kjjYPiUEYceI-8d>;r`4H(}s60bPH?Y0ki4!ypsQGVQ(Q?*H6-|FiD| z9ZJKt^*`U4|GcL`_q%bN{x7`#KSTK&2FEK5=BF66_cEw#V^E&QAU}&iu8Tpwn?a(A zfjO1IYL3Bt&^gi)C;#)d|7Qq)%wY9`LH{m;@m&V1Ck&b|8C0J#sD5P7`5|ibU)TUz zj|jqUS>@6C&!PUGS?NEI!Y>Y)Z!FRun5Eux$-d#1d&@5OnMe5tzvf?1jRUTjK+EmG zwGyNz64eJC4JV-epG)a2gZLu`kuwZJrx*l}F$f%G5WK^y_(sa;zn&ZD=5Zrm(7{An zp8t(v|C=TL*NOfw>hhmK<0*sHcZT|(;tRp&?W_coi~qAF{b8`WB4zwv!x1#4#U%Nc zf$t3i-y3Gh&s++>c-6rxEA&Czi^MHKhb@R%f!6FvIDqDfSuvyJuy9Pv>bTLR3Fo4d=Rw24RVg}9KfF_%G7(ksMAyEH_ zMT&__fr($4fmackcL$Qv;&t%gB5m`jDfNCXX&=y)o z9yw+{1#W&Rj4l!zXm>5M61$QkgQ6FMya#BkOf`f>FICjOOvtTOA!M>b++!bbpYYeiNSrn}g>Ny$Kf5NAApMB+etCCgr6`*R! zscM6B^+uQKE$(&O13C}IO}~({{HA}`9!?E+1`c%=UPU&3WmaA(&>`obt8DSCEgX;v z2+dM(6Ghz0$TP<=XKqmKI=@EHVs9^Stpr)u?NI~TKm=K{4Vg;{?cC$vyv?(A17sP4 zW9bUpe9(1!Ryp%6vgTT6&9%v%XPGhEHfKKQ+VQHj?q$oJKv%>rvM*fXQnu2gYOQzG zS|9K-hJdE6$Oy95&cAUB2sLc=2P41wEs(m&uVJ%q{U+bK&E7Q|A*5H$das)GkopK$ zl?1JWW}Zhdkm{qv*@#*xarXK6nP=hk5u%;~FT;aWPmsN|uv+O7xP}6e@SA5LwNmEd z>)_2wpo=Qf7F|nUd=tDr{6^;Dn~>A(Ag4XRD=c`$l(h_c{Y3WiyU1+hHF(HtA+Xg- z(E2E6HLOZn{Wxz8;@A+-PHpgE4)FRYe>HSu53)@OQWI5ddsn^lW5do*4Ld)Ttb7#N zcig3DrJ{culU5vqkQ)P+B?E^!=zN~K;6Ll(|J*14GamWRe(Jy2rvD7JpBTLEGT5AF&^y4Oxtl?A34_Xf2E|?m>2?Nz zA~xk#$&OdfoBoR)`_HiOGegil&^9IWrwry#xx@df6#SQr`_JO}pF#gOgW_8T>96c6 z{{*%Fi|T`x$qDO&&L#!*mGnX5W@1|ZMKu1i%Y9}Le*;-Rz##UTMfwAq!WT~E?_6qs z__Y2*M(u>ai?oGxLFR}WfVxYd`bgq2v_29(!615yN$jDp`X?o;|AwCbjeJ2!*XO^U z?|;n@P<^Bs@t@E3KZEyw&65A3^Z!dN0iBY~u=YR4mj7Ja{+mtu&k*uX+yFGtCu#7X zS?V97&<7^rkLa#C`uud4rL| ze0KHlxE|V=ate^HFA?PaOIRX;FPpr6*b|~ zP4&oLY?im#Jb#Nz=~lP$?T$&a_zf}`cpX4{f<#={RYL_eyctByLAQ!Ch%m58@ri2k z@XNAtKrW<)IEEm}3Yu495o2JNWCbk?$J~)5D5UFS8&Pf?*`(%Es_0R!7usTxG}9q# zp>5#?^XyeRS*vxjR_o@i(ac(5Ub^0_WSx1@T8rX!7RBpK3s##IuF=n0YFoY~dg`Uv zY3DtgcH393cdXv%RK3lqdYenlCeS9G`W@a)yMudFi)$L!k+OKgNWabF8umQYg8&VTNj#F@}0&Pcv>=<&d zUguP{(!O|^S=L;$jM)}hb1k#yS>?>P&s%7px6rw0saxp^@2a(awHv&vR)bb7RIc%? zUgulC2|fx3uZaSix53!~O!aU5%2rP0BJOBt< zzXz$3vcNM+nQI;*q5GL@?&qunqqPsR*W3pk_^|GAG5EOe>aDNqcfN1h`l@`@{j6zc z{i`<_M|Sg@rZdR8Gw_--u_?E%q3WLU72EE4&+W#4JL02Ym8-OO2xWJdL^6CB;)&U*DAfWn(Q|>FP%xh+; zm(0>{nWf*d$baBe`O2gAOHdnBmx&qu7d8AZZVW;KI{!J9UNA`BXBI!jD0+fH>=c9K zS!T)CVmkknY(ZB@=(+#bb^EX7@n6U5znuGjNvHn|w!hhY{!2~$Z?XBm-roP3JN}#Q z`meSAzvZHzD!orMqTce@U6eNXuW9#RRQEr#^nV7C@2rx4xK%*|hR{Xjw%L*A_i`M2A%*00XNVd zWKMeq4m;3lI(}yc=_p3ER0fSE2CFFy?t2-6PB4UD0ilTN3^BJE;vX?2KV!&vz>xct zJz_6|Od|t>DtP}92Y5|2xIGLl0%2Qw7{F)#Fz|!sgBbW37zMx#K~R+hIub#IfkBvs zLz;zOkri~KgcxW?600}^hm@F_YgAUJe?k?flnI}N0l%ant44xp#00&>RT|Mtl!Io8 zy7Wj`S8-@(F^Ksya5yqBS}<@ra4AMf8DUHvB}+X^*Laq$aV}YH zle@&VdZSO%4v&UyHYKZUOIO;ItgtO!X;-q!u4I)>(F*hY#oEc!-RrkzExM65|6*{@ zK9`zJuC<$;Yd1U9Y;vvH>Rz|qyckdcvxk&y>{ zj1nu%or8gf>_LW*EZ~#?=?L?KLX5*mKtIwjrq{b{xqH=m(6Z|4^{&;>eXK5JDLcVTDQ?;+&xC>B6ok+!CWuDJtP{UIgSfkZ zL5iMVlV{=L+^NU&mOQN92&#-+Hh@Owq;LKgy8d7M&VP|J;A?bNf)+M_mU*xE&$bj) zAF-x?VDP=hV7i|{V-thMY6it63~F=uop(ytJ@?!ApK}}NRyh8x|9Lk3m)-bZdHa9a zP5D+%j^Y9&6c z{~U_X86@wsNu6aDKgA$%hDG)wm%=AGlmDs?pk)kN&i}QX|EoHKswDM*|C~nu8H_(F zr2SW4^j~h~e}?u~40#tBg10l6&Sa1-V&F+;VCZL%-5{y=U)2g!A8{!B=aBi&uM8S# zlrRDvjw%DLjAZTqL+T?bd(gUW3CI5uj{ilRKvj|qxO2qk_@6=VKZD9m2FY^_%2OHS zCo(9fGbpCAsYWx%1TpYBGqBq-@Hp@)gzy>_idk1N*v}LUUdxkkO(^XSQ|2?K%x4Ta zubB(pGZcPdDE+}u^@+3XpF-g^2IUU$Vg_aK-cmO3s2!x7WP){t_`tOfA83q=fd@Qv z#tH6Bae}q*LTV)@Hc4ha1$H47CO&BfPBGA69h(HVw3W7Xl%kOjqmT}#ur`mF9)n_} zyi=8+OD~Ig1A}HEgH#fOXb6K~0O%MHUN;6dTLuO*1_lcTUN2!I~*(5J5;Q(D_iAIw#K1sjZ?)s z`_k108FN%(Cb-vc%~^6QW8vkX?!Ayxbew87Il@Nj+#7a!Hti1TIsmDUyj!-2YCAE2 z>U40pfmXId!kGwv&{hfLlv)VM0A9QY9*O0U5s=iC({eJf2{8(+c2Aq&Rl3Zta+OQ@ z8mH1#_N6QBN>Tpz(JCgi#aQW?R?gc)bzUyT%%D`d7z@o>%q6I?i>Y#oQj|ONim|Kehf_StUcyz!BToYl|N1)+2Lk1p0 z=u{EBK7v$60v4b>caW8M{NO5&+Z430LE4j9KUUhW*`atvNdJlKW%tWAzN*~vHg)^& zs5Kw>*Z)^M_+Reff5j*NWgh%je*WL&DQFsqdDef<1>igQX8dQUd(TkynxXk8L*;vh zq&o~jXBgZLGWeZj$i6Q;^*_s=|4h5UmDdi?S_qE)|5=XwXFd9#^%xi({?C5&Kld>( zI{u&U)PMdX|9KAnSK9esVEKQRqW=tj_ZjpaGN|5QQ2WHB^Mf0*?MU}Ok1l9&IB5To z-hY1m|2*2DdtJHJf3qupXP5uTD*K*Q;UkCg7f#jB-0EMs)qiuU{^V46#w_)KP3k<0 z1n68-Ho5COia(UhLC3VJIsDhK2O(9b|2m%k>ip->I?kkWkV7Mxf!Bk9K?6KT zF9a^bLG$Je3}u3fvjlWNhkFa^gZ4lPX@L$D5;6QQVftUf3bcVp!v4Rc1L&{?N$}+n z(st0gNyzEHu=9U@um9X`{~6qVGT44#us_aVf1KHFC4<3i27_h>n|@)p4hEGp20m8? zZYKtzXaVCIPWQQj{!1CckFsQ3V<`B{Q1YFj{3k=@Z-$zG3=NISy?lz|Ie9dLlFCr&U4X+J|MFAlIz`59S6SombQM3g~y zqH#&G@X9mu$T9G0Fz~1|u!=LVh_P`?3y7++NcoCr#WE;mg02r1iDnQ8XW;V&U8lw2 z$iVKv!0ODv;>5sc!oXm}z;4FCXUinx&!rO1DsBrt)|CS^z$M8iA`d>IK^W{d z=tGa))&+X)N;)-jBIwyoa|h-#It6VXXSFIq6H2mOKl4mSrjcZ&0lO@xYV+E zxlPduyOLE_xl7Em7J@GHDP5I0|8mB>O935woU7M6Rc&yp+Td8Z-m!AMbM;1tigiX= z3zVWJcr{7dPXacjH!rv{}-gbxa1S3=%#JoVELZz~2#XLj9i6|WM<+}UwmJF@0ps#yJ?V)L8Y9UrT< zzN^~$E`Hp z{}~!TG1PrwsQJWG_JObBlVa@;gP#A2i~sAa`_HiTKf|W~%)9?{9Qe<%|3BBk|4fHK zi185U{yx^jps6UXqyO0s{%1P`M!Wy>9{R6+=)cDP|4tkJ8!!B?p8KD{<~@V@X9lHr zEGqxl)Ib|%`Sd}XXZiF%wGyA+e*t~))vn-G-$Gh{_|<=Ms(fWr{KTsGo>lQZliVjJ zxlgPzkC;IHqw{Q1=b5A~v&-M$Q~sr7{$Ilhgw*Z+YdQW`cL5y=Az=QWLFqpO?m$3)kXC!V18oLv6`Y-7CpVQ?(hx301_g@Uo-F063{Bu_sQo`f*MEjS@YLSa z|3WMOyDa?=I?E0;d87g^*ttN{E}-clE?AWWsfO6WTsVnVA3-W90Z4ts$ScFdC(ACN z#LB0@z@^Fny1Y*a6ckKCyh2Kh;$Dp6UZ84-Czt_(c)dVJ_;R~2a5^!tyMhp>4FiWY zXg-wL2sEk0W6v%LLJVAxd)64hj^SqE66F$*;ue%<;}8Nbv4EG@tlCa-L7A=5WwYZd z=lPc}^eLHVTQJWOyou7QYDH-0F3;L^mc`30ik3l;ZShL`($&^^OHI<}sYUj=SFO)m zawB)~_2BOPPF3rit2esVg3fJ&R62HLYmBoODn|BuHf}9ibuW9#wcwt8?)6)pt2a7U zZ*;ER>|C?et$qh&ye_EgKu%2g3peNat6Wg;$YB%2Zz0L(4AK zE~wBor8&50VnqGo#Ev!by_@2DH^udCOX%Mb*S9CGZ%=g3!Lar{KJ{DNDnTcE+ZQc& zC|==M0=k;qqh`HV^#;Fs&@zLd7Eq-V*t9*Uc}Gz5j)3MJL9M$2APlfbQ1cE%O$4cx znzsdlDda(pOmk8 zTE6CK_3G!POCJ`_x)Rg6-!)^NwqJ{sMFxYcFKF2tt3LQrW>wH+D1#ih`7Ou5qQJna z$iSumLL5pATq>X{iAMum8R;WZx|3b#Bj-t?|` z(>sebKTTJDW83y$_UwPzgZ~*Oe_<&9&rtuLZ}NYJ+V2cu?-{(GFeE*boBUsS>wl>= z{}~qjXP6H{j0^rVF8|ND{y*c!|4f_yGw=J)bNoN&DbTfjT*v=&pZL#p420MZgK8g! z{h+(E+4lTr+4Y}gI|xDQFVL-~i~h6C{m(J~zk26?hNK5f)<+mLHZmyhU{Kw~sri&g z=LL_>FFw6Le0t!DNe8^_!2q;k9JD;o0MudO(fZG>_MJ)b6NBt$2Fdpfl8+d~A23T@ zWtO_iAbFET;Q@~d=!`pMd(drts&=5`Ak>^eTUL41{xb-^V&u+XVDJQ&hLC0_18km} z8C-HUiD;}6)%-854>~dkbPPE77Ig_*=(wDu(|<|V|KiS|^HHT7{zH(UBd8`~0CkK0 zGX(r(2>itm@suI*8AIYNhJ>38!6z8J4@jr%v~IcS+^~fSQ6{AXzY&(H&|gC_oGnDU=tD!5@V`#;Zy|3(Y{GZ;()PaZ+`A3-`h zko$=s&3G1YhX`I1LHbIN`Upa@f$IWB1_n+>UKuViRnWx~JWA|BYApQ941CfIeA0|; z+~B}r5by_Sab@6j10fz)1`cP?&5r_Z3_MN@ z+_nsC77VJM4C>wtY_g!*4K#(s3hG?2^Ko#Au&@b$gPMVXK}bN^-l=erUG6-y-1#7s zv%oZazG?PCgS0u?iBm(m_9f3g@71&oG$mBJ(zbM^RnansvQ=3c+myLqQi(+;AL_G>>7(sw9%?!~O7*X=4-vr6iL9SON;0~{nYBp|!PAUz;9 zu)PAHnIKkGMp09CZ667nSY@9g^Mp2+;yFQ$s{&d#hIVa>>)VqsVPEXTLvfQ1M@~2t z+OsdT_h4x6!Ei8&>OB(PbugrTpMTR%pZYC+ji9?8Ak|H9+wS1D-9fFpLRxo)g4tk^ zy}@mJgIf0lG(+nnNUapmyu-g~yMNP8|E8UOjXV4rcLaj#qrj%^pv4JIppiQ0tWe_? z(8&+5{t=`;@~+tkse|CP54`H}1~1I>sswFQ@~T|pS+N>|;PnwQ1G#R3)JO11rP%4G zAqZX{#o_KK#m_t&JM(Njco`m~hC(JGwGw=BIBZQgXvH12DhZ`Nf>%k9*(C@GJ^5h~ zv{r)DM`&c~lIswZwiJ4|MaGgFkVz=`Iz4!OlmouHqF~j7;#H4}S3NFVc0X+>!N!S=vC4nj?Q)nkh&>UJT37COSj1d5Lg+ONn*)d2s^Qnib z+2>lvbcd8KNu6-8a>>=|b&u;dJ}ck+vUKC~{EaX3H@?i>@HltFL>Cj{owv{;NOwFL()5CGnm9&w1=W=TT67#C;r5AF=NK&%E;31}`pu^Di%H=dtNaUQ*=Ou>H`!!wFi78KReZ>+ z3OWW{+5W$h-G3EZ(EO3Q3uv^BMfMv5`vV4+L|A=i(B80G%r? z4X%vj?f%O+{+D<8FXQ@O*6qKvI|#|SK&vEa$N!?P|Ha(?Gy42z_WREe{huN3KSTN# zhU_0)W$)RFUNc0VV~9LwPx_i>Uo^Dz1s2X74wd(7M*6PdMVic z4O$aT0L|PnO#9C;0~}m)KuaG5w*1$h_n$$3A~;<^=8qtg{+yt4o`DHGm;gCCMeG%FM6I#HYe3qQNPm!N4!WARx=c&I1k?W;PyiMj=~9 z(7{vwpv&%sf*FKE7=!{r^G5>SpxTGu12mDu53ZQ_JV7F2J}kvN~= z5;JfI*GG`m7hr=yP6gLZ42+yI7GANHcDeJ+)28dE&9KN{Xqms*B5#pl`drQUsoIHC zlV+UBS#ra_eYb7tO30Y%SQXd6(?)3zpXzkg! z-KA!ePs^^r_PySX+pP>{j}MOW_j^`LylvY~JbJwA&k84f%H*^ljT8 z+;cR1;?dOkm$Q~$_wCpvrezNvuS2enh}>EQ@g^B0I|G9t3$LQ2l9`rifUQTeS6p3G zUSCYZ!i=7^*;95E%-o+b{XoX_196l0$4uNC-w#4jefy*O_DA#|4DUM-+`T`zdq1Q; zitITY*>gC&>tJx(UU)SW(!K|dg4=e7f)Thf+8xxo2ZH>YcEW2SSY?E$k9-@pL)OcI zx>pUL%E$+{p9oYL)o+6Ij}S}dz^mrql@X*X`c%)aEJ&p39pYJ46MDRlzErn{iD?RS0M|+A+--C z38{}Dog?^=9ddn?vE&w{GD=-?9o$R0mAVAFhCFTQjr3(V!51olCX%vN-pN}5LM6-Z z6)n9}Joi#^@1fwbrB<;$n$Ed`+M!J1R-iR|GfBL>>y0lJM&lz~y4 zfl(5KnB_pnyl^Wq@F;^iM_eilpw5vd19ZGjhe1G}fnOVhgbWyj4M80xVKW91V+K(Z z22oQ65fcUxGX_Cp1|DMuAxj1sPbT#cKI=T~h;IARl@Wc1lILH}UwyxL{o{%)FG{z( zEZzj#jSJd`3%>uNa?|V5bQAw6{Li-GzwEOAeAEBSRlQ;G zKgggrgF&T_LA#wnt%*S_l|eLxK_m;bo>`=gL86O6bTg0A0Y3F7LfS9bH9=<(vFm`B zZSy`OgsbpCSD}L*{>mqQ4Af{~4-&Gt~YAtu1W$ z#gTZDJLQZ^=heu0uRpvNH_z?|A$Ef;vRg!5yL* z;H8NR{xd8DRYnYpL5mm-*8XQOo&sCX0691gGWZ5*T|;Jw!1M2*(}Y-HwGyNm94t+m)>{&0Q!L*5y*OA!pgG)cKeFJNG)*ZnUr7=uorCwQ;*w+a8~e zeXfn$9l%pao{ih{mfgu+b|Z4q5udg_ejWR~+V*(2@AdCG?B8`bu=j9K-;vmjAmm~&8d2pRh=sfQCLpGcenTKpU{e65Ki3 z1tB4QB3QQw)V+eNmxIg`LFStvBxL#tIz0qgzYRW571A?;kdP_~(m8_ni69JkP2^dz z+O2d2avg-srm#MO58uUtyGY2@5S$IEhLA}}m6QzKJd5ZkfxAmrQs-ZV)J>Q*5kv&D zK1y431Al##w)7Tcxj4K&%3pmyZ|wunB8SD-6K9_f?>gvIy2d1Kvb1Xzmu?K`+$t^; z24-E*&C3i*;KMDY!8H-+*h$b9574Sw@LmrFF>qxh4LSppSq?OI!6wVVDi1>J$_!j; z3|yKB#H-1`tI5Ep!@#c#T4Kke4;szm(FQFD2Q7rqWe_%HP;eJDjnVL}a!i{XP`e>@ z($S20mkO8PF5URFV#|y2&Ckm>KQG<#vSjlM$eFH?10qV-Jug}N%xBGepEd8fH~tsc z_Mc(@f2O1Vd5`|*JN94d(0~5z|CKiWSKRR5Xx)F+h5s31-ZS{zX9#%05c!rNI6 zC;sP2`_EAHpSkouL*0Ld#{Zy^x3>SRz5f~7{&T0C5y-sgJ>gE=;`h!5yO6{~6|hE2D+~8J2)Agk21}zFc72f1~yP8Eoc) z$Lo|}eIm%(Iml`&$S@sdeFW(fL24ofNL>S}j~Ip71(cYDG`J-7Ii*Zk#SEaQq=I&R zf(lGFULghnGiG5+29aO}iEvP51i9K?IEXfDQYEA9k>Cx*P5ws|*i_h{JST)WY!W`lF}daI&k3K2c7 zwHtGm-O5~i-M4+WYyD=IhHb8m+ufUYdbjWO?cDF)ywkOIvsKXw_xi25OK;~bzZo_8 zs9(pv;NHW&o%{W|4g~fb4eU7@)ORGf|7gsV6RC4AR&IP+wCbKuW;=s~8F8z09IA~1-o+^SanFH5HXfs5RYRIE% zEppEYxf*gWTj^H10)mk1AY?YYK7ugdy`-3FryvMd7YWi`LaUG9TzF*!VIWsV$OCuS zsw6~x1YhI;sfHjVSnyg3`0#Kz3F#$4dQ7R{k-D^H*VC3=Pg;5-Y3YrWrMFU+-b!0` zFMav_%oTStmfrzgP`l__^31a#UHbwmR@tUbH}I4$ z*e3s1-tgaK+kd8({|sSI7$ToD#Q$N4{=pFP0knar z{wqBEul(e{>a+h!&;H9l{x5v{KgWgtjK}{o90m7}w*6;a_8&C#RP>4=^elt5 z3`#v9q}t4&(!`=y#-N_hAnDB@<^h@$XESBs&}LxOU|>-JPrw;6u-GuL6|##rams8L zQa!+{_DWRmlYlO0_>NQiKd1J89s>~KHvlb%;7|vx#}m;1FK+f<#`eFo&3|!l2S>*1 zzp&$f28(wL@>dx+^BEZ8K^L`42QtV-gYNQB%wmwwU=UitrnZMo<3FFle{Q4y0_LEl zbc)XZ6+QoJ2md!n{BM}_Up?->Lim5VfdAs2{~3J#vxfXk?w&Gptk_Rr$`}tfZG3eKd`pq@Y*{xS~#EK!$(kEI3u8EfYXIS~4VHN1~46Ysjb$9${@Lk6MTF;;is!%|aGLV%Fkhvj9eFPyv z6$*G>4A&477pJ%yue26}kUEoyCYz83yRarZzZ~fFZ;-Lf3=CXM+@cJSb0qlPKsOGH z1v7|;fYy5p1v2mjFmQTmE~P=AEjlX$(3NoRNc_OOQoKSJyhsE^EF^!BVHfMNWl_Ec2II<}bC*TWp=T zST|w1xL2b=+N_*K*9unL3F|xLUcc3=X@^(iF3*OY&NW+|YqmPoY_h9ZryM)kzH)uu zvO76TZiV!LIzC=4J0QrXZI@5$KA+Zop3OTQsy4VcZckfqt$fpyf0<=muW!-9@WyS)y@wJeo=Bf@K6%#V#2FXkW}J(eb|!l2$%skEA|{;x zp$W&}br7T`3hz4}4(=93^d5`oJr>>r?iO_(4(m7sLg3y|P{*F2jy;f?2vQjZw}Iw{ z0^9ZnwC;nVwp{^jXw?vCp2)X(yKnP$NbLjf4f!^JR&;|;w*%cC3pyCQ7PKS}bf^Ms z_mEf3ddP|e&+2tvHR~Z25Ts88se>RST89WSOov<Ct)=LmY%5Tt*E zyhjO{4X=jK7|>d2@wJr2*HV{WL#vNcmfT5QdN*z9t@wGDgL@A;RIM{en(OrSS3J+O_G6Kih)fUgjnT4J2=?YK&!z;ta;UZWo^>T!y4RjXT>yc$eMYo za>dQ^b@wXP-!EGCsA%1zy!B7u2y!|Eq$Yxr;0rD@*S=0){Zek-f302rg)aV=y7^!5 z3g`p|rlbEE4*zG^`JZ72xX_#NpP~3AL*;w575~k4{1@r^&ye_@A^ZzN@COFJR}4x2 z7?%7O+VAknFwx@((~&5~My-eEMJF!GD1p|2fZtZain$3$E$r{by+U z$dGi4A^0?d-Bt#(wG4){L4$Sry$sr&pvp)lg@Mb4fk7WU5(T*6BeT^^vMC=!jJ1kpD8l|Hb|OOZohl^#0Eg{$C{dKWp)So~r*W z&Hvd!tFr$y^!{g<07Dc1v(5a^FzLTY&MBFaOW`x`XRiL3vHY=b&k0`Vi3~bb;sNu` z3-%k4fp(Kh}zA-pa~8r zR`75flDm{YLqR>6up;r)j_8@GEjY=g}BxHs$sA@G_6owPajmFshs z+{#&UD`L_K|IYnBZM%Hicl);Q_UqW=-*GUY^N?T1KG*uKZVlU#=3Hvt_n~mjz38c@ zBPN{)n{X_&|0r^O6yARrw0?N*rTle|Gp8SxbT4LLmt$ZMWdR)(3ken?cOh}G@QMpb z87ONznz<9OQK=l!%4uX@Qg$m%x z$gg>iU-KSNrPHzlJ}w7dsIbky2~-*RG=YZZ{2I3**GHi3Mc^}6A(KR)OUY|CdR2pt zNQG279#v~049E--q&|Xlh#+i8eFW(fxtFa(Mv!h1oQ#=t3=PFiIT1JI1cmhxS}!Sa z=GjDyDhW~z!AVGc1R>#-5fbAv_@E)sARgpIRu~DoJRMdmU4;+XL1ZBn6Qn+ZjNPRz zy^*>EbV+*V;_JDKZ{{qzkv`*WWY_+n+I5aOa}7f3r0g;Tv;!E#tQj~p7+91Um}J2x z{6h*!NR7h{UTO=egCL7+xxig}hzPts0#`|(ItWs8fwsnjCe0Z{KvPT1k_;@;AjBxm z07CK%EJ_T#dJK|Q3~K(OwrQ$Ct=1W{1Dbb$y6tl>XDzz}Ibb|%?W62<%^U8kp7yH#;IQIURyZm4D&VQcE|5*hAvx9e)|(elmoAVhDf9)&8GwgSZ7Z~)9GU)U&sJ4mvFH%q3!eDTKL2Vne+AluC|9pm^%kl(FLFyG;{wult zS91Ta;QAlbQE~b@RIBv*sAOa{~0ENmm)wG#m)TBz3@Nt z%>SZ!CzPtM#?HP|wBcj+vWJmV&I`NsGiVgb1kSN5-e=x$#f%fM&otp-;^ ztD&bDF>LtHun~k68zZ1dCP1()JFHiBv* z$hb6UXD&AbXjTg{nZ?1tAT6WgZJD#sDtD1}{sIsxTxwmo)H)A@OtTlthjtrf%*$JH zD}VXz@czT@joUq%cK9^!@oC=U-n7@PagRsy9=FC_W`(P4D>kOgyHd2~QS9`yejWS# zy7qar?C|Z}>({j}u=`+O_d&m|1K_Em-C>iCWiP*-w&+^y^s^C@PK5R!4e2`))^{Yl z|0o27_8*R&emZXM1<-P)C09%f7BC2zGcao~b1Q=eXYcRZ~BL|Fd`2pd`lfgKsvdn~l)CZ9nXC*fTr zto%s)<0S3_-3{BY#@2uTX0 z47o}I9fb$3kCNwJjGlTjwELh}!#3-jg=*nFk}jnz`U#*@efTXIShc_d^-|!PNDMqK z2RV0z3%p(kQu{#G)56)@44~#EA8fD=QY#6;xm( zSmZ#99E1$mRh-1FV{`%vZBzTg>Xs)@I9xFILjI!bMN4nxt$L8Z`eF8(hdFB>Wq}c- zZi1Zb3OUmi!iJJ-p5(20lDX`@>NDk)h}%(o><@PW&7rXyo@CL{t-b?@aF8>!e@Sl74f3g1G450@Z ztYlVf<_?3r~jYZ0EC!q{`J@B5oFV0%+cUfn}f-Akro0RBErs&AD5&;bY#)Co$76DFn}9P|KGNnC(z{ zz`Ws%Q}=bD>3yH0ANB%QR`vJNDi$RWo zk%x&xn1M|KbmuteusC>q1PL@oa9~32!bYp&8NpQ*Hv@+h=z?;7MJ9el2IxJlLd-mJ ze8S3H{PHY((ky(^(E3Qql|j~nQ8|=TJC;o=j!i3$S2KxUJ4Hk{T39DaSSN~0IZ(p6 zET#WY!kkOBJKqsVAdnoy}N&t8~MI=xN8y!)pu!%e;fLB@`^!xI`I1 z*I}@O8%_)i44f={vXaV1N~Zo+zUkg+t+5r;)7#e-PTWyE<7oMulLfQS<<2;rH1$m4 zl+!U&PQ^?)6*lEe@Whjm6OTtuJRUXSSQM-W1gQ-}dXGV>AqW`)?h1iAM14m>dk-Vm zM8RE$gTV7dAzg^&3XsYOJV3V_QW-(2Ay7>O-cRJ$vctD|yFY00_I7YJw8OUn#DLU0 zpvnlmUJle1g4IFrYRD6OC4@)y1`wlm184^m`0P{6`pCU<4WwJ-UbzN*==U0zvXw5l zJ4BE=$fbB097Rt&3PI=<6KtO>WYi8aixfNcBq+*|NO-LjGwl?fiV0pHL8>P>2_L(I zFd&^HWHK3i94qMFiaD2(=72`+z_-(0PMUWGhAu%WBS^)R1gfyEq|UzzsfHjVc;nKw z^!Zm)=3Gjhb~?J}U`X9&=iEh>G2I&8RWfD?YzkhWRc&mV;NduNa8q0mJov^3?g~K$ zM%ZC(Jaz^K$i8?81F}0FQZYelC0@iJi23d;u*0lO@?&`-`s~%;88M&(;=dVIStMBK- zP~OT%c`F}fuYQ`n`f2Qn*Rd;Ji*Ed_d-A`;qyHlJ{xg8@HsZPPpZ_xG23V0Z|AnUh zXGnR@5cG^84ZvVG_^I!JLf0@hw zg|C8;;PwB)H~)*>1=UL85B`hZ`!90mzvx}i8AJkC|1(|zA-x;_r4DNBhbWx$guzRWz)3__E66c#edUEr_5Td&|Cx3EGa7tn(E7-rd6UoPjg0FLDd%qzCZOG~qI&;@!FOAT z=>Hco1D%p8>jpaVPQe{?u!Dm0e|e|>eD43n{Qoml{O9fj4a6}_f~|#vtU!P)MVJp> z5w{35_{P8bKlk$gvQ?Lq>aWMozEiT{W5K#-;Zx7*Brj#qFO>3`=u)=Zq4~7$gc~w5 ze`ziM&$__g{!QJm)e%C zur6I~QMAIcaJfy%Dyza(MwyEYGUg@BI9I&(Vf56KzMcC)eVn#~KJACR+YUmISL*@S zhTRU;+oC3)DP8xxao6X#=@K+ z0#!q>E>g&ZV-ZtMCC)jMx#UXmh6mXzZbbI&3hvkv-MC8DIfsEo8+5mef*XsvuZB;V zQ`&^^>b1%3J7c={rA|4XGW$yEoU8G(u0~J07&{eI0fkLE5ego53!88(dgAfuiN|C5 zkHz#u*CastK9HISROx`bKM)4;h+KFN%Bs2G?t{VI2ZOo}1c7@-NOcfsh6u8O2vQ9N zv}_M(*$%0LAhSZg&D-H)ao&wvy&JdsG;Q-~LK&9>ZC?d%AOh7p;NB2~;ZeN-g50at zLy$-1I?pQTp>rOU>ktUGXaQ0mxq){TA$N!%T_nip9HcUGEm;n)j9iMBK`J8%iCiT` zLDzL4W|1HhNs#&|YRU-+16O?nshi;S4zBtL-eZDyl#nYU2s;s6B|-W}$#X9w*GKUF z5u`E#FKM^}t$o0!f~PIIk+R@g^4!aDGtNa!IOg5F)4pJ-PV6KFuX;Xv*|1Q~yWjLt!3iXp9ZNHqizK_0JzR7pJG1|fLd12hT32RhS| zSqQYAfmNJ=T^4i=w1^(Jf|aaksG(=3O=LrG!StBMby@v;^5>kdTz0c)`Gcb65AwhW zdOg#sM>(q=WvzOY0Xn(pLCz{rr30yYK;)`>d2j@-kFr)h$y)UUQXi=t_-}jjzu1HS zTvz@xocPai>A%30|Drel3!es0npgf|NPoo;^^PI(Ekocn2KO@z0cRPKA27}SFS-ug z?4S9cee-|8z5h+0{Wp62U-AN|N)oyPt&aq+gT?^G?*13K`(OADXg!1A&HsYeK#1Ye ze}?n_jqm$i#mw{CmbXo=+nI85n997&b77onlk^Af)$)L-#+I{(mk#5E9e@9S_c={hvYYKd3U&{={PRh~4xl zpZP0Er?2wX|E0}Al@Xu%e;#$vbxXqLp!!J06;vO|yZx7U`>*KoU%?r)v|B#nKU>Rx zp$Xtc8q;BG;UJ3;AT<$WQQUISG?3`V{{n0NE7o08th*LH{YJ@#kA>@>r!2bV*>H%# zyi(A)!>MGabKALq$+u-^{nlFkpKa5Bo*n<$cKv7F3EH~FvJ1T0WH)G8KJ)JX^2h!w z9RIHrJq=W+vrEdUTMEc%GH{7AfKH!ggLQ}?=@W945Er;91({KTq*cfq6GRq#;Ssn> z0@X)6(hO`eEIjZVV`UjQWVqyPmCQmV>@q~HQ@KqNIgR4kjgt8+v&5{6rR>Y4919dZ z%H>@OnbbqXol5;G*SJ<~O`mhUe9N<#nWq^ftr-~PS-9mvQ>CDT;2?WoA;XCb3=F)A zhJg;H>+Fix*px1Js9bGVvCgJ!ja}Iq(1Aadn{0~LnCCBzpMJJ{fd$Luk(m+$6=rLLmtihTpMmx`tM<7c0Tpp-cmQoz?LfiAFs9elgOb~z##6j{eWX%$Ugv<^@*bowu>mjugq&|X>kopKha)384 z@PoTrBA^jJW<>^GZ3bxve*F*`mpuLOM$63U-qkCE`wk?|IhVHR8t76d@D+ieYid{C zgPaZlsgJV2l~K;BC!jNnR^H25{UC1@2tm${$Xk6EjBe+xzFn~DUcst+Ijf%Jta@Uz z>a*+SKO&d^D?R+rbK^hr`Twj}{&QUaFMI{Gl~rufe}>fi41Nz70v<8=J!bHJzz}dB zgxoJML_B2Z`YyiWKg0U}jGO-p9{jI$>%ZE=|B_e!^IrSUcl|%#wg3Fr|MT7gA^to6 z`C&DYw55?B9w9{SHU|D%@A~b59IVDRt@kvI%NjX zEzc6*!AxT7Ayjpct)T4SwZu-`^0^G^ci4)<=p>^xcwJ) z{m&HiUm^q4Ct{oh?gTA_t%zF+uGyFW2OU7M0!*$1twZ47_@8(Ef3@!WQuUV;=iVz` z{~>qvDzuWYRWzRMV~F#{AJzopM4kTY(SPh|5^5d(H>A` z#I^4~`|khZC;uy7{cqHNg@N6Nfk8@0%#2+`7M#W)M>Ig{8psMy5pc5#w&{RD4AdWI zkOJ@2Rs?q{6~I-wEHi^38>0v#moO`zI0KI)=y+8wanOn1Jlc$+CafCak~Z1=Ho3f3 z*_>AS+%|Hdqy`&6sz)e9N=+CD()u z0~i=27+EF2gR%VJ78*Nvs1UN`Pf*^-&!%{dL-_`WsturXd+N8kG;H%|+~v`@%d2sh zOZ67J(sjwRE;sG@RJ#6YVD~})&iw(MM*_Q!2KF2e>^UCPdm^ysIHY3oYd_%Eejv2x zC}=Ho|IzT?!;lg?sOvy5s8-q+(6uk9YhQ5p{=n}2UM)L zLZ*kn)b2WLJ(wsE84mRx6&2P zrOP15sbndS(!`+q zh|Tx~5BPWo&_X-Y|1zfkB@99Jk+{Ks3FH6LX8)y4|I3>Hm$w8l1l0dCN&aWx`NY8c znMEE{TS=Jzm$&<`;0{7^?x0?hD7Z>u3HdLR^Ph3nf2P^6`e?y_hQ2t}ZR~6gdweJ06lCzrGyv90zi*NhI z=&ASZH~qKW06G#7ULP^-`OgHdjM(=5=i2w5W6yt~BmdeUkzry|U}91L*F-8{QW>21MHm?PSQvyD zxrEsTBpEnFnfRoc`DGY5L>ag=C3Ssd?DOQE3S~U2g$+aTwA?^6fYbIejj$^~aWe-u<1^&NrsjKH-LWX(3TCW3T@ zz`I!?>*c`J5NNpqcw6gUNPPs^Llo2j+EoN!J{N$rhsd{iCvts+G-m{=k08gK`ZjC@ ztxTu`oqY;rfNLUn1%$N@g49Qlp*eVc1X{5Gu8iEu*SM9hL0-a*JT&K2vJ{RW)ewY4 zu8AOQNPPq+;Z+i(K7x$eftSRCDkf0f1RKYLNWm*6NRJ6JhzB9jW|U&5pN>XC@X83n zfLBS$bIv8tIhP1V@pI0_&p8)6=X~rOQ11zRmD1(Jxloim=W5E_YYFqNCCW017q()19qPSgx2Gl=VQEuQb!v^lD8 zf5Obu=?gAF`$R~02xc$44@Zz%DSgG=^c8nAm*3A^ejj|y_yb6t1=_B=;x>d07YA#( zle79>;>wqC%btmD_^*HJzr^GJ!uS3&U;NK?<3I1s|NOWAD z8LY1`SeyYNvl9$9=NPR9n!|Ic#`RJCzm z2a#OY{I{tP49v=)twyX0;08SM*(}7@N8r;2K+}0F4E!N7ev=t&o->#|pHbus1IG&n*82=R&o~wTOB()%*GEd8 z|E0Y^NZ9AUkoSLv@c&YU|2byd{DEA8b$t9O``KzF8t+G)}( zdJ&qzwPIlt#KR}bMfAx>_A5kBQ%#tyoxDINWuZytB9qKTIvI0J3zwS2^f9Q0^B8BE zL{HRCo|(J!cK-SY>C12UWOsu5Xzbt|0U2|Etgm1OPh)Y&Xt@Tp>w{5>) z=b^x!qrp8#Lwb*e_ni#wIS#2`A}5?opLacO>bZ#i6OfWTuLA!j(w0BmSAwf9^QM{zwe-L&wfZv6x@5*zjJ@&)RP%YZp6)IXGwL7S5dqBrlpQeM}jR!oMcKEjM4es0<(RV0z!tv<7qY-^aAk|9Dq!UpSPr>^{ zkkb;P!N(?q_Z<%dxgWGk2)RCjY!|{L!+Q^h_d-YHK!b9yRSK}0XkS<-s9O}$z6V|n zK@L?2Y=`!Yz{deWPgDqK2CbU&ZQcp0lE4SnA+MB!tXqKeh9IMJ5E8jQf(*)eRBwQf z%fT6-3aDzWTh&?+1Kc@st$^-tb+1?psf-}?5oB}@J~Rgzor70Kj>Su`)rCRTGx4*|B+fY(JL^pBtTWNG&c@9J zU800pA0^Gc5;fyO*pxFNy~n(o_c)cVGD)AK6WJx>S;c9Z$eYR`gc5%o)>JGW*`JCrr=eBPqVS&Oe`FS!OeG!OZ1rmUs+;3#AHos8vo(w5&z zU3NQT*}aTq_aLLcs(3}&YoEWpTmH-pO&hUSlQ>p>@Yi=F##_2R$! z)&G3wLG=;$RZ!K#1+J1fum0z}3aXFTu7W0~xUT#cxb$D>%72Cn{~0cV>LZ5IeW3es z7;Kn1jhHwLL6s4cDyTAIR%2k&03lXY26lA@P8HA*7NA`okj674B8Vg*fe${YO(oMH zX*rkMM+WmpoLZoj-ZIvp%7tI)KLgKa2G&;$obMR8-!brhVBq=6!2bh;c)l`ly<%W{ z#=!P~f&V49GJ1U^d7q^#HW)fZ(AknKgBS4g;vdRk6BNB zw*HSj_CN5@f1SA>3}R+6h=qVRG^v4B*RZLvOInz?rN(z|_AOs18#`SmXPHIO3agS; zM!73Y3s#%wuQASEVNC$jd8N%fsS`*mc;ec@OA<*RI1+lTJiUIuYJ~G`Ra(~>}1*&KQdk-egy_mK5a_qEY{_VTHo41E_>&QT?( zTLh_(TuN7=RYtC`<=jpsOC5?9I~Ff-C|V4ucO3E;AtSgTq#A;gMAk=;&Jpq+B}o4W zGH?f(SAthV$PCCj2Q(61AH~c7RYnOjPsf9+q}W+!KwTv8)Dq+#rNr46K?@sZo==>9 zHgU?y=)S|gEn6JQm)qsc(v9xa^e&aMNfXlXW0J9B;8$f}l?JbogESx^>x&q`%`-^8 zKwAn65rmWQDhX2Qz)48A2+|>f9DT+E8ua6kWf0V7RA! z2dj$d{pXPQ%fR-Uf$=^A%OeJ!=M2K{SY`gP%KT%I`OPZ(hgJG3gV+ZKvDfT!pLsPw zHy=UzM~WW*l{`U51Ic;+7Y8HGu>ayIp!Id26M_1{$NMY?FU$t7f&-m&vEx7UcF=+Z zhFzd_aZG#ua~=7wci=zAtXB?Q7gN`Nn{fVr@uuf3yWh^g_CK=!j8s6cQ^nqhzRP}d zo+hsUA-fHrjvFf19q)z|ms653WkO;0v zG+>i5;C?!zC_A4LgMc-MtOtXBjcC?(hUUj&vwm@|{4cl>v{*uD-+!S?{}mqm=Q{tN z>G*%K-TxUH?lQz|XRw~eVAH_sGr_WWdBLiCWm{i4HEvT(U94BM!K{3%amgn0vR!7S zJ57qWo0aagtk_~xz1^mElSAV+%j|g!@<9x`NopaTmPKpR=HE(Lc0FX;ac&JaaQ_HW zS2BV}6d_CdAR9}C8MqW2vgT(`Jq4+5K!?bIDx>3pT}Ok#2;44>2vQS4*pNC1LPGjPnDr5~4gzm$&*h4M8nrK=$8 zy5Zd-cufQug>x=i3_;-IIG|l2p9W-LkA;m1NBcfp|Z4qYRn{5--f=HAqv{laF=Po|^)#cuu=zx$u>3TXa_=f;2GOW?+Q+9w9@>kQ7f z8SJisDkIBFpvnj|WOtgu^aQ9fa^BA}?Z48N|KgYbt33i$Mp9Qm-6BYR#0~2c@qmZv zAmeqQ%INBU{u`hwiT~Pv;amS1(l;=01hNacG4VPwNZ2z-Ixt8%GRe9y$-02pLRJiX zW}uV$Kv%SDFtDqDDij9t>LYM_fPsg>Z?;L>WzNL^46e`M^^u_Fetpq>%$YZlSZ zd`f@Cwf{>RfRKa%=&n66o&Um`pd}4ry8i`rQR*WVZ$y10>HD8E?7u+le}>}U47K1x z80LWbN1&0q-Tzq*{O33fsxr9tf$Ag3Y6P|epnAmO)PI?E|D1cTgwB7~dE#&5fv@H3 z9&{c2oV)meLE2)yl$B8vu6j(nAF}AR`p*rnjF=9CDkD&pbO>Y)JGe^X zIPzcg;(zxY{~1D-GBD{gFsQIFa4~}por7#T5CWY;#h}f=YoS_oJbcYhi8+6im;Yy3 z@}C{D++q#rfI*Qfpsmmh2mY&``Y(Opf7qV?(v5c%6P7qN9!*S{qH?=cO-By)*&C7P#*X^>c+2Pi{$FXU_X#-5p;b*FSr&1kH|r4 zB1km^+B5{Ni9r1z@L6-9%BbTYqz4q(zB{M`Q5nH^5Bin)ZMY=wb-aNF97L z5xhPEt>s4DLj=(Ef>|Hg=Pg98j3WCFqt!>qH4%gj zsi7d15nBHUt&4t&Bz2 zQm0>x={sRobJ)E02*d23Vw?Xn9RJUK>%ZhJ(9{pxCD7!N?CJju)BiC9zh`iN%;0#1 z!R9>ZyaMyH4CW^otWGmn9b(Ym&S1Ght>U@h+W%^YKx^t0?|_Ejgsy;Qi#TEZBgl9i z?^V!3JD#imd9H$X$Z}o%FL3og&&B`xH~%v%d&i(#${?G{X$V5RCdq;p>B1JN{3eM! zx>3w3flM-Ppo^RYO&E9$82GdpxYa>5JE&ZR%raoc7o>TBDGu48EDqA1xY2mZ6W;v) z3|`MUwf_s5fG$E3()!OR{+~hgFSp`5AOt!A)O6~@+|3`W5B{&-{IX=t zqx5CBy*l^lq%8DlIT$kgUiiF+Ml1g~Z2vEI3}gb!A8D@UsPv4`{ax-?t(}dM8($+n;Z{K57xYoLAn?=n5 zi<$$*l?N>9j@q`IG_N~s-MHVe{itK>LD$Zs!Ci+X13DQrQvI6td$phT@41k&_*(qD zi;hVx46N#)o6wm=VAETW8Cyu*1l|oH>h6^_Cu+j6ppJdM9ft#ZjzgAngStpPppH^N z=Mmr5Lq07BLwk-#O*|7h;dCfyWgY0mxuD)-kfTujyAJzw9rA47?^?IrwQhU#l=B%2 z@5D?#=iheFyJ@dS{jR{eUG60tw6oXfCJ7U5y$oUDNL*&3mn}P=JP`gFY z%BTml%n!L5f>cJK;88?Sp9tI=g498fDWQq^V^&U0r-K*ERSFeNYY;}Y5 zfGXFxR<3cWSPeqpdIwSmK?T8gpu2!cXVAjA6)vT#oJ&?Zl`MBgTF4EliCl`8!TU$B z$_QE?VXchp^A<+*9g65Xgj_2@s-%bsM?zZnS|6t5E5&B z6gBM>q&|YL43C|8HhTK$$f+lT`wx4x>~N{tV3xB;Gon+{qmtJ=kwHC>LCh931Oz&P zQx-IC2HNriS*3tkb`dFpSq(u%Ae9jlxX;DTz#zgdpdzSZr)C+f?UQF6Q|FX3&8KWZ zP{a1<&I6GXPRCEX7(3%)(%fsw^RGexGFz0f4AkYxUUolg=>r5>b|+)Wt+WMKlIC29oqjrE(lNh|J|ChQAt&g~`{}Exm4)<=-pB1oSIvY3JI8lpbpz6Khv<2(o2d8Bgg zKSTCr2D3Ur@5$=^%?1%YI>AlaL5-Sz)v`|cBBn{)I+36a$YKtlYX|v_7=-i~1R-^`^_|KvCpIh}mr}BSJ)&D#i|2eh(3mEWmTl*+s{uS5SUFP{~e5YKG zT<}-E>ilI^o^D6fJaewsR=K6n7WmNl^p#yxj-W1SkK(0goB`<=`cxTuJs*HG#{#U;6 zpJCQ#;p_u?ZMVZ0-H%`OJYmIM&xyxus<%5g?Xzz_ZdG^4y!MEF>lsjG)O6gbgv_{-z38?_{SF4zB+J}Yfqj>}+E3@LeUP#IR&vWSMkN=}HRznu;4TuR`^E)} zM;`FrC~bB3N=SX=kF=0G1iX$e7*Zv59QAEI6gBZo?vne-v#y5qp78HF9Nc$2sP~wE z*J0m|gFfvCJlprXH0`pl*l1m{)}wJx_=M9@6VC*99rdW+ZiD$yWJBdKI zJb>?dfb3lj?K=)RIU%C|1Z2fRMDH>9NFAaw0@p|IItapm>>+|zM!~S#A0YJ+s4_yW zk08|$goI2IAu6NB-LRV=KzHO}t&gzQM9>}(xDIlwS_`R+++jDgLVG~q`Uqb8K)OPp z3aAWpy{uE|Dma4FN6sZHA@z|HY_9(>r}YNA%6j`n#ewHp?%)MsNO@!D7x=3 z9L4m5mV!g7CpZbIry#WwG6|`eaIJHI)Jl+=2ts15kD_6##1m$mNuGTkbiCoTQxOx6 z1a|KAtl0=UYcYO`l3%lgO&W(z7?Z3G1HT^l2rqtcIY!28IC7zf%tot#Am@O=>m$e% zk|>*y2EUqAl=#_j8xs&j+L2CHJ!y-btT#GiLI+ z;I6~|ZTr1zx7!u0vQD0^8``PpRViYdBdiKu4syKVh)H%wT;UG+_jqGdj*-bBaN4HG}yU*`5~`%l|8#`fvaI zzv4a69wJWg+z_Nbg7%EStL*r$g3bjJ0C$UcuYpGDxG(${zw_UA^%sVu)kc}CtqM2W zm#%gwTVtEQ$Rus1PEd=KLy@3aDu-?iyRt8XgcAe55eNzCfR5Z`5d#-o%&@h=Ot9s_ zkYI=G`GVI+knR!_c$5!vv>-o2(rUYIsA(F-nZP)|qN@V~SnXor!q)qi!{|Ekvi z6-@uj8UL3u_%EUNUsMM)Ovk7BpHt;8r}9rOm0x`7pq)yP`bft8zmhNLG;g_p|5E<{ zr33!+hk_78&OfF~Q2&T!4(OCQmaU+L2kb}w3!ejBQ)uw`zs7A)|3&!Bf3DO2*-wB- zzVrVL_x~4P`onwftEe?UYqor@*zi7k?Y;Dsw_>JWa%kNuXt(cbaj zm2;Ad=J!$5WVqV^u~YTrT-Z+E-@rOVaR>QQ2U8t;(y+Q|0U0Yjv->$_Mi3Sf9B)= z)vo?$+xnle@Pb&|)5yhllGi@V-}E$O`YD^nLoS`C>^lxSbRKc+J?lF0V%+?@2@CHR zEPs%*_)g}`t8sm&;_5b7M07C-xEX|YM@_r#)pk60?bE7VpAskT=e5m-?s^6_5h3R@ zaDW#%Ko;HcF)+yTC^$s)9*gQf;oW=Or|*P+|4F~T6Jfm|6w-Sl7`(6nGINwQ|5nc8 zdyx}P`*$4lY261ozs?K1SqU;u70|NPr(vUU@fMSkt(py|^;$0|*B=rq+REH|UuD^U z=_CK84*!=q_Fw$qf9<~i3`zg_EdDcS{pSTE7R~=Gn*Z5#{|lJ?w=cOG)czuT!pU&h zS_Q~%Bgo=9(6Vsw5krvb3ABV8wxABQvJQTn0{DD6cy9=@UJf!t1X`+%+Bt&uhFbT7 zCWgTG;=$H$gU_7IY@j9BO^!R<#y# z;v8gs0)zy0jw)6oDx=a>PGu`W2wWdQ`b5y#qLP)a#n3ITNF5^Rt|G|joJ-*%Cs0ka z08$@8svbCLm%G3&cY$rreA}G)$dyr4?;&_)6y0~2lKKczJt5C7#h~u1h15zAGIHuk z$dFysv{R5>O9?a1#7{pRJN0Db#G?Tndt7QZSY$8Iik+n7*DPdL%&8mBAnysfv5rF> zbnXEt!9z9*A=fFCu(8%h5JAWZV631mu3QRClBPnMt}-@>nx1)v5v?}K6Kr$lyOu0> ztJ&n+ywksZcX0Q9&^GN!C!(iaOq_E)W$_JA&uB5IK7v$0kSYmO4K2M5VP`D9lfLM7 z=AzrF^R6e)x*RhZ)Y$WG+Ur=c*(h^~dfarm&^|GbCT`nuE~7LCwI~KDPX=Kp1{OUA zDHrL!Ywo+iOOChvXWIQ=;3Q~65dV450&Ip8{}~qkXDE8b5crP4b~`2HhH{8iA?HY zjI!{Ff`l`?0)i|cf$Tem6l;+EEZpEKi5IjS zB6Ob1>MsH{{~7#VaqIjSGX5`T@?Xi~zq%a=sak{jM+&B(DoNfHBqe3|Ur6&mx7vRu z*>4QuZy7{ib1427(gt0AByIa&$`w%`Ne6)HBfb!n`iONdXrB@5Hc)-Re&oNCBStWR2lJ{1@%vDPW{)~@jqbE`{1RYsyBbC+VnAh630JN|HS|uF;UC1kS;=l66|0ZYu zYaaSoh~^Glr`740HZVocS+! z6ufD7+kduGp!!JZ;(w9D|Jlp0Np`*nnRh*H!^^C-4-yt%bLl+g)qf#+$^G~hj}z9s ziCFR^X!1p?>Ydt2v!#Q&8FVul#DW-D%orH785ne=>`P;3-tz4{mA33&#f}dN6ZTnW z&jhX6V~_?f`4$9|T;O&iJGjZg1zP0KrDa{PGJVd~@R^srdyo6|o$%{B0a;%c(0K&3 z0k-3?SK}V{n(Yz2C(`ELNS<}oul;~q{SL>Ptq#>&e46*T)@*gESZ80l+9YG9VrUz$ zZ=X-k*?{GrJQuvSo_@=5?gP66|84jGmpBMQg1f<4AmKlQ^J@l;{|uV{dB6x#A93h` z?)>$ry%*B?3Ua7vWZ&_q{u7WTUdZ(k{FEU`eFRxh2im>_*+T>-;k_Zygc0~sJmd}$ zWTXxAS{3EfWws(`@t5r_em$&Hot}em61jSc?mLV;jt5~w`bWsC z9U#3VNCgx=@hF}>O33Ftz)u{Co_aD8hQRgF+4$)o6gw4k@}U9U`o&?cM2K8Nf%o+V4YtJVfKYz}DK6WqB!9CYE=DbNDK zxffIBUrbwgIb-qF%q7>d7TwQWcrRno{ftHT)8^kwntd&C+Idhe=r|NqztgjLg>CvA zie%>CUk7 zKiB^M!WaMZf*b0fDTCSn8HztJgnwpmdBEU&ox%1ZgV}KqvN*(Gd6+?G5nsYJ@iqU= zPXD)k`(OP!=s;BdE1>mZBkAVYRM*Z%Wf1tG!9paDI$EB`sK|2MhvUtq&e z@6O9nGakiEyd2hj*1Ku9bN5(|7Vo^#vpi{fn^s1-!caN z-zS-x2&aYtI4y}*) z&Vj0ah1>rXZvU6O2CB#TF8t>?_n+?qXc@fnrT@~W{u^!jufF_C^qMaT8-HeRdX>87 zNy5_m;d8EePdx88_6p<&%~1neTRLkH|Pa7v1vzv4*CXNc%cHCt71_G@1c+YRVxglT=L!t(=NwNxfnR{ zbjp%@F*DBicJAf3&1BFBVUx3CV320zRR*0Q44NW@ba;6|Q+^DhBHEF0rHeDBo{#GW z9R%dle#o<9uV=?zpO(X(jR!pI_qf;Y@~qqLRlg&7($V5YSF`7yi|E=NP`Apva;bG( zx2i{}kYO~ZMi8&23xlv8gHmSb!WRm=|BGz|U5_Gr@IT`&(2XiwJ3)Owi3$H1QvS1; z|7S4#&!zpJ16&#LX#Qta2O&X+hgy-p0^81qbzgzhK9KqJ$O(rcCmf3EJrdOmKH9YR zaB%lQ$id*ClTN!1Ll$gw2q#A;dcHo{7yfU)QnGdgtAPj=_QF#C1@czT_S_#g8)JkFCDJS@-9i)t(6P_6VUtbqQlpgV8d1F}{tc2YrP8)(0!GmS8XoNO_UvLh zprzeh;B!tGAvcsk5Opsa$QctmhOHM)130>T(45IqX3iblJK4R7xO70~Z!7avd z{pM*i9r71BRjhNb-R#q{3v_@<|KW%!Ct_!uiUY4{Pn>fue)^S|DHp>goDJ$Z>C<|^ zv1+SL;VPr_1$xm_mHay;oXUC4b2xP37*xVQXO9b5F|cVfFsU=Fw24v=tc}4 z20^dXIag(l{nxqnU-km{0D%dh)yN#j{&O4y4IDA-`_DN4KSR}bhL|4=J}($tZ-Tl= zX2%&UPBPdWW6<5sV7}I<<(KZE{~mY$8$bRJng_Y^A5tH&gSQ>QYa$2(GJgcGk9cqY z*E#c_Y0>NGnU6y!-wp0K(A1PV=SF!o8X!~Em=D&jNe_@0F%+fy?gg-FIUY4}_ z;F|VN+vf?3{9|^dXM%?RLV$C&|sYexJnWX2G>WK|1j$#)wn=N%WP<#1d>z#E>fasHwJj*Ci-B7ktezLt zz~GQ#V3!6lK*JjB3=CSVGG6K-eUWVk0(*{ow;l9o-{aQ03sf039`J75=UKnUyJ4qu z#YU&XC7u#s4|i z{^wo~x~+p@>wo4={~2cgWr+XJ;PQk)3p7&4rv9Hz{Xdube@-<}WyGrfU(WZPecu0& z&PySkm*DkLWdGra{)6Ft2O@h8NA`efBFLepp#8SsiV3oB2(p_MvSbc2Ljb|q(gn!q9OQ%_NY4l|D@0^{1n(cgYa-{OrI0=myfOlnf%%IZLDNI?A@vcY z8nVrq52=Z4vgg53c-Q{$uKke82u{LlCCt?hVSR_f`VK*ArEt`WDH66U98x!-)k>hc z33bR0YxNX6^)v*%+inRR|{=Z^{bY4DU`BJ7Bq?C z)AnLja%7b;V-VG6;8A2?mjSQz<_DE#pl$n*vI{Z+1S!GDCLxs(oP?MInM>kkU>0ED zkYW~4VwbTH)^L$B2~)94(DW`b4R0_{=yl4T;a=&&Xp zJq9ivP|t{2iGf`ibW1z03InGC1DhzgaD$w`1epqi)cBBdK_O?PbA#Ie@TL2ZC}04O z0wEmyaR!aosk{F-4E(Pi_+Q%fKbO&e2E{K7 zN{7{gzl8TcOqlVidehtZX%}T&&$8-m6Eyg*VE8(UT>LL|{=dwn{|eXsD_!|7aq7SB=Km_I zenxKiox10L)1hCD2YDNO|3&xyXFc+t1-t^B z5mF`X`_Hrww8EU{C}=!Z;tXg$i0|}&&SM~D{OA75U->V60kjK@_X=nPRqQrM0mIt= z43mE{bi89|c+D{J5BJXh(kDS~7P$Cd=-PjUC;x?4{AK8UKVBpp=@<fyD}A$PTSauJJLicT$8jmRamcv{X?e<;hDhpput*v+@N06(nv1H~ zun222^2o6YC^1N-Sv8+$m=EdzGA;hkvGTvz;{SZJ{|goWX9)YxqW_;kO2Ih zhQKGCLh2yo86wE-c#u=(AS8622z*;Bgbf*~0}svZgpiQh2ewuLR0n~sXNBF42dRVL zq<7s`2!agCK~DRD*F^Bn5wuD|8l!`(TY%I=kb}6P^-<|6$ixt2iGnjy=Lpg*f{^w_ z%N+`rITS8)1|yfE(k zB6|)-^@5h(L8>G;iKjk_m~bQvj3D(9@|Yf|GJ>6k7Xf3S)klyKJV<2(A!DYT0^L;A zvDd$8yJy2Dhw_yU`3o%4XPCwJ=!P}x29&9J=Bqj-N?U|W8F~w8I&;YxGm2_4@F_5G zNHH)$&IRIt4e4MOVo2G61SVv>4z2-mdIKA{Y5)%_u_}TVFpFEV$~&@a1__zON;wy( z2ha63~}R?SO&$v2Oq{}t~2m%I)t041)12J4_Fd~O5H zbTBOW&oBeLaVGaKL)<$Czk3XJ=NL>6GguyH2)bt2_Fru4f7K)ZO&|U@d-PxY+JDI# zpjjav@S+7i@KFq$XuYJXp!$gQGU)IJ(JTL%5B=Bfy)IpQ+`DkQb>echkZuXLS{};#=lo}g`oL-YU()%%p%yEd{*>(A5i|lOEf2IS_`iT3;f8JyNMNa$|J^7#S^ndOXAZ0wK|I1wbFMR=YB_e!wPvOdc zu~Yx~HbO6E;#v1!$%`{(8@0r+diZ(y`Q}DPT=%Y;)XGx z)((RVGrzI8qLYSEuxmhNLiUvO=5;kQ4wo;vTDs_R(%fqav#*$BZc>g|=-+uFa`L6{ zsb@2n--pykaZ~oVR4ww(no>XSM9r$R)ho}Ww9W=M7vY(R0TjQiDvTO&N}lx^{_SG6 zrJykZ2`>h2Yw-M(DyVnHqyg$ygT@Jkz}Zcdfm@YPL{D1FiA&Oukw=b0Sd~FG)2#k9 z!*uY)h*Q8fB$j<;$av1+{(-^z34=PQJ`z+0)khpk|JfD)Gb{XORRY~>ZyfR8tLd&^ z+r#jl?!t^<&s5tI$tNrbdy4$>il)H@IoUi-iqkkL6v zZwOKoK}g8WM3CLAkn^`8B%~&S)Io64vt~1-KJo&eeF~|Mpmk8ydbi4Tpl(qmV!{Yg z8M(lY;C3ln3BMJ>xnw0;ePmy>92waaF0(IMZU?T6;PsJn;W8)Cz4Iu$Tp`sEgtSAd zh8%JiK&l}KiCh!eWzVzCnj6*$IwdI_+)IL!@LDOn_h5MML2%SS>n2Ej6o%AAf-Jp* z^q3;RtHg2DN03?xPQu6WAPiW2bP{>xL&T(GkrR)G^&Rr>23=p`(Xhp_a;<&w67%di zMoAO2qB>MVYGu4jq@6NEtP=$cgW1$QS>$aQB+M9vwLqgltm5GA5@gZ=vLg?(C_~N{ z1lgF{;UbVdv+SUDK8qLww=9EzDw~u(pO&k*v7d@#nwEQxs!N`VbFQLehKxC69`cR?$xw4VNVc>r3r&Cv6ochP@=^`L{P8QOm{^#2E~{%!cjQ1po*>lH)6 zD~49kBro44&^g{3H~t$w{jYo-bfkm8HRy^3-YXymq&EZ^ql2unE@#9hp`I>;IZ$QUdTf{x374Tt|a4*!kZ{+k5-w+#PplKI~v_rFH|fBCfUHf;~Qr@qhM z{3w6(qnwquGnQVD?0u#hxrtNjzk=g`Iaknmsp158XIS;0c?+mMVmthw=Pc+vCE=U@g>U{BxdB2#-~*Hd&w;8W$xHtwFM$>|aGnAU zgbE${&*nRcfx!_v?h5w@hyn4qC4n)4F9_Fv}ie~X9z8Kys0p7hju)!(Z9KXW#|PFwvnaoGcxwgdWw zt8FT`dbJ-;oOdU8)r*q#pYvC}3!nciWX|K**>?i_FL<;c_3J(D*l}93V6#j8VgIgk z&TYs2CtVJiawc}}RmfCn^7K3Cm;!@nxn=rE zmc;)Ik^h;3|1-G%XE6BBp!1(mc)p{byGE&!q65L;JCqA>X>K-Zh)y$g6sjSM?_En$3_p2+|?) zgk1*V0p86Ds*k{TsN5x8>% zt&Cvx5oC-GQVl^!c!vnWfK)^FIrEWgBHOIFa1;ivk02zxK7ueJdq9;DB!b|z5^_xh zse>T1evq07jf7N85Hbq&AXW$$QWHT)to2d!)RU04;t^9$giJUR+`ZqweYaoJX3yGn zZe>gC^XFP-Of-n^FpQ|v@Gn(%Nt3pTlQaq9*YXflw&Id8<`7k91a+50LE~wR+~65= z$Xq!jFHo0+4CR5d4`^&$0Mz1Tm15x2WDqlCl6PTIbYqb708Il)IfAxL2^xcrlHk#1 zU{zrN4S-04*6cBWPF)7~dRW1E7E%*IOoq4?BEkap3ooc&pc#|1_JQEh|Ef3t%U}M_ zbKpPsG0<8sgERjb=l*BNe8SN5mwVNJ{&nEnODFti=mGVTMEm|rO#Cl0=fBFD|H9k< z^KS-i!?b(%U+>X>rEC8M!BrAuMK}KXi2pk18a~bS-x(5CXar7?bMF$jE91~lXH}16 zQVCw=W+Zm?D5~Z>wnV9|85grc}#kfy7ET;=0}C=9^|dQn>6jcUeazJU1)tI z@AhBD15_W$`GM*qkst!~k<2)L`B@Ku zswAevpejk=)PJEfpu?azj{j#n1FDQT&x32FbN{){f$B=$bKuJ85~wl~Irm@eGHCIn z*tP$XH$iOAh!e9)+JkAB%sv%;;8H3zKg_M2AhbZOZW zGUZI_f=d~TuO?4F<=3<|b=EPT#?DF6$N$Ed_{C@`NzicZ16`X&YM*R1#e;nBQGNk7uygmx;-Va95 zDk-%4U}(qwuujk=MDPj-!a!a%M@&tGRvAG$L~s&P6G2GW2|u7gIdCTkQ4>L`BuIVa zS+x<;8-g6F;9jv7as)SKeFPbxgV#im*&;lZk!$e^cxB`W8>4e5TxMUe6x1OqSn80s z*dcE*gpD~;2bm;-cZlq==RqnXTs0A-N($-Phm63n0Er7Q*?TY)31N1QAXO6DL=vP{ zf>c9TNl0Y`CQm>v^95ZiKjl;u2!XC!ikNZ&g5XQev}a#X=WhSz zZQk{p+$&ev<}a|!m}!_WQ758JC7@2xy-?gXUBo1sTRVtb)tyz|ky+dfG(g0o%)knn zT@nCyJ=noh;b~|PC$brI8v_5v{~R0tb8Z5i-evL);4f~k78iG_Ma72XDQ$MFMa91Ow3#cgF+RvI4PYF z7DYD}MK@MuSAJc8VeDS*2|=JJFzQYmTtIFy5UCLv=?e| z8~FA9D?9yHbpEgG`d`80zmnH~dB6XP0sn=9K!_prKT82a&jn?U^|_9LJb zZo(HqcjF3O2X&4_Zu}R#23lRfc@|V7h+h6Lb_KMsf&JuvmSg|Hq9UK#1!Q=&T0T1OHj}|7X|-I`#*$){gJwf59{VdCvUjI`yCJ z^nZ@C|9QcP{|x9AKIrLR){Li5Qnj7L$|Ie%jLaa)lp*be`{~)CFhe7@)x5ZhNz-M-4C%v1mh4$?W zpKu_e?@)N}!N}fyk-ht%^F*D8K!b9SeozPal(~+BLG1^isAC^wv0PyLUI+tn)?7$C z;=mvLJtKHE1ZHf5R72oC5$FIq*oM}v;L2ztd<7k3-2!BE4njgl7 z4!QH}bLOJfL38Y~5y&=cHYP%@j6%Bh;jfRd)=iK~3Q{paY9i3V4zQ&T#MDQSQ+XpM zorstOs*Iw*l@aKs`$@;bCLM!RNs!4V&`{o_V=0#$-o4^BMef` zW6c6s3J4N~Ae9%Sc@Jsk@qwx*&{zayDIlaXh1~LoT%Z8Yr;thtGH?MQ!BR{D44}1V z;!Z(@YHR)*Z2Ql=`9IU9|DZ{tNq<0p{CC1yB6QLj zis#aQ-6#JUCfs0;D{eA2?QT5{@!;F<^$6p-;NxZff1g*F=q z31>mjYD3M<$}!LQZT|~7fT|%iyZ?~-NY3^@mnLW#gIx4~h3)?hj{o<+^j~`3bEm#@ zp$i`tt-V^b_GuOID_gVB^S_4le|0V$_Y8&{{jL31p`3~hMDvKGj#t4 z)kiD-GjIIQum`jdj_(X;X}uu$Zd~E(|AnuEPEX>y462NHF8oKUkN7V9=Z~5XI&78! zK2(J@WeI^U(es#Vz8y3}#C`-+8F3u~4b!nf7TST=fwLY4^@#aTf$Agf)1dl@{mg&v z(;x(|k9bf07dQ>tw2NFHiCp_Hat%7qC3FjP*Sy|NP?Z$E_;LK|*J&#rrLA}rzw~k7 z^sDX@u6j+r;W_D^_ml@7Qy+UweQeox)2jcLRsUn#NzV-Dezafx-}cb|pzHs2FaOs( z|KIb%fB9t}wFG#?m4XH5%+jWS?~Rp$owN^$Ur3`1G6M^#oiQ^bc#N8f zhf_#~M^qg&pv}O?z%IrntjZ;>$;cxI8XITkV_@QCU`-d*>EqM;&!q*bk2t}V5tAxt zl89LWTp7v#XHogbt^Hp$=(&B)f1j4?J}uXSd-i~;B+vlee#qSuX!Q}OD+I2OpfyqZ zfxx!?@MUsvMsVA1I0|aTx0@AQ0f7eSAX7zt%TP^5C&R(gj@|lmn*&9?y*89k*lQ8-h+_JD75z=q&~tV;WZTU z@&aWOFRF))fSaLKYs8Z(LNGZ+*(O}oLe=|9UR z=(<*h?Vts9TwDLMZTv61{6Fi={|q@F7`)FjxIbX!!ZpkX7BCl=5z%B+G%z>2HkS-FW8iI5WF~gq|v}%iY zvXbvYW`+WSG2NB($Bz-pX==@i5`mX{;@{XX2Nx|j6k{jp>BvHTr3<>|4v;Q-6{^tZA z2gke~wC|Yf@PEFu|Aj7s#yJJSm60&`oF#!vpacDQFM;YK@vG4O5#uqCAb;!<@H%Qt zUtowZTiF$^&_4NJ{U~TV68jO*z@6Y>(7-I$K2S%A?cjfoBcQ|BA$1UB7K!iFe?IV} zlJMF85F~gGbZIa5`TyMKK@JeShL|PfzxH4B)_<|v{}rzMXIS}7Wy&k_<$p6)eMnyV zI(E^m@Htn4C!F)`I^ooPE@0*zy_xq-=0D|~^-*g1KmMiv`Ir7@SoEKH<$osdfq2Y^ zLCqV*lm8j|J~HsRv1%nr8AmZmJAt;Cfo^MJ0C$cceq;f6ksy1VA+0sYEFok~EJO`> zr3&b_K~@GvZcY(7Idx0WF+>dPjBFz8f(l$B%FO(7ECTWj9DEFn>R2xY)aoaRewsmpEHbo;$C&uzv*gF*8vdf*%{QcGo*WO zXwSZo?!6!c9yJ10L!J8}=bb`GXnoXyvTPf&jR?GWdl%$NR=<`VzRla=Y`?~B;7eLT zJ6j>ANkB+%@ZAuQ6S=(_ws;7|W15G7xAN$XD6m-Oy@V@`N zJN`?q{V%uazsvUj3X}gc#C`+SM`5oRqTVq?JYopC!w__rA?_u^%>O)FKqqQCzWc9p z_rJ>3|NPfL2y#j)W_=`b1-yOoDrotZ=(YdKPyVwP?qXoD11|2qWA!D&m+UCEC>wi@%P<^E7@?YES zzl_a)7M1@De%}ozfa)WLo&UuT|2Mq*Kk(>(jX4kPdk#4C9yClj&F{2Q$Ou#)DLejG zcKk07MzW6o6Zx9Mk_ZEc*|skGPM3>LY>kIO`*^EB_^~fu@t# zPl67$1JzK;s~8x};jYKa&|;7bV3_(`?+mCT#(D^JSFFe}Xnn*4S=ew0R3)(=2aVTp zpZd>r5>y%Sp8PL(8dMpHock|w?!Um<|NLh_{WVB^#D4`;AMsrQMJ4YQP;XA~8fXeq z;>>@YIiJ*~y-8mBI&<~M#AWxQ7u<-Sb1Qb%4cESlX7z{UyDqqI`oYj~pKtajrWya4 zW`Gt~gO2{#0y+?n@gV4U5QP)}8TvngC&)CxbE7h#`9V;91nKWXZk>P+4zPmP%W;CM zCrBQIw80=NRoKAdtPBiX%zP4{{t;-k4<92NXbhZ{Uyg}S7E~W`@`L--0fH)}+=~CX z75}pcxzg>>%??%EU7xf@a# zLh2)E9Rwbx19gkQJtJ`Kvj>etR6yYR$hUbr1i@A*KuS1?v-oc!*P(B$h~Z(J8Wkwo;t{}co{OnRSm)WL0|@GPb;JXf-F&h zR6r0Ctpc*onG3IpAhnNe)@*pyW1TV6Dt!h7S*A^gAXvyldr1WABgjqkkRB7fuLP-& zu#(7?6mpdWVZ%F0a0YxeID`Qiri+>as(s*<5rh#nnPSJ_%(F$%-^s1C|$rZCn6*CFr*7W95wr7*KW)Rf@HR-v* z8+<^sIgn{qcyR}*HIPZXg$I=-IYH~qcof*gw0Kn9ocB+9 z2WTsb;MxB|SN{uK2HhsWbm%|xPH=Z=_J4-XA563U>mB>Au;o8P?JI`R&ma``pCRNg zL+~qx4A2aS_<{d6FaMiA`7eAGgm}Rh7(q{^1GD+B{^tiT?dHA$nrINZ@}K|ue~t_P zT^|0I*!q(}+MfZmXI~i9Hvz5U5Mp4JWZ;x#m(=4^Fy&RU5mI;L*Kpxecji=cVN-Tw zl(YaH^u({tz@-M7&SR1UO$UJPmE&b#;8!(r;g!;15|HPTP-kG~0f!}g0RyC?ggmLl z3@fQ1u>&dJ%9KqusM`Nma{z5SQnmT7~WTiNJ6iv=LQ2YZC*5 z71)FDlm_NuA;A3ww&b{G01Ugcc|KNY#{h>pp9LLx$A9`i@0tJn=Rgx%e3wBr5!XfV!Go~#pxCed=eh|x z;)7xSC;15vwdcHzS^O+`*4>~fH+=doN!IR@u0LRQ`oHg^{|x=_89HAwbpB^({>RY! zpJ6KKvW&e1gVB9*Lhd1^{xWl&^TnEbL_KbLy&F8 zOgOShp8-MEY182dtv-Tq;k8mw*FHFc)IrGgQE>NuVyYxaJw;^2gnX7bq)GxEK!l?{ zf(+t;x=vG0MZi$_#N(m;M}xZd`LyitYux5pv(dd|xkK&(yYy*h@qGqi^_u<_+Md}e zPASSZ5t2rJB3h0-N|wA*ddvbU44e`S%tGLnHe}-xq@Y71k+KwRnT!lfJWSl8TvB>6 zIu2^Kk;YzWCgIg?nZ1UED?_KA72Wh-3~J~9JHGp`e+N_!2kKko9$t`iS>3s6G-nKN*_;f-0tE;1xC7L8pl8 z9Qe=B{0KZPCWVo`rOa_`$2Yg+R>`kTS?FGj8x2OAe4;(2j3zP?lsD zW963v)kmOB91tl!HcoLyPBD-yH2mE7#%>v!NAV_AOu^A09ugHxjU$HH;4hMlJ*3)??$eM;58Ar_Spr!793O)?ec5c1*w31 zo45NkZG#~1#;x#yILO2hVA21IWNtv-TmBSNl^Ak`3r1oehsm63bJTCd7AUa++a zkSYm6LTVq#d=R7#g0PWmA2c?kGO`Cx41q`F7J>Ug2ogL!1Uh^hbY&j?5jj{jWScz` zf*{opgtSSYVVg12CVd9H_Q7O;n*or<0F;DONf0uyb1xb~tB;VYB=~d^qyj=C@z+O? z?h`IQHP2WN~47|$VyJZEygA9-{ zZO9lAq^%B3`n0DQ8JPH4g%x;J9OTV{R9$n-!fH)pI$d*SGSr;YS^1me6sQF*aTT<~ zh36FLOf&WapiA6DxBll?@n2%rf9~G@3~3)3{BAN#{bO+aztpDx4E0|aBL6e^|6(Y9 zBeL?p__6;M&p{U@$X^2$O1$9NZP3OUaD60r8Pp+ykdV5G=h}acOQ7zO?BoBT%N{c@ z*fM|)!QcavkY+8U*x>~&uxAqntt4a>0T(sGu(i!1;2mM`bsdmvaUhLhPEe7>Ar4xL z$SJ|XBMF)>15KPkf}I&`3m2G#6i1MzHDu}mVlNK^L$;L094WK^N_L>r>g28eOS*v8 zeluA9XVCx8ko8+(DQE=(!!FR3;tZSr^Y8z!cJ05`^#2Uz+Zg0O30eJDbo{U4@?XvQ zzcRRUr0M{=4oTJ?v{qce71Tds%KFdH{)>GosNiK>4LTl;y9!oIDnV*6L|}vC9 z7rzKPzLlx}KilO0(#!s4_Ig!h#|U8G4T;B!n-;93dRIXVHUi6A6o6BGQ5)~G2b z!X_Mv=sOhJy+5dJr%%IX-xCv5fD~3#rxdFIIBRRI-hcGzk{fa^X|9 z6_7V&6V+zsQvj{RV}z8PkOC93{s%mM!T_1tplPMV$iOPZCZfo#WG7|ltKg7s7+7Hx z*=8QsEiv`E$5GI^G!nN!lhka7LC2SI9{wRQb#~8%i{ree<1Y*FSsCrTpJ2mVF;=2Il#pWD|onupMg;T)JtU( z232a%K_1939lSJy6ljp)9Z0Rj2^-`|7gCraV(?$y`oEmje;JGaq7IHh{7{u^HW zZ+hv!`IY}l*Z(VA1vNo?=@|L*12{p;@scI^oO&kaHD1P3k5Zbj5bkjug0)eyW6f-|sHMmtHWkC2xu zKq?@3ZwOu!c~ozJ>?eX$M$m2%cxVo?iW^=Td4kVafDFn(CW#;>)2uV6gKDMB>9(2EA)|AU zaXAPH=@VI{PQ$E>EK{ap)<>WVb770_Ahi!BiCG^(M)Tkm6Py9Bn~=Lp@cIbOz*-+c zPG^PGN)fPAhTsgy-G8yuPQ*?-5jFLA^t2ODH1#BS7wu`tvIo$u_Y;qT&NAvd0-C<; z*aKSgP_-U(&2Hfm+pM|P$x{s@yFmxXxtGa0W(%7q^6Q7OtNXDjIxt9Cf)4*-Rb*h6 z1dl8U!g@@Q%MKuGk7-&Z@iB5rFo>!1s+kMx`YYSUGdNT!q|TSv_uu&DfBwh+8Ls?i zIQpOA#DC7SpmkI7hyQ~P?@Il{;PaTl`5uGQT?X%a48hMBVy=qx{FmMfI*Ngz?jJ+{ zf9CC=b7p+s|F^yi8nNNO0b0hu5AG%Lz-EgeeIf|Se)T{5RnVn-%;)}lKL5`!`wjzx zDhPqbSRj2Ic5r0YFQlS@R7nEh`Up~af#-%8 zxfno4=5m79XhCMiA*}+qIm`?Uc3fid+$x`xtp7`y{g*KLFJb*(*5SW^%YO!w{|qs& z)n@+ZJ@B7p`+ue_pjkGFIsX|7pE6i{V6gbWZSr5z>A#u_6sbD@*Kh*OL&-S&mvQ*d z?f##~<3B^fd#1d<46VP}CWFde=GCClQLY1^?WmHM|4ZBgb&eqQkq~&g4Y_~BclAHV zd2s*e(toYa3k(b{;1ZRKlmsL#X_O$f=$FkAaIJdaKhr_bNllCg|1%y0-Bb&@B>f0z z7K!u3f9_MD^9*@UgVu@*oQ2j$GFShLo&)Wc;yd$S^b%-+1n9_)@^1_UZx~V^GL$}H zX!*=A@jKIE&@wr;{r`DSf>!RapZd>n`ajE=|6G?qcL8%=`7eJH)a_zu{=krMmZ9{G z?5h8=d;T-b1D{E?2sB*Guho}O_+s8nuSN2fnAP) zO%{Y$Wf)i`K!_O}<4lmA5oEv|lEvW5dwAG|l^8fBcqG)tlwg|zSVcgGO>uHD@$xaS zMF?nCGpK%LQ2QoidqF4sl|$JPuZDB}?Rx{jqj15Udqdj~hO`}koSzWXx+fUhfpU-= zhyvQ6lSJ^E2-Y)#E>wV5MvyT&$b=ERTLf7$hrDVIQXj!)=Bz%Sl(m8_EN01JYL(x+BxE!Pcf|CyUi)_INQUO6o$a$xbn~5MK zq&|YLQh+d^l@WM95oozW=5(u!X`spowLY>=n{Jgl4PGB%t&+gQq`Q$x~4CWQ!vu>2ZB`Tzk136fa9oEx<)DaL85=xMM zC8SD%48cJ}AaxUbmlnKA;sb9JgS3+&hC?blcm)U14hc}mY6lhu21^DOUj~7DV)}nY z4E_u2{}(j*FJk_m!{$GW#east2O{nN1rCD{`%dt7t-_}a-uoF0UozNz6R?EVM@kO= zRh>ag8q^%1^^vS2s50Vq|IZNnh9T!SL-S9TiP-BSiCh0AZ-e_sS3#qx@cKvueC!hE z1!#Sw-*=UP(F435gA?Kbyd*nl7LA3IfkBOdQJ*1rwdThEtVco9g-i!QRT5}b+W}C0 z#JV4}s)7CRfA(Xb`bgm1fBv(em34yWK=qO270|IoB4?rXkas|6k=OXkHA|^*aozk~mKP=REVD{XA$L3uM)$_{IOS*Z-^R`VXp-8h=PE z1$VvMzB0`E&$#kG!zR!$rs)3v%4h#ub)07qiezArW#!YBRMcZ&6K3O+W#yHD)<>Yq zNQQw~jDblM6y4CuhzmTD36Ec9@C+KbO~NF|z#$G^P$vOymM}B0i*XApaf^ww2?{Z= zMF{KEF{pp%HTe&!jI#fEHJtNqJRi`p4^kh2dPZ#rK$nBJ?S*evMLx*~exy%8GiYK6 zxh8_J!NYW*`UqYJ!5Q$H2tIQJnIVE#L-0-zq)LKwA$=kU3F#cU!KR8lD%ZJJAl5Cw z&zghG3L!7yhF3%I=^;o>1nC*!s*m6uBKy3BHhBwe^B39YErj=m;0#D*1Uiu$wxJch z4uWkU0mI-QE#5U7T}oHkPapyndrT;8fLC2;EUH-3n>p$nd{|qJn8T{`uxIbiYxyN90o5A8b zgWVMdn@bEXr`XcoFf9Dfw*hn-1M418!`0yx=rC}p8~=H)gAn9&AP5Psk9ewNcbKrNNW~Sm_TYG$PzM0m4RGC!NnoV6_~-{ z2_D&F6aZDkpw1VhwT#>w!WD=R2MB`3g*d0n>F(y&{4c2epIhrcm*#&)t^aKL{}~)# zh{gZcT=d^=`+ubw{~0pAFqr*dF#ODE^k2a8zm&y)1-t(W4xsgQO7@^@k`x{OD?0p_ zaQrXr@}JZ9Kda|IhNwGCc^??szOzjNbw$}${%79ypJzYl_&D*4;Ny30;;fJOuYp!F zFdPT+b%~yX(K;zW>bo!INTpz~uh_j0gTRANkLI0<=Sx7kn&(@Hxlf+id#u7ZxIV3_iqq4oho(IbYgUkoe%vv2&*xEoYCavTGl?rXaCH-k+V z1BV`?pt-5NA85%Cg8&1Q5NMc=NenbF%OuFa4C(X2PjQE=afJj68VMf3;}&J$6$fon zV-{lIQs5BM;1pJ3<(FgNjFYwOlkq;QANRng?x#WM zas>oAV*x_CR;&e`J6EyRt$Ym>frsWGtL7jz5rl+vh@4ATKlPrrA^bIweGceEZb&s`pEVmY5C{Nt>LUmjQY%5~BM1qp ziI7RmS}AhUk;q9$AXQT2#G|mT5NJ~pm<`=R3ta*+2~-)yOga_=Vw{MZcp|#*XmroP zh|Yb1P1}9zHhWgA2E|m)EVHBu1`$n~fz@grg^JEOiZ+SjM&Ux5u3U2BHGM+ zN}y|8m;^z^CSsL1WWy#}4x*gP3Ld8t;+D75T=*>Y?tkg~pc4QX&i&`O3965TulyH3 z^PgeHD~7Nq4DMGM>~AvIUT3hr1*(#4FEiL)W^g>tknoIQ&VOF;jW8Vh|J%LzZ}SXP zTu594%}~Sddxeb7q18v+S3m_R@0I^j5B_ss{Ldg6#K52nE=wWx5xmlYFd#J%q>4c! zAqvn0Azbi$Gy^-hO5z2DDw7~+IRt3ABR>OZMAV?L4CrJ6hSM79gNiJuU{?D!Y zpG6t81BF597lZ0w2D7^i4$nac$rgTO2!G7rc$UHPH-pVTKJ)(~w*O_VLG_WmJ*YlX zw1d`1%1)q8lc?Q)M%P~q4qq7}ZZYP*XK4A#IuSHH#kLYO^vAsqG)E(L0aPDJ;qD(n z>LbRJ|G6*!H=XsAK_D7D2rLZq3}_;ZM^uwtK!t%_9)!3Q7y+gGU}9hh-tJ#Dj$ph-L&O1{OXRL1`u)5Mtm`VCGe3 z6Oae3zEdo7h}me7{?fVVOHj+{z~)o_9lQKHb_KWZ03q=C3E&E77pO97+ZoWj1B`aT zyG7vc&raC59Awb~V$~d^Cc^9ofh(hJklqlSgskF*bcH~*Qq4xM>J7;C5u_^wC*fx- zxK^wMRY~y5sC*4%k_dV29CCdGsfHk=18mWP1NZ_6_zV%~5+d*lI(UZ&&VW=#kggEC zA7q;|AATnhgaJ8s4r_g6i?)*pRv%fVO@Y)$SgRq3AfzUOlhD`!pKbwRgO;6uBQCgY zH@sqkGvL(|q)LK!m*5>GI0If2!5NUs2$_V`M93thCc-3sym0AEu**@186axI%vlYsJjH2*2mMd17#<2F(U(mvTDjq zvE%=B@BQb$|DXBtfA+Kg`L6tzx&}gA3;#3Z++%Qk$l!b*bkUyuWzcoxHW#3z(;0^7 zrwsG|3vB_NbfLx08UTl)>XFkopKhLh2f1HY5NbA`lXy22vkEW|6o+OCXr|!ShFuP=%}{ zf#@YjGBGe{u?ssfNNy6;KEk2)i&OnKtMY$V<^OETf7w<3F=%{c*8js{0J;c?&+xyn z*?$=;5RwO9)hB2BU)~OMbDyH^e?>dcss>T#|NOT98QgC&xLjk1y1NAR*`)_vdKf{cd3<53; z3?{5R>Y&~$17tW3GVY2Tn~*6;WD#UGr2h*Vxdffsz{$ z4xQ>>xig^oXh8E(PzBV!Bd}vvV8^atlu07cNj^cXyTCOOs6K+%K5z!)`~=88RyYaS zLj)lqi?_k#Hu%*KFb1eM1gUo*l@WvuC*c(koZ(iv4paewNtg095ad?A1~kWAx(dVq zZ*GOuL2jihAPW^-VMi-KdP9!I%ixufec=-LJQ1YcaRi^G-~_%G55k7rNd#dd&l5pv zA50Q*9|UN62)u^~vN!=wLS}^^Gen?jC~b;m>SU`l#Bdy>_JNaTus#to!!%(cqz*zO z;Z+j68iF(Y+jqmOBsc?7H$g~veFR~^DhcN3Sz=`i@sllhI^rgQ;MQ>tei=(gr2Xch7+%X6{naM6TdQOAOh4~;sfu7 zfQ(ro=O#+n;BJ>nWbT@W%2&XrD1hrD&hw!9Na-GEu`xr-H-@M)po_>I?l9P22Hiz& zbAiG7JcHFa5ZmbtQ|xnw`TzMgfXH3Cx{um3Kvq>pw&B&ss}EJ95RsECCF?M zq@4^|*$XilStWk90{GsX84P0U8Du_isQqG7`Ol&DpGWgQpZ0%d{XeXR|G16+^PB${ zG6l^pNt%NO>*TEe%US=IwfQdxxklCo#Fer9&!GE@LE|ff-3126%M6j{8FC&mG<|^7 zN9-%0^%2jp|H5ZM^^p{8NdsiPH{_Ccczq;#6?DL<@P+?|bKf&a=Yr0);LrzM*TpUg zx^!kN{|mrJ>P~^GB(bZY!@wEVfa{}G|5q}l=R5je@C4{a z4Bq|!+4lYCIRZKzLg*Z*KH@(5U+2ny>BIk7tnwHbO*sX$!JCq}z-a@L0FdJmSE~cE zhz>HEhqN4#g9lV42}?3?3o~=g*tyH$ssbo2%K7v<9X!Q}iGJ-5x zfY(Q$0Uz*n;mDN{5#hs-YtSuL)KYyv7n6E))})w(?hU*tdPnGUK2qpqtt2e z`p7bMGQkSSG+`n#LaT`&TzGvH(6$R1p$+Q=wxf5TAZz=PNl2}PfA|hkJt5aekXi|| zVuJLHkjaRNha)B)4x4y5e9{p}j|sZk0X(S$u9A)+lTcak-!8W}QY1QCI^Z{bQIwG~7bA_x(IlvEHlM4UX*jDg3E zL0~SA%0_m@hl1L#d9?rY>ilQZ`Oj_mU%>Fcu*rWx)Bi$dpe~Y#`F}}M5Rx(fFK_i< z$@;&d)qfdlka`BWBMeF>7@W2-xb0$yJjRfFi=pWQ$Ao_j^Zs+L_|LKh)IZ`s`Cse; zv_6u&4O(RfxwC`!>VIDF!Uo~1pxsHr;2wwI#s3CNelaMNF)->faF{T#DS)PjST#Un zaeO8WJVp#0dZ2C*i!uX~BDnUKVqnn+)kj>m44gI$tm+I5@(c{ZTpVf)nv+>w*E4us zV2HfWQ1zX+_dm~~{}Su|3vBx@wg+@6FxLUlay!0LpgAMnbN_|Tf{^%mP_4nR8oXY8 z6X=`=-h=;HcY@aJu^;-+bs999$A1}AH}PHh&vgM*sj(gT&wlbh2ly}|$PsyMVIFCW+tprZ}=R5JA55YX$~i28Iv@h8_m)8SKKRxy8>5DLfQbe8eC)iGd*-yrD@4d=e4(a%2W|QS1DW zs&%2=`~6yW1h?)FZr$(Gw#NswKn`4jHSYo~S^#nP1hj#UQ-EAQ1i5<%xf((~=@ecQ zAy-3iHmJ!8nixXd2LY>(HhENoY9&aA2vQ|MCWhQ9*SS=I5OPiAQohEuVlBKTg49PK zvTUVe=?cgU5qzxzoPpUVf)CB%nk0gBlpy1B$Rt{2WRpG58g?O(HMkCflUC`oAaxL= zR>D#fK~{06P6pLPDU&QxCRrs-u}T6>8-e;kNfXRq$TYDZ6PYCRfe-rI4krVecffnH zAO>oE6wtN{txAITn;;D2nh3&%*GI?<m@PZ|BkQzo7@FeMp9Qnm5~s*GU5YQKs?|T-JDlIEo2_Zme^~c`iSw|f4STL z8PALL1c*` zkxkPA4{bFuuufuNJ<1?_nL+v&o9cfCwf`(yp!!JM;=hQ+e^JZ-q86YkNz4eeK3v-D zzqHwZd5iy2#{caVvxCPKL;1%msHD8s8(=N4$GMi=QF&5$HxC zSbf9~u8erW2Qk6xBc3z=1uymxoB22Op@QaUzO zaD5~R>Qlh$BVGpvZe0cjMbJ4I3=#|+MGSf~80?RNrU-pcF+^QwsCo|CWwz))|JMJU z`$3595NORd_nH5E=Rvz-1<(8!JP8_AV^{@h$*}DJo%6u8546XYaS!MqM3$qV$_r8- z@m>bqqr`m@bPy2BvHz^cL0A5<9R=Nu&U6s80SZ(ff!9&6ffx9Jj(Y_U;(-o#JqlU| z&v^Vl>*@bNyZ$q{_k!vpWuT z*E&^zDj;~31nLUGxoB^+lAPh*)2)RCjR6TGKGD8F* z;WZJw8iFt&JtIhE1gVeU)ewYXl|Bo(KC*-z@&j2c2O*)$%vkJQ_XmD5Rc>n0z#R(h*3l1R*0P9fQJOMg)sC%z}$1d-tEnbbA zy=ynPRjhU`S!$az7qSvhBcw^erwXzXPt+og&oEqA*PmO}olDLZbm%Qo?vi1#Nd8{L2jC0=0mY#|8?&CH+uG8@hV8Kz;)0}0r!>v zJYd8Hnd|^pMtooepbJK?f+niCZ~srZ`k%qN8N8-W1l%)X1XoGi;88pH5DG77JBffh zgK`vuQ3iud5Ce-g=*9%lDh0S2WWO_l=bJEnjVX>Pf>js1rv@1qXA5wm5T|9KAm7drW0^bF{H2BFjcd5`>O+5I0hf3)d8+n)dYM?ojx zG4BDL>BoHVKjX3g>}UQ9T>{N^iJk`Cw9bDHv}TX>5a=Ey?&F{(<&1|w_oqW{Vq)0~ zI^>*j7wDi`j$@#EHzZDh>NuXQ|HTi0PJb0V{9p0Re}<{w8C0?u80?rpH^7Qv#vcPi z2$y7yfb35(WzZo%Tx$P0)&Fy8{O3{q&#U&IR|SN4)c^CS|L50v&ZqrM$@PF$?2Ul> zqtRU#f;&MMKm@e!3~1j8IYbUZLTVyNs}fSy!Fxc+4Dh`V&>M*0m5yKi7G&gGw;2^d zZfo5LK8SlW_+Gq?9@U%Nt2TnsdU&4*vW*BbNd&2V+%VQEz_+qu?GQo6=peNZ8VRY4 z>~XABz`CauK3imyJrA-<0WvoPxe*VvB@@;yg4IE@Y(PCD5NVw@6OL?BX26ko%5(@a zOP*$yJjE<|ib>)mlSEJ*WR^6+B5{IwLceh$2pJ{x871_Y#Y2&4TrV<$#1Vvq)I<;x zYkdT%p5WCGoPkyoLAZG8BRs=*$Qzf!`VNJ{uC##HO=u$U`UvZtfRKs_QY9hRN1(yI zNk@?TP4M~%>6{|aeN2&)j)e6e4C&q*)VVvPZKr?Z7Qfnc?q$neOP1T^F0f3UY!uf6 zS&66NT`2FAp==v3V-_J};4h@+Dx_e|Eos6bq`}Co2s-Kkx_1e_aS2OL2{yb|9{o1{|z64wjF_PfV~0Qa0Hnxf{ZD_@0H-W3X;%4qDa)DoI3Pl?dcq7x)}Bo1m(VflIW4PqRiyw_3t9<=DyE9{H?Fo-8cl;5ju& zO#~+)RSblS=52@+njls#L?wiT^u-{`Afttl)9oa{LtbVK3>FLw$qWp|4BXooB#$tN zK4Xx5$E5L}#|Tsz30i>aBVPUg+?xMcl>Rfyer6DS&LFmifo}_gI$MeVDrkldxjy2)^xt&je+G?O za5qPpfklCVLxq9e7&LarWx&91%D`a&>L0PFGcYJIuqcCW5azOE;IRhPN8C;fd{zw1 z`V5THpnemB9s{E>gLDsr-FgO_6QKIYU=M@QJ_hd-3>i-uy8kmR`_HuFKkM%QJcs{_ zoc=Fy5wssx@C4`_c!tfOvng11{TDg`s*hOq{%1Z2I_QJpDCiO;0r2pn+MWN>7eVVE z1kQt&c7v|=1J_5)M?m!v3wXSac@Jo_67wF=DX+Yz|Fa(j??0OHjiLE1L(ezR2JKb< zdA9%O-SOYL>HvdW3V0D6wp~{&=@RN)B8q>-RY3I-o6>(a6%gW70#`}k`iN8IKZo*v z5yLmi4!^DAZhGgv^s7DWUw6c}d1pY|9>3P@Ak?x0QW^QR?C@{e3BHmQIs^n>oB+Cb z2woW>_lDp#5rlzU8Tr+3fmcT03TPwjlGY92BTqMaRBmvu+UQof!41rSS4NPP+wR~c zbMQeq(3B8(dI-e8xQz&YzyiEO1X;O_Rue%MFhG_r*yb;S_l6)0kfjcA z5_#;7___&FB_Y>G;r)jpg2>lfK~LytDO19sWXh@d$qz^GyHGFxL zoY-V+m_!XhM?J91f{%LO1s`RATz*1IQB0DXMcR&g>U*7Spq0IhyFru7B3J&iU;EE^ z;XmKS|EwoKXH7H2JY?{F!r*=nG;?Hq5i~|;eVV~!H-q&d4x0lEe3Ka5mpbkHumAYJ z?n7{+_bRAH5b;5E>Bs$5t9GoAf!eD}Zj!T*ff37`^^L5Kl# z@F`?j9poYa$V7l5ld3nj`$UqVqvq9(@Xo($Yq1|=RJ?4Tp{|nss&vFhl@n^XCKZABFc<@u3 zfk}mdQ=5Uqj)B{ef!~fnz@CB6j)C8vf!~RN&yhjE8HD&;7A>$Q8(;tS};J(-f(8Lx) z^DPFwZ1A~us^IRSCHNG?90sOF2EGTvs^7U4{_`sRXI1~tq6VsrKvk0Je|GSFMqJAO zIdxwOn0!zTI%%19*R%Q%c$3jCzXs5hkZ<$$fEEz)Yu)bKvJFDQ%QtXM1UhOP&VY=? z!AVFRgiIpWLB4gHVbu_*Lj>7O1g?qJ!w>d>55XZbAoY=3`5Jfdej<34801q8bfVgZB!sd^yy zhJ)Flt6CxJw_)U5jLs40J_zJ_BKV>O%ap0s$y2S8rdTI~7?3^@gfvNgojH19yGk04x7tpuA_f_IT1H4%h_ z^p7B9aOd8j4(KMPz|Oq^NNflfxsrm+B|%8gToTgx;b{FPcr^rP5YthDS5ol02AKh= ziQsh;vQ)%`BM|e$`VIwk?elNn?boskv<|0!3urA()ml(5JAbZC#x$FhKI7}4 z&mwiFba|_IdDBn{JwHJe2TmCac418hE(HcQQPBPx&`G3_q7q&{W0u2W0qI?uoByk> z2MyLSZ2!-C6m$a-+tvRJ=l*k@{m*dtKSSGphJ>dKeovwGk<|rIePnTh!SVou-BAYp z%?y%rMJBupx$xiM$$zz5pc~0WVf7KDN`llx5E4=!K~^$wUir^*{=eCS{|uD}K}Q3D z_P`2)R$_y0HDv%-sj#zdq#bnw$|OSONrf%ojNKuSvY)y5wm`!R){b`~)Bk$Rf2CY? zRMk8jbU_lxi*Wx!_T7VosDUB%J7iZWq*j7VTCszt@K{0DNHL0nr(MiIrMh4)43(c5=75fK=Qs*lnj(DZzrcCW$*T<8 zK~qR9d;hcS_z#*wItZ$gm=A(FO8jR)lZL8S|BD?1%{TEK19gJ9kNg)s^`HL;XgH30 zFKDk5^MU_t2fzme&Hm5O{Ei|2Aw%YEhU5pJvlAoh~B6Qlxy)Jh0)GIYubw6J^vXowHeR|@Jq=HGSLuVbHA%PxWNRU&RrNP5)kQ27y%dq(vL=72S`$VD^Gx+ad2;9Yx zbe1{qCPUo|rtYr{)BZ3m{?D@NKl|GMvh)8lgl=VE(+ADwfTpuqVJi(h(pep1~lR#vqi&AehP^ z6c0ke;S54y41!S%LeUJO2@K*%43fDFnzan>i&^|uF@zsw&AtimAF-|Y&$#tJq<_SJ z7SunIy!~JLE~q{dzwux68fc_W5ZpO})JFn0pr`I=Ec?%(KZAkYg+aiVLEMf(z?^~8 zmVv{Dfy0u4&76VRh=J9Ffz^zG*#yL9&}U#)W?)hRcPLcAq#9^JJEIP0MFg`x=(r3< zBL>+bal6S3CMy}V7cgk7VbEH~puUAcV=IIHAqLwE3}N?0%R#k+^e)iAo!Hs`VrM~< zO2P+0=W;Wy`OmcWKi{tZLi<3=1^M=Y2J{3^{1>_OpYu3qLjm`3&`cBaPS9dJz61aH z5BwK8{9pRWe}SFRsZv-fp#%NZgk~P293_K zss3kD{Lg7{SK8^lLHud^yjwo?2Ynk31vYIDXxtXsv^BJ8t55T0pXSYeOmW#<2z2r`_+THQxa_}W{kSYl> zTLc=K1Fz+V_lzLjA_xiZ86np}_JvCz6%d4k56!{rBisB%HsEu9Z1SM@LBJ~`yW9o# zIrAOBB)Ct6xEc>}{3+_~c#yqCkhODG=`$b~UMCP{jlrp@mdQ5HhfRH-v#)D+P5xZV~~F?;+Pm zSgRy>^#mEjgOJE|6NC+~kKhbQeFPz~)<=-aC>+~iL=Y)RWrU@31U~-(vI+t&8#U=z z*u+zz6HbQq9}Vd{6415JzjJR;>n_lJTQ%$5E7y3IEOpAAXPY+FG_Kb;yiv=qLfs`> z(JocqJWNc_S6IV|N70&7QkOwM1+@1BbgVIawF6}M3sNF;%2n=5+5cbW{C~BZ{~0&^ zXJ~%RF#R{nuK&!Z|1%u_&vE!a!><3J+t6~}feuaey362nnZf2PgY8++GH?Cu3|gBQ zVy?L@`fq;gzuU|I@^?W8GC=Ahj_WA(5fAK?2F|OXDhYJ3>gE4@*Zxa9_|FhFoq@rO zfmN1)1#+S?2Lq@!fy{z2Fz{*VxhIRqF5?fK#}KfSA@&$^-c5n(XH4B+`DgrRTKu1D zHK;z~Soh!Y@PG52|D6kFf(NM>UiVf5WV&-Dt6d8R3C*M0o6x!A3%3`%=^!_0@QEi*aIrGA>(xtw?LJV z)Sdt0H$X_}256*C_~w5Qx((_gv7i6Xc=EsGbXa}F|&ZL%j)9CGj5vEo_iH{$Ktq_>_hT9~hSYXIu;FF)@MnF>xLIFLnjA zUy1W5XiE{>9?*Of$1c!TBax&3Id*~@NFARTvhOfNon{C=4XTfdUkUd7=U?()e#?J} zz5j)G{1@K=s*ia0|Cc)RU+U0*hS^^k0{CCK|?N<5_xjqVM+ybwUpmh*Ztpw>GL24p+?SssK zR6yQ!n;~@&GD)x+g7l0al@VyyDtId^WcLtcnH;1(g49G164W6oS?N%`0>nTXqjM-( zZjW?ZE2IK~kXWlBNR6(E(7Nzw%5nh0JE!5NVH$RG|hRfN_vf^?A}G3DC;>e#}2Nr;Y8^ES-- z2sAbetB*iyDPd!G(8ZQWTuAkV+<^ksSsl>fJLKvKUK2q&N)Qq<%>=1~FiCi21Ytlb zBM6Bpf>udE>L5%q5_}Rkylw(9km@7IsYKwVchLGMr2j-n|B0Z9#{wrD4emMW*M7ji zeYaN==n_De@-=pa%PeyiTBOf1h@Yev-liT!Sw=z-AM-1y$pss87ww2#9n7u^Iz)3fAd%Wjh_A&ybU_1 zj`jL~wj0p-Bc5xZu_MSx9pvOX#%uo>!0W-dF8p_V^q*<*a|RY81_mWYHeLoM_>Iz# zH4MTG+^P%~brMkv7(#Y11Z-i5J;#u97jzMA*LQ|#e?WIwuL7NB#<1-_M1{OO84sFmm44`WbHNo4vRKVe*!2qg;v_K?iZ>$yrvo>e}9` zINxE&{vbQ)zx1~Mayvmw;1o{&S6=a-A@e*#$uow!4-EC+m|FiZO#Uyt>_6mg3*KG- zMGyTK-SwYmWt@$HE#$AT!3k)$A7~(H8d)ZX~v#Eg)Xc3(n zXz2pC3aE?3sr;Wq^&f}oKVh32D!wnxviI8-9rJG3>D#nBxM_1p^Oo?Yt>H~uLmPI5 zHtYg*g<#bXq*j8JY@qeqKK0wc2)dOOQXgThi6DIRohQk?<`UuiFg3lI#Y9jD;Mexc9#7G3yM9B3Kp2`Sb6B)$y7{sBB z(HTZ{!VxUipeyZ>t0WK`9L11s5enH1TEp+#ydA!d1(N}(jF9_EkctWG2@mk@5~L1- zkdRslQWHT$AZ$1Zsdtb`|#31paUT#n0aMDQ!|3rITLn?9{8_u8FU1V=+*xcC;y8c`_HfvG>y)% z=|AJP|DeM#=ly4&^`D{dKSSv&`Pcc{>V(`4kQ}WJu_J6)T|23}tw|MmUk09f6@NNJ@^FA_F~!GAMb%mId+yPTD;7`nc*O$D7b%Ch`F+lK$V+d)$jd?)`4p95`v z@xJ)q`rv=_#BOkr%LrZr&jwyV4z6uL1iV&)GayIAKn?)s1g$e?;s@2;9O4XIl8n4E ztbz(W66*Xidcq1u!fNIc+BQGaFJ{nM#GtkoR3$;`BaK}Qn!6cvk1-gWWNITT!n zgU-_sHTtg;`q3isuS4lc=ZZ7l4LiLXb_O?X4sO~EsgFYHcZSsO^ljXNh9H#=xEcag zM&Kh9K%F3XH3VV6J4BG}tB?u^PQoi7_|P1@CW2H(kg+(o@-@!LTZdqEkW=vrNHqi} z9l@O=cr|2SybOYnYa&SP1DP8_W<&Z!$YXSnn#ekL0lYqfteu0LJ_o6cAWP;T{Udm- zWR*SxvjRe{i7eBmTOb|84Vf?k9k!4HUBV4n#SN*9AZzE06Z;|kAj9}xIKwc$*C4J3 zQW-(mkopK-0ULUn=wN^szB|+Fh zUHc#m2#K{;!aB7Csk0ze5`=_QN$~mzlYv|xK^8E8MUF*IJPtviW9}v#10UvaJbL2M zs0l}+CLV^M$O)jO>7X?jy@!K)4~K$LP}c#!HqdoT?zJ0T%GcNyEVav?V-!C@H>_Db zpjO?hT-G61#yVBVFq%)tk3-FaNzxRA#GOE!@xzw8Zu>8D_P^$>|8fuii{AJzedoXC ztN+S}|1-3H;F|hhZsmXGdH)&O-ZC`3XXyRK&<$GHz*P5}q2en;)d!BrprIAU?f+FT z{kMDlU;Y7TE`$Brf3~Zz-A3T4B6xiSVqEiF_LE^MxW-%oasb0F(4KwX zQ{XEP&Vp_(5IOr_^3s3pE&oA_?ih@~b4if9&IQ36Y9JT(A)j##7GV-&V3T6tkY(Uj zVBk|_5Y=IlF=kURXI8f1&~W6|b`v!85j2SqwTO{%$W(SOQ1mWU3#d{KuGfrgRg3P_ zPUtg9n{Jge*FI;yS@JYB&w4K1WCk&R23|MNiZzh{28mP#@e~G;GzP(B(6o_2B(y#f zie?awVGvDWkk4aqU(6P;h9PVZWA0^!y60>?Ul|tuXI=$bgwJ~LKil#DyyrmCBzgn1 z1xf4%h%Is*)DPvp4w_dIx%OZB#((iE|78w?hPedOuP}%lWa>AqWFr4MA!m$mUk$T}AMk2(21|)IpHhA_xiT8Cj%G1D(_j z?h`>OBgoo0vlP%&kx|kFBiPU!WQ!|gMK^p`ky(77aeOZtBBnlq$0J(B1nL|$ZiDoK z(JCVl7ko`TyjFrR{91Nk_L$HH@gS8^VEb-=FhU-(gRlcJZb*lVKq@0-5>n~FNl3i| z;UdpHLBx^QKVViy$Rgls=m>-i?>`Db5fhF<7-18Rg-tjHs*ENaiJkQhb`O``PFRO!GZv;6#j1AmB5*O8Q5(}QL5WA2e zcneeHE{4n-tQF50TE4SP0QHPmmxI<*vF`ZKe(*o&uF`X$HCFZ}J5OnNIKpyJie5TA(Tk zQXg@efNCYs_99*I5S$ixW2^>f-z$SAcz{k5EUpbY;EvmjLBxSUzy`D(i9rL@!vblv zVPJP+5Upo0n8l#I4pblMY+}&a1X|djv6Vr4D}(+{P<>>2n89QhgVg~B&%F$BmpF=U zF@)`7uv!VKkGu{s_#S2OKLYCh_#9*ap(CIg%6T1w^Ew9iouEEc+-=^9uYz;`Yi|87 zvH!o&&i`yXK*MxA2mcEm`!BTnKjW(ZJX2pY6rEr&-_M}AkwNwqgZvvveZ(gJAJhYa zE$3!e{m-ces**q(U{ygAMr?}zMU6rCPZ=gWvCMemT6x^P`lNsTPUQN?uVFI;1=eo~ z1YZaaseq7Yf^OJbTeFq__4ohR*zuoz%YTlopba1#t3Zd6 zF|Ys6v=y`v!r|3_!`uHkul?r%@1o_t4%$V_2VS(mbq#dl5D#n@66E|ko?D=MV1usx zXUN|NsuVyQjv(u(A){yDj*@JWl+Q$l@cj(IyBQ*nFr-~%sCdcH{)1uie}=iB5kVG!|Tkn&_u4&u}f6)=hsF^iY9PLZ-tm3PTf^3GQeDA5eA z(Tix*k7+eY>@rE2V4gnNJZF}5!93fNCH7^@oT}Ej*Kc-j-0sn|19C>0SNTSx_?c36 z)eH);pe;NcE}$xj!;yh2m_a;&K{As;G!-;qBoxaa5D8vl7senQ4qELXp3UH}fZJ~! zL&SEjs+*$Iei?24Z@v#SEn;=(zv0gR>f8RSZ2qsl+Jupap=F=f&UVR zK-Z+Ip8l_Q<-f({|E3rIYp(szQ2kas;Ut4%7ie7^n;8SEF6cl8W-ZXY-k@0|ZE(Ly z8&vy%PWd(f4bX9zfO<@9=AcD)EG7(WW(>^647|1sQoall-k^owj3yxUpeZM9&>{y` zS3c!r2Aw$!`tuodH-M1NItCEh#Gto@!C)t7d4tg|2BTdJR$Cd|4l+a?VF=mB;Jk^! zWix~8UIx#D3|>bVypMvaArN~ngU4o{YMh-F@6=zgcN@xSoV|2+Gkw;hS@ z2kld{KKx%{#dnL|Gg|fgyeiMwWbc!-e9oqJg<0$ytHcjB$^U$cpsI&c4b&~-0F&(C z?M0j#AaOS3{{n`e#4UbmMIN`xyzW+W(4+3Kf8$=?`rZBw+rgb3@R=`POgp z1|!ef4G`o}vmP=K2U(K5bzzW zh^vT-S2z|ecPLy2sfHl+kt3{UWM90@u4pL)L24z)xE#D1f=mxt=PrQshLB0j%E&%v zzFqb_NS_FP;1B3PRFq>DKr6Q)br6JvS30;D=CIBYXqXPXoEvoH0<2pEADx5LM;7sY z7T_ZnAeE76TrZ>!LM9lS+`j5}pw}TwNzfrGrccckhQ(QV>CSeFR}()=Cf& zNUa1XA$_ID{=<>|hr|1iVpc{G6OKXZAY>A`J_?_BJZ!=-NKF*dcPI>0uYqRGkky=fF+kSWabJh_kI?EP*316|@BcU0@}Ge_3cR{o2sEXG z!0cR-`jXxgbmNzRn$uB-7}73+ra~b10nG*7Jj=KdbmTSDzW*FY{_~sxZNHVg4yuws z*Lhz8ZT1&A^`GT9==5pLL;v-T|F>NHPIK}-{l&ixmi)0_@ZNIPOYfdjnprCp{F(*r zOE|QnICY|hOycAmvUL3`%pw|1BI`|~8Y~i9ZPU8!Gbg#^O?N4rVlB zm7aBLe3~~0v~P#BXZ$<&2Y`C12mD%gIpwcZ3-0AL%wrG_0dLcYI1|}m0 zE-TP%5t9}Jv_8@T4bX9#f@&X569z8u5FD>1=u8G)OVDx!Hgg6}O9pOh26l5$eI)M1 zAnpa)Hp}Y(8t7v;0{4})85nFBI6WDZCNUT;!+aRNDzV(~X z5N3S@IcFPI>1^_<-sD-e9>xV7ssLeQt%e|TLs)8}a?k~NkU9uLLTVxi>4ba%1h_*4 zuZG|Z$VeTeCbBD9Y74#?0&8UinH$1i6G7@ANIwX|2314g)445CDx+DDJ`rRr4y_Y} zSs$6FOobpwO$6#3fe-%yZFGfoiy$*ZMhVbyIrF$)bEK1hOkzPDBE#5j!`N>9=uRjj zwp%y46N1ocB7?{d{fKt`h;~qy7IaE2X#baY-DdP!sbRBE!)8P^L_vLoRx3dU?jSuT z2pdu>1$6F4M;(auc!8b!kx@|B0Z69_LPF{w2pQbDH>7JH1ci3*N1)z=$aN5ejXabG z={JRgE2Buz`42}R!*mcbtnWxzA4mi;Mi({lFoY2a?i@u-1l3>gDhyJuLF%Kh2}eTu z4h44~4D8$=)PBIPd5=&1Hn++RHl@q0ikCVSEwIj>DHAtCCT<2p!v(eF-$CmZR)G2% z3~T;#Z22#||G&wF|9YqXD=hiXkbQ;0mx78~^36|L48&pYP^>Xnk}Yv@4VMI<#}d4L+g))K>zJ*YTeR z9a1gi+z4u8gQ_7G*e)UPh9e#iS>I0P$i0l=yBVU-Fl63hD1FS({1H@F&i>D^0<=_^ zVJGOAO_t;Txz7FvT@-i&WT5C((2$+LSdHFy4>Hkc}{tF)c zFLvs`=9&L0$N$T0{m;JOKhLcHER%i-Rh?moUC-ds!(iXe=iRRmyU-$cg-hiY@5*(q z#VZ_&7JF2!_O4y$*RavAb$eh3XrDf43wGCm(B8x0y@wTV0UF;a$;a}V-SgDkWOI`OJfjB1YMTLAH^UT0lI5jIE6tr zkHLHzujdK|pH&PV%NR^%F{pMk=uKg;UCa=%i=p5WL)S}|8Q)nJ|KwlypKA*Uac={~ z0^`bG3=2Oo%>BeL@fkzWb%v<@4AvVM>^Ac_t!0pCW?-^rV6g?&N(|bddW6diGzY|I z1*%^lhcoawGVnTpxV&}@ymky+*3d-@oK_56_Mnp)xSbfdoj`0Zd(hF}oHn41w49dU zMR!IFBH03VQyEOxF&M05Fj&iAv>t>^HZquRXRz48V6~gUW-o*DZU(PI41wDjLU)5| zCBGdEKKnq({|H0qDF)x&44&H=LUu6(?O<@3&0sr)!KRPFs)r$CxAms~3WxtI9{F#1 z{J-M*|F%=_Dpw!&tUlmfdnmN^U`WdW)5P1Nj=LG8o-;^2XOjo*X64lc%@DDvfG3aC z{m$ffMUW~9P9iUUfOMLIy7qzAA;8Xp zfb^2!^%0~R3hzG(sfplZME}8v{)3Sd4#COLz5`+X2O;VqT`EX@1i!fnbbSVRokM8X z;gHTl!EJkkTKBltZE&hy>s7VFsc3;(#v-elZIZKIDKGiLumrq(ZsmW5wg1^S{}(^} z-|p6bpBMjwum9(p^qa%?0fY592IE5vx_cP(S2D=WV3423ptyiRaV|sVQ|5L5MfU$U zxbxrS0jNGwx&^9?cyEB}Bc2niEqux}( z%ibWz9WXL5u&@fK2sqT}$IoU6-3h9avhVO!ykuy6&)oNiVa9)!#h}WFWizNU;yC=D z^UQz#OaDc#f{^G{P%laF+<(5)prf8yPlGOr6FB#u>&kz&3;$V9{%6<^I(3J6`+wnW z|GC%w=Unt(WbJ?bz5fmN{rB7U-*Vw+)3$R$$*Z|NdicEijnWr+)$a;#-4)!pEud+0 zX#0+^j-BD1yFz>Rh4&qZ=sOhAcL)*-QJ`tQW6={%Mo%~y)PBIeWUYG01XiOw2C+Z} zPVmeTyECXVVs~X=abaL~X5a_}O&&>SGe~4W>m#8^2H_|M;S|s|C8LQPE{hrL<}f%f zWC+_MQgBUe(htj}|BZKo7O@*0`)_sbztv3;GQ15MBQ<^S-}C|KHaeqQ|4pugko(2| zax4A|6g_7MKEa^e2fjDK5>y{C8ZvNLfTn@?tr_^O!CfH-&=4Htb_ot^1`cZmb_)<< zGY9RKWi^2N+)(osRAjD=3W?L|D*)oW_GD>(b@H;aw=rb@zaGLiq*lcAmTFqd* zfx&zWgT+<`%WVu+yBMr@GdLY)a6Ja<0r?$c@ZSomlKgiv`0Zfu-4CjdLQXP-oo4Xa z#o)CAR38Oy2h~Sb6B*1q84Q~l{1$32`l@mIKktVBOcNe6G+t4vI;32E*t_;nK-2M% zmIE#&8x(wwahR+F)ko4V*cJW@Y5(Wb`p=^Q>KU;p{by1Eoe0FD{GUw~R2lK<|5tST ztRM2p3${Uv6JXasz`I3|+6S#Zg51yw zsgEGng*z85Me7Pdri37)bC5km$QxSWwGX6Lg3J@4)I{K}kTtgY$UJSPB^W`vKag4p z!nR1AW}Y$?LPF{tcx43Pg64_99U?Pu9b}p?(IkEXgoGdXV~Dg|!6*(?4M8d+NPT1+ z1L_$;DkFVZpGZHtQ!lCmf{^PVINKn+4Xr*xo(=S@+2~!r3DQ6EY6NwZd>UXCQv>Lx zc&~;nkSYm6`ZR5W_oN`BcJPBcFxNR?)<=**Ja~PCsFG0XBS>EfR4YO1Cio~HXq9#6 zUU;`CxN|S0PZZL%55jU6l7&Wc<;do(3y3IgS!s~cOQf-nS+zz zeFs3LU;jY}BeZ89q&$RISfRbpv-KeL5$vcvQ1>dd>u`9_(Qt4B-mhhccjH#~`pq^a zE5u878}(mfTK1o5*?&-dv;@3MaTRF4ywaur&QCx{a_N7D#48N$cR^Q6=AC-y_vyj z7K__dv54i46}y7F&V=Qtz7ne&1CK9gVS`*YsFx&^1RA&#j$;r^Vi3<@FzVy= zU(FDEKp_7TU&niudH*F={a4!ZpLf@Po}JLs41_L#X39jbf(~U8zW!h2#(&XU|Hbe8 zm%Q^|=Kg=RlmD5P{^xD_&QS1_Blrx1^%e&CdKC2H%|w{=1o?wlM^)V{o3uU^#)osFOjrg+XVHT+$<@~8qX{~)n?ns$4S|N~l>hT; z{FijOt`+*ix?rbM`2o+`b$(5o{pxlgP~&>wN2>m63Pt2Ctg+$OuvgK}ff% zwQvNfeK1K}6%eE{g7$;JN1j5~Cm@sXaXARX5!MNU)JG7~xoD{q_{?nx3E?{AFM`hy zq18mlYqpWAA>=v;QWGK15P`Zy;Bh&t^jVOjek{{w!n#E>k$Xds3J6{YK`J14?E_(e zj^l>S6TxdD(}anTwF;1VB1mrtGD&0{kC-GfiRm+m=`)V*g&@PoZV1wk>eh?wf+ARD zq#Fqunu83`=|{Bdg}1>GWY!NSS-Sz$5pLMzUI$`8*q#lWys*_rpjrtwpa-vypj8sl z`<49KcOq9wAU1d<9;EjK=?y_@A_xhok09f9@X83%8-i3oSnDH5O#~+))e{;S-g_{# zdq1Qyf{>w*-VK<9R6`LM^$|oRgoGFfxyT8^23Jx?!h4QJ^d5uMM2~gDQHzN!(vdCBzEk-)vNzOpZ}{J|Id(ojlpI;gVq5CjeQJ8`&e8r zf)-FIFXPo(#jdi1A>^pX%Kzf0{_8*bulopEAIaSNFL(#kMMAER_-_2?y8#-q;|AZ+ z!+8-@8Hqgj&)#u^fx#Q;>{D>1#KtdSU1$|MnImEsN5n3Mq)W`jkC~g_@b!IXnDL)= z(SL^3|5-PI)-bRg1l@2gZ~;^qiGV93&|-$mpv$s3Pr~XW@P*x?m;MV}{Lgb1G|j|* z_&?{W{|p5W8JxB;IB#bNKgp1Fhau|{L*hw>?6G36|LN(E2Fk z07JwPwzypk{;L_BW-?e!WYB8`A@$`9TB{gT=Q9|sQ3ze>SavF`<49=x;lT3!_Ni-S z-8S)>En<|r%^-E1LF65zKH}HLcXZ2bm42eL!1T z!PkOA7?8>c)DMDmh)P#L>K&{kq-O-_{=kp-u`gT#semANvqDyDLr8di1n(0;Y9a^; z8I;3X2f?c$$eA(Nox3gB*$aRT(tR+IP% zCh-#>^%0~df{>Rk84q`=+1r?C0 z5K;$)^+A?`gU#$Y8rFLZ)Hwq8k0SbxK#)uKIs3MgjD1gq=6_>Y{GSDaD-2GayC{FglcU-0;U z2GAOBW$>bIa7_dvl=<{Rc|$e{Mr>mUIm(cFjiK~8L(>VN4g|D`U2j_%|=16sqteC$8(DNx-caQ?sWrT@I=|8s(`Cty4B zU*X_?hADp-tamVIuV66S#9*_7!E7~y!BPgJRSYI;7%bN^*luKS+sI(Mg28GEgV_W& z`$nbcxgky4lO`MupKvH*;$g@Ne9#mgXzl%x@cyHa+CQZ0fNS+e!^GJF4pj_l2@KMq z4E$aUJnm4$?a9FF&A{i!AQ;HN7sSBr4;r=Ok6=>BW{^r{5J>?c{&WWMd=~fRGV%Kv zVh(d;Ut*|x&eQjualwC{)&E(xf=Bc{w}Y#Db!+kEVnGsv|w@JBMR`ZI9Yf)JMx z1Gg~)mk|hY8ZvMiGH~fKaO!|=>tol2c99rO7(grQ?7*jF2QV-M@)^$wseBaH{NKCc zmxRw-4x78|4*xlw{&PG2=XL%s;QXK834{ck|MR%~=X3in!UjG?v|1g+7V6c3` z;C+KJ=sxHOps?eN(fgTV4zNXSX7F0f;5?PVx|czIH-p?R29f&=0=F4tcC)D*mbPAE z5U|rKc9&V?HWk+c;uf3vRIhR?U1AXZ!XW$!gd~5kD*or!{?D)fU%=qMuNRx>kX~Y7d9oE9V>T0?zRIJGPN5ZMUYoDhyiIWL)OT_ODf31Irs_NkUPOa+t0xV z%E3EC$h8k-(-34`95U%%w$i<9C1ia9goLczMn2;dzJnFcfUHbFKIRm%QVw1dL3&1* zRTAc%M34#yxiW%mBeKq#YneUI5`1B+b=F*qthr_xv&}MQn}V;#vr3-@seK^z5rl-) zMDTGrI0I4>L8>A6Dg_7wd3p%BTLh|%!1v`rdP5Kr`5aV8pU5z#*D$6Ra@r4k6}Leo zbX-m+szV3dEkf%P8Ah}lgtuWLc>H2As8t_Vavg-i#9y~K z)I<;xtv*8TB0(wz%>EHteFR_30GUMsS4q$@I!Jv4>21L)BZ#r#UHc*RQFzyW2#V}E z7}mKDQXfIIfQ>pB4C-XT`dM9vgS!qx=8ryL ze69Y^F!MXpivJ9Y{xB5pXK?Ig5RGHt3}WEQW)LZ5;BDh~KcKt(zxUDq7EAv#q}*aK zJ1m&=o@dj4r4#>c-~2ay0xAq;ZvPj$145#=|AP?t)Eeaai0#UM=8K@}gXheDtq1=Z z8qR{(46B1@jzCor=oD`af2*iQ!H8`_k=q#(FR|x8U}*fz+54Yi)_;y=pgAM9t)LsL zSq}W?JPEq_6f#=`sgI;CgKpb_TyM>C{6EKO(7>JOrT;<~L6sEGIZ%DXdKh$BgZhd8 z43mF@=G>L$Gw7^f&|3vM6jXfZ8a5>{%BX>YnrV|6p1CpLY$YKH}Q_ zAGtmfy#^YJl(-J6k|b|}x=VtG|1&K5#!&fzA^Qs!XApE@ z5Dj6Hk7HJ@Gz(kfSA54I^_))RZ6*JUg09bm+@ABe{^$4jFW~lH$m745CjvogA`zee zB0m2GeE)O0{bzRn#OV5g!Q(1}-))A-Qw&k3nPU$!gzshuSjpfzkHKjQgUd7qwL=Ul z2N=X2G6>#b5ZlVYyM}?MltH|KLAilZrIA5)27}CW2LAmFB1af_-!SmMV-WhtApVU> z?k}Gfs4@~T_%Estu9CF>3+nt=^trDW`NpAQyHnLp$Q7&5zE3q`2^?f)f@jr6uj);h z#T0ZVsA@f=G6GE-RjhTdSPNMr2k!xa7!|9Lt073$<5ITLt#pMO(v%Qr&2}mDfK#;E z2T}(ilaM+HPTC`l(Lw4XNVf=52SI8a$P5vDN(ep*2WQyk%(sS241wz)3nVtAJ~GRg z4XJ$~RT8WYnqiR&MUdJDGCBugnkJN?>sLfE+KBgira zI2qEl56%t-Z?=V>`2eXeV7-_lknO8MT?fF+_CO2yAiG&1l?$XA!cRg}f`;I__Cc0a zppl{72Sd6Jgmxbc?%WSyK&(^mxNbhWE+MuHw7*rS}GW41*F|7fe4kxQ&mdjS;JQLAWuAW_#KAG`wUSJ7-AnX#64n2c*GF*fHCSUL+~*Mr$r1}T?}GL z3>+a0tey<)-VAKujvH$-1A8h1V+8|8Jp=c22I2V(!lxJ{&NE8g<57Jntp8WXeB)BGmS3r>MtIptR$hBlSq&|X>@FfcH(K*P79HbuvVMA&k_!u30 zjhsXNB6uAHStbXmj3Ctz%DoS{3m|}( z%(O|L0YTPj(=pd7K&l=Hi98}_kvs)7ACx!=UL_$j;FS?*=>oWC1RtG4u8fRhx=q0d z(kFtDhEeTsq!-nu7uBW{)utWU3PCy%Ejke`df^~s5Z+=C-l8AYjEwX`n;;0Y4nh|3 z0Iyns)6%)ADI1mV4OdbGU z!UrcIV}lTLh#}#%B-|ooMsVkT2ny=h2SI_I`+Zt=Y4_YVo%WPr&QJag|C!sLGnh_g z;D}^k@MU1}1C8FWTQjg(fsT}6*JToS0!?0f&h(!Do?+>KrZxZh*ZpT+^q+abf3eN~ zmCpQke*0hh`G47)(E5n~I_Q{g)+_&+FM)Q&a)A$06SxkpjKDi&*)IKOx%!{?-hYAm z!wd{o-~;7gw;D;<=m(VvL~Y{?UeAzlku&c;L+ux?ZqSXi91H(5t^d!t?LWiL|E&8# zl@ZT5P(Mrb>VK*0|0S=1&S4Nc4?4Jj`#7jF;yn4E=lp-cEB{3<|K~sVpX>C0&QqYv z1X+&%XFK|z}^nG1iBWy{SQO&3x>=m4C#*;5}q=|K4OTx2da-E?lHtZW{A4S z5P6p&>JCHXZHCA*3_(Zv0ypybuNQUhXESMFRn26OjAf8YW>U;zQ)=MVZ0FHjAf&%c z$Z)T?*&zvo=aPmmMD;$28+;Kr{x78WM?e=eN5iiVs*i+B|BG3IDkDipXniE>462nB zo&L)^fNp${2UkY$`bgN}zp%r9KIi|uPXF26|1&!NXK=p7V15o%8QCvmuwAa0e3n=1 zDFf#b28IF#hE&j|07e}KUJKBbwE75Ixm1B}{dNcUeIWfHh`3MHdLQuGIY=i6-Wx)zj36^XAQCnl z2Va%|scu|Lmb(-$gOHGgaF7}Yjf7M{Xe4A@4tYKZQU^i0Kae^InS@;ZU=Kd#6jx;g zse>SML!b>rumL)V2&5)5%K+6%Ht93q*W@ABN62*$WONQvADJXh0x`hr<w zoro6Qux4Ex)exjI(hF^Z&j4aFAe9js38{~et0c5q35^S>ong0tZr zB@hFAu_JPQ1e#QWof+=ew#%nw2c&L-kb&)c1KRckb?k$b^T;Gv5VYbBQl~&jNbLYy z7qJIC+_%@i9fbVa_Q23Czcwg>*|Hl_(GVn&T@KNUi-fGi@N7Tm)p3Zs=bp&a=lsk5 zi>&?++T6jP$iS`5z^V$m;(nJ!EqXI$V!H=gA7S$8FKG4)PDq> zk~bA>?<&we3!pip1EBkx2Imb7 zzPmsNi3e<9h+E5;v)iTNpnvz7sNO@-eTO0Yk%RjV1orF$4Yc+i4sPA=SiDvnAW_C3Rl(rC zQaJ7aw0~6doTu+6%cB1R>;7}^0Ig@>Jp7;U1ZWUa_|kvLtN-P0{a3&BU;5mCw&njB z+P*X7ePu{{%aHsCgyJ7EM8j&IhS+2!Fuf^_aosE`!rs z2FEvS)}I)RUNR^=WK(~^rSXkd`zNm^XyG)k=6_zz|2#S%#HRyVB*3TfUr_tMpw@p; zz5jxG|3wTzr|!sD{g=1>uW0{Y#TkUuT>h&%|5tPPui^Ax6^xXfK;nvy|K;ufOWXgK zariF_M&h>rCGG!<+Wr@?_|Iwio5|>lpxYbykYB7C2N)Q-7#O4(7$g{&d0DtbSOi2E zxcM141VM;dl7Ug2flYvcQ<#C>oI~E9N$e80{B0Jw|9l$%g$(}-8~qnB{m*X#>Lm#p z{Z|jVW0v^Zxq7{GHE7e2Th&_cs@2|At0DCcdL5rD2trzB%!DI&hsX>pXpuexf{;(%hTI1Msfm#5AW#Ja zKIg{-b(I3B`x6gZr~sNF0y97r5V#rw)kjD*5%LN;NIwWt9~nn=8iDGdPWb4YL1Y^Q zK{`RYVa<>}k#=~qb~vaSg4aawsz)!hNhi1g4PjP8a1pec2*Sl$A9;co;Nj~ufh!|; zt>oRf6dpB?QY1xH=pcZ*!OnZT9Ox%K?74N8CL-r;b+q-$YSJO88+U;H) zheW16S6uW(c-4P~vMUTqB@A3K47|Fa`iM=AfmIfi!5O%;^<4KKy2+)Jz^$98GXKBo z0`LO7)u0P&*f;-Y-UYhR#`XDst2_UN@B9~njnQ%5`Y-(CzwiUlN^g!!pd-G8Zh+42 z7P#JetYwHl#R|G#?*m82 zFNP`q+2(=HBVyYG>KU=@{m*{%Kkw=PVpl;^d*7aYu?t^sAJMpIX%6-4prvKKP|KD=SXVnS!Jz9=>x1WH_j45C2{pwT%`1|ClkgU=h(MFQ1L;L1oS5L6in2SMv2$yf%- zGzOVWQ0GXcl0mVaLAIK~X^BAeUWV{}p#D+yGoHSm%nMQKBi`ent{LW;%6mgFs?g>N2Cx+~=44K~;Qa&=oJ_FT99uFDp zZ!p-tV6cD1Zu6Pl=CiQLS7DQ{qQ?J)4gU-3f)J!W;?e>QeDP?37Ewd$BT>Eo!aATX zlCS}&K9V$t)<=+fN6iIvE*x5Yr0n!x*%?$H$v7bDBT4)JQV!6*m8k1~0f+x0p6_{` zUN9)`WMG}kz@^5(AO&i=F!3<3aDXQqIT%=Y85jf^KuL%T+|kx%5O81+I>)1M2f04t zHv!d0LdO4vjQ^_z-Zo8m9r-UGtk$c5jFEE1CL=e&q zGA;+(G2~pb(ye4Aq&|Xlh0sVyO#~s4Ya+B>54`(>#(=MmgH%Hh5?%qJF(7r2eGceU zAIO>8kje;N6Tzw>&~8?6S7;`r(y>UNVTy#z)271_T76^*J5d3x4uaQ2#^7sPO~KPc z@M;KBF&V}7LFyx;*gg;&Tpt<5K{vF*8Ibx2+9!%=htx;#a~1R>THy#%CFw-8Kq@0@ z)kjz>Cb!!45CpH7APjhAgvNl5Z-;b1u=ZRK z7q2&hu8H(**y`E1)eAbMwavS6t5?Go*V>IPH5)wZH+zA%Q+YLP@vPtMUc1q)W`kSJ z21p=4atbaIv>Ci+y=UzPP?cS^+Ov9%XZ0Gdnzi_mTkU$6>a{v$>m9ldbIkcHw(7s~ zn*R)8o57c{+A%XrfN$331zm5-z|O!X%)lzbEw0AEpde+NsIu_0#!}FD5&H^I?ZC7i zbXbtd!~g0JK{W#3P0;FY>0AG0ZvR(3_h0!I5C*Dibt+s2;RmJcaS0PE@Sm8hR)v%6aOrH|E!1pa~=jE z*2Di<4})!3xy*sFTZ$Rhn(C&kQ?R#x=mn#Hx z@t7Ag$VGvw8y;^4ZZ8H7cLpv`5QEo;fe+R>5)NVz31$!t0|mZBJcCps=m1r@Tn4!^ z2Gv>ytxgbfUdk4+iy>+sL*`kAnnyhSU)dJ^7hdmyZXP?e*v_EJNZIMXf;|W+I{b$qIfwrcr0D!#(dECK z<9{jp|00(E#f<;+>-=YsImN)Sje#MDfgv7z;;S430}H5n0!=+a7@Q0YBFy~iOrkr4 zR8O$T{O42qFK7Vj9|@WK7c~AaVDewk^uMz2ZIie+&eiK3E7y8dt#+?mOsE#8&U^(RjlHSQ9;^)b=p1By0%YqDygqWsoo5d^Rer904$AecHraEmvu48)yf*}|h9LEhMfwZ~ z145dmPJ=TbeIl#0DOPDyAm?vG`bUuZ2!1;r+68%#3J6jkLFyn#56Bo+2f=F}$W~Sq z5_U_G0r+SI&;mK|A*isG+tB(5QW@!mgC>m7>LdI;BkiDi?Vx(BWh|r+#!5meBe&Z1 zkZK5xM5~P8TrY5a1S#_&Y)Ijc++%{Up&OTARR+AOf^ieZC&m8(eBY$hCUCYxR1!nhmbtjNnkQ#;IbBbLCp+%C!z z-+zYCO$^K~;NuZQK=lz5KNFWYyQmTavj8Kn3U1^5;SDT zvGu?5h5t4$|I6R}FLMpF^;P6Ps50U@`JZ9lUxvyX3<+l#B9AhJon#0-!4Q3hA@?3b z_Y;N%KUt1}E=OQ2I>TVnC+*rT=h3I=UMcHbDj2*-EMhrB)Lw?PlMEHl`8$6wO#IId z=^1VK&#>)3^G?vD5BDihWh8MGG(shD<-gq3|1y_BmmCS60c|ycY>XAU1geif`(Drd zhtx;hCqbn**AY-P#IO-O?zbDXE0+H>xH7uXL9u{AzMMg$fx&nZ ztK$OJ@SW_DyBQ+)GUi-hsCvNB{gHLSf4-IfIktf=nc_bBpXb8tk{+Yq#KZ6OV zGD5D8Al)A>jsKWc61+a*1Mf5C)&#Ai76y;k3F(76QxazXdLMf)J!WlD7j@Nsx+3&f&kTJqXD;fa)X2Op}b`e+}3FO7{OXT|diNy^uCp zz#`kpAm{|1Hev@~^$WR3ksDk$i8Ax6Gm2~%P(H>i{hwR;zkoipJ`ytd&u~QG9QFYLV7^RH-p<{&w+65vgaVzN4P2@NY!JWHr*_B8YY5XkO#S~HF*kXSpw3~ z99lJmIQtYnJrvgu=?%g65WyKJbrARfI;aS^K7w?JAgBGnY9h#nR?u1n$P5vLgjYsd zVNE)rAOxw8kO$}>br3R%wLXFfqSZ&PHS1hqYhd8D5}binF`-Q`dDd@6tBi2fN6>|N z;MxRU8NnIIwU2+(cK;^Og@6HVpi|3yo59@1ZIC0%AvF=`;9u}oZjY+9kjqLSI}0F3 zPC-V4q18wJ0;j@7PKAr?^XA*;%(cy#3u(RBXV0|FoK6tgXU~Kno75?`X;UG{E`1sv zWR*U{JY}kM<|4bEqYSfu@~--?yX`+i>@EgwUj_zy21Y&xRv~r)DOoiGaYb_m1~Dcs zNe%&d@J>~J2H_~f@_md;{iJZCVN#-P#1U_FV!W-^1@B1Z4! z4E`G!!gnyFUtueK#L)Z^bRrO>?%VpGaVKcHhv5)t@J;mEf9dO>o{_?p|MHhXMX}%+ z&`Ck;r$E=Kvz`U@zl5*;=e+Qr<2+QH{rG>56QFCd88(CWF0BULi@~t#Klic!0;m5A zp8YR$4uk~GfCltf4*h323R?WYeHwH$E61t-%t!z8ANkL)>A!N%ZPB{3ZmkEsI*&qD zut0i7{%!mH+xGi4>@m+=s^HVbWmdwVl*Aws0@_=|69C#?#O24p>C3?72V(OCF!1_- zsw!ddNS#xp5b&4L<3EGje>SK8%=Z7; zZT_=Z{%1G(&t~|a!{9%w?teCI5aQ7N&#v>IT^EEnb^deefCl2Yv_M@RZdhFd8J&Z6 zjTknwsI$_|K~j;)14-v_a!_Li+#3P5w)n|ChA^b$L`={;RtFS9Jj) z6-YJY@E;dso4$6r9RdI+d*i zUH=WfcNLPs9g#M%qAgBa zfh(gqHn0i^a!WWe2`O9PHIZcoVt@`_4MA2ZKn+2+}8lS3q!vVQe?Ne*`%M6>{7h4igAEWqm{t*|Bt(g|(Sf=v}6*FF%oR$v_*xmK>hM3DLjjdZDA>r%ZIQb|GTCU|88 zXW*=r!1WQNR)Uc5UJ`o61m08&ses_qOvnuH#;wp=2|NS`R|)D~)q(DRgzQ#;tjTe& zTmxC#=~%oBvMkd&f01SG0?X`q7Flym(`T8a%`{4xYM4C51cnR~CK|*~Fo>U^AJ?xR z*N-di;E@Pnz$02eq8+^21T@nJ9cu-*D=^vmF}>Omo&2%W9Quwk%>KzZ?~ml7|Eztl zMZJ0$*gQb>5xWS7ptQJx7Q3K21A{07yBGt9I0KUiD4R3bs%CHE+X$+UxOV+ly7b@V z$$zzLpvp-4{(r#-;I2pS7tn#ykq;Om?lT0xVF-H75cC{WANfCI2z$Yh_KYFzpVefywlc3Ip;8{?WBzP9IFpvExXmpO_BxofbL>KGv|4c_f*S#^!`5{7zV&i{-L;uB2fa)X0Eug!f7`i?%n8@scMP`QK*;hBgXs?j zQElPc)0v@0Pb&J45bC4=Y(H>DBftKDmfEVDw z>LXAW$r&`YB;)X39?TH82USUsPL`qPe@(~#+D@k>^mhm=mNRh1gYU~w2F)e$sxSy| z=Tkh&Ed8Hf{lB38e<6eae8!;qNXYoVsM&udpF8@IFC8n_J5;Q5t61q)u@Y1nm92zy zc_1~BQ|T)B4d9ULzd@I=7DJCsa41>sSPHu5&Y^e(h*1I>)37gI2CsL}81QPyp=c@Q zxEy3A2u|98uU>`MK@f&j&V0yyte`Qv?0JwWA-n8(kWH-M%4nVqxD$k22SM1@88bmw zwPwx6Um2N!DSRLg+pm>>*jucrZeyE=&5u-T_!Gw2{ASUmck%>)OTHb_#JIt!*I;a3#^Pl_h ze~kzK74L&iPE&XQI!qgM_D!izt0eSg(3P5L)I&Xq(=-v zml*;tF?bweFj>c7w2;AY0fXgY2D=puW=k1t*D$zlW$@n35POcN>^b|y|Ki|^Tfb>d`mZU|6C^jIgS6b>4ADc+}fbd4iC7q1DO?qR3dPaPXkn^ z@PlWFAj){O5H%6JK7vSrDkHuB0$>E0EfUlLO(F4Xfo6_`wf~Fif=Yg2@M1Pe3($C- zg57^5m;cJ({*j6^sCrU#fDX+;W{VUZ|0{qejN~2vBO}NZlAHrb5ZZNe0j+b8vHve+ z`(Mfq)M=7(0EtN1gC>~7tp9Tx{O8kn$iR1;fuWXxp^Ab10E^632Fd?CY9KcY8T=P8 z0?i)@8vPeG`LF18UoZTPL&X}0iZw3fi(SeWdzP*AC|!YBAHfgsaf07-2fp6{vSk%{ zk1B)>sfHl64`ewUgbgPl4OB>#WCJQI7C^c@$Rwo8W0$)CQU_Us8CE&-t#al=rNH$O ztPYw7XCSXtut=W)uYI8Rv?A3mW#12vQU2fsXfS)s2KANY6+gTn*u>k3iKB zxa!dis@DihYx=3&m-bF%V!0RK> z+5m8^A0V|4a@Psc5Ato;3ON84a)b!v#8=N+(5eGhSXaoUc$r=9 z0*mxnCdt!GlBXCYPBKpHH;C(j?jw%r)s5)X4Qtg3Zqf*>Q}eG;@vBhuE>-d>QS>ZU z@F-IBELQX=Rd6qnbt#l}DO7ODlXJ=;h~%Af<(+fooN}aX(`4+@AxO?46AzMf%oelD zU~sCo=sL_i_Xo?opWO5R^Dg`^oUxffA)A3+k^!_%i4%O?0}FVy5<3F}gQIKI{KY#M z%$q`&e`471pK%N5Itk_5psklu;A7Jmw)|r#dCZV{g(2o9L)c}8@SC8GE%DD8q8>5C z-Itp1UuNomhTOLdrQgJQ{+rDCZ#3(_Y5O;hsFMsf%NPu2GZ-%dT@vcFiNSq4L%>0X zxbqBU&p9T6W~q30fJW-rkN)Sm0IH85tGgAhfO|%l{_~#&%^9&C1yx2u7ye6M11)S| zKMSgl*v^3JBc@}Z6Lr|O{AZZ*o1ynF!+h{0%SKRr#C{N58G-8~k#qkA&VZ(qSPnz$ zBc6-@`ObhA@-b}t&oK87L+KTU=v|^|>vZdmLF%KRmaRc8TZ7wn2et0?#rp1KnpN5CW>3_`?|Z!$36=pFgxp;tyjGiewN;0M$nt zbqpq53@!_KVs{DF-cggI7k#3`jKusdtb`mv#C|1wtpA@z|GWU$T|JU|B?rc;Cu**Sw|kz`>LM(`>L zGK&PQmFz*CCTT}d?IUCVU)l~-A4%E$mw+JK|Kc{FDoM)zzn;f`Rr?>x_Irf&*Dy(4 zW|OebWO?X@M3q7=gAR2^s%a@Vtk$K7z~yK}X?AK*Mo}Y6z)50*!kVgYLG2 z-*5+KK?qpo><)6%eEvf|HQm5TtViuZC=3XZt|vBk*E5 zl&%n@K7v$F@R|sD-2$WzGE1EX=?6hb$XW$RO#~UqL9UNX!K>!rBxtz;xPJtxgCHcl zCNhrig;Yau65a`dFd*kDKuCCHq#FrZt^nx`>4r5!_O!zLL@3=NaDAi|46TgRgX`6U zL6s4_KEkYnv;%74$faU6Cc>Ogw4Y?FAaV}cyQn<(|e}QApT$}V6R!LLL;wG5Jbb}5v z4{O#7Zh-V+wEZj9yh>Ev3sl_;l-+WaT(gy&(&g-uWo?sWY!aoc;-#$lQ2o+P+>h^VLe|FeLsRo$RLPE%ZEWD!LE5X>)an~^L}!# z`7gBjKU?$)29ZS203Cw}0|R7^7_y$0k%56p+A`ffwws~yvf@h6N&u!K|FxchcE9pm z`Y(3pKmVrx3>6O;!X7Y$KV%5F#o&JvG<6>Pj3NFpL&6h=DgRAR{|8koOaJrl`Y(Fm zKl@G)65I(MIxBw#I#knc1B1gR2D6o*Q(HavGXx%I2s_1)affljKiM_^#gBu=20@n@ zT>;fNQddEjy~TK$juQtjw}Y&*V?Ont;;W1#gCtVcjcH}ISWomR@U^FROW{|q_T z80=>=IM25#SntzxFtlxVXxr|fmhC|;+e6y-1vKw+DcfL_GEds8i`l4%K{kOwBpg)5 z2uFdcA)zn^L1YvGy52}I6x3bf4r1U5X5b2C5R73E&18@%V9@Shu$;yaw2344EPLA< zmBs&s8a^;YoM3a=!Ch{pU9N&!qXES?NEw)_*ah|Ki5~MNIw+8A10V$$Q+@3x8u*w#Ke( zjdR%|m-5AK;A0aYwU0wFXdn)B!8iD_Zzu4js~{3Q90#d!kXISm7cN1b4??bV(9Y9_ z)Io3(GX4gygCNV~Y{8u%NPPsUeITdFK}bj)1YtwQ{~;u#CPH1pJrhy^S!RGP4Yx*} zErQGw!K)$AB05-y$Q(902VFG>o>MSNoMZ%>ErPCF0I%f+RYUO2MDe|lE)t|G1h0(X zbr57k4%9P3noWRAG(b-Kfovv1?h`?JMvyuPavHF4b$DtJXlOA!HI!2f3_4r=`5%!S0YMd~z@_z9q{P-M4WSci5{qq<+Ms#m#^ zdx?Tep1gCej6;@~Rf>pNys&AUkZClZVK|>&Ft2V9w{`#oacKCmtNXC2d2^_Gv8#Bp zE4#8PIuk@}N-i8qF6@fV4ANGNGS-YT)=aWC_z{Dg1A~MWgRr++_F~q#KNzNe;$8RO z^xA*XtnCa8UZ4&UyS%)L2@9trXbCM8b1^U|s5liUuKcgG3Ur<}|Hc2x_d$ya1h4;R zI`N;O{XIj%35L-7p!&$`3aCnof5DLOk|FUack_RZegE~3{^#5IpJOwqRsxN`Z2(Qj zb8Ps}vEjeQ{QtZy-^ELxGbdhV@Y&B`v6{hfIfL~^2De=do_iVM&M>rpW!?H;>ePSP zyZ>eGfQI9wuYne3OP&WUoZ~zNni~?l2wH0=ei=L&bml+nng5I@Kqsm4od3_g_di3& zGls%@42|y?rv7JI0KSTS8)zLJ?z@A% z%lZ;z1F4UAwV}N#cufRZ%)kvE`r`u+)(L=nMv%%#L?3ie zHi*>wFQNxp$p+edYWiQy0#qN#fCttdgoM{VmazN5A?NwPNi*<`c(85}bUzWK9|T#b0J*0XxkCgw zg&R^IA@_!m>mWEAQW+V6k6AEHm}m^Stus?> zL6F)9KMARloU7J=Dj@Ls2M8Oo!U35?u9zTfP-RpLU0Ds;(E}mjl@S^PQXfHTA9#HP zudux9HoJqnMQ-JwNg~k7OmIzPkv0XirZKKdFRD#1tU)87O2e;01>6skcg#|>OO>}t zma&YJG>aB82;$T6<573xR&(W1a^zNY;8L_@m9b!!GJ_x{NmC|CV+Jt;22p(mQGF&+ zeMS*IqL8Q#gNQb0K@qE|NNK#%RKlma{a&5-T#W` z{xi(|3#xj;AArW_!fr4`U1zBL#xU(4!_>bFbN}=21s%e`vhhFj22e%7wBbMVM$l=f zpwnz1$BC~1PsexuW2pYXTKAcu{0&3cSq94u47y7h^p`W3Z(#C2!H{=TWBPxU{r?S4 z|5v>VLPF<3M|N`@2kmkdyZm4L%74Mj|9Q@V`b3;({<9wk9r`VG>c7a5{|rs98RCvJ z6uw~Z_|E`2MJ7U`oM0V^9Z`&T;wllJ0cVOdgx3UeoadSjmn;Fz|8N_2jJtNSF z9C*SAGHoOf#2^#`LZacI871*(&}j_9;SBr{p!>@Cqd|jp5;>syNUwvzX%0iu5uwJ% zLeu_>&HB%g|CGUE9fR~@2BlRD@~4;-&#=nB;#U2@p!$zV{XehTe;(C;Tq^&$RQ_|S zf^MCI)H}Qy|KaruuQq7097G&K3TXZp&;%*qhxL!Rwf^&fl|dKLX@j}i|GB`3SLZ)3 zc-e}bKd%93?2F&vKLiOGf+{1(`E`)}v%<#z#mqp9*@R90i<*KEpqQ7Fs!2 zRV8T)8oq+1MUXXe zkh4s&u9Smxh#>V5goM;bHqhz@v_=lHGQl!?4jfrz&4wd*ZwTBWLiB?mOXeW84}?Un zk05Mw@LF!j%|wuz2tp#Cu>cvJgUmZXHn)N<4F?aEZzI=7paCV+!?*Rq+jPTPAxJ;0Sr1kpLHa=uQYW|p(g}i-T0u2%q!U!5 z6I7!eSdEFa0;-S^WVIw3iM4uis$A_Sf$aK$Z2W=LN^lZU zD?!f31Fy$}3&Q7~JgPUiRjzX>UFA>+nk2H!m~D|h!z3A085zX&Xooke1=XqfRVaCt z$h+ptIOWLLrHfl8i5W+U8bu1}hw$qJa;SMQD>^gD+Av64F-e#+ikmTr8Zrp#GVp7I z5T6#Pj^I{h;8tbeQh_24B?e5yrGhTVsldP?&%h?jz%EA|Qf6RPWMI%_Pz`2izQ{G_ z6~nyW40C@e7ad~Yjb&h9WnmE)leS=BRxrt)Tfgl^)Us!GU8kIue{x^*oo(xX{(Ybu zcEoRk>La;_|M_9=13I;v13XLzxq(9L z*nfu3rwqQ^7}75=RDI;>1s}({2xL3suK%pZp!Jc^IZ$OJa0*l(aUA>4egcGePyFXO z4!U=Pq3jWZVs2yNLK+Oj>Wb5B6SF56ttG6qJY0tSgV z2EGu`a2#(418*Scd{oF{Z*E@(E+5c}dcFWqw@5S+R3C{)GYEl^Kn&;zQ^{OVeWcyW z=scGpOX$)ifdkV zl=_GdT-`usf*@54=&A+S{1I|}1R1R3)&h-dK~zFWQ1t|^jJUKxl@TxafFOQ-(A^Du zdjFB@BS;A^Yy?uqtqrQJ_;tZEO?v-@jsJ_7f+{Rd&HwD`pp_C3DGs&&?Arf%4gd3- z{ui?NFKh>@k3{VL3)_ND3W6-66SoH)90VGs1Fu+s*GKa9DD{!FJ)%Amw*}Ql!WRF9 zE&j_}fsnWbs6LV~|1WNaQXfe<-`5CyYgx1!QXe^#E`+R4fSg}ySF{weI03Z3uW*?| zA*eFKtdAg-5u`q{&0ho>FacLa@MUt4U8|7Qamb_{_*_#+O$6!tz-l7M$^`WK2wn}r z8Q^MYHiU!^%0Vh4i}V?gng~Kd>LBFa5Ts88pBsWt4?%iEkV{(O9U{;?ICxyn7_&D7 z?h}EkA@Hsu!x+RhdGMMDd2R?&`@l)$ZjnCt_Hg($5Rk*SAvF<%gzO=LkdT^4E4TqZ zHw3ASAiW`YWu%F%8iH3$$d!?XUj_1+w{SM{sqC0+cW@WUwGw*12b=*J%7c)QWq9t@ zpe;*SswePPCTQ0Qc~}o=n=O3HEu_wZ)I@F-s~wA%+2${@2CZ0_VVE${D6U5@szWCf zyf^_|6Dhjp$vS39+oecYCrX+{3+e?6==kwycyTE?F-u#s$(l1unJ|m%f%-B6stkO} z3_J=9oU#lYG9bh$&A=hWz$U@KA_hXN;vhDsGy|tJXn_H%I9O1GfmsNIm;@P^_|Xxg z0BF}LqYwjOB+9@b%)}tf$u7kZG0k??4Th;-8D{?$nfHyM>>Pt$Hu!W?QwFzA(M7){ zmi}W{@}F}fs0?RV_n&DWXn;%T#(%ym|0ThQsrNra^hJiiYYbtxL5t`TA2H1PCx7O@ z@}2*RxBhb<0#`a~{&Q^k&$;P8+j>xC1nnQK|Ie@%jMn@I^_A9xml@3b&(QUSq3$I^ z&v%9i{~0FyXKnq-5O9RSU^#=udIs-94Dr_(%AWE}{LjAnzvSNkG8g`<-2%-h34<>s z=LOd(tcO9HScT5~7wP-V;IWp$eXDT6J%+{~j6MGuW`h=xGi>?Kw&y?p@&Cf$dnkm? z{}(y~(!zcObS4AyanO1^&eNbhM?zEoGelovQ14>k&f_yHam!m1(7eO9VXI%$=Absv z9yH(P?M@}@)FLJc*wiqnrZR{|GYCb1hUo;t8TbPjczhXnyg*m)akwzBJAksRv(xxfb|@M(f87x3Y7yx?&*$Y>jg z1lK#rs}~@fjCeIbE9C?r!*}5AMFQHOEv}HEJV?C;SziY^k5tG2v_gR2^go};e?GJS z+{U1VeB9cgh2k6<|Jl?*NI(~KsDiZFe<{=dQfB`pOhBveIMx5ND*k7X{LdiwpIPHS zgV}#}hyScj2*hRgpUdvQsNH`td(fDh6nGhf6y!i5N6>lTGLE2w!KEFb>lq}$)s~zs zXuFfN?SCno|5BF!B`yAoTY->>#eWeC(BPq{`G3gy3_^yWDI^7#$2tM;?F&~s7O#cu z3Un!4?OL=3QXe7rgCKiXosh1&gAB?+NJupV?+`iUFLKOV2uF}zLXh4NgoKDdwhkfp ziLAlLCP4Z`)>*UR42U=y39f*kgL06zF=#1W8uY+z%e3iM8PhD&r$T0kAf#F9471c3 z<|)&`cSB4=n<0YKM3CMPq$Yy&i6E!_7{yO8il1N@*KY``cR&*p;OmD#45YoQ(6fEO zBKpytka;3V1q7~rK&PHUcBX*2?fT)J`r)01;hhFy9q`J?AiNDyA3;dH&=xcVseK@% zR$!fWP(5TsPA9NVJD?UgV(S4Lc42;5{{S*uW;O#$rpas|ryr5Yg1|IN{K0IJHH&~nxrdznomB#4tVWuA%`S#%-x#KU7n%1>dgf4vzhx%1GkMe}QYD1=Uhl!PnvCJYons!Qg*|A@mkQ%w2}kZ#+9ehc+nP{?C2@ zG(g9&`9Iqx5Mtl(pJO9vu#Ra1c)V^sxT=D%H-L;{-}hf|ALtrxksbfJ*8S&O`d@Ix zf5VOcEqDI6oBdxf?j6v^$ZS68KO>r>Z6A5tP?;3gX}B*Gwt}#bpTW!37rCUtc1>k`bYdHL6s59 zLC}pt?5F>8oB}N}XK47&;JAZ9q@01bK+dhjzkI8I({|s6t^Unh0$M=((cEh{nxrid z^Jrqy%VUs^W)KVqA^tGXif$fX1}<;VrR40c3@i>{UNz7? zr#$LV64J-v10R(D8DWF;bU@V*yjB9&N01Rb2nnf=_%uQF5p?)Y6SN2%QXfHjPrTZo zszk`(zlafN2u{KRG~fmrbF%?e3Zl0E+4cW3EC1!t0M$pJT1o3ak1l9$thfoNlOk{R zUk;3f4M8`wg6bow{|t)%x%B_D*n;ltV{`t`?gFZkSRDUz+JX?IJ_1!nj-XzWB>2Ep z&@p$g%1GJ)Bo19-X9wErBx?hju>w^|NcEAZ<$qywXniDX`d`o(R2c~y{g-oksO|sG zI(NB45oj*hsbslJ;cDlCRme3FWQhXgDhD_T8mk9i|7~Bm#3pYcWU2_#4}uKPL24Yw zyoHeC;~a9K%ef)7k7M3KNM&RXTeJXK8i#yx0$S|@=Yp#t(9j&H_5lsi%rZ}#2}aXF zBCz@hv}q`9ibdKK5CgWK$TVfTY07kSD4J>pUI1(YJ6Zu+?|@emUdNA`V$N;Zm{M0lahpR2hNC=%78Lum<(OYBj%d1&=~mmn?C+ z6mhFK0poCfgCJhL06uLmZVh)%6(=@Ddp22fMhPPZ0aXyB=}5PJU*zC_spJ3E&iq$D`(JI^ zR|b#e3`SEK+*UHgon$Gz!_fMhbqeT&aOTzj8F&5X2CrD)ISm@_<39)5PRo7lKg$6S zVn6<$`!u*`x9UH0`W*(%c?`U14B`bk;S+;uxBJy^325Br-wZ-NP222CS87Mi;I*k@ zP|9EsiDKXlVc-vA;PnI5L>z9Q4iTFZxF)h?V6kOjv1VYk0dEl@O={^`u{}@{>wN%Q}_93mA%rwV4Yj(GS`x&_65uB3zkFbBgn`br0)ai z2SIv3Xs2pJ#^oT@kRABI1bAHoXF%#5$N-%!_|gY^@Bya~(hfXf1fC2+94m)h0YTXC znh3Ib$|`*ZmYD_1UYLCHZBLMk4#ghn(Uz zR$~xWV-iwh7FK5yR_74aWDvAr5U^%&>Jn``$GqS_>!SY*bN({S{Kc^7KkG`+-9SvM zK&Qs>9{JC87F=bV10CkZavW42F)aPh5OajV>Hve?5eAnd44x+$ik@ok1g(M-xdK`~ z$FTA*<2umdYPL0?g#n!F{LE)+*dKgo?tJz&(QVq5UHjP>%*O%LA#2 zASC3NQ%J3ZwLap8t)SxoS2vv6|2eckRT8K6e;&R6y!xPJ36KLwL2CfaL8D%v;WDTH zQqCYG=?GqI0A6=3;P9W%?!SO7==LHhYtTFqi}rs`ga42)Mac5MkmY|iYY<|z{?A|zIzSJ4jGpO#7L)(%=7=gu)b+nO7)d&T>O@I! zl>}P6XA7#8(CQ;uYiNB08oRRrRYu~L|KatKupVeZpp^X+Rj>C}*()83Hn^28b17Z| zs*DO(IG`S$0O|gKDyF=JkWLV!4uTB6K|~;I+no8xbr6JYmpuSR*SSL=Bbk*XDL8dDS$UcPBQ^j zMxd>c_$wnweFPbxGekWr20k$aVSwtO*lx(CA-$*$NbQ46!fGOvwR5_l3yDAr72y3K z2m`qqg0LYq5rhQyj2g9r8bO<0k&gP&45)=7SbYRqR~AsEfwW=)tpY;kYWP*C`IJG{ zu{f11N3MpD*~m2!ob6P$5_!!7ygou^KsrjeY9g%SXcd!l)f$)TwQf~w?TVIK=PxqP zo@bdc+bng4al%C1sBYb`R?VPBwSX!WzY0aqVi}h_3A=O=t7H-LXd%-`A)`nEgK%Em zAPx;*J}n<^bx(FB2SynyRw*-f84Fe^6J`k`&?7ASNk8CTUZ41uH_xhE3j@P0p50&X!%lhC|VoQ^}4|%8Wt6UC26%y?Bq_ ztjAJo{!6X~^^f>B{}IX{@VYl=l=`u1&xd`EdI~9;y?2$(BVRyYe1C|=Q_}BdMq3NGj9FQvg1Gd zLC`>%$kqQMS3#E|@m&G+R>ZIUSGxOO;Wp?l389Psd9MEFzX3V|g8%G)<|ClPy+LRG zto<*%>_0=x7Y6SW4Ek#r%=a>c-C->G#8dy5A?FT*;X(%K8V2ni29v35E=w6=PO_Ff zWN81vJ{45MvaSEmv=>zSaGVAmZO4BGbf7oy@&6o${zK{`zLTKtk_hO^hOZ2+n;C=~ z83beGY}#FmR|Pd}hipmnY1-jgztyf_jatMM9`h;&`6LFB7zUvzQ0Iszn1RQSffKDh zvIE^=%4!4ZA~9Gpa5ypu2QbJ*Gbm(&#_PnRL3@$7+!#QeDL0T_E-walcLweN1_nn4 z!!FazL(uw2@-&P5MM!p4YYc} z;=iETe=b8%&BdYjpF`3t~k5%Rtv_%0#Hm>cN09@r=xyfT9H zeIOMOa^DBCjR;a1*?SQzUVLs3!eZZ$ESb)amrkNy8fgq#ANrvFl$sj!=c%KNF0k4ce zb4GDJke(5w4ubcFAY*ipqZJ@^5ZZaC@X81>Mh98N4X%vfHxq&H4G(SC3u!|^p`bY< zcr~OG+@u@Ss0$jY1Jy@}4pBfYT89W;6G0eS0aeIV51g&xSAmR>>mzu@gr{zT)Kf0t z0}LTc@{nsK%Lb&X=|=Gr zv?DroLR&P08bOtjcd5L4p_EgOgk8F*HE6s})FMXAELPMsM#w0fUq3{|AV^TxUqIWF zN8OcI$&OFio=4G|L)M%{(wI%!ltb2>L)M&2-jYk+l3URNffTH`!3a&9N70%`(VAP? zibn-X@~But5Wl*efTpvko(Cc1C93Bos_P@F>m#P`Evn}wqU$NB<;I{M#vtX%pq9@N zG@GI6Hvht3yc_=uZTm017gQf{>;QFVKxY)}{LiohG^-+T96S~mdzQgw4TJUy2K6Nj z#%n+~s|FooD0#uK0#xiX%m(juT>@gWEdS585>X$qZ3b0F9D6{Okp%b@Zoy0cxi9|b zzY6NJFzf==7XskBjX*cLUj8p|^S|VEP?aQb@xQ=DP%Xx=^S|=8|7tt`yRP}K(f^+z z@;-y}Ne1_$49O1}!jCiP&SlW)WiXz?U^t1vWGaLA4uXIXC@h*^j7? z_)deaOoG%$tOx&d9Qn_C0(8hK+lv37m38Jz8F;D~#FKQrr}=o!lYZkARPyq`Vou-jn{$pxB7ri0%UW6_K(<{7+CE=hz&es$70JM96zcFa(HDs_$#qPhV z{eLC9{|Yw$WvxL6{zzJY)=2XjfClbZwf-}y|7TDIA!fz@9BThL4MFvVuoL1B_{#Wt=F$Ast z3t9aa0WTVtasX9EQnvqPzz8&v1g?)Htp1C_>LY#w(8+s(+MxPK)$_Au)(ZQAb)d>9 ze}O~ZQir^yHhJ?Q^%3NV9ta6Pr3YRgL8>GO7cv$HuWOL|KJaP?(g}jAfp8%;k#)vw z5XzhmsgFRl54a98PoD}wAQA8d5Rf%;kP$ifOc1CVLaL9b_(1JXHy)JKpt+wf`#!hlsqZIEq5@cIb6p9nld z1gng6f?9NfT0j+3XbWU?4l+Qe4X%uIgBrC08^EXz+986~N08bFnN;_$RL7VkLau}0 zY*p`4T=fyWZh}-mXe4qk39X}qStTJ?Lj>87DhWb5RYDJra4K7AmAB9;XTD|jJoB`f zMu}65V*7L>I<>-D)PoyUe5;haD-^xTW!;LUopU7}vnA}(#ck8Xty9IUQiROn`Hf>B zYa4`&BKh^hg!O_2bpi!-{UMzu9t|%69Y0WQr57NmA1GuPfItR8LIyz~L4zQKpkaWp zVX&}au#icxuxSW{6gCZpAQ7_=QS&fy%Lx2P(ke#MDn`~SQO-I^)H(%(EaHSrqXjL( zc#H!XH2oP=y%}^9S#1m10;kDkuH&e>Alv_xYx*C)+5Z_9fjT=3t3l&z^6UOHVi zn#G_wkwK$_L8XO3a~gy30tTyH3=ub&Cj4ia^q*lm_#}kIpeY-Mh5uQX|7TtEpM4!@ zB?HSAP)|hQ3}~iF_%i4eAf5~Vxi0+Yy$q5CEvxGS>)rRC@7#Z$%OE6h^*{fW|NQ6w zi(L6He*M4bRnYz*wu7MSCm1&W=UDKcq4OU@_DhD?+YFIc7~BprSgd3)UB+OrfI)2< zgT{OYm)#8ckHq`_bFTa^vIW$?<39!(C*(Z|T2{w@@;}#M(CQ18ga6r&{1-hA+Cs}S z;Xgyn1qP)q2DUs#wN#7PIgs<)y&AT8HEeUQ+i9M+Qa)%3yHyQ?awce5oj@dLGZB9n z15YrhgTv+l8n0t_1(BeN$q7`~u-P-P+A;8YFe=6{Yo;-4rZLDTFi1o)i1;#ayD+dD zGq4yiFq(o8vpxfp9s`>x=wfcY2BW+K3H84M7t(TzdaGb)jdaa%e%TA$HCGEE@k=HUG2f{Abhs&t(8wZY^O28uC)M z{jY5MU&HagvCn@4-~XC!|K%+}NY?bfxc+}p-T%Vc{{__lbIAW^k^0ZT_l<%5H3Qc> z2G#$<_W!x<|8v;<=e7FJY4)Gf5rmkWJ~KOiVQ~M>;P{cj?k|Jme@0u-G2?u;{~2`t zGbn?OGlFcGl>?szq~!2l$>G16<9{vJ|0;I>6|F&5%bEX|GzHxN$ffb0UG+bkG6*q9 z|7DQ)#i0FP*!90i@PEO8|Gb|6*`5D$+kudf4XEn`n(Bfqw6g&n-5_Q4AF>NczyNf_ z1mu=m$atN3=3?9Ym5}<#K6i24`CzmVzC0UJ^+2u@f}Fq)sfN%xKJcms&OqMN z3aNmsAzh)Yxe%^x)?BOfSyt&YtTSdpR>Xm-o{VXbY6zLMOrHu`p8y$!1J4bCsv+2M z3T7#jK>Z`so)PH0Q^+ZEU=p%20ZJzJ!;j!LO6Y?ed5T6tIz$i>Pqzp@MhB^Skn1C) zYN!pilL*=^(g|&XtlZWKZh$b5?`%arYYueiTyU*!Fm%-%WZi;pKrIM?PwzxK-m|g`7HwM#AeOtPFVd1nD9nlW5%~mkQ99C3sKA zxoQnE8*B9hsf-}~BZz5^r7Pg|ky+{tqr@r3as7IcUE1NTp!uV~I`x2B&@ptrRZ3nZ z3LZuBu6c4Uprhnu9kV5E)5NTjrR`FrY?EbdKuFOhQNcPv&MHR6B1+mkQrbLH#v%&D zwTh9oj+L{GK_HtrS?gGkpiLY?&^AWSCQi;KPR1@)#x7RcHU`Xyk+B0IS^HQyhj=BI zWCF-JRoNv?-6dPYHAmhxPu?|8(IrdXDMQ{ZMaDT%!ZBUKF`dCUok2aCK{boPsGPx~ zkHK#VL;4wp+D8oi-x+59XP5`BUz(mWSk7UPY-A9xXOOI9kgjG>>|;=!#9+Q%vg8Z% z#QzN4{}}pyf~uss;F@Lre@5`w9m{IanmV>^pt=Bbmcm8QQK($!{&SoGS4PJ`D=0v^ z7JzzH3 zAGE%X^VolpWB*yU{b#88z~H)*LAaWMvrxda#5HTBU*k5P`Ym1!+q@gMJCtwIh@U0s z(#D`yz#yLrTFD>~$sia7-Fd|A%fRjlTFk)V2C9$PUBQ)+69c;gsNRu_6gJM|H!Wh* z$z)JS1YO3*>BPWb4!&JZje$WAj5I+T#Fz{j7!(;aszCLT{CWoI)eKUXS>pjCXn{9mK8Rf#ROzs2{AUDrWSG=Jb3>q-NbNtPGN@O=rUg1nN6_NG zu;qUdE6})>wC#T-8&E}{?(kpF{lAXqe`SaNvS$CKP5w(8gDN9IO;F#5PxU{$+cB-2Phx|F?_$ukQ(}85OPmL+T?TJN4*fEJ}g?ze^1M;2L2>|d+oSEKAz zuIN#!2tFKD#wAzQHBa8XP}VhH-Zfvzqe#iCM9Hf}*{fX1vrOK-5Po{9l6#4gdx?r? znX*Tzl6#4wTQLO5yA`7$$T_P@-c?H8Rf=AfaHQ;22|-Gp6{>#KkjXxcz&czA+~Ean zC4;C`^RH3$t5))^Q1&WUa4%7GFO+r8m2=3Fv`!T?O&2!LWVWpm^zLAYT`p3zN38Cq zSlvyIfHe%Vl?*(I41#G4{4or|2@JAz4CXU9%AT1n{?E|+ouTOsL(dO}{@&nqFK`KTB?QOC|IBAV2cI$=`_I1TKf{7Q3|;>jW`X)*tULZo-}*0n z3AD(8@8W;S%b*D?_AQ|6Xaz2S`c1s&L8mbYoc}L!`9El>_I}W^K!(}}41ot3{PrJ#?ZI}AR@8JzYqgq~xo`!2Tezs6xuKS<=*f3f5L1rGja+Xq6d zNB?u3`Y*osKf{v$+!^;7j8-!6r!oknsrk10m#y=z-R@nx-J^PwNA)Jd%oP#={oGbf z3~G4{GARt=@t~SWI0|%nD&#b8o&W|;PY~j81C83TIf3Se*qs~Foq<6)%f4_2gVb&Y(QOPO*O+APu_*rH z(frS$0Y;kt*)>4x*%(zo{SF4De++8B8MOX0=>BIg{Lf+z>NSYlfaYuXHU6_JfX;K{ z(FQGnmbCp3K~lE=K_`@g_ngYwfoEPULAytVO#XuwNE`j<)B;XFL-VFFSz_a!<7FF zx&IlWKQXxdXK??|ZVlRDByI6u#`3?i^?xO+|7te>HSPcFdw^;~4cq?;rvD`jK}bLk zROfPNfcDaIDF0`Y{m-HD3sj+)fo73}JpW7h{}&7SFB|<|C*{9h+JBAc|1$pn#hw0( zI)FCGO2GO@(pDfJ@#uqY(}1kCP;kDj<@?kkdx3S{V(m5i}2*R~Un*w1#>mbOu9GC=EL!eoqDIf+?eFQpo8(a;UB~5_TL=Y0P zW*bUk>k~n$Avg)|6Cu|`kf|c9H4&r&g6tuJuwi{7(7FY1Wz>pX6B&dy=><1H5TqJ{ zoU{Py5TPB!4L)E&C#YIGunJNKLHb0H8V8fq^{>=H?GtJFRcQE>Y50_>dzY$vmui4J zN06FG#j8Zwvsl@)7_zR?p=c?*4#H&Ms+HjN5rl!?DYd~g} zEC-$93$BuEz)KoH*McNZF-e?c6h8qHHM)`Qx{>WV5p5bFObW!z!8+3!}4MLj?LYs8K8@qIa>h*#f;Bkb; zfV4y4F|HTh21x@@GP2DusvSQvj_L%Vm`>xEPJ_rc{fJiGux2gru^S-w2UMwem8g3b zD>~=dYW-}NUFc{@C7}YRXHZyoGVenYYpxVg5k;1?j#=soGz!S+J7{?%2&*ivC ztmTVr_iu*U=b-wi`#VGTcZQyy&?;#*sAI*r0yJ|Zb?(2!El`!je&Ij+1yFs&diuZ6 z@&61{J}^}NXXpUcTx>i43!De7eB(L`S_dw0>OaG>-wZu(7-s!q*bQEfeHL^;kkCbt z8j*AV#V-6;JMo`y!+(yZPYiMA7{X67q+aJpy2}uFhQa@wX!a}h=6~|l-xyNwF+^Ws zD7?WiQTMPp?H-_%xqrQ78c`b2IXwfQf+vBBpL}i)*EuBH=iE^FL*yJmj?rf z3#dNgac5JBm$0i=@oJZKXynk&VUToTV9^I}H{%6YK+Fsbj9>&gH;@%X$t2rD>Lckr z3{rR46drM?{^Qm9&!Y05N$Ee6@_$Ap&|V$}rT@&DpdmPJ%l~37|HXX%O9g-mXBmIc z9#R2Q(B@ed+5b$6py?#gmK-b4w2>sZG6EeF=Lk9!PTB@E`66fgU&i{sh#44}{uea* z&#MbM)K1j!Kbz8j2D#S^A`cmOw=yuTVqj#djmSJ?$ZhHL-J&;GAi^`A8kw4$5A0JK$B)(To5sapS6wf?Vc^Iy^GzmoZX zWeZT(Oxyr;PXxa%$aY>`&?0TnQf3;^B|3Y5>rGx*gg#T9v z{;%WLa_HS#~+IY_mXB5@enTQXe5#J;-d!4EPZWplS%bcXbv>5LO?7m&t+GCxA!f zAhi#qD}+WOS4p6*5WF%cw@Y)B?(2wng zAUxF&_&x|wxd$nWAS9$jG5{AxklF{C)Cp|?qb5+51TNelH4%h_6mgLKM3DLjGA;)p zVcUo*;q{SDKow>+1nK^0`BmtE5u6LDi6Er9cd43JiJDi53JgIiBjoxBUNPaXn;^9n zq=tgrgaxULAS9^z0QZ&P42SZSmSOYFh3NT3r7 zK?_qd8iP*gW)gqd1OM0V(fw`qsBss}ehdra47`pE4EhWV zx(r;F3?j}9^2w~KnGCvZ;%WOCnx8Y&KW1op&(Qjbq4PUK*AIrie+*MWEAALqfEH9s zU;i(08?=C&`S5?ii~mJ0{g=4a0f5}sz{t?@O|7=G;r-r`WX$slu)nBQaui&_SaA_nx0v7p7?B2f$y zF;FBG#~>NOAR59T9K;|L$iU~#zzf=Q#H*hp>)ask+9YLN%pe=ez@x#yEYH9U+2G8A zR1<+JA_fK)5G9%Hn7@-j>NtbkX$HAx4D!zz%;Ud&;D3Y2|7!mKwLC#Lx~khfl{UI5skwc(+)fx8xXFd8~ z=)ix@?f)5O{AZ~A%MkgW!RsS~{eK4M|AKD+g+2aDdH$F4{V(GJS}&(+^IzEt)KO9} z{jX#OLSi~!nIs-E2y`$o6oPIz;+Vo9a)UwkA%oNl2Fd3Pa@QH;Z!oCcW6*fOp!SME z`#pokHwL{w3}*isEdKMDfR+LZ7=n)DW7PwNfOgmsllZe%dGjrE=h@`Vx6Yksoj1oi zZw_P=E4ceJ7gGB`NO*l@n+>|y4syvIgoIpm2fFPpW2SWmXum4jB5gPqykHx&i4|N8 z&4l!fEK;X|PE<&nVg*Cc)!J#GJ*~*q5QGh?j8Z0nvN>p#0+<2L?~tAmybgjeKs8Z9 zA83*&4s^N#q|7ml?}b-Da0axng4IWmwF+<&-Y0^zz#$}ZO@!GSg4aignh5E(R!BP> zTKj-&C`f$-se>RS^wd-Eu|HY?p!x_hG>2RtLFyfC|4MjG1g(#J%Cx{8B1laH?;Jtu zBUR60&}|Ez#i}rbTn%AmLntdcM*BU~czE|p8gYWK=D5ad#}(yep_^bkex zF+n!4jn&p!b1l+mL00uc&U!LSnqZMK$s}n4G^v1-he^@|Ncyo%ooJal5wsluykitsgX&@4g&CYY{;f;CLF&Y(m21g!J;EOSIm zGB`D3c_iE!*v%Li4M6i9Y+ek^?hL{?VyXKW%5E`~-)5+K4mwY}`6EN?XNGR@z}+m+ zk_OJB|3xqU7rgchSJzvyMq-9}tzK{Xe{ za`4pBED*_i?7#FC(5@(k86O#DeP-JHUj%#r9mC%LjQc?qitPFSI%odN@Bc5o+JgU*lm9c+|7Ixv%+T@*&iP6mTg2Gx8}eI${{ARGg# zk3_+zq)NszNXIkCB{9gxFvvtRNJlY9MS@Oa5D#IHixxC4khHH6wk+e+N@C!31sCZu z;QEM@fq@e|UI(gzV3--ifYe9I7Z~(zbL)TS*8k3=^_|1uFQ37G(7qw1|C}oSMf5;R zwMDJ|3z>tiBWKY3&!GN^LF7II`(+01>kLBA7?i(r>4R=Z)KKNgK z|9_=j|4o+sm+AShp7ozU=)Z{Pe{rAxiUI#sL;fpy{g-g~uVDUPP~$&`?0;s7|6Fqa z8AQG_FrQ&y*v7z63UZ9Z5;3b|A};^c!u~5I{MXL>Z(Q}?yzakw^M8w$|3=mSwM+i1 zr2UtS{?Ft6pTX`wv-W=m70?kz9Gd@G_5O34{MU^*W}0x$GIyRu&RomvIaWDyt#W5u z<<5pJ)t+OQJqJ|vWY2`uN7mWXZ9v_nSs=lz*^rtDPC{-Mf{>6yrj-1}exADRQ#N8sje zAG{7Si0eVChIAp#X7Euz$dwVK0zxApeIf{{8`i25+5)eGAY5=w1nL=S1%vKsg^-xz za*#8Bkn1C8O$5H5NDI1+s8TDS5>g*QY9B~N1F4TRe99p8krucbf|HN|Iyecbh9EVO zibs*Md!e#>p{i%GvU?$9*E2GSwN}Cu1RXF{vK(^wA%uikENiwI__$p&q!y7iOay-Hp+y#G+qF&h9GJ*li>$ej6BMCj z)_kk%1(5ku$XY&2h)B*ttLz1s$Tn{=8gj^63`b5#r)()>N1>v7ft*X8ltY%3O{$bls+@I#m~jM` zx*LO-DFc@osF)IoHO$z^P;!$Y{|ZCNU54r>4An0g8s0HcW5C!=Szu!#q%(C3y0` zq2e~K0Tmz(%swBtWR(|?B2zYLAvSmykf-uz!^)qk_kbB;N(RMj2GCJIQ4AuXpbL-0B0-%a zP|pZlCCMa!Y9)n42E`=Mx&`SN29;!PgM2~L0&e{*W;q`Q79-Hi5tA6WKH_3vDdb7`3|? zq^lTsiy1ic8CVM$xXKu~${EC4q$mFfI00H<&vX30%!&WP`~U0i|F5>=zv=4#sWJDRZ$Dj$>9vrUm3WbFtA+$A)&(zQl}a84>LGlXGnX(Q}au3 z+JDtm|Han+m)`VW?7)ASWB)}?|Cc!bU*Z~QYXIc@D8)`+tdP{~2;# zGMIj6F!?5C`CifGms#p+tIYL~*&>^qSzrX3A+pPw4?^ILLXhglK5Mo;xB>!IH{ey- z5D^Fo=@vnD20>eu=`%p5_kb@Of}CXvAt5bK2piHCg^*UM(=1b_TBS{aR7r3WUK2qW z=%XC4&XIA#M0i&SUI)P$pj+@DqjN~>x8cwn0aZwAwktqgB=GV%NKFJG z;q?)yNX0COx_?f^bm z7>#rUpMH-_x)d!(pyFkaYsCnXke(*wPBF;oC^mWXt#amCWX(2Cod!9}*C?(BlpwKmK3f#D9i}y$p7X848~;wEbi#d&yAzjG^P5_@4jL zS3y_4GHm?Maq>U^ssD_7{~Ug3#dL)$O0XODiX>d9D=BiAVYSLE|Ov@gGwrsdIqCfI)h>ovqn0bb{4B<3WIVS zgQzp8UkNJhMHrZPK$Q^_4|oa*TF`?jaQV+49@u@EJN_?&%OggopPXL*d7S=p+W+UX z0M$nvYX7-3K%0ytOuHJ^7>pzqCe?~9R;268t zf1%+2qGA8#WB)5B{?|(Xsh9oNEaRD4#5GQXnG6gu3=9UK@izu#1_mYYh@1!m=*nPj zu#?aTP<_Ouo?+5B;T3;0_Xk z2jbxMku6vRLPF{wNPPr8#%DUD#(|Tdd+yRef{^+M`D|^-cnG9Ef>%Ru2D~c-XCPNe z@bz+#`Up}18N;r_gL94HLv!GTSo$%&AcR``=tOluY9gJe4mg5O66r>Qj#fafjL_;M z?eJFZaL}R!cy9=6eFUk7Acy=QpMa_ZK4}5Ip9oY9K`JBv8V%I?NYlRpGHnE@iQq$U zkR=Mp^^vA`sit?S26(CnxekJl&Z&45!RsSPtpqKN!H2X%N@GkCQa+=Rpfv{Idr2Uj zBTN#xqeLVdQ6+(w)xql{m$H@cD#;OiJu##J2VKHdvc$GziGA@B+rmY5MT;RMsD>(B z1Yz43FR?3HY*)0{z8ItoqQRq%0t5H=KSKi;HT`Gk{LZ`{bi@$L z;r|R9|Fa+Z&v)cM>+b);C;!V_0-gB)I-&0Df02v-6)%IXixr;xpCRKFgW)p~sO|iOhB_ z42BI1>SYY_>EM-h0iYQokx&M)aL{HV$tVV?Xa?Ch2E`Nxm2?J;37&5!Bc;+Hp>flV7UY{^+#WH6equ=b&s_eWr{OYLosmcpcFW*=`np$S!xDP4*lIFtW~>3PRa4twDz*%(Kp% zXANGD4HRAL2e^52A{=k6x#C-siICTCL#iRjh#Z83^o$^E2nnf=KwDr!nxIt@cs>ZTPXwuqAg3#62GnW= z)`~5^~*!N7Y)->U9vdXZ1P=f-Kd5usy5SxmB%& zEa8AGCWKT>_QlIQikG?MFLKG5=ae%{{=7oXNWz? zAXv+wv0E|m3#dM7{?E`3UjH!hKkLT-;@AGO9|Nr?XFK>`^z?tB6QD!9Irf3Bc@;Ph zIuuCk@_)Gt|E13Sw>>i-Pxe;D+hF<5_Q2>Zkk{~J`=+n-~QoyNe?$iNWE zzz!O46Ev+9^JtOv@0R!P6m_jm$&B7#nK$g9)^x zfz!pR{gBLp|8k4|>&*YJIOV@=^?#9~|Dx&txnuuxh5zRa`!5vrUo`eVj}K_28N2;| z78_`N#BB40!T2?U#$^VX;|z-18KgEam@d%Gxa?SV(mZFsebE-X!p)9FE1im0xs}fI zty$`mGm(K)4_w#qfGbB%FbP?R&H`JA&IBH~V+M`XF@WnM24@C=5C-8~5aQ1TRY`m~ z47}M4JQ)n!=?pAcpdORndIpoN3>q82Xgh=M9tPv1T#lCoLmtV*zm?DXZ&3eVr|Z9N z_kY#y|4JSIh1&iLwESmj1tEqu@T$umaAnjFUJkJYbaXu9j{nTNKzpmX!IcsB>Hj=u zL5TnSe}N02%1Gz}XyK*k`Tw$K|BD^_uQBC6L-1LH$URoc#~|lvL&n@dl~K-AP-T=o z(=vSy2tlt~1uafMu7e=`AV|*$vpzBh_k$qy5egf;aTQ)GAy-DAOYxGYSf)%xu8Ghp zAP5&!2Z47ILDtJbDji5ogt(qHt{)w+XVhD7RXjB`71Pz8@>kUEb z9S8}pkKj{9@cKwAtVJuhNeguRDRg--yiWvSKx!g530lnyzMfS(pcYaMY5G+o*F>6r z)f&E_8%`k`S2e(uk%~_l zIF+q*tz6?!z7oDI7jjrDWWhOPA1?f~A_y09dLHN?zRGoOXs8NwupUSXl>w=J&`6J( z^=Kj>Zsi(}s)k5XIajVlMh<1G>`Is0mMnvImvR?aX3sNCpJkjn!#H)CVe%Bi z#D4wwUftMkjfghQphoq8dJX?N74IrFk3uDvTp{}mG3Oj^mo|QnE{2c|tns@UVoot9 zU0^7A#?bVIdEPmNbC6PV+!L zBZiIty>9$x=z7h-zLJ4^ErZKh-ojs?nyC9fLpQjjn*N`48|dP(e_CL?|{|pKL8BFgmX#Z!>2c6*wV#r=% zVA;vQ+sm!COw6`Z*s6`iFq_3No7J$8SwEjat$;xxi$OM%K{6F|vmU1}1D_j%fCp#= zop1o6N|K3XP)uY{PG(R{2lb4UGC-t44oH_sDCqQWMs)_zfEe-xBHZBdI`Etkv|3^Z zFLE&TYu(4Y{y)pI|E%Dn*SWX-=i2(8ck6$iE&ql0fY%4^|F3ZAzuwaS4Ax5-R8}(R zY-G^c#Gt>G!F&q{sV!lUo5P?nmqB?3gHfw#)?U!j#APep$`-kmEpjef;#{`Gu6UtK z<#N~j2@Je?Aj_H9z%>!1e*~$In8E!cMvyE5bAXhw+H*?=GYHp%kXS2&P$Pq20|Q4n z15+UbLkx|hp=`+v}q=rHpsWXG!@&KuWAQcd_ACxlHDs`G!@)U?5X!j6!<+fGQ z6sx2u773Fe$UJ@`1et>SK@bvN`(Ukr(CQ#aw+Q)S2>qyTC;~56(2ME-BY0)hid-KV zg0E_Y*FoCht&o}sUJXGQ&<;^(3#c+8u|5KIk-+s4q#uMrg6|}P^o$^tks7!fQt>H+ z)JO0t2~zt&>LW-U1eqIB@hE~Yz{?dtlSIfpBL$aS2vT&-Q*g$aM;&3j-k`2RL9Q;acFbkZuTksX3$?f{^gq2a^Hm zzo1t}m7snNM9LL*FCnOkf%S&a*qHT^Th&^3r25FE3Z+VNs#s%Rw!*Fyv|Pj1ViR+hPn@ITmDO*{LgU#R3C|6{m*$4Jmj(rJU}-Oe7?|BSOa(h zxE`4fokHR{{@-x_e+IpN28Q{J3Wpey-$>Vki>ZEak<|-20)lHds6Jvj0O}tJ9Qn_^ z`#;09FQ8*0rh#l>-48k|Pw32l;q(6m&iv;%`=9;Hf2pJYRnGr+Tn?_1)Il3aSxo*j z=>2EXf5M=8OU!ViuEz~ftG6UZu`%* z=fC*L|MuJdGlcGEFx^yHG^b_kntjwz~dIFx8*87829~Gp9MnFGyluZ{?9-8 zKUd#>(4p&)cU|1zI2RodX@m!GG+(&hq~ZdB=?s)>x!%f~-}5oFE5TlK`)ekQc0? zv2D_4Sf@=#Mvz${$VxfL=p2NER5!>Zygsr_0bM@?U78E7j7$=t)sRKPBuF&`Cz0zQ z2pdw-!23ZE21F2z#9kSJ`$u}v%BVv(vK0w|wo*dsAjnQ42wNutv<(D8LaHGM39pQF zLR&Pz=K~=3k05;_I0^3-K~^k4CXC?SA`RavoI|DlmM@MfDHE`S4I#vyncZ*AQcU+iV3a^&Oob=oGMpC zDkEePQXe^0uZ1IsF5>GW%pMb{>H&|-L8RQP*MYi4um$aKlU*wi19zaYyUNw}Wh)?l zhuk;@S^H;^GuJ$O4rC3Uaq2Yv#7PDTeFh1AkiL>maD!$*t*m#sv}Y-|T`_}BB7;gA zvsDd;_e$P?bqrCbI4WK+ZT>HD5;Tb;c<#U0+5c>N!NYOOK>LXp)`4!65IhMwM+kIe z>taw916tuQ7c@s8dG^0d>~#jlY6iijf^N4!86W_sU-j~Tr3;|TVMWgTS3CWmX*p<^i^1TVurX-Y zC%+MB)-CycxJWL6=8>?tXP- z;B;c(_GIAqWe^Eu5DSJ)CxO@6$;C0q#xh7Ig3bWviDY06Vqi#NV8~)%nZqcwidF0+ zm-IPK`6~iyw}rKzi0VG$(Rn6d@Jd+i5(CE;28IqfpIfq9{tK@H_5K((|A*Z94!J{$ zaSN!Mz_j~6-|_z*H~up;-(fJ_!r*#@-Tf$|<9-I$Jq!-pp*4}-VsL%bWSY4X(s*+z zUg8QGY6dqy@>jZ)tP3hzz#wc39**Mz&l#~ZFtCEg48he8D!~j^CJA2n?ajcS&Y&@e z!F)AC@Nv+&e>o3%T7U6O`!Bxmzu>a}A}jxME%?tg1KiA*3hn|=0Z#)>MIxtx+a%M$ zyUeD7Rl+Ny3E=34D4qdY*}||I6bP(4|8wmF9e&Gy3Uss-AEa}19@P2eyZm493V0_q z_%si})1Xaiy6gTilpeK8U1*oN6kZeALYB`V%@D!6JZQZh(1LAn#|Kigz;73_NSy{< zlK`FxLau}0Y_sGkklP<1B=Tx)2-_@ik||PG2*QR`KoAmA?|{g7(54K-*lx&?KFB2e zPzCsi9E1VC9S>3sq1@0K(F(aH4{|3FqymDkSb*#*f{)0dty|Cu1r5+aPT|%52QX)Myia^>LX?MLPghn z2!fuC=9;GnLnPHl_Ru2@AqVY3j(>-e@cPIWTrt@fE<&!79Euh@qSi`~wFI^$%WR65 z+LkOsL&)7FNPUDxLW*TH5?;{|!+=y?pn<+h#F2Vft0d>DHI7IKq7hVCRIEiIVVAbU z`$v%K28|05gphD4$kr+d2{F^9d<}?ExyGdedZiR}P!T+#=3Kf0w0fj)31|Uc;UX*W z0z6Attz?=y*(3#2E9u5|8%MS4hc#=4H!BC$O8YenyHzu2W-}kmuWNpGt2}RVw*u1*9x2iE%s*E3o4pG%k%m{QcP?9a~}FH zaPmL@>Hl2E{)=4%t-E8|@L%}gf5DUg`HuaUKk{ELXEe z&|RY9HZLV?Ux_*HVA5T}pp?fTmIOM8n^99vM9N@DT*c}+y>_JBZakw#X zdob{Lf-WoP^#e`xOT{tB#DH#(6^~~Sj%E;wXOK!~SLhPdnj)orSk>&Dn)xeDtGAl= zU$h*)YB>ByH1-d;YWR{qJ?_zwFxo3^uzN%(gPv>|!w8%wV;V!F(Nq)_exF*$nziAoY=X_HM|y zw{y{AXV8+HrJ%lW?sE6?jlty$8N?kybr5LrIb<0FJC6E@0n|C-G80wulX95hlyShK z@vBSkf0McY9ajF=Ujag53;y#?|Ia)FgczrTqih|DG6NJr3=nZh^#qZf0*YTeO z)knr#{xj5_vrb!No3X?;Yc6Aoqsg zwUUN^jk;eoq&|XFMw))rYTgx)Dhb2@-$?}N6G3Vs$nidqQ`aDs5#*>J_>wvN^^qKG zfDY0dLL=pzvf*o_?Q-Yg>L}U3DkeAs(p|F7UjSOkmcIZ}J;6z|x(UvOtZK6|^6%%|V1H5j6tac!tgfBIR58NR$Abll=%GD0AY71H)A=N&R9+PY3 zDm3I)wc4!;TARSstbx}-5E1AY9k_D@5rLB~70~_~yh?)fl0eP_FS>xNc!$i{*%d9e zhOJ$<$eL}EHXSlLVUaSyIKEpys#_<#Lp`KcEx1M_uuj+}6ST%mEtA1!ra;V2&ei`l zPlFad@tyn6a_~RHjGv&nfF+;{EI7~n7rF7D?-FP-TIlkB#f$%W_JH>st@!V9?mt7y zaR!D829?zeiMIsWz{S)wa0hDse}*NXsRV|7|7C9dXE+X8?=5>4w3~^!|GymgCPALV z|M|}R=RN(O;S30Q?D@}NGl{`wzI4@HhGn362hJ1!dC&Y8tN+hn^F&bZKQH)ZI$nLy z(QW*O|9SO5_inSRJY?WK#lT(7z*54%70AHh59$Z;crkE$GjMr<&P`=^Wni;oV6kCf zb^z_%VY3I_aK!7zARfRV70DnU#~_i+AeIOk$l|MI;GNGPwt_+QIbwk3MMGwxG#>2BrTDT<00M&Wld?Z?qP4t2@J%{|uW!)eysu z|BO39US`_!pJ^ZHDiEQg|CO%(XGp%rpf#7lauWzyuV=7Y2bwtojnpk+P@BPO)@hlw z*R^P=OW_i?q6Kb63tWpAx)v|AE}rL7vplqV4ug_2c%cF>Xm1pBxdkT!XcUp3fx#Gb zE0b6@gLw}__$rq2Q(S%DMQ8r!S@>UQ`G1z>;12IXaOZR;c-6r)aQA0AxE?_xXM)=j zGr-Cq6%(Z21nC4#0r!s}6Gr{u7>6vYn+dMS7J;?^Fm3|ftH^QiKhMeke5XMpbpjVb zM@R`?23=eNx=ZUSm~r_(_l5u5CqYPm+kb|JGgfJHZ8H}@&Xj}HK9GJ8WE2ik<3LEr z$^=LXhmer@AV|FfAtAL7gbiAj0PY8Y?{}DF3^^Kw1%$>nN$3Mz!-}*Z0a^{k z^}wqhc#VUH0qGFINk|<8A>lQVVMHqg8HBeWBgmzA@Ub{ZO$4173PYJ8f>cJJW!vC$ z#%0{b?kn1L7 zHoO`_X4rx&BS?K@i%})P>n3D|ZQ&Bz!X?&4AOxwKAoUTXhJujr`UsiffV#^G!i7{x z@O5`^2BdC+bKwk#IJ`c>$^aGqm7vN9R3(8=I)cwpKZF%`#>}+KpCe zlPppum?cgE9f2O*q7w-|s4BEc+Ovm2uaH5xfx&Vb!<4U@r$LKydCq_e5Qe>=o)N>^ z|4c{!^IiZ~Mi)Sn(fk+wvu*`(0>Nq2@FCrjpLsQw*MFH22UbDrj{0h zDkJ9Y|AjAtZbxF+4=S!TZv1Cxd%~bNiJ{@6$m##`*FYOmK=skl|9rRpvs9d8VDMqk z>XoU!$FSr-=w$EXpatPl&Howfp7Ck_=hpboq6WGnmERC_dYiDxe-6$64AM6k825oH zBZeFXW-kUNPiQ}g-5pduLk{s~wPj$oW?--dAr@OueI(+`t{lUtlFXo(0IH9K<3RP1 zP$PrDA_j?73}QD}6do}t|727D%b*H6Kmc?BgyDaFLuh@(qYt`Nh+FSJlfqL5mXi#H zFKo7hY9ax^k4PHf4#;38SFQKR;}A?Ww6}{8bec^1?m|Y zEMicZ#%0}SowMJyXsJ{FB99Ufaw%HiRzY~Mx(2x# zf{e&P>L5sk1z}GIS6lFI(G>9L9HeIiAtCdLQ^5w#gpSu?t&fDRfT|>ZNKJI*zrZz6 zUy18H=-?^tlmGR${%5E?X_YqDI(?o+`Ya3ZT@R3|#}YOx1mQyJ9n+MlXb5sD3uMd< zLPDp35+^~quznC)b%TowIabanegblRgfa~Tu8$xqw2`YJ{pe0KF62Z_2npXk1ZTk4 zDu7SCM5%@#Rg!*Kvwj%pc09;23y`@Xc((|$oEvmXCwK*&Cg_sZM)=$iygq`Q;S8;V zAh)%GFK>m^L=aNluNsap>mv>C3P^nfuY=&dAr-F@&@t3Vm5~y78xg!`1h0u;l~KMT zc)0?+K9YCIg4aioY6wokZW5Vmn====4nk&I=gvbzkctU3a0Ko;LApzD5>ifLl92ic zlZ4btcu3^B37PFsvK+FI4_@g&7!W~Z5+d$girAcswLZcWgbW5kY6nyjR3BkhNst;1 zjRakE3*LnU8Zbn?S{GgifuvxaGDtTKVhD15Ub5~1ZR+Pc`d{+Gf7!183=S{&wElCegLaY%>wr%321W1@ z=z?(!JUtA86B&3nGl=eHka)x(|B^xR7mFI`fNpLb&}|69#{b1lKnQX>D!cK2Zu9@r z`u`a?9x>RQ^#k|#7&e3U2!ldtH>hjLya%*$o_qg)-h==74*!?E^xyNue}GeQ{;2<3(2pd@(K3E5-caSHHAZzL%D;Xw%NARY>>Z3*fL8nV?0PV45*$+Ae zm-FO*&a>c)v@ZYWx(G^LTvz^cUHZ>+Lz#(2$=!t2SG?k1q3G{tK(qH-s**s*N>@SZAY>9!8L9bJse#upX!usC zc~^kO=)g6RhIfU!S2^-}IryqMNLL7%gmj1?B)BqyuA>93TX4-&a><1wMdutvmn=Af z?C6J+@{U=^^${8yQYG2u%tfn=;9Pip1gV}NB)nF_WLW1f!mNzYM6lLMkg^vm37HW_ zu6nStG3z79ay~Q?QXe4;Lh2)Ilx6!d+dk)GtB=7niX0CT6f33 z=fBv*zYKY|_*(z-wSw1QO#l~7v%#Zzi$O=HDqZ=naPdFyK2Yf-cK5%;gnJB30Sru* z;6nzi7!+a{q82jrUSY`Ez+hIxte(QaV8s)=E*X3)pTP0|d`JIFUHmWI0qP$K=z!`Y z4#oe%I{$gJ{xb-FXW+QQzm1g07R`@?zk22O}3y zp9nOCWDlB1Vs~KR_h66;231Bf;S2(?AjI1Xt&ha_GDtoK)kpF_m{tF=Yy5}QN1~?x zCCxxc$n?LE>3>es|6FGOmCgV2Dg9xPTCLRv+Ug6rsF(G?f40LQ#C-&`l0oPgXeNbW zBWNItulF;9&n5<=^`Lv?b(b@!F9g*`S_>J}W-zEu*Ns}_T(H9_Z;pNTOvk+WcG+`` zvlm(yuQV>+Vqbs2sr!QGr0d~}?q_WN;I;9;`?~+SOaJRE`OmlLKie#Dm_xRMfv5IB zCpgUocZ(q8Y;ZLM>G(_s*By{*2r>i*sf-|9BshC2I5sDOV-_+lHyJ!qHxb+m?gx|b zeiKA$8n`L46122~W&eMU!=ULx-n0L?&;RGY{GadYf8Hzqxvzo{-!;(Y0=~=txz2-5 zsbW9(YAh!=dDkI1OIq>2H zILNb2vie!m4iswLELDS5v1yYlaLj3kVOj$E})?~Mc9f3Wu*EDR2jkQ zBL(M75CWetqu`V+@0bNakn=ntBwBrBn={uYdk*~U9mokh5E5P`LFynIq(yoT`3rC% z$ZjU<{6(OO3F$-#t0X7bITaAE!B0MSDqP}}zrZPffn)w` zyPO%;ki+`YXIQ09gRH^RkMGrw@70d&(~j*^kL*#8?2(xH$p6BB^_!rh5|~c>X9aJl z=RO0vAC~{>e~x|s8D{(e)kiBpE9>}=|7V;3pP}(HLkoD1Qrmyf*2rG)pwMzq1tfR_ z)E)+P8CL(dI``l3=znhAL{JZiPmzH~UdldJ-6spY%|{bVN-{9;b8x9~7Vit%^I!7P zf1%U=1&{uhKL1~$@;`(5D>1GAd`kZX6#h$V|L2hZ#=v%$fn_NJ^AZN;RtAP@28L(` zhFAuccn0<)2F^Ip^-p{W415WoiiST1w7h}aA2fW&>dL_C2wHT<=L*G2~MeGF3fSQVbIDE;72`@^a6pI7I1?zX=G5 zn*L{y|H;6$n%Dk;$=3h&2mVX!`!99izt+zGiW~mRF8$9j;U8ngTZZHt48A8A>~}Gk zYycs{^$fbJ8MKx%D9>Y1nZ=+mjX|Q3L9JgPVXaiwc8>II66MDP>uyN2-skFmFER5k z-@^YQ%l~sM|IfA*JRk*GG`yJ4mersc0aJy?c?WB*@}PNMC6Zcz;F zFMsC0&V~Pam;P&C{;z-izurv{(!Kp(@6LbSTmKF3{g=G{U;5^M>C2$yD+W9NGc=vm zOIc`?vD7SehAF5@nhNO(K?dUBl@Vm{%_4blWGDFm2#-`J(+d=gaXn6k$XeOHH?0)U zKSTLThN3SFwf{k=5xiM)2559n^x}W1hu~!lOF^^wE;s%&WS#=exwBa@FpEiO+fChd zBe-H3sCr`103Svz2j0;m!Qj|qx%i#X>HqvE{)?RWFM9GnW72;HtGD9X|GDKs^^v6J ze+Kb)3=EeT80IrDECAO>jSQ^m4D6WKVi_PLk;)*J%pep7UIgv~ zTHyd1zHB-ULUc@KSr*P#0)_7k+=n@KH@k1&ujdj z*XTdD!GBiO{|vm(8Q7LHu&-b!ek42ZKSS?3hT3}!8CMx1jxu;|XRuktV6>dUd<}!` z4p4n$yAxC&X)I$A7h<>7nC-_lEO+m@fLG zu<*aa!vA~=|MSd8>g>z~jh8Y&dP9pql@Ze-a6JX7iQp>dplX5FN01&6q$YxpkeUe6 zZ-NZdO$HCWLdNC#VfE2u@B%!D7DzQT9XuX559A)kh5y+Xfe>hM=A!=$OaFn;lJCsR ze=*E{!!YkX!@{==i{CRW|G==~Bh&idT-*P%90ILo5jg`&OD22&Gjv?mOIfIwvJg@q z!7Cs%2BbcM)I<;xQZYf+znLaXgfL9vCqQ~V5H7N~F?dxrXo*5R=n@&oMGv6kwc|h+ zul1Ru_0X{2np&C!MZ|Fc0iSyUxk`)mAYTGiZ_U%=3AxWU7_k#uIg2; zhQv_wECcn42-Qc()ewAhE2J_)o+nar$yIdDfsl|YN!d9IQVqdLw8}`qAste7VI?uE zCrlB@m>wDluaeLhxc1&c*6Be=%uzmD=wb=bYG?RXT*yLMT(uHh9L|7OPw+uMcpZev zK^Yz{;aQ;b&mpIvn`O<0>|HWVpJA9dTRFH_IC-&Q<3W`bZyb;QSG)dS=`yH3 zVmtGn<@|pR@PHogng7yfL1)d$9Q@BP^$SDMBZkJ042|CyI{z^&293^1-1;wZ>p$qg zy3L@gUFEL*S6cg@!E`YLgA)UTB?F6!g;!Sfgf%jTzTkCYA`GnjppyhZdrmnSq!JCP z4l=C#&$17+jg({8e@0Kxh1ycO|9KVub1MJm*Zj{S^%!))0s8_5_5}<)6B)QCFbK4O zkZ=uyNG*e86$nXHGDuZ2$doWh6flUUFbKpm2!u26hk^FV@&tf-OngBgE^i0}dk6z} z1Os0?gK!T6-#P}#oeZ)!85Qp`Dt}>B|G};STE!<~^k2dZbYX;$`G0<65E3@}&#LmD zLF6n0*I@?M`3wxx8Q3PWYj0yP*v6o~mO*7LgT_V%wT;Z0TNw1WF`DdVu-wI9e}KX5 z0E72Y2H(RB-Uk_ik29p-W~h28-1|pj!GF1B|G5`}5Zh8vR|qr>1Rr^u4;}}Dbav)| z+4DhF0JJx>0L-2b?iN8*LdNJI9Vp0RImp`k8L$cka$nUHFc(riLAphdiV4DosDV^c z5D|!hu#?`oR{j@Q|6g|4K-Gq~cuxsgE>#t5m(pRXoc;$g4yZd|NBzfCb3GK9JEl2&v)$?HQ?g z6so!xsJa&@!^YwuT_MQa5Tq-FT>Buim7Q{wopRvSkdjj-W=*8vkdD7T!dgio3u0DJ zcos!~cQY-t&Ru|6D?#R=D5;O&^$souW_<*&n2;He+5~w&6uh?uVIYx9ol3AEhzZCf zq<%poAqqgJ)qz*>Ih3!o2k(A`ETw}hhF3#y269aVua6+U1N$9%=`(2cBlvUA z{wp2-uYBS^?;_A17S5&r#W(+#Is0Gl{(q@E{}~Sb2hD5m1nq3)J^G(*;(vy!7X|^d z83bb(7!();4H-lX7+4j-eIg-7c0o{;#3&3p5<#=TwEh^wd{BKPeHOfE+#XaPiKzb< zRQWHg`=3YcKeOZ`29Apie9IYxR!spex$tYC+dh$dxfj z7c)p^Fo>ryh(t38MS^ZEhg=aW7|OsK!oU-ZQXlcIh1W+6il5;1k&wZE5u^Wn#vsIR z{GVGNbhkJI&r1f5gA5Ei7#L=P=CD}$8Mvl0Xs!nx2c)!`L3SB~)KUiNWekcd7!0;E zc%R{mxyzaVQnLQ1TJL|8+5e3f{Wn?sUuy~I$|tdT|3&A5_NB5TS0#`R&TLo(1gUW# z_0c?V?E|Taz#S#V1^+?Uc0lSZWJBQf5xm}kbZ{Uf^1vNL!Ax)^1=)o(9W)BZvfw}0 zvj410!2^S{Kzz6)OXEqv*}=#}r{m;Os%`!9VRvg(7^@;4q!Ul}iYp+E15{Iq+L6K)vJdnDd+ zLO*qpPSSix?E~413Lzo=AUFvbcEcngl@6SQ^p8-s3xRt^$g2?`^$uEn1gVDLq+x6) z_~On^(84*?%E%xRaJkUkNpGV&~gteS%?PJqt`!5NSxb7~%i5Cj>5 z!&)CfY9-`42+}7~bk0$5%2sj&BaC4>NF9VsLgpHnkC1;g=Rc z81RY-R7mE{hmi0IDU6QNBFOk2au*5C1{cN9J_3F=*7Xu_K}g*M-w$P13c4yCQrbgE zGU_9E)dR0(K-CatD0;|IN<- zm%j*FH3zzBV(WkIl;WT>Gzb{lEUB|8lp%%f)wrX0(}i|Chb`U;G$&aaaF; z_7(pncY#Q@^fe5EVc@Ic#BTrq=e5rb4ZgG4HWcpPZXNH_}AT@nZZb(Ahz)1)B{=! z9%lpJatJ*V5K`$tDjN968>D`LkdPV&JVeH@08~dXE(DYC`Uo<+1X)K1U&a6#g@cgr z%|w&IYq=rS5TpW{3#$7-_1Z$vgcQT#|4hsNGq3v3vKn+iAH(c_0<-_{&-$&^{(?8_ zG=u8`2ID>k{Zi+R-Zwb2iym9q?>B;@fp1YU+jLc{ei`-G-pzHBYOyysFyr zs(kayvQ01Y*FMc%{kUM|liXzw6BgbLo^rvd^RQm&daJa#W=S(GlBd9{9%Kgcv89lc zOCd8s5E9bmfv(Mt?}gtk1gVc8vdA?NgblBB^kci><8qM6AV`-7%x;I*N60-OI2&B= zK+aMCT>t^8kC0FGF$iyiEZBySdZ8_F1UW(hPC_;cK}bkN10lg(p+>F11~96FFIRxb zs>Aw3kSQVPBoVAz1nLmMY9igj7b5g$nTHa|+Ix@=h7ZNX{`G8OhkEVy%hr2qITZkP#usng_I632R3Q za^el7GD0RH4R}bI39pZk8Q{7JS|34bB}iclua5{a;H5OY>cPrDtB-6;mO%=6Y~&KG z{UeA8$RxZ%L1w_uKS%D1!RsJo23!|JCD!@~;yZ{V;Uk$425h$%=x`@+FhWn*aLk+U zl)u0wZ?PNb)~D6>g{x(A*GgutWJp}ekiMRw?T*M=&?T)xNB*<#1>MBQeH?V5fY6Em z66gNQUidF@8nnn6R8?&T^=>3C{O3Ii9!i=5T9wDX;=kJQ|GMY?XP^1+zu~KnXC7z= z4FfaygaH;XDWqZ^DBSYQaM6Fhga1WO{O4c$pP~2#pVfZ`o&O9H|2b6ti|YPoU^vad zFp)uQ3!BX)cH7I`R!12$HZv$JV33~8AlDB<3SA5eT?~qy3<@0#3JnZ$bqw+)3^IkF z%1Af?)I}1E098i(;7KLkKyYOg&cNNwAk@Xcvx-4t8-v^}X65?~@}HU2ez9u&=h6jL zMnd3CPkaX8%830T1LJ-MhCSfxStm0vv@kHVGjR0?Yp#~_J#Ul!)~fBl$CUr3^Z(l{ z`>(P5zv6=beDnVE&i&60MxYzpA(acH?tqYxE)RSl4pK2emLfnD&j!y2!RsTiN@!)Y z5Y#yW)gzE93^K3=uZ$qWbdWJO__Dgm;FTwHz!N}_j@bPFObh>Wt^kd~32gk&zu`aI zDo}j{I#ze;UxrzK*;f2#Xnw-rx{N`gk%7N}L9UX)pqs(0hu34CX5@0Sv@MP`hy1%P zgipB?J>y!!>}zQY@8m4Km$&>u#l}~SyFT}y{NH!tU)`<`RomW_ZFy6;{(145=S8cZ zr7eFPH~+Rr|4H4Fb>_*lFzX{kKM3gzZFn67uYlkTNY#T}AAvRv#q~fcAhg8^`mxyW zT?JP|pbn89=rlRVxu&2_5ac``w0j@G6%eRCLhb>PQ6GWF;y|ml(dr{`byKeq*q|9u zuNeSc$PJkr0!x7g=pcO}NYw)&A+-{;Lj+#Jtp;8-2U@}nz8C^t8G*LAg0IGd49!7$ zMv#6Ga!mwb^kwBeDfBzR{EULQdikW~{-g^LIx$N~xYT0X4x5vCxd zkcSlb4n<2LTri2{m_EniCD>GAuhbxQ7QDKFFpz5^csB|n1@8?ZGr%qZnGbgx+yM{< zWS24AU-0{$Tnm@E7A}K?hGXdp+rlNzg)3Z(S2-3hvCN*MoxDgZX`yiGHp|&}w0Hft zJp)=91*&nj|7Tba-aoqfC?Hv;W2G{xkT377}nMfiAX_H2*KC zzJ-CImVv86(PtM^{7Z&_TMQ0o7!3C^D6L_TUdo_29kfngr?!v3r9~Y~XMMk=&jPyxt%L>Kwu9Bii5SLUj>OqFlgTHZ^~{O_{!zw565%4Js>R9hH$(;37Id96D6th+=#x-?_v*%z+$ zuH6~dzAvu-Sn`yUsnbp;O*@%B`&{Py%Q=g#Wi7s%GWUGU^pkNjPp2)unzj69-pbo~ zD{tp4znQcAX7=LiS&ObE&N=5)yB>0kwn@?iV_4M#=@UU}A7m0z>6pa!n8x**#P*oP zf~JJvH4X{`X`QxlOt*0iq8|k55W%Y+2*VI#L=N5$LaT-#=lMWLNKFK(gV0EDpQstU zr?puxw8;RB@ZZo1Ib{yg34+u+5E60^5!xsm{7?l*H3T6c{U9~iIX`eNF0mBM#ovin4@T5wy8APr)S*(k+72L=Y0KK9YCLQglEd6m}iR*GG_A2~Ii{E`qZm3@6w*dyolv89KV28>= zDhP+7rI1C{^)D=DhS_{Xz=Re1O z(4h+aCqd_#g!_8WuP zD+aE63?la!Us_{yQ%I@3iE<;)?$g^Zzr=`NuTpAH#f5 zhXu6W08-{dsu;-nw*~(h=YtTa-vOzKAS9%IK~@Rr>_B=?u={byMSt`u`lD^WlVwYdTp5WGlLCw3u z+V@3t?u+R?lrZIZ%8b()^DgBrxlywAe)X1TRa>8zZ+eow;#TIe8(B-QXDq&&y5LgM z-1CVu&&EzY88!8|Yr`h%Z9jAcA0M$g0yIBoFoAijd6c4#F z((tc=*GF2|=7u135S)bcgCI*3AXO5mPXylCid+*x>LWE+&j_+s0kbl~sEHs)p`z4B z;LZ`KK7!Oia1yODf(+J?R3FLOrl3_rXk17g1V3F0%mCd#0zLw9wsqEAtjH#N9uk@Z zuc2TJgsg4Od@ur?4gs&W;8RY>)f4LRdJFCI;0XPEa!3V)i-go-$RvTWzJjHod&~=# zBHveT59Y$fAzX+8&w?V}Vmw_%%hxO5*g5Z^P zaEsuAa1qFAa0m&Jg|HzcXyX#-I@$%cuoH0|!3W`j4sXw2;8486rF@NH{&J19c~ZG+ z%sNjPuK3`8?7!uW|ArU-^PU17%F2HBKj-=X!YBSStozRZ9;{>8{$K1Q=tw&Llc4D* ziBtcD{bw>TI503+F&btwtzgU+?%jb&3WXOQk^5Ncx(oXa4(5VXop zcr$~{K?aqR3}zP@obED&-Dk`B%Gdi}cFupPr644*{6EiP@Mt8YmVwLxL24iH0t3)` zYzP<9%>l1NK-5HVDI^AV*?G`D5%W2⁣*J}C~=OM=ZJF)acOwJ|LO_d_99 zCBjIiMgN%>|L0o{TAa_l_&@u6@N{0^Cx)4y+1CB%KJZ`i+<)0y{}t~3SGf6K`ObgM z^Zyx^{A0*^$l$gYbT5YCb_U%Q3@Y;(q*@uon;7Kl7?hhBG|Cv%N`x%3^g`Nfk~`gU zCqvHJ^Q~AN-n2EceNS}fzUaQAiBnFe&c2wp>Leb0>fK zmGt>%QsDgm0i-vCNkV!+klF`Q6M+ZiIt(Jw5pwMV+sE1pzXQTBqSYY01#;dgWQGV*AL)jH zt_X*adZ8`)VXcsw2ucPw!0RLAQ+*&65Hbm=gOEw+jd=c*S^-s>Fj5nCArYJbx|~saxK2fon7YISN6hTN8*iItM#oN$%B2b42dQlO$G6J2q0I!B1D;D6D z5qw+@vVu;*J`)+q*`^~SS(`L81UdH%jYO-Jz||1wI$$tk7UW7SYw*>LSnDH*AgoHl zS06#tz-uB1gQWThwVpz+o8Sx1;nfpz-wLx)s(&K#YQu@VX4tO)Fdi zsdSuR8sS_x!xnsX2)v628Q8QhTx6ZM*sgGydFfhhNL(~k-D3ue=3EBOS_Y;*P<>={Ny_JqV#1gV%H^$Wb41M8KX_M>KQ?l z!AY=Q#`*u5zzj46%-}IPt{Eux5$MVf$QU5Z5Kw)z>_5|r|BP!uh+!q@DkApf|Jj%S z=UV)qeF3O4VxIkw>E!9oV`B9q`enkXyoagX(pI>LDX? zkopK-2Z3(E1E25%?+`)8;vl3t?3QrI{ooK1vMd2kLMtHf^bmp#?ioS$7NOQeCE(r= zXlEq8G1Ysz`R&;}h=1^)PNPUE>GJ^DpAS)IWA)|A+>m#(92*QP&TZoGUS3on6 zWsxt&vPho=Ii43%0pTEL;~LO|b(cUD7NjOZBa!<@$ZWgZ1rTJPw-8wz+pN=k@K_%* zi9YTJ*`*7qeejd;ngUZ9M9LmE)(5E?;DY!Wn5H3%K=j(?FM?Db5H6$=gp%O9dLb1P zRJIVM<s|RTbRN`QVm|>Io#Q|IpZny0#$%uZh~y9bXGlBBpf{ht;34~>|595)n<6=n|7Si8 zI@3)3_GwA&kQ2#He0h+iG*8R^Q{g8p-D1**A$&_adRbLruzB7QTr1uO7 zcNx4-Fj#G4&|eMe7O75W(CT4OX=9K~Vqo)UU^Zl6&;@mf7!<&Jodm%v%q789k}7C_ zE|VF9PMt~B83wau3_?u|`in)vPO;a&lb8VNiSWz?54(ZK$`}`cXJ^1{(1L^o;H7bp zt`Ma5fz=Dpm2$9}2(5B~a*<^r+95R&oDE^X&7Ak23A#SvFT>2g40HZ6%mWR(aW4BW zy#Bw~rvH*#|I6(7FTLl#>^=|zF?Rk}-v3{A|9^!;pxQ@t?|)DYwHS29o#MIwYIpz3 zUimL@5rjC;gZi^P=l}DZ|IdH+Ki7%>oLm1hO#R4^dWFG#8-w{`29pI0ma|x#7PC0d zW3ZUaq*2Mh6U)F5%)k%;IxUbXh(Xw!LBxkaGMqsvfk7vO-Mm=9rBTACQ!aFZQsfl1 z_!*i>v$fM_>u1k5&Rbw!w8Xk(iCy_}*P3-M)oUH{7n!C^H%XjgoG=-(bOCa39Aw!x zWObZDY@cCVKV%;(a%BXmi6He3RuZ!sGK}hk)IOM`L1Y^Qfv1E(qjQE4?S>KUun{>> zR|s-!f>uO}Rs@Kv8`ca#I-#IyNH4TSCm4E=0_eI25VrwR>A(-%)(xuH39N(ENAQ{m z!a(i{LD=xABFI=Afa#bswRProS_9};r&Oz34Lx$$y zogg?v*}VXn4L@T6zK9OKr3f-l1SgRzBec#DuKEbMPXuQ}Dj*0cW1R{?kYl5eN#xoG z8+#_~2*jD@X&?lzl^}H!oPjWW@D?-kN;;q{GapUfBx&B#ymsfSqAk_ z3_9P1H2;g~{uk8#&m#AqLE;{R_+19?v&wlN7%IOo)c#fHfidkdL+}L#$GxES z)oL>s6eck!G%<)(Fz`n(Ft{-=XfiOUGcd@5Z>AFf?QdcbW?&FwU{C^gmrNM=oEV&@ zxOU$ah&sh!wv63#uXOB1hLXp;tzQ@>{AGb0TMa)Q4pKWT01wk4Pxrvt5Csqt(r<#W zv62vTkV%L+kU1boO}Fqr=p;JuDhRgupgs=g;{RMr{f_?G{Y+y0Ag{Li=Q zKhFvf;#~EgVG|4fY^c!KtU>Lau14B9;mO0^7P zg$(=&p!!HCn@PHaO+KGTwUk*cj!89^K{=6GJCnnrM98U5+P_CBY_fLJT;r^zHl^#F zs<*f`ZF8*M=v2Gjv1XlP)he6PrG^=^G!rIigm)AR-2lZQz;+GAjhCk09f3kP$fu38{}X!<#k3 zp*4|iSTlInDwg_4E4T?>8NnGkfpt28hy~k_nh4MFK9DL2QXfGkgEU|_5TTFD!R`%L z_AZ6oMg%3n^%0~df{>7PbnvPN&VbZMkZK4*DuTO3@M;KoEjMz#1F4VT19T7uXqXOm z1|6h-1X;QOuYDj4NbLhBA$1TkiChz5vLW>mTpY|rsgEGX#iNn%DhbS(38{b}Q&A8S zB7#XGS4=k9bI@uc2p3!*%>$3=fqF?0K{$z7A0dl?4=z{$ThxbTXD+;Af-^`h&q-BJ zd?o0RzsMmUebGCq))7?iP%saIoQf9ML(x3jf;lcl^IVGNITp-xESTw#H^(7wj&0sz ztLz1m2{Y95mPz%VvEA@q=hT1Wi=e~0g}~=LGavuYbnHLtNzk3Mpwmb9{^vXas;vah z{Wm%FpJx&HHdwVU{A&OCRH5g1vnl*#5dOd*{D58k3q#a9j)MOTmES<9;sZm$3x;PvE;;9FC985jgWdvY0+8Q7gz zRN@(;wktN>X9zyZV7Gz6XB%tGF@};`%pLC-rv71`1zuteS&as%ARtTT(CQkrIt5bu z%mc3(K<)=YCXyhskg5l+5pFm{1hP_Y(SOFJpeZ4~mH$Q7|JT~`Uvc$+jfLN&r@Rzt zzaiFnORe>aQsV{1+LMa4Cyn}Um`}cIGwX@%?59qPU)j!kYB~F<#jGbfv!3eAda5|@ zo$jh%;=BLL9s19)8?-x&VJ(>44!XCO?Kr5-AaNda1Bmn$(7>JSng0^U|Jxq;ufFQP zM&2(5vqKCb8yUD)FfcD>U|h(+y@Ww>6NBhs27!GH;>#H1*Rh(m^4aw%h0NBCUuIFT z-mPwLK<~-OX_pe`-%ML_FL%wug7r`G*FDTz`yg-K{oJ+p@>btXUve#Z_W6hjM}k^+ zfkuGSW*R0;hTMe*sgEF}K^*7|A9#HPsfOSrr0PMdcaXX8ItZBonF)f7!okQ^z3>)D zWdyn60hxrsUe31w zLgKHOupT7=ucy!$Sj%}dH578~^A?g(ACY9KZT=#Mf_e7&b0Nqke=Zo!w#lCj8nP>x z1F4Uk3g$T#%(Kl~>`=H&EqAG0>Rg7z1q@j$89Fa>ul&q?=)dS0&>_#<=l?Sw2O-|0 z|M`!DuG`}}`CsJHf6>d})1*DOG041P(Ecf*|6g41Key(82AO{h0&f{OpD~C$6fpS9 zkoXl;AC-M(DE-7x_>v*}F+;*FhQQMdPP-XQ)-q^J1RYGu7s@tJvb_VOU48GghVvjSF-(~Cm$T;Ia`#jJ( z0ni8=q?&+`^TBvDM^?ZZX)LVAVXvtbT-D zb}xh2HU^>9ASAGlLFfRp+EyO@eNtYtG$U79=WcN=-{;eEIHc!v_>>Frb8e(8x|6o# zUiz}zxoaNguDPGH`d-eOyLs#G7p{9yyx~#Vx<{pJAC|4WmofW7Xv=o{yoDx-lZ?Rq zBWN`Qsf^-!^kYDmq`<2oI0I{agk1H&*|2&C)a$`oA3-W3eeeSr9h<`Uq0>z)9pfNX9x9QU@WE<|&Aib>VEhbrYyQvPgxJkctV7gw#{8 z(YzV(nh3H?0?x%`pw&%i-6b?GWMq%X`Ut6>g6@HWS4Q|5kWoQQlB9B;DvBXX3L(8D z2ni8^kZ4vw>PIxMVfq3h0#OEG+vUx%&7BSD0omovfg?zT4poyo&jEa#IOI@qr~C!> znF}nEW@;qQ(@9?>)p*!^-UHQL|IN?**F5)M8+-w}#D)KwXaB38{;$3CKSTZ-2K9do z3g7wE{|jn?=8Pna{xhijXApeP!1aWI|0RR!TULkv3|T+fivBZ{y<;eO!;tfYA@wdp z#3csL!=StEjFy46#t0QMa5;e&D)509lf&+>L(qbtgCH2RKu5U>M2Xn8Gh|)jEWXX) zw1dH79fR*qro^)h<@Xu8J~7Pr&kV10AXN;c%7C!p)e5{$Sp>RRb{+lNcIbW0 zj;Ey?9u%*;SGeY0?uuKv%WkC2znC=TsAu&WlcWg-aXor*U2tR&*QFoZ2{}ap^AuCa zngn>IgT_FvgOJ&J;ca^1ZIHPk$YrdM$_UaAf>%cRVW36Y`ryh4JR%1=Sq@eofv#l5 zJYxa#d^zL}5xg3L_lC3sYPI}p;0SUB9vTVVIs~tbyg{8KNCgBYOO(A!zzp~p9jH&F z>{*0_iXj)aDti{g_7i~);)c{g5E4=oA(QYiI!KiS?-Ripkg5kxLhB&-0G(r&oI@rE zfjdXgkvh9nIlEMNO(biRMzAuHvP_28LAV%@x(O}|uZbWGw3-OQHBXyqmO2AcADO33 zw@3%=Z-O%{(r1{bO~obLN8wOXu0=%yr0} z=a@Gibksx6Vwb$7#(66>lIJlbEn&@B$53;eVbWcWwZFx7|7Y0)ItH9|nSD^YR z@g{@c2?odA45lj?H0Lr%S23`sfh!{|@V(xwh*}8BVg*ekG0QP9XoKn_`AjA6sT@VO zgle8JxbBA3N6bm*7^)tD>Z9rZQ0gOi6$7b&AQcUyZi3W7ka-^n8?G3x3?j1RKkF*! z2|S$Z{&TJW&$IqN-^Tx(>;5w>2aoUde_`nR2tuXT7=pJj7r($PcPF!#`ecA2&bq|X-J+0gOq2u_k_7lGw4}GoO`Mz@N z>!P*K3RXYKT=gt-)ic+{Z!M-hV3_oPeeP?9X>S=?ZZo7`VTe1;;BuM4_z;84CI->f z45GUjgm*9qTwoBp#vpK)LF_Su*gFQ%w+tfh7s^7wPTjr=56?Q6Rkc1S4N=vNH-dES^}g3f{>6& zAH67)McT;qj($Wtr0PK?;ewF4AxP~5sfi$@L3j(KCW3c=;FXb1DCp)pNR^}!(gg1l zK^TzANIR$z`FuI#`UtW_0n!_Sl77{Y$_T9vLgqs1BT%;peDSXiO$rfo-z-W?1+9Zp#DVFI|!CZ*o6!Y|{ zaAcM`4S6v+q;A3_;gu0G1G5f77J<}9kUhD$>LWC9E6`D*^DMKW2)RnaWTTYQuoZpK zLOT~R2?bF`(%>GXvVf2fjmWhOel~1*256Na+-hqK=Rpjy%UxgzM#w6$vaP@y!N9(Q z%+1Y1zK0QX(;xVbVAwe4Jgc0!HhJ@H^Fh_LZO(j1kXmQWwauAtmA%L^YoT7+9KEzT z3OP$v3RbEVZ%`^+rF%@|IcawLcIF_dG!8sYyTHC z{LgInpU?9@L-JSV^q&lQ&l$3xFeKe%2tC8#zKg+PHG{!q2IV#e(Ma&7S2+d-A!z-B zLUA#$DKN-HF(@Z8NaQexWijaXvxV(osC>ZR`GFx|KZDgu2JgLGiI*9wo-lNNW|;a9 zw4@DQp}>1L$P3{hbO04%i#G!4YC7|dSypAo!SiD%w_o_YVJ7XKHT z{aH}M&N_~!pt+4|r1*ngd^|Cu}&Fz}`@Fobg}lylhhaM<*S_;zR|&vUO@AJV=j zy6g!K>b!G_vrZ+?Ig>Q!Lh{^;S<7x^FTb6$>`u4NGqgvL*wr`ASSQuVCC$yl?H>1rZtHrOR&!>2TWkRh(O0QSmbeG&Yo<$4Yt5$f{ ztnzJG7umBbb?U*;){Ryf(+uKzjAJ^DV>*rEx{Ttwj3Mi{!Dl8IM|B!Sc3>jNqBsZ% z>FL00CFGh2d>eQhxEg}q39cX345^PGq;7a41nGn|z$+urK?>ltb2_0dTCkf~H9|lY z6Qmk~kg&=KbeIq1OgVT>q!~~PIadKv2SLW=G{H3ya=imxI|p0T4PCkbKj8LVGubU1?4N3ymlAOzlFgw`_xjnr8uNh2XCt3(Kb)IoSiNUa1{ zU=FU2VB|zdeS};YS)@%dOGB!X(xxNVM-VngI<(=O18vSR}{?DNLpGg-~83`Kym$dvZVF9X-xZVCU#J*=p{LGN~ zh#~y}L-ZvEpTnR71WcAQX!kM5HZTZX(!klUvRa&2GvLQ>#)~HkWLVUgj70^P7rL*DQJui()|HfOiW8a$G)+y{Li}b zKg$YmO|;-Y!^}SnQ-6RKiZA=my$pOqYvXH%^otBJM;OBQGeqoRh&jNJdx@dpDMR~h zhMWTo8ubjqSqv=EJc<>vfeYn>7we=hu&>zc*SIaHbyrCHzQ~@#;k^e#y7om*Iutkk zc+AwJY4gu#FTI(!;!ghR`+2MGWi7dtw%}UI;%kWuE{DxJ9Wdpn&!po)GtR0coCMv< z!LgM=_yB|a3r4Na3@U#aWWIvVpb&Zn>K2JzW0gC~t#Vqx@PLTv5m|@b%5DcW{CDXD z@6(UiU=+PsKWe2;_%d;~776!O-Gu2D*^A6FW}BwZGESRqSGY_gsD?!^f0T}7-TFNxHTA9bQw4dnY5#1?H)f^+Bg7 zfa)X2soJ396u^~{A!=m=8*PKG$A(oykfm|0kopL%GJ@AfkaY;iq+VzfXk0G5Q75bc zg3#(C_!0&6;6_Mg1nD2asv*!d5b%>u;q?)`LjcJ3t`LNT*GKT( ztSaDTa>`!i;7(8pa!sV<1?m$a*GJ%Ls1Qa%>mzVYgs6|Oc8lQsBL$aSV(KFqcs~ft zKvX~unQ*qWZ5ncYgj^FL&l5>oCqXJ8WKz;H0dmGWG6^{W9$o>VGg2m*rA&g?N9L&$ z;oT)T!wg&{nL%o$wCSL|Z>iHDB&0sVBq8+?8i`ybVX`3=lUc@0T$K^BxMenImy=EQ z9E+^E7FlyKt0A}utjL89>%mHC(Ao)PHkvYc{{p$j!Kwz?5K`H8xeH*Uj_~0}C<8K^ zhAay)99|z;f)CAr2*OEhE=1Hv@C$+=hNF>o*>mi3=Gy1XwF8AM=*$g^tl4k}K%~G8 zfI0Sg3vIIJIc3gw%A9YXH{BtBhHcg|i`0eM5qnjGwu{uK$PM{J(%H2r*dv zXR-ay5OJS5<}pLkH4utA!Qiut!EQN&;VcHtRtBju23}9_aX_-*O-9g42tlzqu^Hwv zIIm>z-OS*>mm%r|L(WB}y2oOz&l$4MGB~Yda9qLQznc{_UiXBd;}gTwf1tzMARU|~ z;A#b40WASvgs}`vE&<w9O<2>0sLox{UAtrY4kk=GnzP_?{pM#APW+jE z{{MuNzgzcxsM!3pVD;Ukd8gxNpNN`zG`MeHKN8CQ9m)+Ei$jo6-FlH-1`$055k0V;QJYaDeD(*uK7x$7!D}MO zzEn5~Ilu=}*Ffqc2pd8|Y9c5JJ<CtPTQIMv!$2NYxN@^#XED zgxoEHOch~PL%4?KApIa@61fhNu}hb>O@kmAn`8t6Z!AKoiNH(gkgFjnt3*l51PDT| zi6CsV#7Uq-;FG7I*F+Eza5^-Dk`v5QCcxP6N@}uc>SRcjWSTnJG-ax3%2Y^Ygq0++ zZbGiAkn1Kh@D-?V61fgSV?*jMNPUFAN`ll)kg^(HbfYmKvdAPv0kQ}t8={615?Vsjz?fx@tv&V%m39b{x`hy-{{bPhMsQ>A?Fy3*D`2MWl-)1 z4LQoxGe{OO$X77Pl`$w)F{r0A2nRDryD%tuv*`x%n?*^x=4%Gm8pd|pWK44{S?X1} z%DZ+$c;}wPsV6e$UdUQ>EqCe7eDI`9$@+&Co1f;axSF~2QqtVB;gb%z*X&b`n!~BL zh*fKzsO}+g{Uf4A`{b;TE4m+4^Esv-vQs@|r(VQby@<8uQ47tZ78=D(HH@7Cy}d4> z(>S3MvIbi}y3-_~53{j0aOz~L|}CgXoIRAbU7U2#2!dx1mCX;K0*PyC=OB$!AWRM z6xN^>QU^Ip0WuZ`sgEG*H4&s+1eqj4tB+*Cvqg}B zIC=Og1#oW&YfS{Hk|5O(oJ3ScHp!6w5%KjA%{0^_3;S5Nv z4KWSE#zWd<&w(vFnG3FuKz&b0ePoe6(;|DOZT3v-%;~mZ22^FvRNK7i_IcCoa;Mql z&9cdvVUarDB6YrI#5UECO(M>Bgq?1)n?GmJzt3R4lfmH-L%=SEz}*Z%J3u`hm(>iG z3mEkKKv64Z)fI+pI-Dw7c-$sVSBMjB|*}L8`O#8(!^Do2f-wg9W_ueur{|~xhaQ%PQ z&HvfA{^#2EpJ&H^zU}|HHvebe@SkPvf2NiHxi^6j!_xl@Gk-Bm`O3ZZzxLJtlGpwV zp8L;r9)Wny{pUXWUt&Ax*lnitn+%q#8B|&s6q^_n3m7!Y8BFTgY&+P^x)`+U8DtU| z*t{4Rv=~4edo>t1j2I-H85F%4)FU|!lLf5v1a0yKY|F)6>ZE-;W&FCN0(#{_CaNaP zHq2RWRl32xYO8boRUZhH&K7lRQSq6i8#>!Ga;{m_JoDID=CQMkw+x6Hj3>tjOjIx>$ObiH;?NzitI3m z=rWG(wN9TJ(y*>@?#YHVH@dbx>Dclpt?xio)0UdW7bfj|*1GC?`J9u*GmaNbJ(4nE zUr56```p<^34M@y#4xhiC|kJ0Z0XxH4)5PXj?F zgW4eGz)A4(Z0z+BtX9HaAAzQdz}+J4pn4rB12hl^sfi$~6jTFiA(fF@P#wJ9L0&JX z4ZfQd(iPG`s*E)Ksv$Lzx__lQ?8G^EH3X@8K=(etj{1SrLEzd4bXzM@WmH0BWdx~` zz+|C3aWxU7K0;p04X=ja4CFdU4m>Ca+13iFh9D%o8j`X}g&;ijk(6aJ6u~BpkUK|` zmI;uW2tq>YBT#2NaT26fLavFR?4$`MN$@@qN__;bp(dF_sv$TD?jJ#FCHx&Fm#ss&_EUy z8OU`ItgZpwOAfIenS|6w(Aoh!N{Q)NNHt`ZG20B(y8+cEcxoSLg#xSDAu261W?H1r zfYkDknFZ(wYW7UC%<0x(gjOF}=S&4vM!C~$b7$J-&a}x~Y?HazC~?1j+#WgKXObQd zxvgI_=-p$`+rXf`mcecXgWCpBW#qSo!F?Tr;|f;0xeO+g85I&4_yZW&^%xiw!L<<+ z1A`C?w=9FOkBC7gL&y%Pyh{ufZx|}yGE~1}?fS_-_a8&$Ee7)r27yEd)oKpsnG9ZQ z8R8By)IMZxd&My8ALn9F$A^8>f9{?C1^53KIPjly=YNiE|G9R6>Ld2e|GBsP7vA$< z?)ZO=i~n`6fROsR|Kj`pv#$niW#!-bU-metG7`V?U-Z_0!E68dE`!kp(EYjm>;5xL z`@@z0kimT?gKi&#atni2C4*Tjt4%wDc@u+51%q%ps4`;nV&Jr7;Id*6wiVC_;?fCY zQt@Mu4`fgXV$g_Y(NE&GC=z$66Z2@1@#|6ypQ08sQz3eqavX@1jhHMG(I*|=D;3xy zm$%X+2H!fEVjoYzRxtK+b|M@^rJg*9BKB_sE@Nl{UpJX@YKar%qG{XtPgbn^9B?a(x88O9-(`NH?MrQXlDs zb-?N%$Z7<{4R_!Q2yt2hmhj5YEXe;#KRTaGc!dfvQ_n07T>x`Lb$Tn*>K?Esvv69GzKN1^M z0oi5GBNAC>%_Xw7g48C+br3QeQu{zih-2)s=h%WT1cCPrAtpdbcnt!njLZntN02at zS4p612yD1n#!PSnV3uk6EXZzXaP>6P4BQ@o1~|A1Gs>K8l0DZxYqo6$w9{mrJ<~dS zrfuc|5XzctpEK1cWr0D`JdLPT3PFnmU8gdbv@n>oGnjWVIL&47UdG_PnjvTtL)322 z>S)Ue3<~88ykX#zf(#g#Ef_d`8Tf)21acVUs~H?OF(h4PsD8=N^ns!KJ;$Q|Qq$ft zgsx$bj$_~rV9?0tckW{FoX4HKgT3Xh?5xj9%l@mZ_^+_wyUd)=V*O88YA>->o?~W1tH_}phJG-`o1w_U*_@N%wRf~L3t{JY(InO33rhM zYsqu(WGs4^w)lS1{M$+MZ^zHM5jEpN*pxHA9fxfzHXCIwQH`0d9Wz-wX0l;izd>vt zq~6hw>(`6zgY6A*Jn7KnONrByEebw+PGV9IP6Gc8)BQ;WZJ2 z0jZ3{&Ep_75rl-CY6z)`AS9$FLM9N85}15Y8rdrY9!4v>x#gfvN>f~zJ% z7B@?shFk|BvmrGUW~~ITkKi>FoPk+GfeKQL`UqNDqKc!{P*`gsyn>)RIBc`#5{V$S z1MwuJ4ubW1KvOmlHnOuJ&Vz_RWFaEfS)frhNKFJG;gu1b0k3y38IXzwR1<+ZM9>-s zYkg#qKEot!x@kJ-OnA^N61a0}lsVfd6I2;N>LcsS=@uE&EHb89rOmZUn`@If$tH7> zaq2>ov_%FXN%pf;`L18hI+DcXh5aN{H%_0U~bj`bk zfqNx`;Cf!|X`;sSbV8MZ4 zJ~EE!HjVBzjqZi){Q*}-osiig$Wc7TQJrvP5Y=KB-3l3DGl&EaokcY2MKqd#k$Gg3 zVQ`IRRI^586Xc{F-7ru!WEj~DsgFR@J&;w_;BBXnxgJP;1S3I%au9LI3IrI5xL^o& zf*feSDy&K(P#QW?oR=fG8=LMkI!TX1D$mxA0a zg3J>^NVNJ0UI#%Kk`_rI1Y5u$X_)}8jKt03AaxKXiK{*`OzcNPMoAMOA|@%5Oj0Hp zB~5@M6X>!B^hydore~5e2{F?I9^!-4L|92k1%yUgq)tNM97z_kzLKzJ+Amgkuk>4>XIpciIk%!V*5)8|;E&4z_MND!jYCVd8|w#t|ZimdcG zmZ`H8L%PL08@Md0S&T{1mC`Z4hNGiVgadrV??>0)>3VsLEb z^zIi4pP`Vv#H4VoUG;YVuA@;?FN9COn6Thh=JNYlD<0&oew?@FN$T>4X)7M3t$C5S z{BiV>=RvdY*-gG@-g8}i(tV4~|BY_^*SYjx@AQB9J^$5ae`Uxx#9-3RAXUh~mj*(t z=?q*s48k1@qFoFUtJoDbu}GZdmcPQMa+6!}8jIXPM(KTw%1hWZR&pB8<+E7C?>JG) zXO3*>B%PFn<^?O=>UX#|>kR@;iksaU@vq3j{!}bTYfx9;CkmYb-L68V!pDAR_4YE8NA_A*l zn&AxafLk+UD-eXO6WRiD{A^3fH@LP(YwGz@D;&R~4t)RmeAX|#yq)nQ%RSKjIf|J-QBcv5{ zkPeZgWt_Nq3>p$Mi-w=9i_CymPmq2SGHH;|2dRBTwnuowYV zTM)K&+H~u*>CiSt>MVgQf`cD$HZ(=ZMV{n_n(0E&Y{ePvM|D`tm*O>QT zxbuxj!waRpZ!+`#t1SC3x#_>+uK!9q{_}yBz5QpH_?Khxe};*_7`*2&NM?a5BR+oy zg)DZ1DhAbT2Av`X$5xTBS(-U(O)GZV*X(d>-tXUaG-C3(@aY$0=3L8Ib}whuqk?tM z>i2%F*!87w^V_^FA5vF8k6-aRe#L9Q#qXSFJQkgBS9;oG=9Rx#R{aKD_LOso!Fwr# zNJQ1FMflaji4LZsu$H}9Mx$W(`_2lX%OCG7y+t}AS)1H zJsr@w2l`>4r3jF<2k;6AjRC2HNUDz@OW<(TN8kks^+VY6wn3>LWGZDtHA1VL*CAYOqrj;I$9r;vonZ&W6-PXhU<#;L1qR6MDQ4 zq(cND!G|g!DkJce5NJ>iTn)h)`Eo9K^3J)C`Uq0vK&m7N8&V(1I%dHckje;o1)ZD& z%4KXJJ)e5Xmw9S|a=@DQ`VHF|OMr6Mb z!-kwa2&qjlNwd@$W~nn^^$Tb?4tn}i`cz9alr|k6@^A(u3arv+*dQUeAgHO3KGhn0 z)`w;KRNFKV($AQnpE1EGeu7S9m#}XapGPZ0)@IYiZzK+aF7;#B^Pgeof6(FKJN`55 z`Ok6ybaOiIzW+R{{xeMd&oJRP!_5EuOaC*pykIbFXW&a=5C~=94q%XoVbRE8(28fb z%9jgjHBO)9P`TQtb$d|PzK97&;%A&nn0Y#J&bgHN7c-V#%~^S~bkn2e10R|WeXQB@ zx_I-Gymj~U);`Kx`zUky!<0pLe5YPDuir0~vrag5xmeUZ{^02g4}#o(2dQfyq!#$fAxIsh71*F11f3*; ztdRp9UA_u97ASApp0@X)O8IXw~ImdL! zS#uB)-Vc(qONG}*pm`$jY!Rpef>lW}Hp!5+3ZRn~z$+HuRg#otGH5u?B1r~};B}Cg zS*)aG9IkGWm{~LgA>T6sncsxmQxB=3AbSvyN&WaZ^iOj%NNkL|sFiA-7$tY<8 zq&|X^$Q2MSHgbK0rUqFGnGG)z(aK~<2MUcuQ$q}w$Z7(!iosPMA(zr{HheAyGKqty zZpG?m_!>nt{~~jd4j!BdJAn{!8X-g$a!Mg2NFjRBE2H$O5LsL#q#pz!k=0mZ)JF!H z6ZJDd+c3?Nr>MlumI<585VKUW{Q~=j{}KoPvmE-*bl^YdG0^$dtOx&d9{n$L_&?kB z|DX$1+dnb%{bZQ^UvT+L<=MNSbFEH^m}$vRTY@^Vk_?@iWchXBx-O zHIAKY7`4POYKckQJd?P2hHjk1a0LXa zl5`{6A-AULM6~EcK$aOGNJxbOI`;rv4e5k}7Cb=uK5!{mb%UsAu!`seHzC(Ypb7}J zKGF^Xb$>KrBmWEAbjAW~ZU|H>!N%f{D;nmVj0tL8gbq%wi#b zX^shAF+muZBYS8fu%jVB$LN9B*v=<6Pnn8b86mUbt0N!`NO6irA{XXpQkYz1#RS># zngz~)^c3J+Tnu=jjcf(HenA#QV@a*njS0|Je_LF85(P_@D0(ctyed{|uAh2C<3Pwbq0=b712Pt2 zlF+9c*{U|rga8tO zjIil}FA0LwK5!Cp8IX23&R!2>dAXiL~eW`F+ zP<0brtBnn52G?o?)o4Kw_}V+jnF+A7wLyI!a3`o1`7}95eFPyPwGX^21mQxKC_pBL z)KJfqgI7c7br9@C1++mq$PAI9XR(4u5d=Z%Bgo0ykeUcy4T08ayCZfFLHa(ZWFB~Q z4ob>9=fV(lwg^%s$vI_1)+fO0AP~bTL)Ixn#xWg%>_MF%jGW05$) zDtVGoTE9_RziH}3Fq&+XG(jn4j#|b7hWwr49hVpue&XHupJDfZmi_;k4uKHQf&Z*K zLAzF&R{v*N`Ja0OxIU`9!C*RxL86pFD3XCEm_a;*Nhw~$Dn-@1R4b^)Afm}AvCArR zl6}!UkLu;#b*uau*GBYgO`d+BaM9_e&3C%?J?}sKrsvS>rX3H9R$b2ppMd~6mZxW* zN8@J8{Ka}HGxSoYYb8$Bi|x~o>o*3UZ(x$xX_DBfo6rZYgL>f%wED;}t{sxsjN^Ot zqB_(9s zL<_t!!mN+r6$)~F1gVukl@ZdJr8>dj>M5)NUgKaen!v+upm`r~2d55F8A0nENbLhA zHG*m&^^s;^HKg8wpP2w!fdCzdtATZepbSV&1epecR6vluLJ&5*8bUC@r<+1*B6#-) zodF)5gVZ&Ug>w)#q&|YwL=X~E2f^zjcx9vru8F`?Mfqssa-bD)(DUWs6%dpm=ad6R z+3=bOvS#AW5@mNF9VsLf6=UhnFF%Zj58P(dr{etpp+U;(Fj+B+M!a ztuiveHUkBjjWWborw0*1tBEkVkje;7LUu)=)k<(Ka$yKz!)qc419{CkoQ+%`L6qSk z;q?NfzJQRp)I-Z&c&7kXEJJDvOj$G$OKgYJ!X1E>fouZ2+Xx8?I2Tn7v>Gx`nT{WU zs-cvr7RggU4AjcVGIcWC3W5wsutQoGmZ_61U~4k1k|$XtPOwRuW|K4xVu(q~1XK8V ziHWmq(-vqHZL(}WEI;j@-qN??TmLH^_%C$mKmY#!Y`Z{+d*^?aP5-&){$Z%U$KbJ? zL93HNDuqELjzQRufyowhZ!C*CgMcxMv>m6WpR8??qE|U+pecTWY5EMu(q-QD8+@8K zdpB+JYTg{&vnOHZ>4cf5<7b|YpLsT7(uvUiqi&7646>IPp06Ak`3rl(b0zO&EbIBdkZyLFywJ%LFO& zI0y-;kHFOstR^yxkN_h|^GFB-nG`n-7dH)uC|l^`V6S_v))seLd>NbQ43 z!aGoC49toNDv~(K1T;c21zrasGa%&{G6^aD@RMe#(+DcStZ|S_bxc`^2(lUo8`4c6 zvOdD>-$9JULqcqUknjo!IY2Pj_^KhZd1H)J>*FHkjm?Tdz zNuB~}V1N$ugw;oo{*g`6G^>OukXDCbVlQ~V+a%E7UgB)C_?cn}3uMxkG30M&tT@2X z_kelfXNI-^S+;;GBaS`)1^54F+wz}t_AiFU2kgO{7)&NJ$Y(MLMl%R_GjO^vFsp&i z8sgAq5HV+vc3@HQWYh|0)r)2@N@g}qjsOBzS!UU8)LRDP$Ns3A9!rc7jpt1jtR?df*%A z%wqaLyMDl{tdZ*@tJofs@HR!eWFAdFCNXW$5F7(LsG^VJrJ~)J3a^axLYpAf zkXA^YW^k=$a4o2>2dRBRpxqpJeS}&EA=gTfQ1=J9CW5db)ewY)48%d^gCPAQcr^r>DnhP?lp)tWz-u2c19a>*r0PMdeISC^ z7R$lwBjm0Sa(x6Df>U%wU%3ripMa=|(CQ!v7hDr%gGuP!5TF?%@HQgkItbDyf{-%y zsgTM@+75gr1ZZ^~O@v9p zt07DVWcUtFLe33FCQTFjO%waksv$TRTKiy*=7H-YNO^@!BG*dDZ1_SxA{me}6ODuv z?^Ge-^$~W{z@u1@z60d=bVywZserI+LF#;2q=I@D5bvXTfo!f>@)TsfX5ho#;nfhR zXOs$>+JlI|T@GhJf*rzz*FK>6#KcLUS}7S+^_V4103q<9RTjxpAV*i3CQO9vJAfn! zvy=%&2|b31y@rXsy73eA5++I~%+kzTA=7+Rf7%U|#c%Z1ep6iiOK!z?iMbyc+io$& zY+Bb-zz^)$2svXOrm%wh8DPUhD=2fd0+MyFa%_M!E zRnbPLs+}(NJ6!5_*fnmkuG?r_x>h%Txkma@x!Bpl0aKhCj<~lS(au_-6E{&WVKV3f z?zn!)+>mKpw@EDY>>cEDcp$YB{!`@o$TF_iCUJsO#zg1LN%rAo47@tvDu)H! zAz}f~6LEmMOUz(#H~~3fn~Q;gPf*jtEw0uewA&yQbTBFSY|>W9^pFAQUT#2{}apvM3HtBKLQ2 zu_4DPU@lZZu7==i=iq%JST&RfIc*!#GlJDbIgs5$NMmuJt`MXef|KBy2vi?II!97= z>G1jpQu`p2kX3W=%1GKSMZp$?@lP_1N@h*k~BTE)v+#miX6fe~n) z2+|=!?h`@kBT2JZFhUxX19y(#HIal_1iTu8FvLv4AbSvyNkhn;voeBBn}0tE3)K(=N8pAhr)u6G2FLePkNbX%X9F5!+)N*<=yd?vYT>AfOMfgLuFs zq&|WS-?6}}A{c`eG_}OQ!>8uz7FTN+)?*mf13N1lv`PUoItQcJy@-~Tb)(MC%l2trp6b`vh1g?f4l@VxM z4y_uJG>e6FizG~Az?D%f^4t)lCW2Hwn539V7`Q$H?L>g=N5DpcFAk3FhSW#;u*wKh zB|%7dHAFl^A3T%+SyGNh!s{b62Cj+;R6r$6gw#ryBxc}_> z;SAwI%56L(L>Vz8{I)6V)db9)5NDf%kFYRJMa<3+V+BDq_-hzf+79xV&BQ{K!XqYe&SsWsZOd2Kh8OHZQL?HE%abmAgLXR=H>tviT*(4Fvy)ufQ zpboiH7 z&;l%~qe5T>%~Y!uyZLO*<1e<6PW~a|v@Uq%63Sx%6hnk{c;YZzM0d9=rHP z$gK0pi*Nh)ANOtFYgxQfFMf)4%tVv;380yy*dFlK2~e#BsgEF|4Nx+s*DwZtJ`Z$T z5af6Sy{HcTux2ChQ9Sw)tIO1T1ZILp=O7F1AY3@f2p-mB;Zd=6iz_t> zZvmq=qwqH4h<0Pp(Qt^lA;_c;goKW^f!Dl2+2FNs&}t|QG(e{j(tw6QRZmd8E)s&Q zZNpymfGZ=dpc)-y6j-N&R1JaViTrCoCuT!thyp6e_&C!#4gWew zjRV;e1R0Totk6aohyzzZkRA|(g!GIc^$}#A2)<4m&VW=mm?T;Sq~uWqS(5;%jF9Uf zWHzKeQgX>dLZF%ma;6-Fgs;|yEKYz-34!(v!6t^lM<`^;IiR6TusGssJmhgXcufQu zqk}A#leSKRFyyS0;0RJf!RsJN%LGWR1X(hNTpvl9$4Hrj>K)V$QLMCCtdwbtlxdU{ zc*z{3N)k7Tgx5i0CSedZqz*zRAv+P!NTiAhR3GU^cOnqBDhX0CK}ce1B|P;JWJVZ` zM5~)nxbYJ(t0A}u{_+#KEJbFUrA$I_VMQ^)Qk^^n@cIb5cGUU^R)TYhVjzv6LTVFSB*FT~0M_?`)Ho0lx$g`~6%Y}u^%1yA0?j9C#Lv)( zpCOSrPda5GU(zy;m<8-1Ga1~v84L>;q#_yE92o?>1+^0ttn$U2%T$9}^%JJr7cBK| z*cse$Fueb0!i;m-i*M#Hzgx8CQR(`p6s{Thcct527H)c)x&BG|+DGZj z??=wO7~Ff4q!H2pxfKY~AtI(ef-gVN46N2hLXi3kxlaVCeIV0A zpe_>Z?r+Tc2r?f8BWumymmEXXJvq)I})kO-wRlCjGG z9cl``G#qlQoV0D4lx;F{H6(490_heZuav{AkL0YAWU*Za0k4lB)exM7^n;K|wE75C z6PZMcn?&NNhQv(5M2$m5jYH9DA_x~>eFUqUAQ=r0327PPs-7TgT%q@WA$6A^WeysN zQURf@*^38l-iORA;O{FzDk(%ogH({>6CAku2vi+lIud+BAV>=4FjCnB`a7t`8YfJ| zG#@g01_@7GwGV!ANL^!|&~Kj553OIo@d#nVNz+7_EG*Iu!N-P}B=i}_cN@odnLTcEE+*vdJ&w4$^7P- za=r~_sWUuk)`oQK3~1dL*|#fs^8U1Ghtj4WNS(GnX~LeUjxABm8xp&=XHVLnKkabl z)Wf+mk0(w&mN4aL_WVovi?8L*IUCZv#Uf*xZY<16v0ad35DcT*AsGbGoQv+%kLm^? z@CgZ!vwFaX^Rz)~B?ud`PTL`_fkDCwyu*ka-1lJxuj>X?I$#E*R)XBh$_-xR0G5)p zvk9-!k7x(o<^Vpi$0)qTFs#`iw8>Ed!tNL2v`Av;t7h=Yh-z!D^psI0Ldi z0bCi?K>9(DRoRff4+KzCPsdsRZkeUdxdkD@( zu7l)U^56(kAAzf(Jj^->QWHTA+=kadkj2}Oz7H}98j-V02Td4(t0BmJ5Rg6*WC=I) zR3Gqi1xUREsfplyA1SLuSulchjv)OYv;$7z=lsZ6CV+?IKz$;l$_RZkk%TFzTLkGG zNts5$D=;$h)s1D=|g`W+{fN&DhX+k6Q;-LGR z;I$H*fmRb?a?$E0!?=EUWrWFqmYK){dbpQNz>83n>It=c#w?sE6oJf=;4}oIK7wr2 z!l@pcYaxMvM+-y&)~W|Cgi(&essH0T$gbS=okb@ zl?2Hw$TdY6w#M$b%0)m2t>K?Gt6f>mx|*BZW~PN!z7Bb``;o zQ-IV#kYx!nHpx<`OB5jWkqmg#5d3T($fhA=5_H-Rczpu$?jhuTtdK!D2q|S6C20}~ zLGb!W%s2uvE(fWDBuv8Kbr3Q`HxhIj6&i_DJ%Q>YNKS#1kcON-tfK_UacCs|`UrA* z31kohUK2qWkX{uUiCHVbMIdDkoP-o#ct}XWNCJsmml0y)Df4kj;i->|6DC6X5fBpM zW>V`TNQHvMc1TT^I0>QvR2jujGEJBa=fWmsKy!2OiXJo=6+Z!)gwzzIlE@JRk4i(N zZZbSVAq==0czpy}y#`syW*pxQss*6hLG_eTTo)Xf#dScCd3>iuLZ@+bi*8u6oO_A9 zcO_`P23#)~$F})bE-IbARm>s`yefbdeEI7+1-XxgOoul28* zVU*aZ9o1wI*JY75$s%O}sD1(85CmP!7702J0iJ90qB?ZI2qFT>anN2&L@W5ZI`D}- z5nYCn-J0P&S`ob_ksS=mK@6-048nE{N?|-kY0_>DI^h$|(iYg~uXHQh;90rZziFpm z;|}lI?Y{LpT}n2Xq%6>gn4}okE$!VZ?cJ&x(5MmI0_ojAYof3Q(4HWq+kzlHCg}P% z#3mr<0&Ga_gGOqAr+$ztBgjx2eD()2<_1}Q0PhgNYa&qh$G;M}8bV?FSK_IU)ch(S z2sHKMU7-v+t_Qk28@w_B(gVUImAp!npqGG`Kx!OFl?3Sop^@-&wjq00mE8;B3}x66 zbI1(H#1Nc>^n(<@N1nnr4S^Y;vlK9UM(}C~!hqL15C-x|r=X2RNF5^h=p3YH1R0cr z_l6)0NM(dZN`VLEWG$0rEt4Vj5uAj~2Z2dwZwONLfOfE&MM#=QLXenwq!{jra^5ekuoBm+o{ z!xylb#KNl{NT(alg)&TI+f8HJ&Eh(Yqg#!lnvJ7dOyj!M{A-l_YE%R3RRe2H5_&v} zW_y>;leLd!U=Rb(*TB~gK(_oKZT4Xjb&tuQ*{mPg z3_b1uqdwA(Y}W=KPpS(dLG3`S^^rkDmu7g6Mp(D1SEYbyikefddvc#o(XxoTO>v!v z(fpZKaONRdt)38_;sNn*^0yB3dPh`%sNNL7Mf8G%=_fs6&u+CbDm>KCX=n5RH$ zU~@M{-~$KHtT2r4#){xI3myifW`RU5gbg7LW4j>?v|0%+2sx9@C=RlW4N?t3NyFGq zaL)*IXd7fF9h?Lm6b!l|q1!yJ+bFV4)vs2`yGqfk(jc-UxOz!)_ZG9*Y6eMj@L_R+ z;0_T#1A_nq10T4@#K*uO%qXL85Kw9yT%qh0t`L^gK&$BOC?^L_qy=jYA$ARF9 zCnKkykDPu!e(u%yxmOF;KPlP#B5TEi)Wx?UC}Yu$j72vx=Uzyjek!DKlUec%V0BsHe9~=h}L9U6Qb3@?yAP5_ITn^ITK_=C_O5h06Cqk}zR6UDT zJ&RR5iXfy4c(pd9Cc;_=L24Xi5>oqMlBhKis80l`gP^2yrabsSIa#L+c`(9QEC*Vy zAY+#Ssfi#Y=I$ZLas||(xddrzD3Y^IfFQ`GAutJ=FoIV`@cKx~ECGaI8(P7O6QC1A zpaD8?hX`I1K^WrT$_P>)q1Qp+mD`}o2*!Yn&Pf`FN*afX8wP`rQJ^?B1mBMUXF&21 z8VRY43_v%bwCRE`K!H_8ZIJN?WRhStgscEk4Z%rxO$1>;3M)J$T1|w>h1Wz72Bdg| zw71bnw6d2}E}qK57`)XNvogZ7{}ev;RBl5|ep6stxo2FyTVhFIPQ7btmqkRaesHa6Bxqqa{DuYSNeIv?26WFU zBnLriA23gGE~-<(vx3Dig~c$1->Fg~dXi<{O0W7|9?e_* zyY@gTqo^4dqGw)An0GC4{`JficS|S$9S8}zmjP1mAd`^l20}vWAWgq& z2m{nV@~?r<@1WI0pjrk}*FZ?58V6GEKuHW)NIwYDCqnM{K-u7v6QFAnkcQ(RwGzBC zQiUC!0I824vqJD{2r?FjwLU_wec;%AF#-VUFoFNKdu3#99t3DDl41gebHH5~1 zc`xCBYGItY^sDS_eQ z5H95;yn;Yu;5QjfF=*Z&T#sPtLQ@H^0w5lNs)tlJ;9WcraR>=f45^i%Dv@kKRs&Iq zOd{7u;4PnBxJYof1+?bP2)vpFQe_Y%A&Ma+vSLUbL<|Yl71ar;ydW3Mf$r&!=`@aN zH;QV}3v183^er=5tLa4325LBF~+`qAy`!Jx_rPkn@3<6yEOwGV_;Lq4Skd`<#rJF0&rE)r5J zAy-CfK4r*UWHzMBgGNHuCqM*ojmSZ2A25yJpcqd*7)PNG#qc({eMFD}uZi$82-Z*#1&~sgBoede#v_7OU%n8-}(Ug}3PjHJV1Wnntu5hqao5H<=now1bD*+9Bg_klcYx zVy%YYf{@)nphH~4n)SmQ4I`Tsy-Qj3BW1mc9rLDpR;=`^UFXxh)w^knbL~d=#;wjx zTV0yB`Su(PnS3H*`l;wyXXEBxh?;pOde+&fnP=kWUWlA=I&#vH@CkE(f`c6+Su#se>RpTajxb$O1W}Y6yBaD`wRL z5rM4MM&8g0?-@ZmL`a97f(}ox%aB6q7DCk(?Vf7Kb8j`e0f=m@j+9W|H zhNP{NzyonOt072zBo5ldikKvV%@f5!DkBNg7zywqZAg72X&Mfxk05hH$Ru)o1n(9> zsvcxg*f1E5Ae9lk>cPrDIyMAycnFAu*GjM(xY{5L&|(JA5m+tw>mxkfCFGh2D;u%~ z36sQCJ>jp6Fl8a-GcFQR#zNK=L)0Lb$&|2>OLAm3elw9(LfE*}LuAoN_-Gi!FStmE zuSq45YYJ@aPFMwm#sKXJ#aflX1tF_tAY6EzLXZI&QpQvU5r?#g6dR*>osEfO)_R%7B6?L-yAaGNW_#A(KAlP&N>r6=Un2v3kh@2Lr~1j(-Hj# zBPSdR@7`luxX2`Ff@*N1dO$5?k_c=62zFZ#=#n6`S_x7Cpkq~3vde9-D6NWBBEhTsh3ItVgShg=6i z*vLb3(D@+Pl|zt92OLl2aLuqADhAoUTrCQ1a8p!x_}^;pG;TVc79RnjaPynzUG-YKLSf{f0In?yo( z4@sItN`ZSulBVHU5oVtVvN8dgge+HpS4P5y!B{IG5yJps1AjQeS|7m$wZmJr!&@Ph z5rotY17%4_)`K_m&=|<|5iT~?%1AG&4YMY~C4#GlLKDZVlCX+E%4$fN3}Hi*LD*PH zTvY>BakQEO&P7&?X$!N^Cf zk78z?hSW!i^DZRKzX&0NyY~1rZ%vwZTsE*t+Ou3MqD{@eMm?ZbBM{t0LRvwGRuf^? zK3Hoa$m#+}Ed!~KAZ!SUToXar*s3AqAvk#L17W~x9|!|d2SM0ql@4-!1UbD2t^iUQ z!FxcEP7rdh2f02%o(zK5M{+KpW91+f5PaFToKp^@GJ;JE<-jW)JPhb!Ii&gsUKzpb zBT3tIDR37_!X{1HHeCubOlO@6sfi#xAV?jA%$BfDl(2?WLvS)4QW=462FFn!fe-nK zhSfx%o)Ki<5QK#89s<`y5+-42^$}#vHs;VAq$Yx_k%N;W20`%pNKhX!G$(A}53h!V z^!?zKk+6Y3_-bIth4oMpa`-%igjY;R48-Y+$hi!yO45mFf#h7cIFS_-WKFDgWIL{E z2riCR?~u)f7tZ<^!$(+4UAQ2mxQ3IEDh5(JV3KfU#53R)K^RyyLZtAJ5Z~ca0I3}y zvijh=4mW$K9zsIu5hw|%k3fg&fYuclgtusgcW6g+YK3=bhJo(s)(!`a8|a3$Leev&RR*tF zkQwm$1tO>&-mDb{JqZCWg_U6t(FD09NZGrTLDidq*O)=VmR;RP%r;pouv|ZJibK&- z$LjUI?YklMQS6LUaWhZH&pMMd_k7a4OG)!CB`yS^fX>~TNmKlrwr+5<;FEk_2D< z02`MBRY0iq5uzpnRYvA9ko&d-162p3Wf zA(Nm5+u##^Kn(EKA$Tme25B16nnKSrb7-(3BCw#k3St7p59C5v=}#)FuSyYv2mt42VYJNR;{r zUNJ#x983~Y*FeM}Y=cP9j1OvULSSQpegw2nq#FsUL?EXeKumxQK|`BKdZ2}Hh~5pj zbJPK;oAkq)4Z>SA!`d}M+cd-4HN)Dq!dl?2I&w%rhXt*4}?UkYqY@U zk)jDAb0M36G=u6PJ5qH*>I@^BZIe2T!fVCNW4N^gBGDZNE6oj2wwZZ8Ssh; zx;xdsQp2wTQsZcX+3@NHD+9U4fvkmtkjV85u2pc5`Uq0dK-drxQsY3x(MTm&9fY+S zf{(vJY9df&9=A3-}o;Mxb74cRV)Ai?z!WQ`o; zHgHINBx#!_fm97asw7G46i9CfLP9Pi0uRc8DkCW?Q0GX(IuWv14$&Z`M`UrBC0;B?xG!2Jzjv#dq{A?dMSKK%h()WSaKFAD6H3TOi^$~J4 z1ZNBB`ynGp?SqTd32lKRNEZo`l^|IUxqpPrhV+bxB=sXe8w26B5~K`)v}}<{OmSG* z1=~D-uc9Zm%{oF%7u4&*Kg2@ z>sJn}R|{!U52yuAAq9YDh~TvkG6OUv1YRHquYDj4;_D;i$_R3g17snbl2-|GeFUjx zAS7g8Duj(pLc}3GBS?LOOu`4^AY*Y5DYQNjyfOl9S#>Xj49$TD;_~EO^JI~bylbAE zOD;0PRUaYOK@c`%#RBp5k(5oUv~8NCT?!b1xX9DK1WK;X*pWO;{16 zhJsf{NDRc_1ElQ-A(87Nv&-5;RqKVT)1MAwLUnB<+hiCzv9SS~m2bPKui3-w@K}(amVa@QW2blp$_WEJ1(DWS9 zs1*)AU?&W84G^XwU~{n7N02H8Qb|GBct}u91Uah*d`^OPV3nqSg@9QCn@*UpO}18W zlWEFSr@}=pH5&pt_JmD10tD$_@%n_*ek#WhDamkf-&XIP`k#)|2R7SF3wyZPa1|mqM16d#kXG6G< zng~Kd>LcV15rhrt5J3)4fYd|~Hn=iMk+e&autmKS9NamA)I<f&|w` zkQpL439paf^$wgNtRD!PInoaV)kF{yykY^;4-(P$6V~%ZtB-{Bd?5%jT!2aHfRF!$ z)HNt1xWx!*H=>Ax$0Z;l5WNr*YZlfEZ-L}@%+do=0b!D8)exKunG3)}qE%JkA{0xJ zNK845Uk#*g!moyIvPj;9)E8L1jHHHGSxC19nbeB{?T<#TbTHXE5iLl9h#@t2BL2-@ob-@>35)&RL>TR*%J z*L)DAjl9LgbQIqNJxDIsdSJ@cm;&UK(2b=Y)I9EMnZ0a zfYd%{XUah;BM1qphTtUZ4u~WvThPtmkjw8t_pw5HLy$ViG8wHVg7l9d7qg0+$3TYW z#LZ(wE#g46l6f5X>{IB3k(e2DTu#gkbW1p7h6qA}t072b1R@~=bRZIP><{eZZAeW7 zs(MU<#o-9PtrdLK4_YS(UKyd+MBw@eUK62JLy)=&QWFX5`3mX!Kyv{+JHQx_8Vd9F z6OGV12m)0}NOcgTCW4TVng~K77YoR2NPVOo-i%QtVd)>CRZpnp7-*D+7WEONxTaeT zLS2g>jt7&7`UuauH8d$m2!pkwR8^3g2+}2itTaHXXi&yo(Zu1g3^5rkLUkfg`aZg0 zt>8KcS&-QJ2sw?x*_ic_PG}Rj8fu1-p!x_hPynfqATP*F2~J0Ef?ACRI=1GeTIT}4d_T!f6$F}kopKx^}tE^!UG5c zxdMW)F{>W92xR*YybeNUV6BhPcBg_C8^HFfLh2(32^o=t^nK7s0prJW%l?1AZQ0pV)EAS-2vqF-VagcrxT787v8y!(haxH?6VO?6kR{yk0XpP52+l_IiNG}xgo{}p!RsJohLElgWUdpF zgzTQftdF!p>fi_>3#qw~^Cng{yl{Xp;Pnv(1GK^%vd$b%VisR;5j-^!Rw+n%h)H6W zy|_fETX7(phRa6E#3A(&h(z@@qWXXbltE-0ysiN=2-NkExj%SaLp%f0ok3O)5rL5K zfm}#Z!B4`67z^E+@>t5 zM@5TmvgasySE++KN1!u&AgdJMy&-k~N=O|9A>p+Tg5gu9?p=zEAe|aC5q0opRpd?( z8XMBJ!Ahc4H;|eLLPDglovK}+?2dvIUGpGF!6g^MP;^BsOMuikaxS?L1~Mt*lr8I= z1F44KB)l4eR658cqymDIkeUe636il(N3MF1*^ntA1PMOx6tXe_a+ZR$bpmK04ps+2 zCWF9ZbcvuVhb%!Gh#(yz8Ou2Ez9G<=+mQMQLc*${STVC$2pe7jiJC@1M&zJp`#`E8 zaBm1&6M@e=h15Zi!8&M71nCXII!B$Q2Fp8F8S?xKTI$VD9}<3#ifvs?i7n)ko?fwJL#C zLRKk|`bagfUMFFqMb2D@s&$^t+x@%uhfFvUJ?%u?%+s+m&nAHSN0;O0U5=f1F>d~) z*cqoJL)z@~=G*1YQ}V6?9nj}nr3O1J4s<~(?8*i;$a-uD7q&|Xvp!PuDpB_?g;YjZ zNwf+G&V}@M&}t$`MT3=u)I^Zw2ap{@knuNU5?B^SOZ4!HtCX2UBW zGzPo^g4a9949I;$5E5P!K^Vw&5HcHH6G3K$AUjwg^^ufSJfupJvWl0qNCNeYtl*2~ z5~R%I(CQ<2Wh4b#K?kXQU?iwMLavDrm61s#ygmY5{Q$l$9K--uMxsWbrF58;5vU&o zJKhIZeS}>5z}e6`2woZK1qkT+2*PHF;B^p$0jY-IBxLOkG6}7rf@(nZPH+SM0X<0e z#6u#N1ITR1$R1{uq#52yB+`m#(*lk25U7tJ1s$A(mw{*lN~m0Tq1{fO+6S_!3VE$N zCL6UjLAa2RIHKM`RSDfn0}l{L5P`cz@G1#WA30~!1_nvY6#p30`+K6>LcAC(CMR~ln-uwLE2!D zDhZN1;9VPSaD4>NJ`i!uz-rLl4M8=Ufz_Bu!@ma7A%d(f0FmHR;2<|Os0UU_X0=-1g~@;T;#40 za>WF%kI)$K?hhIRUJXIkCm;@0z*xTxpDL2INduj63f?q?Tmd1o!9#PP3m{7s z2per5tE6QD{CZYN%LMRz5NK8i(p{3V1XWM)`UtW(LDDQ5QWHULg+QJg5(RB6ih=fw zz$+FYY-k+>z8oBW#scEJQ%Ii(OoA#Sqz(~$wg@6FW)LK15G1M}h>1k>0#FcmDV>n6 z52W@%CgF7ul%eG%q~!(AN^k}w$AOz_HJ~gB&T+&r@YG7kQjpREULT<`h^&m@${^(% z@g%(Xg)>lVB1q|tue7Ib4F#zqFl!%75r|4sNw_z`4CJ{v#0oU96zG;Za4-^{wSiX( z$U|gsLGl>T^#h@x4hy8#fTRrh%sL49emYPEgmfKJoK~9HL53GjNN1VE0JfHO#-B+10m6BBD6XPnG5L| zK}g7;oT6(UoB^qTAoUT1M5~O@svZazt=>WA$~tCYT`vb2h(jhJJs`LUWONR*K7y>H zgLH_bZPTP|QXy3j8VOmdja(l=DjhfpuZ-XfDdz(ZJwZUR9r@%06yCjwEAS^Hp$KvY6>A(N1)e@wq1i-0Q{l#V2_ zEb{4UFqM#62~y&8N&zJu#T`nDj*06t$N@$(*;&*!+JxId*~qj9XK1rK&yseTy_6S zkZeE|xZ0{z_pgAgf|IaG5tbD^!4S{++keUTXB5zZL&IG}vP`H?Nkg{u@qH~UtORgf) zI&FDp&^_Ra&N+}fSmm5E5y&YUQU_sHL&*Ijc)bJf3PI{4I0@+iLFyw238{l1B=Wc% zWFM;(_$F4!X+QAF2vQ%x>mXzXyc&WtAoUTXHv}OeHIb-U1O!3uArdzYhujhlAtAMo zgmDA}i5rDO5x55gse?qp+gh=nv4C6+!PzJ^5%Ojta0LXflHfIwpq?)T3F!J@MSR*` z0!Rqbu0tjv?LA1F5VJmlWJb*SS$GDAGawZca!~+h>xDKUS2VcT8X*l>r66?*=_F*O zL^DJqT7itrMXr?yRzPS9AhiZDB&1G(sD#u$kP%2+B&0rt=z`Qi$Rx7+G1;(M22mwJ z)Ifp)nS|9NkmGR(51PT$Lsdd65%8oxL`ny#%LA!0FiCjAKxRPF54IGukhp3exH!DtLGBqr1d;nbFgA494XMTAd9ylw?e?HAvgoACPJ={Ag7)}Y9dgF2)vaQR3%}YbShz;C~lPi zsgIEN4Z&+7c<#T2ckclC*`Up}1A=gLHng~)QA=O8? z&RBp~K#h+t)eTp#gkc|yhqFiFS(j2eM;@Tv+@ zPr)-L8UwTDf>cb%`5nTBlogP*5bzoY&V}fLkf81rxMV{v<L4YLLPhri2!af?p^->65vcov*6Tr&g^b84x#YqTygq_gKyU_pEDn0SHteP$$OWu$ z5`NtSNW?J<-W!5fI+zSl1!R{ZW1k8+=Ld2p5u|4XsgEFgS3%W~bqZv34*6Pe(DFGe z(BX2B9Yc`H2x;FCXr2hUGJ@1W@G42n40PnFq**MqPXwunz$Cmf5&>Th4!I9P%p^?8 zI6?}uTLil?9C&qzq89+El0aP{@Z1ozRsz>YplS$G8R_~%2Ia8! zi6DY}+FlSOpzX!4qQi?+MGa{>mE5OQtmH&vI1%44c^#YptaQA@Mo`LEh`1*D{42bQJ zvzw4fG{fQE#1uypfd>Ud9KJ{ZvOEB5m_wFKqe;OV50J=0CLuKrrU*nFjYMv=fY`yc z+QGGu?WhnEQ~`knArm?v61H>=>u4LiWd%=@$RhCc2~XJIK{L?M8HfR{Qy}#STmgiO zPGamch1NT;x(2kq8obU8az3`2Zxuv=nr{{4oHxj7H)Zcq6`wNDAe&!>oI@6iS^%S3 zu$Xm*if0+mV7sba<}^QXfGoBWb%7 zNPQ$@oeJMH1YyAIBgj}BWS9=TN&&Q82(o4yQW-%>=ss3ZeI#ZU3#pQjt06dB(hPC- zDdws<5z}yC@CG9I5(P+ogwiJh_l%HhA_yB^AHgdi2m@04V3MF}2we4m7~uK{Ix7UL zh9Fgvkd7Z(eFUj`FiBomO$4cru#75zTY6~K5S$Com1qpO0(h>6FyQ$ejiDA)2S>8=M%E?Q*-S&9I!j9@1!q)6MPK-SAiTcyCeMUa{Z(mz5z zc^g~}!I#V-_7K5pr8rRww9^$3V|0*eNCI>CJB*+kjSNkYGAEuU@g2#f-@ks7^DRYuYm9{kn1w6Y|Q!yE&^e|i(;${Ncm4Z ziB|HH#Kr0WG(kxJML(<+v~x2Qv5ON;nO0a6F}}mp1*v5qH4&ulAc@3UA3^FDfhJCG^~Q3t`qK_Y>5kZB!=7C0M{>L3Ga(3%Lmni`TyA;}w_p5cRX=#^1G z6^a_aYDi@S*D0LrheFRySfLRm4=70*pYqOz)a*(bN zygov!gU~7$OfG2ofeYx$Z>%*ArXZx=K_em65S)b6N0{}FoMXB)c-Rem*F!3TMBUAb zsENR5%0cQN$bGDkD_P-n5TqIc?HjU825le$uUmk0i{P_D;KkdZ6Mi6jS0N<0G6K~{ zBIck;B1i>m&^&kM&uyfBDDGlttR5rbjMX6fkwZECZI7O^$}*3gj^p%*tlkwlIDQ7C zWTrKVRxjXk60Vzc@KqV$^KH<~hjTG4f>$VzK{;F|Bd@fC)B>253z`Tdz91xgU=5-d zPU-~K!4XstX%`Qu>Otay=6AqzJ&=`bkZK4t$Oc}h2A72}P-+>Z{thG!L#|qYR5V%v zRaya6kopC3&jM83zXo3W!0QxHt%P(J1E@lQEvSZ6KxiTmt_HY8 zL8xrZm2#l@AjpglX1#+Zf?3hv7eT9lAY6C_1ZSY^8$#+6!K)s$`bf?$Rn|5If*?2F z;UXa`<)9?^U>``W1gVUWdqWU5tUdyb)IsVXOcJ>Qg0La`h9G?+ag%5WL&7u$az`Fy zdPp2RRRkZJgD~KGh#)l)sB>f(4DAgeZ6gvh41gnet%Q6#9{eC~$O%8lB%&HZtm5_& z*7Fh3@dhIw5gjOkS4Qyq2r@$i=@ubZLvS{vGUC&8hfJYjl8_#gs$Vs7<)Vg>ks;Yx z14m~Ht{$y2(hLA~j^I@igaPRyLE5em5?&uc7?7e2jf7M##E=k;aFv)vF;)>;*F+HW z;S~ggfz>4tL5O-Z5?8GOsed6FiL8hrp#&k}#zMGoQVYCG8eZK%D-J?O;y@x1LPE4) zrV4G?@@XCL5FAJWxW<9hDFjGJZyh|^22qJp2Z85zpd#3kvASP1sB!`Kbl~?aKMVG*LMj8Yzp^Km*zZNF4<497$Rv z!D=7q)eqw4ao}zddSwJzAP1?Kz-*Mx5u`GL_l(3%qTvX#ejBqUf{w+(?!^-acZ(pE zkvM971R0BiS4Lun0g(C#UI8I9kgFj$8!|BjCBb8KLb~2Ux{%7q8(tZS=zum8!MZ|T z@SYKPdI)87j$g|IUJbz+m^Bgd30JEAm2d>9q2TRCWCr1tuHfDlv@`&-RRXJ20;^Pm zs?iY2xC(f51-kM)s0JE_T*XRb-V?~gT45VfuNJ2U>7Ns#m#2Sc^&01hcd$*4Z;1()ta; z>dpQ0WlTbNqzxGu#Ti(HSUANwc*Hq*B{;YxIk+VmxWrlbqy^A4p9Et8S22W#@x0umcUUq4jzczzg9Z3$U@0 zkd<%9Qn>0JG;w(CBjcD3L6BA4GDw{u!~h+*mVwkYD04$7cj1BNgCIR4S=$syWrV8& zf=>*|StrR_B_bo_nh2>H0xeMhuTKD-xed7*4_pC3PgH=gA*;AyognB85qMkMV zK84grkggDfM6QXTY;ZqF+%Oo9M2!MP3PZ7UbOsqfBa(#dB!w6PPjA>6Nc9n< zu7S+;!0Q@h2B<3W22cHf>n0EvHZF(EhSx8U>PF3{3{3>9IHWRykg`6d(q6^?^&A_V4z`)PIz{&suj0_Bn3=B+QE*k>_ zI|Bm;0}~IkfE1^M0gtpXgQ~NlTMBCJ0~&1uRYtG_5FoV=GKty`BDOw4u4^#akopLj z#9Hsb1>w~YoB^qfkV!~21n>CB*n_$}kjhBTE)`NAffmTwrok7;!5Q$H2>DuY%vuRD zB?LL+6uB~zwSpcu2d;YHbr6&RUNVPrDIR3OHl#8F-x3aAt^k@Lf>b?V5>y|F8ApLp zgt$p0bUzVz5goMl0r!a@yIJ9DgcLULhg3Rn z281i5?+4jB1Sdh&kgh*y;T&x39Hc%H(((i&=m86m%1B7d3pN%9UC52UKH}8?RYv^k zZjiO6Xe7LnQt_*RBZwe!CPijLWRXeig#fIoQVytuS4PMT$ZB&m61f_Jvmvc($jBZZ z60JUhEWtu9LNVFMCP|z2vY8Z_gzxCVWI)a%#T18#K-OJD7PF~&l_+@?D|i%&8ijLf z_%ce{N|}WDXEb>x)z}54i)h#~aELQ7aDl2IVlfK?12==Xfx2s)rgy2bYd)k_f{@Ct zd2j^l2Voh7!|Vh>`azI`5ilzqxCnB+gNrTelmXwe3hDAdNaPv^A_A>45^Q#t&@=JBY5qD$&j@Iow+SznE*$SlM^uO zBS@tKAtlTrB+Vlw%pyR@6x0ubio+HyfXC$`!8H+-1aD=9)<m$g51jtraQBy>>2-FXP)I<;xTpvL?N)o1^8(P7YQM8ybs2>Ebh9Y1jN_`}56b`S6 zARQuP5^c>ka(x7?i6H08>HEPeBS`HdsOJms5TV_U2d##{RS#(49Hb_KkYGWq^%1-# zf-vycN4y$t@SYK$nk%20D_YM8jjQBS21gJnNPa{kLG4EHI#sM$9;+avGD0RH6%+CZ zA*2q16irx2aN!22bciR>icqZMONhc5TwFt@;Nb;6eg?eoCz65HaEKsQ64O|SYq2WU zfK?w*HK6MZYCzYIf+yxc)p|fJypD#34C?5aW?(gZjR95$WHmLu>INK5_!NLw8X(sr z$ZSN_1MAD6h`_2L=md~oHD+Ri?C8N1hloIG8BhX7AAc)V^MN8ryamB7{DND!NjY-+)GtFiWFQB9U{=-Z{VAQlw5M*brW)}1gVU0Ejhqi4MC2KL#}Rc zvE>{xAySZ9243mlVL()(RYp=6Q$mm`3DPZslaSg+(k4L)d`mdwEfY(G22BaF2GLMrqisdkFBC`m{>NqGH zat{%hgw;L~qDJ9Z5vZzy>}Ca@HHTac2^$7ut%)FqZ^P$YI;CwA7oNM!$UyB1Kch0;MH&kGe9f4 zA$=kU39pYJl@XtsE953eG!jzvs328FNGQ>LbV~86+g(BxK<>$@c$ZSZhr0iLw;#I8TUn=R6#i`=Vz#zxKAi%)DNpd9w(;zLbYUNdA>szYhUaH|- z2wsE$t&bol9zbdw2pd8|L?9&Ang}8&=afxSCkRsYppl>@a1NP}S_W4&gk1YTR>UET zNZO@H+JTlOK$_TkW5=OlM z0#Xe@NaQ*QnT=c@3F&&{ua98W5c~vGP<;fhj8N($NKFJG!POA7R)W+)e46f%Y6wp9 ztGmJLBjj8LVM9(V!Ig_4YS2hESZ#%=3!dL04N*83Qr94pke(5;2&8C&6jPWaL_K6V zx~g9_E`(NLL5f5)QWLh!7_$oisk4Z$@^IA)5OGp#6NoMdiAyhJMhv0`ZEzAY76P=a)R8X0A6nZ6+!RpltIMdeH;iEl5Qc5F(t1O2m`XN0GWhTF_5YU(u;u&yA`Q= z7D1{ZNSy+yW#DxXoC^_waFJC)R$oK*`#{8%yo!~*OH^HRL`8X8FPcPv2r0*4}^p-Fn}|V>m#&^24xcv z>_8r9?SsUHtTcd(wjtLt_}TDE2R#0k3?bna5S$@t17EETx)vNVE(foSAPfoXcnCt> zF$7w(4e1BLZXJq)R6}qQQu`p2kQ*N$cR)bc5K`PU9NHm*T*(R}Ar+9AF-D&Vwn_m$ zH-vl&H*|W)AXG#@1d6~%D?oZf&~Z5fe-WfXIZ-edGBk%qLi$9=B=VYV2pf4P5v(!- z)j_a6kq4|YLNEk0+>sHa4ieDt;8%CYh4|Fm_|)8Z)IkW=8Z3oZF_2t|i-apfV<4+X zu9e^!AB_R(a3L+0K(2=1Y+~vo72hfq-zr>{5oG@qvaE(b=cGBfU72gYz_rg zJ>V(^odK?oATbCbAypEj4kD&Lf}|o`H4(}P8>~K3^R9p-OGsjauRG#sgEF438Zp?R5~P)N*+au?u8I73Lb?D9) zvIcnn2(9YDteDUgKx!puO=Oo2Vt@zS@Yg|7_NkDr5L#~tUi-iqD5ofZHxNP2GKHLO z3aN%5ce6rjA_xiT`yi8$`UqYDAv2JxA;__E;04>DRSJ+g2)tlB5<)_IL$DpJ5U!X> zn7BzKa`y+phSf*#t`MXS5;6#atXKeTBZ5~(`XP|BeV{cFq;mwWfJmx_AXSo(E_635 zv?cw-e#2oX7R8kOSkdh5TVv0b-Ng|QE16bMc4h4dNZ3{43 z`46cnkV$wor;R#Dh+NA+`a8(Fkl7HGkmc66OhZ-=Wuwjtq5Bc3mjta=z_kfvWj>ye zgakg3B$k>5a%>LrDWi~@1*vy~ymA1O4ehgdm8-%KLBgxoXZCI-YrkE(r`g8sPef z2~yV(LkcL{T6m|byXPr8XURI{gOGDBs8({xl1D?}Y6w!};3DD6t>Fwv)dQ(tkV%L* z8VOeb=?x*1keUcy-N0)cOc6+>BVn5iM-UNY5>LWB1 zf-I|m)E)TiBZwO0nh4T=f%a~YY8Dg`urDB9!&1wDJ2(*a@cIZ>P?IVSk4MPv9!Spx zLV~7yyem|Y5a>uZZl5wC)knP;-Pd!CY0rmR!Gj3a2s4zoUzbIydu6-c2jD&R}L=avdA@_P9r)q;Pdaz0mHwT?52baZKA3 zT@AtNAVjwaUI8I91VLL_5$7rptbicZ4M7sE8iI2nISxX?vm^$i7&O!B1tOKaiopo` z$Oc3sd?+4&eS{_pDY&o-LZq;gka`NM6tW;>kvZthf27$8JXIB>=!P#vhRjvrCn4oL zq*RBje1@~}Fd!R0kx97Km<(iPa5kng{33XK4Us~waWFfo$g;?6m@R z*vN{}8aArlrI4Bhq8`GAlaQ(hK|*R7@Ja(ndWNWkD1g*N${wZ49;I+Lh~ZTNu3U2l8L3eK4zSr9i9xyC_e!!N3X3*xMgz(*y(S0+F@K9Kcq$Q2MW8#EbYn+Dyi zipBshRDkq-AcrSFNJw=9ziSmTPlP!E1mC*~uW`T(&^g=U=AgyckbV%j8iHC)B;v~(W3+EvHAmr^r5H_SH5;utit%w7!Sb(4KBWfH9Jy!u-AHjP@C>0Q_ zN)pl!5jF^goOB8y1@%D(aYIJu@Yh7bdcKg#2s{c0Su6)Sb{ov~MP4a~Rv&>k5P??8 zAy-C_c_P@(R#2^krw4?*jR>(TiM!Vr4_hWH<>?0~syEPeQ9aziK?S z52h5nT7fh0*Cx=jLy=Y;z@3Vf0d*C4{0-t4tjgemkXv{lDv{Zc(8H{r;Nn;r$gK{{L}N8T+5vUXb8BTv~QPth$0guv^oA>ybc==1|Q=K?wB0vVS)8J9fB z3IrvmbanR}^6Dc75d&SPI3?F?P>q9HAA$Gyq{%v^L2hIKb2DTevmg}>oRqRlm$FNj zama))AS8q?wc+&!%bOp#s3m_8S4?^o7;i``?DQPg76|27Xwm6LuB#SL=ahc z$xpCCfp@zgR;Ysy2~-cPhV;L%j19sE2~mgLAle}=#7`2h5>f|Yu0w2L-_C6?O9s{~TfAV`l=RgY50d?-TDvlzn0tcf6%i=t<-f=3Z@9fZn; z)Hqm3MR#bo2)Pb|)I<kVxPQb9 zCW#~XHFX`L6kW35^^vq=mb7D*tWyT~%(ql|SbYSkm1G>#CGAoq?9!ziGLfq!NPh?3 z*?}cXsCt6W1WB2LIzHe!2uuG6yi^<1{ee_Tl4jA!JtJf`as>ooL#Bk# zNO*k&KY1InQcljyznVFeb1s|#$-MANO39;08L2XYR7sd5N%avP#aN}VmWRaDP!Kg( z3wB72gP+8zk)R-?rhwE$1WC|Qe&C)CL>Gi4Xa&5YL{o{`*@09>AaU@)YnZhUvIwNk z!W!CeLF5R6S1~H!AvHDFN;XV2$Soqwng~+AV3I2CC6M|EvbY*e0i-@saxXz3P&HJf z-~mFAWDFr8^$UyytucV)2yg`iS_r4;4iW^D&^v(SUGpGy5Hbns5J9RTIEmChg5Ue2 z;F1gJ4S|*%IOU`DkCdG=^xcveL=3={5GnN$pN5Woq@oMxS~?l~92xr@DTho@T?0OE zN6s-7jMAaoe_)3mh}))tsw8mF2+|>f)h{TgBB0er@G3?IW)8eElCn?6Qy)p$q(W*U z$lVRdY~J z^>Uyk3TX8acq^-ifQCC-&j?Zt!AW@k2wnj}7?4^9lZ4ktm{kwAvJ)mk88?7kd=JZv z1>j5vt&AYK6+a2dvyeOvCo$_IWD(@z2sDC+I*gBX-U>|%+8SL*3mg{-FDlU(kjjW| zB)n>X)F}jOB8UPN@Z=Xn3));5+(0x2sB*zL0u;Vj8qPqjXdrA{0f;6J8F$5u5lj)x z`Uq0jAd5rTSnDHr{Q~LB!?|zIoE7?w;To6Yy@&aoQaKEALYt| z7rDthgBE7Xf)~QcI_D}nW+*vj>bs^eh?ym=398 zWE?@manMS~4%9i4v`c}ktcD-*CT*7lsaa$|)k-q5xQtCA9Ld@y$=bq?hJ$QSg>#c2 zXJ|v}EEyZnk#Ugod?2-tlufFnbqb_Df{;>{$smSx5>f>e4_me!59tlT>Z3$)s|0vu z1nK+0Nk~lu=Zc%hz;3|cHj60&ay(g}k0j0}Q6RgYdUv>MV27SIg>vGoE#wU53Z^7d5(8!RiR>yKO? zq4k5%xUkB|3$=d)nls|lbm!Od;MMZr(RAkpGvE~vpC+_JB&6Xkq~R{8?j{I6h#P-3 z#0Tyc38=a9tGZxCyvj~k5q0Y$NKS{>L=XmY?L&|a?`pvrkSYnhh@u#?l!J&s$}v17 zyc9*NhTvSf)kF}B;I#&XtKwGyKaT?53xiCJA(N2$2(A|w18OGHCS^z!13!WQ&c$Rv zmQX5t7Q+J*9&}h4knkslgc}I&AHnM*nsRE&LtO80b!De%E&bv*ZxyE7ifJX=adbp zj38?d6dcl(owE$wQW?d~NUo1Kl(cNak?SL9)dSnh1FCv#(1gU)>E9D@hq$OzT2f5yXup#vkysm-QMzBASd}Ctx|x@2SG?k9fVB6>mz7oq#pvRjPwIR19adjNkBIUflhA z6BQse5rhuZ(nmns3sMas)jo*bLl8m8`UK2=5Raxi9AVZ+keUcWLMkKVa}{v) zjIdTl{HiWkDr8?C`5aVOujt^S>f@vK{J<{rEK(fbDZ30w+jL3WbZMJRX`4(1n>0n+bP2n3aoaS=(rbC!czN4+ zHT!5a`)C!1NEL@jRr@ei`!IDdQnrs$w2hFri;=gBk+hAIw2hOpj+M5FgRDJ(^n)Nh zCP;5c!8S$S2DK&vb0H&gkeUcm2Z65qwoHcAM4+w^wqtxC<8q+6AxM3M-0Oj}Ma-ii zwGw1y0;E=g)I<;xUiF|c;A;{fRT8}R!DK+r@IfXaH59Za0-vG)sfpm*g+vX5A@vc2 zgw{mhbqfM|0fPF0keUcW!s{bw1q7ZR0iP>n4mgF80$M&o;QJsTRSy~o zJKYqtQjVDV2(n}jQU{@tXtP3?!*N(@A~h#|Fyd8lz>0X3?UCDW$fMs-HnLW-Eg=`4c0u!zPQh#BR1gj_NDuCBD@cmYx zx(2C716P2~Afi4}_9}-X7gFNGjjSafl2T~WYG`xNxo&m3kAPmg< z2vXNz)hpKL|3K2dakTZIb0t7bie#BFOm(mdTLH zNXjx9ULQf$!hyR&pn3<)g)$(u3}l_Qs6`wYfyUn;H4&t)fsl|oNWwGjgs)yguUB_2<|1 z7tjlU)I<;x()SV2^?_DB81<2Wu8)9@A98&ppydNa+UR{E$c^D>^$~nB2+|wk(RAn5 zaD#M_AcxCA`ax(l5rhk=k1(qtxCo>^Lg@z~RYN?=_E-_}W^H)W4$gpQPe`^!CgD=> zfex@NIGZEYN8llP^!^c~feInfDk<>bKV--SN`jAMfh;n|(m%ph5`xzF;Sz_`UuYy| z^+cWsd36?OiU;XL0_2_xCL3OVLAD8FwGr9znDq#x*8>qpu8Gk4L6881D8ouZ6u?Qe zc*I({kRb@!H;S#^0o5;%JLBN8;Eko&>m!ItNM(ddf~JZfiCNAi7mWeo%Dd*lr63}* zpot#%Tu%;+3$0%udwd`#yg~98Ln%M!3p+VMKE7+ze*rq7irYPH` zs@Wu|SS1+P1R2-_Iomrq**OL}nFTtT#W?H7IP0gmX{Nht<$5XQ`>2%qE0zT)RR$?m z1uInrsnmq1mG~JJ`RJ#6nx%P|`Z?SAINBK5`WxE?DBH#>gS%Jqw(*d;AvxrCcQ8pv&j>OT1h0LN4?BfaJqXe? zTm*)odxgM@v>{uEFzq()E$-H%fW(M9A!0*FR561hr3lY(%G3`}HYXz@r!)dSH2OGwaR zVOUB4jkdw-9S8$b#bA(#8|C2FD1h%)0PpbuZ|wmkQOFW&7nI5f($j$?X*dZHL?dAp z5aM&t z)WXu#!qZC6$JQ{yUOU=JFV$58xjrfnREE??H6dy>A!=m-#>Kt{8D8dT9;W^-_CAg_ zhBkhNHhxMraY{CEa$qEBog!hCj8-3kdP7!;kjhBPDpA5ZUfc>Y1cyd~Iz*6zhg<$m&=atU&!Qf`-5qU=)&AdRUMu2~JX2A3@4m<#8MwAxxgl9!1WPmvyTgChzvAJ25wd%kE%h^Hhi5AvJ_;fO#!_808tG& zXUc$&H~`f(NDSoD4^X7Q^^vS&wyYy`o(Qsr0n}-N?froqh9GU1BJY%;;$@k z9CRZ&!FeON0x?AcH)V%-Wruh<`$Sp01XYJPb*BVlrzk^*2y?pt19MM3ZBsR6U1L>Q z6E(SD9iDIlfjmq0B3tfyXU;}f?rtxxULT$gf9|e8-U-26lR|l>hI3C1=bjwKJ2irT zMmXp6FpmCUzTQBdMlbO?cacN~{X_?S9qV8n>tI=%Xjz+RF{@-TE6{upWEu!m2Z2ZD zAXk3NSjNeKPLKmtLlCx=r$q)e@JBn zsfHjVXsI@Q%{F*t0(5->r0;`T0YPdaNDl}^!UyQId{9T^Al)L+JQ2T^C;ZX}czwjL z?gp6~g49HCl26SQ!iKC*z^sV`RbAi+UK1f#G`uPf{78sT*`6OfOov$kaVyzjBIKMR z>ym|zz~d5VjY9|*k{@NAb5MseaMnkVMk$y#nyl%e39 z2|fM5DFeX(kIO;oBPj>y;oG1p34B5VNuoH&S~ye^`z;TUd+xy9BIr78c-4cI0jZCmRSdW$g7ko(q;Z6h zQ8)xa>LUn=+z&!N(G*e>LC)}jjK9H2VLj+D9jF2VcY+WM1AoXeInd?c;C0%Nu{cO? z2wwX@DkkWH1W08BUYP)?gM@TJH4dl_LY^A}*GK%C-XNp}s)P7I$IU?*kXa#kWyGfm zx*!jH^@AIv(m^I66%ZPUTpvN^gW#2spsEXz^${PosUpmp2qFRhgwqq3`z7m0`gct&06C@#u@sN;h%UBhl1s6ds?(ipE6G387*}VV~ z(L|7tY6y}-@*!0W5{amZAn6d2G9fE=AZw~+ok3|BE(_lTgk0A^*bu!CHk^cK7I+2$ zop=DPUqG`zkaMjORgY5!Lo0`z7d ztTo+I1$2GDw;;j$K147W1r+t|G#$e<971&*Lp1FI_03&04efMQH54SIRmFrA1ocY1anOeWuF|% zJ|&D}N*KqCFxKgzEK@?5CI&Ke__DNmF{V0*r#ngNTlpB;_(A$d64r^3{XtSz@h~=M z6b@1oK}g8@1k{=+2FwNB!72&8@D6c+53D`{^^BkukXeK%5<=_yfG&4{9j^_ogTNzl z!p0H8#-N+QA&cTfjl!XOg)qkDAhi!@X&iV;2)PpkXNwsG!4aYw(hr1O4h}g`4njgE zhD7uNAhi!@EgY=sfmBS8!8aitKWN1SzV`ugqz`&!r0oTt7=kk}>myj72viv%S3nRp zd~pJD)q}}~R7S`oXrThQsi52C!25t8>#jiu_}C_ExTXl|`7$u@5m5;tE8$nvx6^ixP_YS= zHgFc!uojck7LZVt7nhP1l~5GoR}|uh)JHZl><$XtK^ma?D8+~&)10xy7E~X#da|^6 zv9|hvs-(U^j{YFdNg-^I`eu39R*pB4e0^F*wFe2 zT;qW19klugvIqf8#!8sRV6Bg!)e!77IYb2nTa5q}F^L4#I3{6`1qou1Y6wO``$S0l zh9LD3a?c3DhSW#!nIK3V1mVJ~9!O;bIz&z{0C_|XQU@WE@IDb{eFUkSAoUTPgw#Y3 z5?&v{YacWQa#sk<2Hg)1?hv^{NJt$7At7BM%&G?>f?V|=vq9Amd@Cy{^$}(T#G`17 ziQrj`BnC8xf^#|~zrzRmErJjNp zznBaK@cn4Slcd%TkTuK%Nyvg{Rp{ZL_!bbNIRMi~a1pGv2}BmF6q+EUcY|Dakj;j4 zMzE5Q(KZMPQ2?n~FiEH^Y!i^IOO`YeLQ@Yq4*`;Tok4wk$h~#=N$AKn(ntd2L^w&O z3`ua8N6s-Fauyu4Pvn>-2b&LqthI)b;L{Qy_b)&c*r&p4C@H&iO!l5M0t}e>1CBbDT z!)&F%9-_q%p~sMJ!jNapRA$dumZuzAgXKWp>OR4ua876K+{9uDhX5*K~^R}XNAlX zq|6c^BX8ge2-FFJEI+_Uf@gvt^Fb135s=|HOcK#8Lb}#L$T$>)zyoyfZV_lv9C&>K zr1pW6pnJd#14Lj5QuUzRdh2XW&AjqkkxUxNJu{jTn%}^M(RNI5v2A(BO#R$pPH)>4um`|hs?%RAMvX=K&FZy z49w0ET786B6G22ss)pc-@zh7ME?JP00Wuf@pIt&L5O7sY)T)n=>nTO|LO2(#P=#ALk(io&Bf{+lEXbB0< zg{X(Li)5TLk?SKF*KA1t9hro*qvV})*qsR}*2=6z4RRX0niD z4bni=N2PWQ)h=wUUd*jt%$+{0oj$CP`e;fpE2KV}5z0P2m}`14*TfKJT=kKZWrVm@ z0_d_K=s~9N1#+-~IMaAZ(|F|i2(uazHvwI=3OO|nDvP@gf?w_cnJt2Ji{SN+s8Kkm zGBOT@_l!U_5oA#uYJCLh5W(vs=u!mmln`cpB%&7px(E+^3m&8gK{Bk+$1zpqb-C0uY!P-5AfoGNCwh$0CXq@ zGVZ50FSsJx40m;j1U(H3G&DY@_-m1E~l(8my8gnj1Z@iC(-=rignYj$ zyw^jphTtSpeFUkDL<|EU)exKknF)fE@Qx2M15y)- zAl>_bRvAGmAP}kL1+RuMYa;x05Y}o4QYG=LyYp&55MnbCa(x7;gCHbi^g|Kmx&_N7vmWO0~5DA`#g4Adz zWd*pN!X*MJTS%&p6kM|*x-dyd$qCuX2rs+9g3tj#{5>XkF)j~2V3T+ftvVpcg>0oE zo`gG$NCw=SNDR_yaB!fZXn}+_yvBhuAaw_%UkshQ0gr~kMIZ_wByt6Tiw&R0fiR#c z$|Vb)#*pW6pxa9!vd)>(P8kp+17^!QW?q zkg+&fr!;s?1YuBKA3;vqk+MletB)jX(!{J%lm`vFxkvZEWBLYf_(hq zg1q8_ykY`e!o2L#f}Ao!T+#yEl6;(!{OnQ!9MXaukjh9#h*MF7Ls5hSxjxD?1Jy@m zwhVQy?A?Csoj$DnJ`5B6n5G6W%nV_H)JL;ISs?Y%jBut&VW9e`(~qIm3tAuPS-28e zABmdAh?>SA*GI^;3>q7`x`9hUdO*lh$eV^B^%0^IWDqJ~7z{!Nq44?$dWf7s5TqJ{ zlaOi%Qole*gD?3qGAHgdo%%M57-VmfpLaUE3Ya)mUN=<}Z2O+b;^${#Tf>y4= z+3@-ZnSq@5Atec}0tZtZzGMQO0dDakHtNEr2_R)7CW)oibIAf}2k-EO^tm7;QbCPg zRe_~Yiek*d8QNKZ9rOp*g4G=Cg2dNE5S65w4^xlN7x2)-^c_S5vN#Q{3{s(>k!Vqg zRv$rRu?j+@;3T}(fG{B95H^GaRYuO4@R|tDfQ+6&>L5%Kp3dP36517V%0{k_;I)r} zbEZ5v*~>a*$T?@qIHp5a%|TXWL+)+>&;CGGEI|4}P&RmO2z*!^=sr7$6oiBbf);fn zsev473hoxAiGhypNfWh77qvCrwgBZa3W4T#aML0Mld3ogo z`Q!xoWcayc__?I`xg_~GB?P#ng?S|fA&6T-fJ;)4TT+l)T98v#m|IPRLtT{9Qj*0+ zhCNu7AzGI;%bX$KhPm9Hq1Kh9-J7k$hrP>(y~mGpVi5P#F#Z{lLNlX;=fsN4iWZs? z!#*W~c|sUtUkF2&A497rLy0q2i8Gg>m5Y(JD`c0DhT1^D0hTtT;bA;9p!Zi&9uZEC&Mvxi@lZ4knkTE(wMQaG+Rj|T^AlU?k z1mBPX6@jq9BxHCTk}Ki$5rm6ILdp%aS_vYEMnXy(WD>Gh9SaHWf?cKE`Ur9- z9i&QvaOOygm}OPKDG*qE_jU`bfeo zf`LmItv+I6VB}z77vkm-7Z8-@=LaD^ZW(?a8GbHV0d5&V9#FSPkXuTKM@onfTqS{O zA_)O5P$dQKH>rtmsEKeu*L$mh>Z43EhHMLl5*vmJ2Zjb0hGuuB4lmYDZ?;LnJhP%j zmZZupNtK%yFE%xdcXBw>QnY#TM+r1geTzN{IxeP3w4Xs=ttFlGRV@1rN z19afZD28Bt1h0S~40tsJXF#eUOcGubL8~Fy5FE4z1iog7Uq6H&%s>c&>m&F%a&QJ{ zqY&&oZCt$}c+~@EAgUYWt`N3;L-4sFv_27}CW2H<{OX`Zai}#BXw5d(Y6vnY$EOCW zfOynhxxk1AqdtOHLy(JycvW3+)kj!+J$%ackX=IDN_O!22(uc3i@@t62m@Cgge(rp z7bqn7-eIT+gpEW(Y9%-cshHq77g{la`&JMsXnBHdV<2*u3eLt-6M;|kf|NoO)<+=i z(Ao;J4+V5FG^E5vCXvg0%+eUY2(n&c*!XK7YRKZ8sexB3@Hzr6g2{lZ!ODQdGFB;w zAVdKk5>oqMl8}UfoUTyV;FJc@h0;F)--Rmam?Z)3$VfTB_N9W=%Y#?GLFR)L!1WR6 zs04>BP{rg3B0++n>r>Md9YF{(A_u9JKuZxEvSdID5;Ei*vfv1O!4PQSoD}TzY{>kP zvSYHMeWId$qOxPMvVEemeWH?mtdf1Kx?2teBT5E3$s17Sl+@Q5Fj4c%lh0}+A9Vx8r|Kf;E!eu1o5!zzVG5SmWF^#uwWIn6+1 zAtDeq5(%k~q#V`guf?WdS`UZL1cqO|8 zMca5qyBI~g7&ZH7Rl6u-0bUtFK6yb- zS$=k9K~^;p4l`+XYXu%pWj23Jo@gD0NG*m$U4~?Rh71FS9216e8-`jZmU2h-N@va{ zFM-ZL@%~Wh?m+R*0Ff4N{&p{}CRf%9d*)&*hCFkITr-9wQ^rgi4g(8UV`~q{ec#A6 zk%&n&7=iW#2^&Yi59)!}N61wV8XHm*K}bmLgGQp&M35~)5E63TD(HGUjQR*#8R-W? zY9a_JpzkMu+B3ph4PosHAy-Cx+FpDZl@VmS5U%$O;EQ>RCCNwbIj0jOwo2oHgHNbbWXBxjx%?PHFJzJbBwcc4!3X&g^*?r zVHQpi?oJ+Vj_!U=PQH$gzV`OM_V#`b4*rgg{*I1*4h|vq_96E6p^kQ;j&>0amQhaD zu`ZUeE|v+-W(m$_$aFMfo zhNWntF;BEEo2$BzhnA?Ssi%g84`ijBuvwIlX`~3+HdWJbVbc&1vrvdQ+9GY_ng~(> zp^@;K2(1o+b0O6boP<<5(1AEu-v?6lz)8qF5gLhWWdfvn!df3e>L6qi*A7|~8#YhGr)Z7bKY~;}SV{8gBN?Q+3C_k-Nr4Iz z@FWy616HJf7DzzO`vQ&ofzO$wU3~#&RbAp9) zf|YZ)wM&GxON5nkxS2zknL`+IeFUk7{2d(woSYyi+{p=qoE^fQ9b#N;V_a+#+-(xv zZBpE=Qa!BG-L2EytTNm!GCeHwyo~a^j0(N=3cd9Td^8JuH1oaH3VhV7y(O!>Bx-$x zYr&Nfq&{j65H0hR$+i=8RpBubWKiV+RY|hEEV8^T()?`F{A`e(5u_%P7T}f^;09Gj z{Ok}BNPVOs%%&^B?I6$Us=^hm&y!*ynqeiJW+51B%rE2!rSuaB@+J!pcEdIz~Cg0LYSB1{rbH3U0I0kl{SvSbcSLaQXKbr3`l zQsY2sA_xf)fsowFPFzZk@ZCc^N{$f3rEJed%K38ON(WRQ;b1^(B1n}4uaEEy&>`1K zT(aiUwrS|d4t!e*-u@B14nk%~+od3LWx%`6A(a%Y^hk#@5S!V-Cr?7UQQ-OrbPgC= zVFluXTf&g?1V(}uK7pi=25k^raDNJ|%7a!$&Y6&M5~3KwhLdQeF>;@a7&b&Z+%$p= ztd$NPL4pS2QUGxnCW%!&L>ZbCdVK_2fd(Cv`vHT@qyGz(oSj8PHE~6sTz)HhW1fFQWo4ivlkTWL8KD+$Rzj;t&_&fYw2REE<9==3)#s(u|%84E|~i z5jqU9hK#8u>{*sPu_inL>I_yg42HrC>Rb#;tPI**43-k?eo73Xnr!)Ij3w4=H4Y5T z?x2lG?Ox2C-V8l{OcTMV$CtO=gCo&eF4kPiR87%TO;JI|TFuB&+$31iEEGB-huJfN z9M@wSB5WE0S(^>-@1QY|57Wk9A3-Z1_(2Nb*&le{2Yz}FsA58rf>%9o2BZ!`CgBr9 zXtfV$st98&4pJpS4%vo~nAH$?vnuG!1jsE9Xe9FB8z!3%cF_amJReLFQtv=WczpyJ zlmpEi!8$~|%1)4)2ufnBjELzU;jfR7Ya&R`2)W+jk+{wMAgx?538|+rN#t?~g$*8Wfz?EyQ5tAngS9ep%!10o3Q2S>k}kAT z7mZ6y9fYg~Ye|k~CNdXUFEMOnz0_r6t(_rOK%_vo#W`fc*^pWW60?wA3?z^tBFH2> zmgSu?P~sH1enDb`sv&SU2gCr^N09SHL0ok4WEsa)Df?s)f=Qv(M^cWdQjV$W4yg*3 zaSD2#lIl*{I)>Uhh7NiPZYFAR7ShSK3I$G5#jdgy?ozcr@~yrypeo5%s?Arb(_b7^ z6Zwg?`G^p#kJ^2CT9Htv4^NjbF9c2S6P(~DIN48dq7VN>AHm6f!jt_3r}zuOY9ek( zebnd2395`>_0eR1wyA*}eSus({v7c7$WjLR^Z% z+}a}Sdg5G;3T&<_Tp`-b;d*TG#>`1(Y#HWUS(ZGZIxH^I40-|#n%oR(oS-U6hnvAf zn8{z6AzYgy-;A-so~yxyvDuxa)q}Clk)hd@q1}@KvO~7hi>uX*Ezw#o&O+K&Pt(Cr zN5jNf+uTjUG(^%YRKx^4PXwubFiF!8A(LQ8KM1WF5;qQojJzS&K@c`%H3GEu0k1|t zW+N&fq)G?0Ob&LQHt2E(*u)UL4iWvbr7s`1mdFBN031|Op;5{ z7J@LVA;`WV7-_|=07Z~K5u`GLu#u}Itd$X$thtPJDmnt+ZZB(-212$evbYhJ8Vb_e z!ssX=S6gs4yqH2^fajN?GtS6t=!ul6a$tr$X!#T9NPqa6C=dgws|6_=ags>&5n8h! zQUb$C$SgTzHi{PY1uMfw~5j;wvs^kU~^UDiPmFIGUY0@VF0fd zV{G+c?DApg3t*WR%sMTYbwVIVpFdk)08?K8(}W-v2%|56X<`7=1b;?M)a%dK>(AKj z&)DV1&<$pE`!n|huuKGNfsnoa%zXhYeE}>G#)Kd?kQM%Hef}JM{-86od;HlTsLP)X zfr7ZYg1BnDg^Haxebm`4LGwop8vG2ZybLNl49dJ9q{7Fb!pEQ{$e=F7pef3rCB|SV z$zUSG;I7Evt;!Uu&623kk!H$~X~CUq!&qR?Qe@9t=_Zn3Cg>r@U@ppFz{jA;4nn4U z431Kav6>8tx=h7Zpi>%Jy_nm)S*AqDZLPH5-x)YPQL`_YtHYP2&yTa!l_AttDa=k; z)!a|n)JNPn6toq{BwEZkO41}!!Z-r5IS6ytD&*WAto0G3GJ=!HH4aWTxMze}A3-Y} z-1QN>>VY%h3lfN_eIS()WWhGP-a%u4DLa)~y!IiM0U0MmB|+5_xEg{~Mlcd|Tt7$(UTq<* ztiUW4AtKNk3cRZkxunI#hSx_#GBC??{36J84KZw7?xdEu3{npW(@eOmj6)`*mVtyS zeiEV*N`fN`TGt@g9Y}1*{!mF<@D?6$or2syLKO%1jF9Rhh%AJZvQL$?OHs6o5j6@B zmQ`lu7nK%ZQ4r;{k!5yP;fmH{PB7*uwqmGuVr%taZ1-aB@?q%pXPz3wGBt<=xjyO( zV1iUPpt=dQGV1qZ==Woo0IrN6^%1NxV(j*3?)GPf)I<;xQW?SPBM=u<8KKul-F~c{ zFx2JG))Bzb?$2KBC0OLh;ibxICCZ>B$e_*#s*hB`l@X*$QsxEKM_Qtc`qE5Ba;(d9OZ7@)gEGX-jYQg%Gr)Gp~gZUYOHoLoMs}7 z)*=kfGR*PX45@}}g%%9ub_{jS3=OUfkostQqvwPur4C<~386fbf_OVUSwd}l zwef5nf>cb9+6P`438oR1Y0skYJJ43U<0prFd2~A z2bqM|IB*81j2R}9v`NB>pl6uaCQI2SL+)WgCXr?Fu;H~5gn_k6f=D68`cP{k#BvM! zWEn7mRYuT`610>rE?SQOt^f}MQv2XhKxZk4Pw`j^@2bc- zWXhv<8zH(N3P48+z=qTywFxAgkx5AWf{v=jw%i3Wcni^si-gxia0aAKf!@Dhn+6iM zO969}B<)ZTBq2dsP|#6myCfO=WOe&6VQnW#5h+0)Ar%1z4G|_s1qKgw<|ti;1Y@=$ zE2c6#wni70Ru7JDZ;l=x&WU~?)Dyr0L6B}vw?8waN}Axu+V9KK@5j^+nnq!mh*lp3 zFm%HZL_MS?njFA18P>7t_hXuXsHa#a_%lO#TM+SHf5sk=nIQAK{Fu7^nY;WsI{nzI zJOv>2k+m3uwh*X3(&T5*@h9TFODbIvC&73F6m?KV)Ax4KG#gH-6jLpeZ)7?r>!ZbwOBv{HM zOv)ro*f>hqI0`Zf2dRC~W`f{?PzJ{O1V}XmxxoQKLTemw?E^hQ4t%OMlx^TIYTysq zwF;?mAZ+NSA*8EUAxq=XNC9o=QF4%;j*u2Q7b1>aB_Y>7kopKh!Yd$N4R;6|ZQ&ev zWddZO0(5->ay100jF9_5$n_B~($E~dCW7{TzzgKi)@j{B&0M$ zR6uAI6Sy*hR7o=S&@Cb0x(VD%qM$y4l-Q6`mvRzPhSQED*@fWQI%0G|>LZ9si2EUI z2nnx`;8h8vCW6KqjtU4<7Oox6097%NiUunQt$o0C5JUk~1f?>Pv`c~{Ge~^|ISWo2 zB$%XT7b>LXh+H2zD>1mMF-7X2)JM&3Y+YXLJwBWhd^slga&-AIK@g;y)8)t1JSDf$F0^Ur-kbQjfqZqwWBZ;$DA7NPRTPpK%hbV+E;qCP1Y?^%10# z)#uCD54HufKI-!4sPYsjuxIm9WwjIm)kj)_3_3y#y21?lLJS7N45s1?7E%nZiVQv) ztU;Qbff^j1N=(i&3^pPRwxSHdstmFE?Ahkbkou_Ai?7W`s5e-$Gf=$6N2nuEv?EZo zz(F!vo5f#|AxwiY*OIl=k+;r`wb`4i#gn_vnYF}%J0`5=@U2SLZ&} zDVtP?I5G)t#HK*BL&^%OkeEdrX(EvF*)d(pJ{4LMfkhzuC?TYTeX0aZ7G*pTJk~^A z5lCrII}&u|fn7SLo6&5Pf~i3c69^kRR)#!ohQuZ<@a;iIhC(|pkfB{{q!c(sL6u>w zrIE5th9o4=b>23~kO~O0`4rBUw@H=LWD)1}za5XL$w>6^3YC zhD2k=JZq*BM~+%omO3}qHZRsrUyfcF>hWQRAc#o6FI&GaTaPbGw-56KP-Vn4!H0Pw zxUU56<1qL8G4+DFQOr=%4|Gifq~4hjzyL>(TCCq6D$?%{IT)M1Fw23^Tl?#kcnDOm5y+vv_)@5)`}$X8*{6K5_Ss>f%pDrqP$tgWDLqG=(c z=d57l4jPm*43;(ymN5wtHwqUuKv|O@ZWs*e<`@Qu8wJ9Vm|*}EA>FJhsuv7GkZK62 z*8{I^5DdKl(B*fqBNQOwDAf?gSqeg+!{vO0^?Zc&y@d2UA@vdTkZo;05p6#qO&<`_ z@`BVvpejZidb$E~1%$>H)btR9-2;wX6G6_|#;lJ7)!hZv-38R#kdQ0#03Do-T>C)S zkZT|WRU8Dt9U@3IghoPY9840q4#LBRS3PJ9cumBwU;{>0{II=6yowe)3g(bb5Ka=@ zGlGxQK^T~Wb(}J0SSurGYwUw~k~T@=U<9wNKn&{?N$V7dxVTLca+M?j9`1t|An;-Y zQc0na5LqHgcri$%dbl!NrK<$E4nig&DzWz0AcZ_GDa^tjj|f`5Kz%N_n~CWFW#JJa1HRrA!i9%676!HoNZK|HQU_s@a1+o{3xo@)hTx>EZ3=`9 zsgGoA5;5x|O<`t8eH5hws*eh6SxTKaYh0LXT|w8OcKUMk`EmBb$Q~bdNM+RP%Q3-^ z9j!i^=*u$6j}_M6VV(f1j36^ay|DVI-;V)O`}F%W^!YJN@CBiMe~=)Aoao0i!I!be zo1w>>q1OjQc6u{(`7m_*GWPf}wfQi#`7l(tu@~4u>mwr(P;baun!!ef!A6e3PJzKi znZZ#3)KOC90No?Y4!*rllz~B=9aJ9$DKkRqqY7(|W+$d*C#Fs(h8|aj9#4iYcZPNk zhE6ZWY8U=0XTEeR-e^6>Bm;(YQ-)G&=5kxsG8?8+YsMl=wtO?@C}ZJBBOxaP6=y?L zOFdg_1ABQxH$`I)$h?oFVW6~eFsL%ZsEZy{Z8P-TQ#AAzbNNM)pnxP(Yh(-W=g!ODd#P5_;#z^CZ}sfOSr zW_=`}=7y_2f>b&JYOW9l77~0R5n6qOwvQEF6A@YIpeaMEhTvR0^%1BtqF#L@WtE5@ zp;cItHVGIARC$TpB*2k`Z6X{&$_-Gp1)HdXh+rk9?NZRHGYD4_qmhiin8Io}q*{cR zrD)|U(lRaNDe`1UDNREXHYr2>nh0VtxIU7xONVwhVATPn4nig&euRe)G6T}p#U!P` zbr3u*h+@DT0I7rEuEMN|pdzp)4c0h?)IN|TDr1u@ZJi_sMheym3f2iSw(&By@yfRG zlE$HuVhRGh;wl0R>cWgpa-jZElrBS}2}6+$ONA3xtqWV7D|?$KXQvNOpC5OxFV_S= zc5ns6GQp1(j{1FBdiPEWSAVxG%=8&4~!-Ru}lbJ z=?!4$4FD~B08JJ7u}%c7=;od1$35AfW0D{H1bKWdmK>2RY`B zk%57WfkBOn!9;>1ScNHChpWVdq1u|g-I<}&m7&gsxze7Y%7LN5jkVpAq1}_A(Sxhr zjibm(Ak&gP(~2#_k~PnUBhQAT(2Bj#iapbuKhvDw-&EP(R9VZ&UdO~y+Q?7J&{xPX zK-efy*dSckAY9xqSR6bVgf{pF;X->u;K~Rx?*kD5U1kTVT;L?Q*CVFq2S?DEAXt|N zQ7eH}BCkn+uwk{24+aBKA3^4WAYCC+>myz*PhKrhZ3XTSf!1$>87P$zW=#Y+X#uHQ zG2@gq!HPI!jNu4>Wh7;l2+8bl5~DIofYdec z$_R}Ckwqr47A$Z5IiSxBi)Jc(Q*(1Hzj7#ag})&`;gUg<#UBRCh%kg!dLvq5z_Z1fDR zGQt@h;ITMpU4z7y1oyI$NvIm6`Up7{LD=wQ31^_yM^e_YkorhUT#=tgOo6-T;A#fqZ>_-0=Dcw0gl8R4aihkw6a6qBzhhZkDMb z9MeNMW`uHrE=vtzn-Rt`BaCHE6z9xvwkaXZ)5F=OM{!OLV+Ab{4`!Vl#yTY&blLZm zVAd%?oFEj)H#vZJk{|mdKlT~n+|$FjriSrO2;}JYW9<*(oEXB}AH?1023p(J=)uz9 z&Rp)unPJ2bs?HFkf?gka%Ys%iScx$xaWhCUGq8XwBvu9n0VW1@ZU$3H&MXdSL{ zbLJWwjtM@j{oXA7p(0zWt#;Pi%}!L962>(lh^@t&x5b;c#6_skk-x~9Ki`2T-g+fp#wQZUFuEyPMg&&*lR%vshXK-MHc)FfEcB$&wh2vQB9llp<6kvDi{ zq!$3GZbWr`;Yb)(6JcdQ>LB=1ZCv|?z}+QKeI%gc4np9uIL!J8J|85g=?ST2Al)Je z8$$AEdhlpMHWNW_28VZu;0$(h3p=Ju2=xi7C~-n z6;QSpfGyI7S3__H+HwVWO@yoV!4$`=i6A17`iM{78jLK_>LbVy93}}_!vI;aK(I36 zkTE7$AE7Cbu#U%y#H{1t2)RmvlpDm5QntzP@jysr1ShFgA4$R13QB@IV#wvHq#a6q z1gV?gv*c)nt+X9zUJO%^G!bx50o4uY6(y!Dyw1X`P_T+XY8gxtQ!zvhvW@Vn2blqH zAwU?AdIu69k~W}41rQOOBvKs&t#QC2$TblP8`iCcCm{*zLz*~LrP#L8PGN*G5;ODOU4h^YxMXp1ttDl&MgGQ=7%B%3ki+cK0mu~xgX z)O&KY`*C#z@^<)e_5|?u`SW-C@bm`=^alv^`0|2|BkRhoXz&uM_Y`dK z5NPlaX!7Q7^x~`c;;ZxIt@YrkcH^pWWCb00?aW^7#8&G9It05Fd>VGM3(M3nvAMBI zot}J64j@!v%T;Q{o@v68pv@4b#t^8;;49DIBgfz)3qpRf41Tf<9?}f15)5v#Aml8^ z;Hto2A;VxO%Ag_0pvKRjz{8-(%b+dBZ6Gb6BPXUUD=I0(Dager#l)b<2|EABLxDL# zn<34Rq1>FI$_jK!YPADHl|5H)kZgaLR)f2At(RbvzgSD4M4h)#sUvs3HEW?gQ;`F6 zkt0KaJwu@lLy;{*sU1_P9aDilTY)`Wyp2eby||5yzNMv(qKT`rnY*Y_peSr^2+|u8 zF$hAe$JP%7txo{Y2ccI+2LAB#6NK~vgduAWAhi!N8zQ3TE2QfKseLd>0bOr664C|L zJD8^^2x)nOkhZ&!wmbaB2T%p1;VlS9NY@Xct&S7WfY(7F222*TmK$FCpfO-I5n|ak zzp9IXnk!@`2(ot-rB4K}j9@cF$g30}JtMUG2(lCbP72BA z!y>}Qz{d{iB1tiUHYhnsF?cJprRX!H8-XgLDl3LcJBBKIhC(yOG;M}tErvu5&`LRf z8HNZ|hGYYVY;(qZYt{lg#zK3hLI;LI2T)}MuaEL0K;i`|2huzR>BBX8+qCOJP_7uB=EY2HOsC52+-iR41N< z)IN~O7f20)pTx4?8GIrKE?szB0`IOOhasdIg3R3@&*s1f(9juHDblD&$}(BXGFj3h z3Bo`lk+nl=9|$R7orqZ<$$%SI;^uMUW(gnzZIeaJW96;lC5$5!r4&SXh4sZ4EaW-8 z)EIp=S;F-hql{P}=jY^DvE`Dt+bs&jd%uzM=AxvR3cDl_?LatG=O1ZnXFY4OGCamDI! zWf?ORS+lizaI|}Ib$GK(3gw#-B|0gDe`2Umf3QG{k4Tf3P@#iNwv||fu1KI7ueUOr zyF8PtJcFw|gNq!4vn+$N41*htZ7j`XEX^du&cqBJl!FsY970?IQgALa1A`GigRK;( zY70?gC~@Ga_ZIK;7j5+xEO+BBb>XaY=Bsz%FS6oH)@E>%W^fW`aFSr~R%FOD22H4y z*fN&bGM3sil-h%;q(TRVe0zpGJBDHh5Gt}|D7Irrv*k*&I1KO&>6h2 zaXI)*5WlJmv_1ll!a?dCOcF8#2bm9okdVrVU&)Ri+a^{>O$3nzAD#d@KLK(dE5U;l z;Jb%#E|7!I3V~P3L3%yVnh0DaVI7O(1b2(L!Icr7`Up}JK}b;MHIKtZN?InMNr_s- zKoDB31h0(H7~)n5$Ox^vA;?85^D(&uO+Y9BH^Pw%RXAJ1HW{rlf^!F3eFW*z;Hph9 z#o_f4dMyAOC4*>@v`&FAFzX|DC}CwlVg$L?z|TgjZs1%{^#QJ!;37~4j`cRU>LX}! zHA@gPjhC=a6gG>Ivy786jZ&6Tk`NR(m0`11;_*^r^wD4r&|vUWV+d9U9Wk0@z>s6b zQtrTA<;>UOF4W;I)*C3iMJk(gd zG}xULm~13Lqhxks3_;2a;TkMi#tg+a?2RrgZJwO19*jNy920~2CI<8M2MhEC@plGF zw)u&bILqhRN`~tQ2deRUDzdo9FgU@GqcnpHxJq(YWC+&fuvh0(;bCB9U|@vTJs<`n zvVa5+i?O=Aiz2h59BAiJq&9Pr9ap(Ce~lBzlqiKQ%^uU!v}>Jso7{!+&DoN*8A7!f zBlTD#^cW)b8FS1S3aps0)<^mF3`JlB8No}n;Y_jSFfcPTFf-&c^x-q~5!d$@*GCzD zL#~_P)exKkscRs8B6xiS;)2KJgmnBs2((Ds7x`RM%=(B=+mjEYGJ^Dvz*9dS@cIa{ zHXBkOL2d(w_J-8G1mFnPIl@u%leFUj)aMedVpfh}IVU-c0K7v;_1R3PjN4P2@V(KGEH3TQc&0|PHQsyyI zU@elCal|0BYD?TI4%dhuOdNfD4^g3D4E>QNg}Xiim$ZoWf$+*m9I==Xw1yB;_Ck6D za5fr))cz5qPeD8h>9tAOCK8FHtrM^!SUrorS`r)-&{h3tZ1^}TvX^nyK2lck5F}|C z2VvkRG1C$r^>D?QsSCM261PkeHjR_Fj+HWt&{dRFkr1#|W4BRdGnVJokzmsnVAbMh zah78AR^m+3W6m_?s&e3{bKz4SGY^17_<4Ruy`pldMPn@DKWS!fcivk@*sw* z9D}PIgRdN@f{WDSi#8CjkzmkbV^HN_(BK9gUZ>0p8k$q)W-t?G@Ymrhag#4~S4cGA z2~uPTQvsngebD$}hCV})C0kdRQnkBqq7g%)5krzO2qhXbr~H&kquJ`7?s&DmDw=mTeId{v*y|`=h`sa8SB{@>%poRUlDy@7~2n% z0j+>wlR;3fuD`IZKcqf_kkHx(Rv#gEj*!@p86s~;R|uKp*8){QU=lQB2WCL~M}k^z zVB{j8=?qz?Eu`TNMjk@2Y6#bnKG3yrsEZT8l@4Sq&J9-exME{~wzWbwu|jGh2#HxA z!K)!S15y(~Rw+O#BS9rQK_xrnc^?=Xbk_r}4iT=322w-el7)*yCWDaI%fZL%@Ki>S z`iM)$7=qxVb8rT{8iF%Os*J?Jl@X*qLe>Z`A21o1B?hE>1tHkJjW#tua9t5C}`qXDg1>xyO+$s)|qQuN%LC7>-+$=%d zGD*ZN9#S9aDM@L`iJ8eW=!i0iGcyP=Fvv17sB$qlh%@*ob3y8(3Olws7oHAp&_cWZ zAnr+_eEmV3{Xv|)fqWf4oK4;mRn7tdI-Fi=%$_PNUaG9#T5RrWOjhy?`XUTsEDY=n zpq`5W1A_)HgP|CcrxHV;7E_KPs6J|NVD9kX?(t-4cVTF^XXtfhm>$Y9DS%<3FUtfU z<|0Rtcs)jUIR<+%1_ucca+YCml4fv_WU!Y2Atz}D4@m|uX$Ds*25$uhS4DO!F$M!+ z24gV>U2z5jDF#zn784m}e;vMb7qv=nm0Ev|d^@RlJ+2fZ?o?x*3`2%&6UIUd)_gPO zG$V#0JFa{Ou525&LMOg#8;%q+wpe|J6hnq|6NWqsh9VnKeN<}8P+$i_MAk>(O+cXf z2wKaaRXW(X(8>i|@4$LJp!>d&#i49nAMnT<59wA$1V9vNME%Msh2IjyZ+Y zK9HISLP8Ih!?9QnvP1zg3I`z}^^t(0t$-rR89tcx5u}zuCgC*;eg?eW!Nq`#)Ztk# zhg>~zz-l5+X(LY93=zEQL1wT?8A6blSu}oxTD|j6NT53gHHw|2ZfmUw73`Cd)u~)(-7CQp3C&X;4gAD}r5YeRI1vQZj zY$Z8p*%&TyNZ(D;CIJ=**q1xPWzn1m=fV}BF_5citZY%F$_TV>0mpb6#3FboK`Iw1 zt9WD)WHu;ikr0*W(Fd=OM9ibXC|1-W4pcFj#VXjwNSTMJ8@cg_YlCYH&&g#$yqJ6oGKONTF8lPg1uJ425j1LzFZ5YX{K_3mO74*V`E zY}V2Y=2A=+(#*y(YzC6dBJ60RXY5Lx44MMWK5`5ZT1+`6%mo(gjSlQxo&qh-JRR-= zQzBJ5d?jWEvdj!*>Gx&m_GHL+;z=-OP*o69RS=U@5R*}slu?pJLkbFV^7683GTcfM z>=sg>{jd%apu6)xm%7R_ILR?M$}%{~GC0dI1gbJc>T<>E3&rROW|<4;+elQ}a8_7z zG&*tj_=@!hOZEjxbO(vFfi7|aEepTYt4$>KN8gQ5!CZRtB>Fn3Qh*NK0=iR_j8f2V5Dsf;vZ<&X5{MP~8oT+y&Lq#@}4QvqGS{ z2DMf~)JKqm6kwxpp!M6R^%1B~1gV3NNj@bjuX zt%s4aN+c8)psE~M4X*S9Q3I*1;3P;ftV4v>)B#sU(IV!tBIclulDK)CyiK%}d6=$+ zkE(%-yt1y4m;wVMHv=QLG&_qDAGe(}qCTp2W@`54YV&64^keVw=jiffn*hFPq9=f* z)0eTzNwD0W&ry-tMux#kmeod{-Ata-Qi(@JoLg3ag_}=?lShh|ON4`!UzUYIjTf|1 zE=q?b-<++?j<>~`tIt=Y!(CuyhT;E9bGKEyEr{S=7|Am$f^BLrW1b^-x)qDPm4UsL zfvJs=p@p7-nW2$|v5A$bnT@%rwHb(EW@2h;Y;US!q9tjkz-%D}+7Ih24O;NvDi5xV z&OGfNg5Cb29X>*x0YYuQ{B8ci?ExZH z?xK}$BBhRE`PKrNCQMnT%=s1!g;orCHXsD4i6G=4sgIB=9r)Z3To6(Lp^>E4N061+ z@R|t11(S&3I9N>txq--)SJ{~x1tI!G$XDRODBKGlHBf2dRl5 zByxQOTP6p+cnJ9}JgmJTR@&fNT!EP9f|OC@k+8Z9qk(N5BaRJW zwFPTCA5!KM*)77`vjFvv;Omyr7`W;YT;f>WFKLP4eXBT#N=Pk(hZMDdw*)}-J6cTy z;fk9>O@o+;OhRHASp>p{q!u(1Qg%XZz!3M9EK!kewDM z(i_TM>M5S%z^7wmqi13#XXGhs=pkj~DQoH@ZW73=Mv2DMnnWMoifj95s&2jcy#tjta5XQbHy{VisW{ zrr`pH!NSH7!p0FoM&Uw6;i5*7U=%KD6fS8LF02zMui-4C>Y$^bq#_}uC&8gF$!Vd$ z>ZHYQBFhfik;KNJz{X%B#_FcPAF0U|uP;z&##nB{-s#TOaK|~;Z9LOG3@MI8VaRPJ&0=NPKcXPl;CBV6; zH4&&jg7knO<8lxZA_5_i`$6#eAP@uGSAx_;JUEtcL&oSJbr6iS=7rsP2k#Jp88+NV z=ba)~G!Qn{Dux(A`iMiy5Dnp}j382wOf3c;rh{i~TnxBk zF|%k$iGfxhp>fgbBd`|2^%1D>fsO5ewIHf0!dx8d)v;Da;?^;c(h`$|TZATsrzA&{ zf)w4jNEyontO#E3;H+3tJ&5K!I2ThpL<-H3m|P<3BY5~wg@IfXLD*=MLy*l~ks&~fA=waw@=QV7g^H~hiYytL zT}A301#y%dbhu-!rG!m`g-n7VcLNC+gu;<9_%t42gHR#;5HbA_af46|Lmw?;KYL?SV|7g% zH8C><0Yga!JuwCuRtA0s1~CQ(ITi*Z9tL|!jxaU$C@t_^IAgA4W9Bq7wj4{&Tq~|TE0#PfmV8Twd`pI6 zONJ6FhCFMgJZmP%dIo~^5w1!Hk2rWyoF~@G2z;g-=tu-ey@QK{)JKq-2tq?eMIE?ZFuJh(m#UtgCGpF z%7{bCP{K5dBqU`Tg@z#7;e`Mi15yE@35uCPyGsyR5pz(#39W>o5f`}zL}sJaN1~Qd z5CnGyq_~EYaB&C&A&$Df9kV=#i$H23td)+8Wr7?Af~dr-M{tQioCNU~gpEvM>Vn9^ zorlH{wTMGQ(3NahHVcr_JOFop#If~jAZZ2~yco?Ph%6=vNsVw4(r^+ti4*}NQOhVn z(=a8=P%cGVB>^Ta2`*1%248iCG((0QbI|tHVk?FcYql&yhDap_A5jKZAqFRY20IZ3 zA0-AGH4#r^MQP(8Ny7jkqi|v42*_zUkb6JB=rwauNO)s3OffwjtpG0Q?Q!;C-0 zR5-;{IN6dn%|;;GhCkbeKhK&w*NUsak}cPaIoF&a*PJ2Kf+5G6Da(dA%ZAz3SliZE zTU6IaRM$s9&r<+)PATTu2)HUB$gB{A#8n4D#9=G6(XU&DtVw{^P0*@G9kwh%!v%6W z9HcUWut6m1RdW|}2&F!PY+}V#9}!a-amkv)>m6JSE>N8%=t zcn~~a!x?Cmkrb>t!z>vfBEn`-!e)peK{2yvNUIfI6Tuk}^+b}8Vu}nBxeh{M!zv(1 zH3TQ&RzQktI0+YrFc9J-)<;t2aafUzc^n#oD1+A{D2|713PTozI1J(u2pgG1RtaH4 zWFeJ~h;ffX^f*LN`lH6JnT!GpQna-ju{=(I6Tuq+5^=@2Mjy$C{?73$A@fwW&G7LVF z3=U!pj^Yev3VhzC%F@O`(#Amo2BDylG?QR4(-6q5+&pIC+@@jj=6>S(ZZZ4Q>h9pCVWK)Jza|TO& zbx3_AqT>y(k8o8zXyW+$LC~E+h-KK&a}wb74x9n0kI<_jaD4=-Wxz+3f+`?1h8lrE zIe3)>sd11=NKFJGA>vf2k6_CZpljzKH4d(=tPoi^$t7=zr#>QB8R4pZ;Nl$8rVxZ& z`#{)Os~$8#NWFuLgjYsflKRMb7C#%3$1zFBupWej)IJarSp>p{)I<;xOL>Ai(*&=l zureS;C}<%BtbvVO#DUlrQQ~L_s*1Nq9JTZoh5A^DPsQN=B%q4Zmtw#qaJIm7Hh2* zWv&)(qU@}#Vy~`Xpsa4Bs;QxAuBB}$Z|I?5coyu`YL#5w~+dc62sojG#ALvTe7 zT*Z#u)h>ed9-<9iLJeL*jh=$_?)-HwJasNSg*IGSrp$p_T;3`yzG@7CT8xHTiWY_% zl6oFu+O9&-Q{x~zgFs8;AiW+mQV=#1gr5O9ApsXDq~QU+NC>hz4xNP8O`!S++z;YW zcST0rYA#Tu?!*m4kaOgS8H{W`HJ%lb%2qszpuP{J z0)p=tLS{e)<#-jW_!TYS2wwX@RxF?m$3eK@iV0dbamib9$eLqC>@sE$1g~+hGN5&k zv?-UgDWvvclQCw4o&3WtWr9FR%ef(AaS#$$9Rv~Ql+Z&%kUWJ<;;)r3Wg(>jgoM;Y z5E5Ag!WK1+f)ogFF1&=n9Kpj{4Iv9cN??cySV;3oF*F2Sral1m5uREAQvPF-khRK~ zBupjrlnh89;2|Ns3`_?ghZwRfd}SI&y#oq2O!pw017Snz4oJ;{Ov0-h$b=4ri(K_U z*l<}07oJoIGLY*dNs~xP@PrZA5S02z#3T|_ahQZDnguDE1^HPRc$?}58_NY7%Y~aN zM3^Z?nyH4HD0>=ec^PTj8kjg3o10s>nOnH2nE5Iidm-0HqDJAu2BD&01icXz%mxV> zg^L;ng9O31FvuDP$bu^)c^D~e7$60zR{Y`h5%dgF)Y?Z#-w(Maf^Z>c!9nik7S!_* z0_{Te71r~S)^`Ifv`|#$;Ni2DWpGpmozD=e&XA%Bnzza}0IjA-RcDBnWeAjIh)`jO zQ~(`eS8mEsVaAYQ&X8^fI;5i1nYYSWpw3;S(ObCDTe#Umx2@doHA5I>-bNoP>?VA;#-?lq~VrNAQ{m!hlys z{E8OH^%11@!6Z3k&B0ZYHHR#EWdx~!&`7M+5L^&feFWV)1g?+RB#l7`wt)yz4Z%rB zy@N)wOBz5BT4ls3p(hGjzyn&k17{%DO>j1R1YX2694itx4M#%~W)Tu*5txca%_2n2 zB81JMNXRT4&K9wVM6Pj&VMEG5EF@ZG1m}WlB`oSm5l1e-DP%*c65@OcTEGmck05;z zOkW_I1C@n%oM_)6*GJfDACMbR#nJb#K`IV#MSx{$64IW5EHHrdSuj;X)IdmxX}Cy8 z9fVAZn?#BkM?mx*PFw)`JXy+aX>-mU)yGVk1KD=6nxE@%%CT8)Fuh14=|68Gqw2U<-8;esn8cktFBXy*vtCxY)Af~-}5^m-t* z5;6&?ec+W5rz)tM1M2BO&XhwUA!`!2lpL`lcy9<^2O%@yJtJ_32r>`{=@}uBN|roI zmS6=SLF7sY)DME(1`Z}6QsC;w5?&Lbjm2@unSu~Tw+O8of^#8N4~MiVguy9e2C0LP zNl@nq)<1&OM3CV)v|0wt)#s3cB6dkVc1b-hNka(YlrVrZI3)~V2)%y0uE=;wq^iwH_u3Ev1PrpNUrw zshEgY4^fF+gWzXFY5`b%fnEz>ng;2oh?z%cigBo@aj3XSu!L!dq)9NON)k5;ls5{MH3*gjS4ooKh1L*_5EG!8 z1$_M+a!mvpaD%NnfQXBOZ|j!O4+0_ZiK7w*0pj}pkTEyV#E@YCqyoZKA3^#-@M;Kr z5(45h9(a8upzAGc;4fhsqNHUmBBN<8$>65JUEw0oBQ<`yr`W_Ok*H|9oHraC8v zTsy8bb7mK9Q3rKFeN`DF4S6Lka}^y+P-Ud$DWc^Gua96mRZ*77!D}C47?64g7YVy! z)fH0d2!IFWARQt&3Atz$YkdS7Y6Fk65v-3Ol@Sk)%7{zJk(l}jR@WfvBS_7{tpFOh z<5si)A?!=Itsu97Lr*RRO%hpBw?0Cycd)YArA#;>2?SsBr8&VlTY9a^; zsdu;}4IvB+Qjc9s2N{Wi^CL1TW)g-6K`J9;1>&aRaIU0jI2wXfOt|VJNM!^kh0P<7 z%QTAE;9eMJeMFI&u)SK4qMJezGM0tQ5KMPM3Ug%jm}-z~aHuS}iwb4K)iJqd*8-6uRpNy!Qv3N5oBn#X$-JM2(}zBaauX1w4#G-8sv#lp6{zsaNJQUH2zJCMWY-Tw3ZfWtLV}d8gSeKh zhNhmHx{ig8ij9FrpqWyzg-W=ELb!#3tDdrhrks|NjJ&v@f*`BB0E?*@gQF5hlpaI8 zF>{eELxnR#xdTIuD?_CNL#+!#vnx}xD^r&TLx&qfu04O61&5KAu923msHUB$rk$X= zn~;XPh=v~sX?clgc?oNJih#Bbc|f)WA(QYG2&fF`LIqq^4@4ZcIS6_@0=yc6GvL(_ ze2pA*HQNU4(gv9I4y5)$CRxRGSjBbVBXaCw2sW}Ha%IFWrUS`S$Ru*z z1ZTr*9L)Mi%p?r13^Fp0OhPn5bU{cF({N#vFhNrgg4aZF2E4Q($N&|9rl1Ka>?(1L zD?)S)1`=KiAUg))7i1AQ8=@X<0uC+A+JkdXRFP|p{v%U4j(N7leuM&HT6(9FQl%*H^& zQdiZ-P}bW(##c|wQ%lHIPuW>VQD0qAQ&C!mmr+5G)m(zXMwU5RpCQSNt;C+8#+|v| zgQ?kvZC$D1!VHzZ0N!q2u09{;HWyHRlx4$Xre|oPqc5syC#Gc&sgH!!eL)CZ6A5d2 z3WHWGps&hCt%Ka*!)pQ>*hl2hDj+m2yc&Wt;MEX(P!3WbK@QDEu7)6Nv}y>>MXP|g z6&(m5*wqj4O+%1X3NR8{AA#x}1j%jl@Wvl zO&B5FJ;W|+0z$Cm3ivA{Ht?!B$eILD#RRU8;MEYE0k4c8^${EDh#b5QLaU@8^%11H zfmcAB5_%93TqJT$1YyG~7c>TRr=DRjgaIdoO~T-8NbLh5A+-{O4XKHcNl1N!ObVNT zHm8G%HE@9jVL*x+WD+6{Cy}L4*cjuA5ZmDziDaOapAasj497}B6yPDzbU{jcXsrS6 z%V72;AtKP8CU{jdR2R5fK{gOs98!Z|m4*0S7&gy>>{^Hvy!Qby1nXFth)J{vc-g8k3er3yByHd)A+aW89139|lkilB%m7tAu=yR#`bZR9D?wHu zKuPeb1IXAKE>gfK5QGea1wcE4K$AYuwb|gIHb|ccLc%K-2p2*k&;CHVMQ{=_LjQ}M_AuWSl>$!j38_=T~Bd6F9BU|9!+-vZBHRx zZ^$koF@0}QJud-mJ6;WINj^?-9(D&w(D~tEx(ty93>lUTdG-uBHlW*%dLoqCgJo;o z`O6$Qs-2mtoEYP*L}M(3RrMTH^&EIK-NDF%R||T75M=!}ghZ=-AY6DIgq4BZ!6C?o zjKx7}95j+w*$E;dsOH44;sB|SAS9$F;#P5h*FKmG7%@8BZ26|9ge8ZLQDNbQ46LdNTO6cAV7@yJ{9Dp)~!Mp(``l`(_Q2SFI1>IU9F z05H>^{ULHaiC{+x2ngAjWCovU3MR3$dP+721 z#IQkyEx3TC76ZA0kbw8^AUzAbB&eu24ack?kR1jO7C3`=S%^xwMtI=BMKBmx>LXC) zg1ibEymA0ESp!}SAYmK@_k%Mabr3QMQ3)Yo zXYY7}D;>}^K;Y^L%7D~ILb_mCSbZd9;4Na{Eu`lurVB!_Gkm<^=e!AOxryj{3h6rV zY1+sN@k;S?IZ1TmJ`zy(g49IdBehXFM_6kgOhJNm5V8VDjf0hh47h=6B^3wE z`UtZELKC5SeFR#QpkM{BjG%oV`1N*3Q$o zH~4^4ZW%M^8KR%Q40Y`D-m~<1TP}sMG7WE7;FgWoGx6o2GQd15{y^|xU__f z=MY0eN@)^Er1n3g2Ek84`aUF(;Lam1_h3{Rpv}(ku`*0q%s_+pQsMOxCIe~?xb}pu z96+vOz#$Cn4T0AWAoq_jZG6VtGj(6Wb|mCdj2CZOTYuLU~82lFtF#fMK7y>xhAV(@kxA745$IwF%sL271X3A+DkjwB3TU+wT4h91eFVw%{02dg zh9ab5!bJ*!r+~2*BM@^S8sU`}gaPRykwjv(0*@dubr3-{xT*tC;f!1|V`HOLPnb0k ze(yu-0=NUvoQhm`KvW_tK$eBDF>42??ZosS!EFYp`N%$n)klzO2${s%KY~OSBvrtx zAvgmr3ui!7LRv=( zfUz=QLB~)>e}-)KQYqvX|6$ zfYe8j8V6nz@oBoFoEHZ^Bmq+I2&%im>l$PRq}PLI{0&|QAv2I`BFJ4rplS%dNSjc7 zgjN$lxR~`3kAgixL`;2zwE{vD1XVOB)eyMq0f`V*A3>L7OPj(gAT$QNx`8v$>LcU| z2#t+aAF+aV4}sQlqt!vkTw>}Y%*u$Q`UsNKA?-ut5&#byQihH-`jq-KHC zAdq#}q6U8G2wWc_*Gdpw(7Fjymm!V3LFyn(60$rSQokVAJCOQFK*vu|*B=={>K90D z0%1dHB^c=~q60^u+6tmZNXrYX050V%q#Yon9RR6`AUApoYPkq%x$tYd@oBm8Xt{D} zI16dm^Q+p(Dj6uK8XIb98fj~pYAYLSDjI7j8>uU4tEp?Lswt^zDXVEqshP@ZS_o;_ z3TxW&XgTv}IrC|`@@ct3Y99!RIRpo(iI8g_JZ$9ZhA>;%2~-V%N96cb96;Bng6kmU z-Vo$$1Y{978&dUfqjr%Xf{^|Z!D zBqU#Bl8~GZ5l1T@$lyZiBQz3T6QPNN#_e&85F&~}+&wb-s3Ez`2bbrV^$so(i1`p% z2wRNe`Uq>tfEG-G>lA1;hqQhY646+7K?Fex!0Xu{^$~QM1~x|nshH4eA4s}F6NGa? z27;GW!x@kih>L`nhDkzd6JhWW9Fz^NQy^n&BH$x;L=F7l4A5YmUVsP~3F!sE5hSNT zM&U4PA50Nq>LURiKL`@k^@r9kNVn91D;ghojRW0qs^tk0ha3tAIg`r8CQwTRQb!PP8uI!>h+h08#AGxz5JBuDq)wq~H4P75>^6ee zMZ?!ugDN9fO$4c5ApIkFS|Wx4I_w&Jlqfu1;b%b1grsW72|N%IB7#DK>m%6e0#N@* z1bpxglmypAkorhOFF;t=A5tYjO#q*q0I7W-H4cOgCm|goI0>m`AY7DQ60CD1paa@i z1W|@eg2w2yydc#LnDm6#M_3smI-tc20y^#jnttH=$dgygLsS!lAUDP#*GDK75Ylo5kT}x1IINWpyvD&?x_~SNuZhs=Ajrvbkei0!H4YjBQn_G~@V*ZR z(t$tV5jp6QKFF03ygp);)MFOcK}N`R5UaQjiefve!9TpUt3!xxniX&|INB1$n?^$}#98}jT7#4bG1fvgy$4BS5gRYnG( zkZK6AT@=Rv8l)1zO5(45;Ifcziy%_73R-bs6M@t$keMA=4FWp<4O$aHY9*xlNLbfj z7+i5cHR=Vx^+Ku`NDTsEVmxo*cjWqrThkl4KH^mO zV_3G9A^L_k;ReOm^BfkR>D;u;i-u*D zAm_`0j+=wjN|1U7QU@`M>%kGLxE>fGdPb1`5vQm&C+ZF($k+wE zu7NYCTOWZ6G*ZTANL7zkep1Lqt82*MV)jSyI|*9Xz-lpkiXr_J)Dbgu7b4X}kh%#z zJ_a`#62ed}YJCK`JPs-aZbg6;6I~y{(;Hg#K?zq-KTv?U^bG0az^WKfO@yaDf^0#B zR5T*`eh>yGiM$R0A`7pNKvfcI9R#n8Ablb{^%0~J0h2`4N8suSQt5zh_lER|AV>E= z&W?lBGW;4Y0-CNu8lVaYvp#~1!okE{`P5v6zzA}21ElW*se_QuMj*95f>%bM9Yc^& zIAteDrGtkARYppVcq$`)=r&d8J$H~Q2}fmQ3){PD3pzXjHV_A?ZU~aF3J7w*DGBuv z{JwAGItZB!KiU*>f*hyLz09 zBS;;DOp;n3L24PINbpfwLV5wh`hk#HUyTDjhKC z4(aB=N$5-vcwsi=XgJifvfW`^P$ea(?gkmD6Hs##P;=u^cjZ=tUbhPA8Nmmsdtb`NF4-U1&8Z;JMba|8&GFQ-i}w^4#a>i zQ~(Pq*ud%?P>sWj!bQ|XT(XuJ$Q;r$;+8Rk^pD^hg%Au`6D}EJ1Q)z$0aW`)n?kA^ z5F1<*aZ4G2kpZ`q0hnRLEos6nX~HF83}+y(mqTSkkKN{w(#Kp52dQfy^$v`LS4Pki zejukSK>A0hB&cV^E~dvWriXuY4tao%LqrQJBB~~WjHQEfE~Gv}&a=?$kI^Uv6-xSn z_-h}^Wr-;s>8XZb1%##mQmudn?g-U%5S3IW4MHK8Uqd=aq6VQt`XPdP!N|*6An}MC zh7b`5i7XChgQT!kF_7qnCK&K=87?;Bj2+mzYe+&xR7?i`$cYei&?wkINL@n=iB=y$ zxM(D#mO&;_IyFdD5@bgzwDy72N}4`m5QerNWXuiJMFKC(M&d$de<0ICAhv)uXb28U zYCvlfaJ>WV86mU5wG8MyQb^5$jf5_HgLHQI)j@R%q>2Hrhy&dy1gUEv{T&z^RQo_g zV7(!b2;>$atS4r}>myJ%2Rt7HshH4e96Veeq~SPtrGw0X*GHh52)q{+x!&Q0UG@!U zKx!fgNlbmjBX13=hRCgtFlr)5-Govfp;bCiF1S7dRXX6oH)IBKeZ(mZsf>)kl@X#o z;*u~%MyPcV^7-4)`UtHuf*hoPt3HApf(lua03jhY5u^tMTeqOgBBskG3PC7`{~%XS zXhU;ol@YtJCIms|YKbACMTcGhXjLjW??XkvYyrIhv~dzpp+txJ2x%uH3B4*xbm8rh zfd;Hd8E*t#3<^Fd0CEKn9un$1ytcz@1acVQTd<+AMoe!PtOj{qG)k2L?!Q3$I50L^ z4MGf8OwSL5bOXd-NL0sPRL37A2$6!3klqcXCKA^3MefKTS0c!4wAu%m3$KGO+bUXrNEgbabZlntuAoUJb5;VD^>H@E8AoUS4373K}xFL(N!SxZS zVp4O092|$0gfGknF_avMtdF3T5lYVpdfOdr*v%SJ2Vs)nP7wOSY`7rgL^x2Dgw!(v zcZ(1`ATSryMFKM*;-Eu&;FCmH8Qd~vT+*iS+6OWy2O%NzL7dV?5C)ut47lG7o3_mFEM2ph9L zg3b)|02>jX(+PCtYA4(bxTwv$>7A@_qss8k<8 zYbA{S3o*41rW#1S0I!V1bORuL5QsQ338_s$i`I~;4CLAeD;uH%>*XM|40!4XR3-6gVyTHBbr7Vp17Q=aj|9{} zm64#j8)Sry4)qa_k|VSq1m2?xsf>7(?coei1%$R^2v#3Ks~br72bJVkbl``JAk|0k zY6x89J6Ic1^}tseU}Zq+BXD(tsCU4%45Ytf36TQ#hCqEEuoQL% zr;Hiod^u=M1RjWkS3__HybeNUz-uB11707&sv&(C1ABc0>F>Zu^s^M;H4&Tv8Eu1+ zNEZ^pDLX~~gftot6$dYi z<=6G+*Y$_iP&BTOh_B-yr8*5saDRbd1%#%6`1%OyaYzm_MTCTK_vJevbyKr0$Z1q7*9;3Ry02Nwf8 z&W0F;L#>aXRS)vvJZS5#p`&f!D}f-L8e|eO$c9GpDm#HLutOuk_u0Wu%!V*PH+O?; z7D#;rsdq4!WJAP}NzefcN_KcEBVI)Z$O<}MMF-H%AaGp+u6iI#<3J?%P6zM}ci?;O zcoeN6B4849*8|qN23{+HCVD_C;^5T~lmV-8p!E^BCW5fR)ez`NANbim2nI?W1gVK2 zq^t=yWPlDlC4^iDAurnoS3?Gz(oh7ck6?8WyfV^<+_(xQ!M!AQ2_p~!9q9vKEQeAb z!RsJo2BhPILPBdI4K^_iQ0GWY3yowI)kK~TViwh67S&=A)&x~UqMEEQ#38B)LLyr1 z!kQdl5?TR?Xs}~LB-KZtIb3j!Lri@HsgOetLT5oDz_gyhxofFnrdf=NPFvLRJQC}&^8t06c8S2Y9?hfMuI z`bQ9w*7XsjW+9_K0v(hM?gSC6kGSET9eDQ#8v{}wfhr>fd)QbUs2YOTO5l12>0Tj} z8&?U}N1QTd;JOJ~4Po?yAaxKj39XMHOSRFq5uw#bkXa$nz#Wci2(2;#PXj?`d^n_# zDgTp39i!FooJDhX5{A=f@=Y!)%lqz_7EB&NeIt_`k$u-8Y> z>IUhYA4q)!s*FUmpt7QxEbtl!Ha!HbjD$6@Rz~b18j$=*JP8>OL9S(>MGCmAK<*_$ z*%a3}$aZ0}p|&HPicDP*+~u>ckUkm$?PtNQ9$i*I-%k)$(TGxMbPKKI2lX$&Rv?Q* z=2;*jkog@58$uHAeuzqt71#nFmpBz-N7oLy$EIu$l-mA_uCE;MEY9yd|8C#=tpJ2R_M%Pr;l|0k%I#9#kK}IziwX zN7fv?707~9#)4bMf=AYpN7j-X#5P0R?|`v=719ZU>|KShk-I|NQbs(|h|NUc-jES8 ziK9LOkIqSgNNh7cpt=cC6M?6J40t3!h)YZljC8ofuprF!*lc22u*yga+&O|&NgD84 z2F?I=d_+OL9u9cL1nw5Gh^VoMsG-z2$n_DtlETKo%%ONhKpQ2%yUQW95`=`529Q=O zv<`ywp&;ZCua7WmK$wUR5w3<0l35{RF^AKttA&L8=2{NRorRN!R^5H^B@j4D%HA3^#!)T)o*^$3Ik zuZ-Xo3Pc=E!fPVP6|0adt)X&X5`hMq;UtFe@Xp`UtJp!wXx7 z0Ga5491RCyL+ToWBqyw=1Fw}p^$Su3#G_~ptA^mUk{!H?0V}qH_JGK$k04bNghamQ z4&3p9ErH>iTQtioy{NIwWvAAu_)Gfo+3eFUzFKot|D2L$OHVUm#A z2XZ_D=5hrtNkeWalnMyBGUAjr!e1Xjsv9H{vH}57A3GSWk?j39LoG6|m; z0x?jA<}_GE)nN!R3J0l@FzX{05e+aRTpzIstFsBKqtsd8>IpU{3%VhN7&h#zdv6Q| z;`Dn+Jp~~_O<70*LL5nCc{OO1fo`IK?dS)wA!RkB+$N3$b+V9a1pI9BJPt7vt6qo{ zoJ3wBKnxqJnfL_}H4Bb91+G9?&sS8}2l<##FdMYg4YLP?yq{D^%L~4Q0KAP9w5|YB z0iltQ`UtZhgi8due*{?t2O;703mOA!ox%@35&_b|K_hXghlqnJBQ;k6b!Q~xD4_0$ z#Dz-ngZnuA;1zBVHe}?DSJ@eioM2oS18MC6qz;1AN02?I5E5R?pfMn_0?PKtyHK(2 z27=T$FcMTH@hjQzDOzLe`#^d?2ojY6!ka8+p7A)F(o!fWTD`Xum3m0Uo%6?pg&Ok^tHY#18Eiq0~pnbr6IN z9+X3$8-fc$>K!Nv=>&m$K+u{9Ha7%cG6$}VG}u9%BWV8!Yo!CJjIff#)IrEEyI_w&2eI%p>y`>aV2O*Pa)ewXWUipS)s7*k_9a0~` z84#|ZCg`A1$W#s*iB=5}?C+peLvSvn-hr;oMp~5(sfi#Ya_z&f<^V=ckV*%Qgw!lh z61*52Tp2l|S4PTM7G^{4ae&t~0AWOBu^GAl5l@X*91STOX;y~39p85z< z0ilubst1h$t%Ja|50p))K4KRJO&EdajMTtY5@N*y)~W}qAfC}V98#XhTgc&TTqTTv zmJdk?mpMa49MUT!SQEh&z|ZhRuGH|dNvo>h+i-}i+Te8!@)9*T8zKwmf|d;s>f=xv zaj=s@Asrbc61w0Gtzv@oU*OdToBCLt9HTmgK94L+oXl>yO(M)GQa7Pi5L&Y;sb zkSYdJgTPP2hF3QbS;#^-2pggut{xWyG^z&c`+#d7ob?g38iI9vpeMqC&%}oGebBl& z{3;HDDh}{^2U!+UD?wMm!3O1cQBRG7Z$yPxKxhm|bpuxhS%(0xkNCioL_CV%LwjH= zwBeNzH`1OUaD9YQ6QNBH@yT2AfmX^P2IwGl6NH4U!G_i~;CUZtj|RL&2v!q;M(3bv zz^9l(Yag&0(6R*h-c_Xa3CQ&kqymDR)dTPT!0RK(cpbNt5tk&qH-yPR9Hf9&8F5O0 zM&#H;b-|Sp^!x+}8?!#*6x9VI&^dBwl@aJm&Fo1LMbihvDG8s@iAY_N(Z?DLSvKR z4D9Jd8%z2D84f$Y8mT4%?H)z$`@q@Y{t;;N4y1B{)HRT51+4~wbCK0RRN^P$8X*kG zf;VF7BeePjQu|<%Xv)yIkhR(1Y6#vT!d@T2Dm9fVgn_H? z0}+RiNEHoYkPT8LL24pMX;TP-*GK3Ky!8>gv=OVM0eln=xjup}(#BmMK{`he61hHt)H|po zdjAM=S^~Hlf>cJjm^BeZ1i4#8ph_aQKEhh>;1VRhCW5HMsD|L(B`oC$sN@-u^%10( zM34lp2Y^;OxCcVvBcc!n4XY&3(mR|dt$}83&?+NH{|FSz7_|v#=?jJ+R+ab#(W)T` z7hH)Tj%$P0M}nHr6Rk1p6iD@fNkUZetGhuM5E9OXGa%vupz9c*N4i03A|7?<9jTZ# z3q%C1LVDGO>sV{7W;62~+FzX&dKaETMFgU}Sf`$vPJJ`w^g9)Q+6XyafIE~Gw! zupuP=(4$-yQUSrYmx5~|$czv6`UtYR0A78-8JIN@L( z!*pm>47`3pW5AW6?^i`Sp9gDY1Ui!kHUtN*dT`c9yebaJRT3HX5v2BkkITWiunNT% zYrVsxh*KP{5zGMXSA|q1B-KaYDhae!0iqX9LMj~)SJoU-A3;RWNc{B?xEg{~M&^*7 z5rjmmh9D=)fk_n;cufRhAlFCmItZBoR{*PqAe)8^AT<$$g!GTFRz?F@ zA3?_AD6fxLh1B4P_?if&5;PuyJPr?I<5fs?kus(v+LC7RrFYydmLc)cI4>aLiJ zt06TGq)tJrjNmucVd{dY$3;R`XoIUEc!!Am`UqaXKp1?myM!QIWRhC_Bgj}BxH7VZ zY!QMjhXW175!pWica+fQhL9^A$VuABBFJnC>myJd1i9Y= 23 is required to build the TensorFlow Android demo (though it will run on API >= 21 devices). - - The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The current recommended version is 14b, which can be found [here](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads). - - The Android SDK and build tools may be obtained [here](https://developer.android.com/tools/revisions/build-tools.html), or alternatively as part of [Android Studio](https://developer.android.com/studio/index.html). Build tools API >= 23 is required to build the TF Android demo (though it will run on API >= 21 devices). - In the root of the TensorFlow repository update the `WORKSPACE` file with the `api_level` and location of the SDK and NDK. If you installed it with AndroidStudio the SDK path can be found in the SDK manager, and the default NDK path is:`{SDK path}/ndk-bundle.` ``` - Android_sdk_repository ( - name = "androidsdk", - api_level = 23, - build_tools_version = "23.0.2", - path = "/home/xxxx/android-sdk-linux/", ) +android_sdk_repository ( + name = "androidsdk", + api_level = 23, + build_tools_version = "23.0.2", + path = "/home/xxxx/android-sdk-linux/", +) android_ndk_repository( - name="androidndk", - path="/home/xxxx/android-ndk-r10e/", - api_level=19) - + name = "androidndk", + path = "/home/xxxx/android-ndk-r10e/", + api_level = 19, +) ``` -Additional details on building with Android can be found [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md) + +Additional details on building with Android can be found [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md). ### Build the source code Run bazel with the following command to build the demo. Build the demo app: -bazel build --cxxopt='--std=c++11' //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo + +``` +bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo +``` + +### Note + +Currently, we only support building the Android demo app within a Python 2 +environment (due to a Bazel bug). ### More about the demo The demo is resizing each camera image frame to (224 width * 224 height) to match the quantized Mobilenet model being used. The resized image is converted into a ByteBuffer row by row of size 1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch 224 * 224 is the width and height of the image 3 bytes represents three colors of a pixel. This demo uses the TensorFlow Lite Java inference API for models which take a single input and provide a single output. This outputs a two-dimensional array, with the first dimension being the category index and the second dimension being the confidence of classification. The Mobilenet model has 1001 unique categories and the app sorts the probabilities of all the categories and displays the top three. The Mobilenet quantized model is bundled within the assets directory of the app. @@ -95,7 +101,7 @@ The demo is resizing each camera image frame to (224 width * 224 height) to matc [On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html) is an on-device model which provides one-touch replies for an incoming text message by suggesting contextually relevant messages. The model is built specifically for memory constrained devices such as watches & phones and it has been successfully used to surface [Smart Replies on Android Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html). Note that this model only works on Android as of now. -These pre-trained models can be downloaded from [here](models.md). +These pre-trained models can be downloaded from [here](g3doc/models.md). ### Retrain Inception-V3 or MobileNet for a custom data set The above pre-trained models have been trained on the ImageNet data set, which consists of 1000 predefined classes. A model will need to be re-trained if these classes are not relevant or useful for a given use case. This technique is called transfer learning, which starts with a model that has been already trained on a problem and will then be retrained on a similar problem. Deep learning from scratch can take days, but transfer learning can be done fairly quickly. In order to do this, a developer will need to generate their custom data set labeled with the relevant classes. @@ -104,7 +110,7 @@ The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tenso ### Train a custom model -A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow’s Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model. +A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model. TensorFlow Lite currently supports a subset of TensorFlow operators. Please refer to [this document](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for details of supported operators and their usage. This set will continue to expand in future releases of Tensorflow Lite. @@ -128,9 +134,9 @@ Since we employ several formats, the following definitions may be useful: - TensorFlow lite model (.lite) - a serialized flatbuffer, containing TensorFlow lite operators and Tensors for the TensorFlow lite interpreter. This is most analogous to TensorFlow frozen GraphDefs. ### Freeze Graph -To use this .pb GraphDef file within TensorFlow Lite, the application developer will need checkpoints containing trained weight parameters. The .pb contains only the structure of the graph. The process of merging the checkpoint values with the graph structure is known as “freezing” the graph. +To use this .pb GraphDef file within TensorFlow Lite, the application developer will need checkpoints containing trained weight parameters. The .pb contains only the structure of the graph. The process of merging the checkpoint values with the graph structure is known as "freezing" the graph. -The developer should know where the checkpoints folder is present or checkpoints can also be downloaded for a pre-trained model (Example: Here is a link to the [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md) +The developer should know where the checkpoints folder is present or checkpoints can also be downloaded for a pre-trained model (Example: Here is a link to the [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)). Graph freezing can be done using the command below (and modifying the arguments appropriately) @@ -155,7 +161,7 @@ Here is a sample command line to convert the frozen Graphdef to '.lite' format f bazel build tensorflow/contrib/lite/toco:toco bazel-bin/tensorflow/contrib/lite/toco/toco -- \ - --input_file=(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \ + --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \ --input_format=TENSORFLOW_GRAPHDEF --output_format=TFLITE \ --output_file=/tmp/mobilenet_v1_1.0_224.lite --inference_type=FLOAT \ --input_type=FLOAT --input_arrays=input \ @@ -183,18 +189,18 @@ with tf.Session() as sess: ``` For detailed instructions on how to use the Tensorflow Optimizing Converter, please see [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md). -You may refer to the [Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/tf_ops_compatibility.md) for troubleshooting help. If that doesn’t help, please file an [issue](https://github.com/tensorflow/tensorflow/issues). +You may refer to the [Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for troubleshooting help. If that doesn't help, please file an [issue](https://github.com/tensorflow/tensorflow/issues). ## Step 3. Use the TensorFlow Lite model for inference in a mobile app After completion of Step 2 the developer should have a .lite model. ### For Android -Because Android apps need to be written in Java, and core TensorFlow is in C++, a JNI library is provided to interface between the two. Its interface is aimed only at inference, so it provides the ability to load a graph, set up inputs, and run the model to calculate particular outputs. The full documentation for the set of methods can be seen [here](https://github.com/TensorFlow/TensorFlow/blob/master/TensorFlow/contrib/lite/g3doc/). The demo app is also open sourced on [github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app). +Because Android apps need to be written in Java, and core TensorFlow is in C++, a JNI library is provided to interface between the two. Its interface is aimed only at inference, so it provides the ability to load a graph, set up inputs, and run the model to calculate particular outputs. The full documentation for the set of methods can be seen [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/). The demo app is also open sourced on [github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app). -The [demo app] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app) uses this interface, so it’s a good place to look for example usage. You can also download the prebuilt binary [here](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk). +The [demo app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app) uses this interface, so it's a good place to look for example usage. You can also download the prebuilt binary [here](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk). -Note that you’d need to follow instructions for installing TensorFlow on Android, setting up bazel and Android Studio outlined [here](https://www.tensorflow.org/mobile/android_build). +Note that you'd need to follow instructions for installing TensorFlow on Android, setting up bazel and Android Studio outlined [here](https://www.tensorflow.org/mobile/android_build). ### For iOS -Follow the documentation [here](https://github.com/TensorFlow/TensorFlow/blob/master/TensorFlow/contrib/lite/g3doc/ios.md) to get integrate a TFLite model into your app. +Follow the documentation [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md) to get integrate a TFLite model into your app. diff --git a/tensorflow/contrib/lite/build_ios_universal_lib.sh b/tensorflow/contrib/lite/build_ios_universal_lib.sh new file mode 100755 index 0000000000..cbc96e6edd --- /dev/null +++ b/tensorflow/contrib/lite/build_ios_universal_lib.sh @@ -0,0 +1,31 @@ +#!/bin/bash -x +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=x86_64 -j 8 +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=i386 -j 8 +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7 -j 8 +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7s -j 8 +make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=arm64 -j 8 + +lipo \ +tensorflow/contrib/lite/gen/lib/ios_x86_64/libtensorflow-lite.a \ +tensorflow/contrib/lite/gen/lib/ios_i386/libtensorflow-lite.a \ +tensorflow/contrib/lite/gen/lib/ios_armv7/libtensorflow-lite.a \ +tensorflow/contrib/lite/gen/lib/ios_armv7s/libtensorflow-lite.a \ +tensorflow/contrib/lite/gen/lib/ios_arm64/libtensorflow-lite.a \ +-create \ +-output tensorflow/contrib/lite/gen/lib/libtensorflow-lite.a diff --git a/tensorflow/contrib/lite/download_dependencies.sh b/tensorflow/contrib/lite/download_dependencies.sh new file mode 100755 index 0000000000..778d618361 --- /dev/null +++ b/tensorflow/contrib/lite/download_dependencies.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +DOWNLOADS_DIR=tensorflow/contrib/lite/downloads +BZL_FILE_PATH=tensorflow/workspace.bzl + +EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" +GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" +GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz" +ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)" +NEON_2_SSE_URL="https://github.com/intel/ARM_NEON_2_x86_SSE/archive/master.zip" +FARMHASH_URL="https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz" +FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/master.zip" +MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_1.0_224_ios_lite_float_2017_11_08.zip" +QUANTIZED_MODELS_URL="https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip" + +# TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64, +# so work around it by patching the source. +replace_by_sed() { + local regex="${1}" + shift + # Detect the version of sed by the return value of "--version" flag. GNU-sed + # supports "--version" while BSD-sed doesn't. + if ! sed --version >/dev/null 2>&1; then + # BSD-sed. + sed -i '' -e "${regex}" "$@" + else + # GNU-sed. + sed -i -e "${regex}" "$@" + fi +} + +download_and_extract() { + local usage="Usage: download_and_extract URL DIR" + local url="${1:?${usage}}" + local dir="${2:?${usage}}" + echo "downloading ${url}" >&2 + mkdir -p "${dir}" + if [[ "${url}" == *gz ]]; then + curl -Ls "${url}" | tar -C "${dir}" --strip-components=1 -xz + elif [[ "${url}" == *zip ]]; then + tempdir=$(mktemp -d) + tempdir2=$(mktemp -d) + + curl -L ${url} > ${tempdir}/zipped.zip + unzip ${tempdir}/zipped.zip -d ${tempdir2} + + # If the zip file contains nested directories, extract the files from the + # inner directory. + if ls ${tempdir2}/*/* 1> /dev/null 2>&1; then + # unzip has no strip components, so unzip to a temp dir, and move the + # files we want from the tempdir to destination. + cp -R ${tempdir2}/*/* ${dir}/ + else + cp -R ${tempdir2}/* ${dir}/ + fi + rm -rf ${tempdir2} ${tempdir} + fi + + # Delete any potential BUILD files, which would interfere with Bazel builds. + find "${dir}" -type f -name '*BUILD' -delete +} + +download_and_extract "${EIGEN_URL}" "${DOWNLOADS_DIR}/eigen" +download_and_extract "${GEMMLOWP_URL}" "${DOWNLOADS_DIR}/gemmlowp" +download_and_extract "${GOOGLETEST_URL}" "${DOWNLOADS_DIR}/googletest" +download_and_extract "${ABSL_URL}" "${DOWNLOADS_DIR}/absl" +download_and_extract "${NEON_2_SSE_URL}" "${DOWNLOADS_DIR}/neon_2_sse" +download_and_extract "${FARMHASH_URL}" "${DOWNLOADS_DIR}/farmhash" +download_and_extract "${FLATBUFFERS_URL}" "${DOWNLOADS_DIR}/flatbuffers" +download_and_extract "${MODELS_URL}" "${DOWNLOADS_DIR}/models" +download_and_extract "${QUANTIZED_MODELS_URL}" "${DOWNLOADS_DIR}/quantized_models" + +replace_by_sed 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \ + "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" +replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#static uint32x2_t p2ui_CONJ_XOR;// = vld1_u32( conj_XOR_DATA ); - Removed by scripts#' \ + "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" +replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \ + "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" + +cp ${DOWNLOADS_DIR}/models/models/* tensorflow/contrib/lite/examples/ios/simple/data/ +cp ${DOWNLOADS_DIR}/quantized_models/* tensorflow/contrib/lite/examples/ios/camera/data/ + +echo "download_dependencies.sh completed successfully." >&2 diff --git a/tensorflow/contrib/lite/examples/ios/camera/.gitignore b/tensorflow/contrib/lite/examples/ios/camera/.gitignore new file mode 100644 index 0000000000..9e8962f4c6 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/.gitignore @@ -0,0 +1,2 @@ +/data/*.txt +/data/*.tflite diff --git a/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.h b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.h new file mode 100644 index 0000000000..55891c3ee1 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.h @@ -0,0 +1,21 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +@interface CameraExampleAppDelegate : UIResponder + +@property(strong, nonatomic) UIWindow* window; + +@end diff --git a/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.m b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.m new file mode 100644 index 0000000000..128266d53f --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleAppDelegate.m @@ -0,0 +1,44 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "CameraExampleAppDelegate.h" + +@implementation CameraExampleAppDelegate + +@synthesize window = _window; + +- (BOOL)application:(UIApplication *)application + didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + [self.window makeKeyAndVisible]; + return YES; +} + +- (void)applicationWillResignActive:(UIApplication *)application { + [[UIApplication sharedApplication] setIdleTimerDisabled:NO]; +} + +- (void)applicationDidEnterBackground:(UIApplication *)application { +} + +- (void)applicationWillEnterForeground:(UIApplication *)application { +} + +- (void)applicationDidBecomeActive:(UIApplication *)application { + [[UIApplication sharedApplication] setIdleTimerDisabled:YES]; +} + +- (void)applicationWillTerminate:(UIApplication *)application { +} + +@end diff --git a/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.h b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.h new file mode 100644 index 0000000000..fb5800e86d --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.h @@ -0,0 +1,48 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import + +#include + +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" + +@interface CameraExampleViewController + : UIViewController { + IBOutlet UIView* previewView; + AVCaptureVideoPreviewLayer* previewLayer; + AVCaptureVideoDataOutput* videoDataOutput; + dispatch_queue_t videoDataOutputQueue; + UIView* flashView; + BOOL isUsingFrontFacingCamera; + NSMutableDictionary* oldPredictionValues; + NSMutableArray* labelLayers; + AVCaptureSession* session; + + std::vector labels; + std::unique_ptr model; + tflite::ops::builtin::BuiltinOpResolver resolver; + std::unique_ptr interpreter; + + double total_latency; + int total_count; +} +@property(strong, nonatomic) CATextLayer* predictionTextLayer; + +- (IBAction)takePicture:(id)sender; +- (IBAction)switchCameras:(id)sender; + +@end diff --git a/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.mm b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.mm new file mode 100644 index 0000000000..ea398ad14e --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/CameraExampleViewController.mm @@ -0,0 +1,506 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "CameraExampleViewController.h" +#import +#import +#import +#import + +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" +#include "tensorflow/contrib/lite/string_util.h" +#include "tensorflow/contrib/lite/tools/mutable_op_resolver.h" + +#define LOG(x) std::cerr + +// If you have your own model, modify this to the file name, and make sure +// you've added the file to your app resources too. +static NSString* model_file_name = @"mobilenet_quant_v1_224"; +static NSString* model_file_type = @"tflite"; + +// If you have your own model, point this to the labels file. +static NSString* labels_file_name = @"labels"; +static NSString* labels_file_type = @"txt"; + +// These dimensions need to match those the model was trained with. +static const int wanted_input_width = 224; +static const int wanted_input_height = 224; +static const int wanted_input_channels = 3; + +static NSString* FilePathForResourceName(NSString* name, NSString* extension) { + NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension]; + if (file_path == NULL) { + LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." << [extension UTF8String] + << "' in bundle."; + } + return file_path; +} + +static void LoadLabels(NSString* file_name, NSString* file_type, + std::vector* label_strings) { + NSString* labels_path = FilePathForResourceName(file_name, file_type); + if (!labels_path) { + LOG(ERROR) << "Failed to find model proto at" << [file_name UTF8String] + << [file_type UTF8String]; + } + std::ifstream t; + t.open([labels_path UTF8String]); + std::string line; + while (t) { + std::getline(t, line); + label_strings->push_back(line); + } + t.close(); +} + +// Returns the top N confidence values over threshold in the provided vector, +// sorted by confidence in descending order. +static void GetTopN(const uint8_t* prediction, const int prediction_size, const int num_results, + const float threshold, std::vector>* top_results) { + // Will contain top N results in ascending order. + std::priority_queue, std::vector>, + std::greater>> + top_result_pq; + + const long count = prediction_size; + for (int i = 0; i < count; ++i) { + const float value = prediction[i] / 255.0; + // Only add it if it beats the threshold and has a chance at being in + // the top N. + if (value < threshold) { + continue; + } + + top_result_pq.push(std::pair(value, i)); + + // If at capacity, kick the smallest value out. + if (top_result_pq.size() > num_results) { + top_result_pq.pop(); + } + } + + // Copy to output vector and reverse into descending order. + while (!top_result_pq.empty()) { + top_results->push_back(top_result_pq.top()); + top_result_pq.pop(); + } + std::reverse(top_results->begin(), top_results->end()); +} + +@interface CameraExampleViewController (InternalMethods) +- (void)setupAVCapture; +- (void)teardownAVCapture; +@end + +@implementation CameraExampleViewController + +- (void)setupAVCapture { + NSError* error = nil; + + session = [AVCaptureSession new]; + if ([[UIDevice currentDevice] userInterfaceIdiom] == UIUserInterfaceIdiomPhone) + [session setSessionPreset:AVCaptureSessionPreset640x480]; + else + [session setSessionPreset:AVCaptureSessionPresetPhoto]; + + AVCaptureDevice* device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + AVCaptureDeviceInput* deviceInput = + [AVCaptureDeviceInput deviceInputWithDevice:device error:&error]; + assert(error == nil); + + if ([session canAddInput:deviceInput]) [session addInput:deviceInput]; + + videoDataOutput = [AVCaptureVideoDataOutput new]; + + NSDictionary* rgbOutputSettings = + [NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCMPixelFormat_32BGRA] + forKey:(id)kCVPixelBufferPixelFormatTypeKey]; + [videoDataOutput setVideoSettings:rgbOutputSettings]; + [videoDataOutput setAlwaysDiscardsLateVideoFrames:YES]; + videoDataOutputQueue = dispatch_queue_create("VideoDataOutputQueue", DISPATCH_QUEUE_SERIAL); + [videoDataOutput setSampleBufferDelegate:self queue:videoDataOutputQueue]; + + if ([session canAddOutput:videoDataOutput]) [session addOutput:videoDataOutput]; + [[videoDataOutput connectionWithMediaType:AVMediaTypeVideo] setEnabled:YES]; + + previewLayer = [[AVCaptureVideoPreviewLayer alloc] initWithSession:session]; + [previewLayer setBackgroundColor:[[UIColor blackColor] CGColor]]; + [previewLayer setVideoGravity:AVLayerVideoGravityResizeAspect]; + CALayer* rootLayer = [previewView layer]; + [rootLayer setMasksToBounds:YES]; + [previewLayer setFrame:[rootLayer bounds]]; + [rootLayer addSublayer:previewLayer]; + [session startRunning]; + + if (error) { + NSString* title = [NSString stringWithFormat:@"Failed with error %d", (int)[error code]]; + UIAlertController* alertController = + [UIAlertController alertControllerWithTitle:title + message:[error localizedDescription] + preferredStyle:UIAlertControllerStyleAlert]; + UIAlertAction* dismiss = + [UIAlertAction actionWithTitle:@"Dismiss" style:UIAlertActionStyleDefault handler:nil]; + [alertController addAction:dismiss]; + [self presentViewController:alertController animated:YES completion:nil]; + [self teardownAVCapture]; + } +} + +- (void)teardownAVCapture { + [previewLayer removeFromSuperlayer]; +} + +- (AVCaptureVideoOrientation)avOrientationForDeviceOrientation: + (UIDeviceOrientation)deviceOrientation { + AVCaptureVideoOrientation result = (AVCaptureVideoOrientation)(deviceOrientation); + if (deviceOrientation == UIDeviceOrientationLandscapeLeft) + result = AVCaptureVideoOrientationLandscapeRight; + else if (deviceOrientation == UIDeviceOrientationLandscapeRight) + result = AVCaptureVideoOrientationLandscapeLeft; + return result; +} + +- (IBAction)takePicture:(id)sender { + if ([session isRunning]) { + [session stopRunning]; + [sender setTitle:@"Continue" forState:UIControlStateNormal]; + + flashView = [[UIView alloc] initWithFrame:[previewView frame]]; + [flashView setBackgroundColor:[UIColor whiteColor]]; + [flashView setAlpha:0.f]; + [[[self view] window] addSubview:flashView]; + + [UIView animateWithDuration:.2f + animations:^{ + [flashView setAlpha:1.f]; + } + completion:^(BOOL finished) { + [UIView animateWithDuration:.2f + animations:^{ + [flashView setAlpha:0.f]; + } + completion:^(BOOL finished) { + [flashView removeFromSuperview]; + flashView = nil; + }]; + }]; + + } else { + [session startRunning]; + [sender setTitle:@"Freeze Frame" forState:UIControlStateNormal]; + } +} + +- (void)captureOutput:(AVCaptureOutput*)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection*)connection { + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + CFRetain(pixelBuffer); + [self runModelOnFrame:pixelBuffer]; + CFRelease(pixelBuffer); +} + +- (void)runModelOnFrame:(CVPixelBufferRef)pixelBuffer { + assert(pixelBuffer != NULL); + + OSType sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + int doReverseChannels; + if (kCVPixelFormatType_32ARGB == sourcePixelFormat) { + doReverseChannels = 1; + } else if (kCVPixelFormatType_32BGRA == sourcePixelFormat) { + doReverseChannels = 0; + } else { + assert(false); // Unknown source format + } + + const int sourceRowBytes = (int)CVPixelBufferGetBytesPerRow(pixelBuffer); + const int image_width = (int)CVPixelBufferGetWidth(pixelBuffer); + const int fullHeight = (int)CVPixelBufferGetHeight(pixelBuffer); + + CVPixelBufferLockFlags unlockFlags = kNilOptions; + CVPixelBufferLockBaseAddress(pixelBuffer, unlockFlags); + + unsigned char* sourceBaseAddr = (unsigned char*)(CVPixelBufferGetBaseAddress(pixelBuffer)); + int image_height; + unsigned char* sourceStartAddr; + if (fullHeight <= image_width) { + image_height = fullHeight; + sourceStartAddr = sourceBaseAddr; + } else { + image_height = image_width; + const int marginY = ((fullHeight - image_width) / 2); + sourceStartAddr = (sourceBaseAddr + (marginY * sourceRowBytes)); + } + const int image_channels = 4; + assert(image_channels >= wanted_input_channels); + uint8_t* in = sourceStartAddr; + + int input = interpreter->inputs()[0]; + + uint8_t* out = interpreter->typed_tensor(input); + for (int y = 0; y < wanted_input_height; ++y) { + uint8_t* out_row = out + (y * wanted_input_width * wanted_input_channels); + for (int x = 0; x < wanted_input_width; ++x) { + const int in_x = (y * image_width) / wanted_input_width; + const int in_y = (x * image_height) / wanted_input_height; + uint8_t* in_pixel = in + (in_y * image_width * image_channels) + (in_x * image_channels); + uint8_t* out_pixel = out_row + (x * wanted_input_channels); + for (int c = 0; c < wanted_input_channels; ++c) { + out_pixel[c] = in_pixel[c]; + } + } + } + + double startTimestamp = [[NSDate new] timeIntervalSince1970]; + if (interpreter->Invoke() != kTfLiteOk) { + LOG(FATAL) << "Failed to invoke!"; + } + double endTimestamp = [[NSDate new] timeIntervalSince1970]; + total_latency += (endTimestamp - startTimestamp); + total_count += 1; + NSLog(@"Time: %.4lf, avg: %.4lf, count: %d", endTimestamp - startTimestamp, + total_latency / total_count, total_count); + + const int output_size = 1000; + const int kNumResults = 5; + const float kThreshold = 0.1f; + + std::vector> top_results; + + uint8_t* output = interpreter->typed_output_tensor(0); + GetTopN(output, output_size, kNumResults, kThreshold, &top_results); + + NSMutableDictionary* newValues = [NSMutableDictionary dictionary]; + for (const auto& result : top_results) { + const float confidence = result.first; + const int index = result.second; + NSString* labelObject = [NSString stringWithUTF8String:labels[index].c_str()]; + NSNumber* valueObject = [NSNumber numberWithFloat:confidence]; + [newValues setObject:valueObject forKey:labelObject]; + } + dispatch_async(dispatch_get_main_queue(), ^(void) { + [self setPredictionValues:newValues]; + }); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, unlockFlags); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); +} + +- (void)dealloc { + [self teardownAVCapture]; +} + +- (void)didReceiveMemoryWarning { + [super didReceiveMemoryWarning]; +} + +- (void)viewDidLoad { + [super viewDidLoad]; + labelLayers = [[NSMutableArray alloc] init]; + oldPredictionValues = [[NSMutableDictionary alloc] init]; + + NSString* graph_path = FilePathForResourceName(model_file_name, @"tflite"); + model = tflite::FlatBufferModel::BuildFromFile([graph_path UTF8String]); + if (!model) { + LOG(FATAL) << "Failed to mmap model " << graph_path; + } + LOG(INFO) << "Loaded model " << graph_path; + model->error_reporter(); + LOG(INFO) << "resolved reporter"; + + tflite::ops::builtin::BuiltinOpResolver resolver; + LoadLabels(labels_file_name, labels_file_type, &labels); + + tflite::InterpreterBuilder(*model, resolver)(&interpreter); + if (!interpreter) { + LOG(FATAL) << "Failed to construct interpreter"; + } + if (interpreter->AllocateTensors() != kTfLiteOk) { + LOG(FATAL) << "Failed to allocate tensors!"; + } + + [self setupAVCapture]; +} + +- (void)viewDidUnload { + [super viewDidUnload]; +} + +- (void)viewWillAppear:(BOOL)animated { + [super viewWillAppear:animated]; +} + +- (void)viewDidAppear:(BOOL)animated { + [super viewDidAppear:animated]; +} + +- (void)viewWillDisappear:(BOOL)animated { + [super viewWillDisappear:animated]; +} + +- (void)viewDidDisappear:(BOOL)animated { + [super viewDidDisappear:animated]; +} + +- (BOOL)shouldAutorotateToInterfaceOrientation:(UIInterfaceOrientation)interfaceOrientation { + return (interfaceOrientation == UIInterfaceOrientationPortrait); +} + +- (BOOL)prefersStatusBarHidden { + return YES; +} + +- (void)setPredictionValues:(NSDictionary*)newValues { + const float decayValue = 0.75f; + const float updateValue = 0.25f; + const float minimumThreshold = 0.01f; + + NSMutableDictionary* decayedPredictionValues = [[NSMutableDictionary alloc] init]; + for (NSString* label in oldPredictionValues) { + NSNumber* oldPredictionValueObject = [oldPredictionValues objectForKey:label]; + const float oldPredictionValue = [oldPredictionValueObject floatValue]; + const float decayedPredictionValue = (oldPredictionValue * decayValue); + if (decayedPredictionValue > minimumThreshold) { + NSNumber* decayedPredictionValueObject = [NSNumber numberWithFloat:decayedPredictionValue]; + [decayedPredictionValues setObject:decayedPredictionValueObject forKey:label]; + } + } + oldPredictionValues = decayedPredictionValues; + + for (NSString* label in newValues) { + NSNumber* newPredictionValueObject = [newValues objectForKey:label]; + NSNumber* oldPredictionValueObject = [oldPredictionValues objectForKey:label]; + if (!oldPredictionValueObject) { + oldPredictionValueObject = [NSNumber numberWithFloat:0.0f]; + } + const float newPredictionValue = [newPredictionValueObject floatValue]; + const float oldPredictionValue = [oldPredictionValueObject floatValue]; + const float updatedPredictionValue = (oldPredictionValue + (newPredictionValue * updateValue)); + NSNumber* updatedPredictionValueObject = [NSNumber numberWithFloat:updatedPredictionValue]; + [oldPredictionValues setObject:updatedPredictionValueObject forKey:label]; + } + NSArray* candidateLabels = [NSMutableArray array]; + for (NSString* label in oldPredictionValues) { + NSNumber* oldPredictionValueObject = [oldPredictionValues objectForKey:label]; + const float oldPredictionValue = [oldPredictionValueObject floatValue]; + if (oldPredictionValue > 0.05f) { + NSDictionary* entry = @{@"label" : label, @"value" : oldPredictionValueObject}; + candidateLabels = [candidateLabels arrayByAddingObject:entry]; + } + } + NSSortDescriptor* sort = [NSSortDescriptor sortDescriptorWithKey:@"value" ascending:NO]; + NSArray* sortedLabels = + [candidateLabels sortedArrayUsingDescriptors:[NSArray arrayWithObject:sort]]; + + const float leftMargin = 10.0f; + const float topMargin = 10.0f; + + const float valueWidth = 48.0f; + const float valueHeight = 18.0f; + + const float labelWidth = 246.0f; + const float labelHeight = 18.0f; + + const float labelMarginX = 5.0f; + const float labelMarginY = 5.0f; + + [self removeAllLabelLayers]; + + int labelCount = 0; + for (NSDictionary* entry in sortedLabels) { + NSString* label = [entry objectForKey:@"label"]; + NSNumber* valueObject = [entry objectForKey:@"value"]; + const float value = [valueObject floatValue]; + const float originY = topMargin + ((labelHeight + labelMarginY) * labelCount); + const int valuePercentage = (int)roundf(value * 100.0f); + + const float valueOriginX = leftMargin; + NSString* valueText = [NSString stringWithFormat:@"%d%%", valuePercentage]; + + [self addLabelLayerWithText:valueText + originX:valueOriginX + originY:originY + width:valueWidth + height:valueHeight + alignment:kCAAlignmentRight]; + + const float labelOriginX = (leftMargin + valueWidth + labelMarginX); + + [self addLabelLayerWithText:[label capitalizedString] + originX:labelOriginX + originY:originY + width:labelWidth + height:labelHeight + alignment:kCAAlignmentLeft]; + + labelCount += 1; + if (labelCount > 4) { + break; + } + } +} + +- (void)removeAllLabelLayers { + for (CATextLayer* layer in labelLayers) { + [layer removeFromSuperlayer]; + } + [labelLayers removeAllObjects]; +} + +- (void)addLabelLayerWithText:(NSString*)text + originX:(float)originX + originY:(float)originY + width:(float)width + height:(float)height + alignment:(NSString*)alignment { + CFTypeRef font = (CFTypeRef) @"Menlo-Regular"; + const float fontSize = 12.0; + const float marginSizeX = 5.0f; + const float marginSizeY = 2.0f; + + const CGRect backgroundBounds = CGRectMake(originX, originY, width, height); + const CGRect textBounds = CGRectMake((originX + marginSizeX), (originY + marginSizeY), + (width - (marginSizeX * 2)), (height - (marginSizeY * 2))); + + CATextLayer* background = [CATextLayer layer]; + [background setBackgroundColor:[UIColor blackColor].CGColor]; + [background setOpacity:0.5f]; + [background setFrame:backgroundBounds]; + background.cornerRadius = 5.0f; + + [[self.view layer] addSublayer:background]; + [labelLayers addObject:background]; + + CATextLayer* layer = [CATextLayer layer]; + [layer setForegroundColor:[UIColor whiteColor].CGColor]; + [layer setFrame:textBounds]; + [layer setAlignmentMode:alignment]; + [layer setWrapped:YES]; + [layer setFont:font]; + [layer setFontSize:fontSize]; + layer.contentsScale = [[UIScreen mainScreen] scale]; + [layer setString:text]; + + [[self.view layer] addSublayer:layer]; + [labelLayers addObject:layer]; +} + +@end diff --git a/tensorflow/contrib/lite/examples/ios/camera/Info.plist b/tensorflow/contrib/lite/examples/ios/camera/Info.plist new file mode 100644 index 0000000000..f3d96bab16 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/Info.plist @@ -0,0 +1,44 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + tflite_camera_example + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + ${PRODUCT_NAME} + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1.0 + LSRequiresIPhoneOS + + NSCameraUsageDescription + Capture images to detect object + UIMainStoryboardFile + MainStoryboard_iPhone + UIRequiresFullScreen + + UIStatusBarHidden + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + + + diff --git a/tensorflow/contrib/lite/examples/ios/camera/MainStoryboard_iPhone.storyboard b/tensorflow/contrib/lite/examples/ios/camera/MainStoryboard_iPhone.storyboard new file mode 100644 index 0000000000..0f10a22e41 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/MainStoryboard_iPhone.storyboard @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tensorflow/contrib/lite/examples/ios/camera/Podfile b/tensorflow/contrib/lite/examples/ios/camera/Podfile new file mode 100644 index 0000000000..4ae6fb6b94 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/Podfile @@ -0,0 +1,5 @@ +platform :ios, '8.0' +inhibit_all_warnings! + +target 'tflite_camera_example' + pod 'TensorFlow-experimental' diff --git a/tensorflow/contrib/lite/examples/ios/camera/data/.gitignore b/tensorflow/contrib/lite/examples/ios/camera/data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tensorflow/contrib/lite/examples/ios/camera/main.mm b/tensorflow/contrib/lite/examples/ios/camera/main.mm new file mode 100644 index 0000000000..1a9e542f7c --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/main.mm @@ -0,0 +1,28 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "CameraExampleAppDelegate.h" + +int main(int argc, char* argv[]) { + int retVal = 0; + + @autoreleasepool { + retVal = + UIApplicationMain(argc, argv, nil, NSStringFromClass([CameraExampleAppDelegate class])); + } + return retVal; +} diff --git a/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj b/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..c98183276b --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/camera/tflite_camera_example.xcodeproj/project.pbxproj @@ -0,0 +1,419 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 1C3C9DCC1ED3AB4200B8B5FA /* main.mm in Sources */ = {isa = PBXBuildFile; fileRef = 1C3C9DCA1ED3AB4200B8B5FA /* main.mm */; }; + 1C99111C1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 1C99111B1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard */; }; + 1CA5EB931ED3ABFB00247A34 /* CoreMedia.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1CA5EB921ED3ABFB00247A34 /* CoreMedia.framework */; }; + 1CB47D491ED3AD1700DF7666 /* AVFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1CB47D481ED3AD1700DF7666 /* AVFoundation.framework */; }; + 1CDB2D491ED3A9CD007929E9 /* CameraExampleAppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 1CDB2D431ED3A9CD007929E9 /* CameraExampleAppDelegate.m */; }; + 1CDB2D4A1ED3A9CD007929E9 /* CameraExampleViewController.mm in Sources */ = {isa = PBXBuildFile; fileRef = 1CDB2D451ED3A9CD007929E9 /* CameraExampleViewController.mm */; }; + 1CDB2D4E1ED3AA35007929E9 /* Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 1CDB2D4D1ED3AA35007929E9 /* Info.plist */; }; + 54DC6C3C5F734F3A58069F0C /* libPods-tflite_camera_example.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3BA8BF92C84895BFE59D8236 /* libPods-tflite_camera_example.a */; }; + AC1F82661FBA3CBD0052BA77 /* labels.txt in Resources */ = {isa = PBXBuildFile; fileRef = AC1F82641FBA3CBD0052BA77 /* labels.txt */; }; + AC1F82691FBA3F930052BA77 /* libtensorflow-lite.a in Frameworks */ = {isa = PBXBuildFile; fileRef = AC1F82681FBA3F930052BA77 /* libtensorflow-lite.a */; }; + ACA1A4CA1FBB6C28009B8D86 /* mobilenet_quant_v1_224.tflite in Resources */ = {isa = PBXBuildFile; fileRef = ACA1A4C91FBB6C28009B8D86 /* mobilenet_quant_v1_224.tflite */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 1C0D73481ECCC41B008C1DAB /* CoreImage.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreImage.framework; path = System/Library/Frameworks/CoreImage.framework; sourceTree = SDKROOT; }; + 1C0D734A1ECCC460008C1DAB /* CoreGraphics.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreGraphics.framework; path = System/Library/Frameworks/CoreGraphics.framework; sourceTree = SDKROOT; }; + 1C3C9DCA1ED3AB4200B8B5FA /* main.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = main.mm; sourceTree = ""; }; + 1C564C0D1ED3A92E00087306 /* tflite_camera_example.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = tflite_camera_example.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 1C99111B1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; path = MainStoryboard_iPhone.storyboard; sourceTree = ""; }; + 1CA45FFE1ECCC356002FA6A4 /* UIKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = UIKit.framework; path = System/Library/Frameworks/UIKit.framework; sourceTree = SDKROOT; }; + 1CA5EB921ED3ABFB00247A34 /* CoreMedia.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreMedia.framework; path = System/Library/Frameworks/CoreMedia.framework; sourceTree = SDKROOT; }; + 1CB47D481ED3AD1700DF7666 /* AVFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AVFoundation.framework; path = System/Library/Frameworks/AVFoundation.framework; sourceTree = SDKROOT; }; + 1CDB2D421ED3A9CD007929E9 /* CameraExampleAppDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CameraExampleAppDelegate.h; sourceTree = ""; }; + 1CDB2D431ED3A9CD007929E9 /* CameraExampleAppDelegate.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CameraExampleAppDelegate.m; sourceTree = ""; }; + 1CDB2D441ED3A9CD007929E9 /* CameraExampleViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CameraExampleViewController.h; sourceTree = ""; }; + 1CDB2D451ED3A9CD007929E9 /* CameraExampleViewController.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = CameraExampleViewController.mm; sourceTree = ""; }; + 1CDB2D4D1ED3AA35007929E9 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 3BA8BF92C84895BFE59D8236 /* libPods-tflite_camera_example.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-tflite_camera_example.a"; sourceTree = BUILT_PRODUCTS_DIR; }; + 3BC5BE4BBD09374D3E98F082 /* Pods-tflite_camera_example.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-tflite_camera_example.debug.xcconfig"; path = "Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example.debug.xcconfig"; sourceTree = ""; }; + 55ED318E8D29C8AFEF03DF1E /* Pods-tflite_camera_example.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-tflite_camera_example.release.xcconfig"; path = "Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example.release.xcconfig"; sourceTree = ""; }; + AC1F82641FBA3CBD0052BA77 /* labels.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = labels.txt; sourceTree = ""; }; + AC1F82681FBA3F930052BA77 /* libtensorflow-lite.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libtensorflow-lite.a"; path = "../../../gen/lib/libtensorflow-lite.a"; sourceTree = ""; }; + ACA1A4C91FBB6C28009B8D86 /* mobilenet_quant_v1_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_quant_v1_224.tflite; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 1C564C0A1ED3A92E00087306 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + AC1F82691FBA3F930052BA77 /* libtensorflow-lite.a in Frameworks */, + 1CB47D491ED3AD1700DF7666 /* AVFoundation.framework in Frameworks */, + 1CA5EB931ED3ABFB00247A34 /* CoreMedia.framework in Frameworks */, + 54DC6C3C5F734F3A58069F0C /* libPods-tflite_camera_example.a in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 24D7686C331131624F4454A0 /* Frameworks */ = { + isa = PBXGroup; + children = ( + AC1F82681FBA3F930052BA77 /* libtensorflow-lite.a */, + 1CB47D481ED3AD1700DF7666 /* AVFoundation.framework */, + 1CA5EB921ED3ABFB00247A34 /* CoreMedia.framework */, + 1C0D734A1ECCC460008C1DAB /* CoreGraphics.framework */, + 1C0D73481ECCC41B008C1DAB /* CoreImage.framework */, + 1CA45FFE1ECCC356002FA6A4 /* UIKit.framework */, + 3BA8BF92C84895BFE59D8236 /* libPods-tflite_camera_example.a */, + ); + name = Frameworks; + sourceTree = ""; + }; + 3E9FC355632FB928EA23BEED /* Pods */ = { + isa = PBXGroup; + children = ( + 3BC5BE4BBD09374D3E98F082 /* Pods-tflite_camera_example.debug.xcconfig */, + 55ED318E8D29C8AFEF03DF1E /* Pods-tflite_camera_example.release.xcconfig */, + ); + name = Pods; + sourceTree = ""; + }; + 591157921CF4011C00C31E3A = { + isa = PBXGroup; + children = ( + 1C99111B1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard */, + 1C3C9DCA1ED3AB4200B8B5FA /* main.mm */, + 1CDB2D4D1ED3AA35007929E9 /* Info.plist */, + 1CDB2D421ED3A9CD007929E9 /* CameraExampleAppDelegate.h */, + 1CDB2D431ED3A9CD007929E9 /* CameraExampleAppDelegate.m */, + 1CDB2D441ED3A9CD007929E9 /* CameraExampleViewController.h */, + 1CDB2D451ED3A9CD007929E9 /* CameraExampleViewController.mm */, + 59A3CFF31CF4E68100C4259F /* data */, + 5911579C1CF4011C00C31E3A /* Products */, + 3E9FC355632FB928EA23BEED /* Pods */, + 24D7686C331131624F4454A0 /* Frameworks */, + ); + sourceTree = ""; + }; + 5911579C1CF4011C00C31E3A /* Products */ = { + isa = PBXGroup; + children = ( + 1C564C0D1ED3A92E00087306 /* tflite_camera_example.app */, + ); + name = Products; + sourceTree = ""; + }; + 59A3CFF31CF4E68100C4259F /* data */ = { + isa = PBXGroup; + children = ( + ACA1A4C91FBB6C28009B8D86 /* mobilenet_quant_v1_224.tflite */, + AC1F82641FBA3CBD0052BA77 /* labels.txt */, + ); + path = data; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 1C564C0C1ED3A92E00087306 /* tflite_camera_example */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1C564C351ED3A92E00087306 /* Build configuration list for PBXNativeTarget "tflite_camera_example" */; + buildPhases = ( + 66DAEAAEE9EF6550C3A061E0 /* [CP] Check Pods Manifest.lock */, + 1C564C091ED3A92E00087306 /* Sources */, + 1C564C0A1ED3A92E00087306 /* Frameworks */, + 1C564C0B1ED3A92E00087306 /* Resources */, + 00E875C3B066535AE6B77101 /* [CP] Embed Pods Frameworks */, + 5C2D02120E3E5E09567AA946 /* [CP] Copy Pods Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = tflite_camera_example; + productName = tflite_camera_example; + productReference = 1C564C0D1ED3A92E00087306 /* tflite_camera_example.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 591157931CF4011C00C31E3A /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 0830; + LastUpgradeCheck = 0830; + ORGANIZATIONNAME = Google; + TargetAttributes = { + 1C564C0C1ED3A92E00087306 = { + CreatedOnToolsVersion = 8.3.2; + DevelopmentTeam = EQHXZ8M8AV; + ProvisioningStyle = Automatic; + }; + }; + }; + buildConfigurationList = 591157961CF4011C00C31E3A /* Build configuration list for PBXProject "tflite_camera_example" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 591157921CF4011C00C31E3A; + productRefGroup = 5911579C1CF4011C00C31E3A /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 1C564C0C1ED3A92E00087306 /* tflite_camera_example */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 1C564C0B1ED3A92E00087306 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ACA1A4CA1FBB6C28009B8D86 /* mobilenet_quant_v1_224.tflite in Resources */, + 1C99111C1ED3B0E600A6BFB9 /* MainStoryboard_iPhone.storyboard in Resources */, + 1CDB2D4E1ED3AA35007929E9 /* Info.plist in Resources */, + AC1F82661FBA3CBD0052BA77 /* labels.txt in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 00E875C3B066535AE6B77101 /* [CP] Embed Pods Frameworks */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + ); + name = "[CP] Embed Pods Frameworks"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example-frameworks.sh\"\n"; + showEnvVarsInLog = 0; + }; + 5C2D02120E3E5E09567AA946 /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + ); + name = "[CP] Copy Pods Resources"; + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-tflite_camera_example/Pods-tflite_camera_example-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; + 66DAEAAEE9EF6550C3A061E0 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-tflite_camera_example-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 1C564C091ED3A92E00087306 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1CDB2D4A1ED3A9CD007929E9 /* CameraExampleViewController.mm in Sources */, + 1CDB2D491ED3A9CD007929E9 /* CameraExampleAppDelegate.m in Sources */, + 1C3C9DCC1ED3AB4200B8B5FA /* main.mm in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 1C564C361ED3A92E00087306 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 3BC5BE4BBD09374D3E98F082 /* Pods-tflite_camera_example.debug.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + DEVELOPMENT_TEAM = EQHXZ8M8AV; + INFOPLIST_FILE = Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "com.pf.tf-camera-example"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 3.0; + }; + name = Debug; + }; + 1C564C371ED3A92E00087306 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = 55ED318E8D29C8AFEF03DF1E /* Pods-tflite_camera_example.release.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + DEVELOPMENT_TEAM = EQHXZ8M8AV; + INFOPLIST_FILE = Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 10.3; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = "com.pf.tf-camera-example"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; + }; + name = Release; + }; + 591157B01CF4011D00C31E3A /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + ../../../../../../, + ../../../downloads/flatbuffers/include/, + ../../../downloads/eigen/, + ../../../downloads/, + ); + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + LIBRARY_SEARCH_PATHS = ../../../gen/lib/; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 591157B11CF4011D00C31E3A /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + ../../../../../../, + ../../../downloads/flatbuffers/include/, + ../../../downloads/eigen/, + ../../../downloads/, + ); + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + LIBRARY_SEARCH_PATHS = ../../../gen/lib/; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 1C564C351ED3A92E00087306 /* Build configuration list for PBXNativeTarget "tflite_camera_example" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1C564C361ED3A92E00087306 /* Debug */, + 1C564C371ED3A92E00087306 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 591157961CF4011C00C31E3A /* Build configuration list for PBXProject "tflite_camera_example" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 591157B01CF4011D00C31E3A /* Debug */, + 591157B11CF4011D00C31E3A /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 591157931CF4011C00C31E3A /* Project object */; +} diff --git a/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.h b/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.h new file mode 100644 index 0000000000..94046d9728 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.h @@ -0,0 +1,21 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +@interface AppDelegate : UIResponder + +@property(strong, nonatomic) UIWindow *window; + +@end diff --git a/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.mm b/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.mm new file mode 100644 index 0000000000..fe26ceec42 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/AppDelegate.mm @@ -0,0 +1,47 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "AppDelegate.h" + +#import "RunModelViewController.h" + +@implementation AppDelegate + +- (BOOL)application:(UIApplication *)application + didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + UITabBarController *bar = [[UITabBarController alloc] init]; + [bar setViewControllers:@[ [[RunModelViewController alloc] init] ]]; + bar.selectedIndex = 0; + self.window = [[UIWindow alloc] initWithFrame:[[UIScreen mainScreen] bounds]]; + self.window.rootViewController = bar; + [self.window makeKeyAndVisible]; + return YES; +} + +- (void)applicationWillResignActive:(UIApplication *)application { +} + +- (void)applicationDidEnterBackground:(UIApplication *)application { +} + +- (void)applicationWillEnterForeground:(UIApplication *)application { +} + +- (void)applicationDidBecomeActive:(UIApplication *)application { +} + +- (void)applicationWillTerminate:(UIApplication *)application { +} + +@end diff --git a/tensorflow/contrib/lite/examples/ios/simple/Podfile b/tensorflow/contrib/lite/examples/ios/simple/Podfile new file mode 100644 index 0000000000..1740ad6457 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/Podfile @@ -0,0 +1,5 @@ +platform :ios, '8.0' +inhibit_all_warnings! + +target 'tf_simple_example' + pod 'TensorFlow-experimental' diff --git a/tensorflow/contrib/lite/examples/ios/simple/RunModel-Info.plist b/tensorflow/contrib/lite/examples/ios/simple/RunModel-Info.plist new file mode 100644 index 0000000000..1a3eaa8a2c --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/RunModel-Info.plist @@ -0,0 +1,47 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + tflite-simple-example + CFBundleExecutable + tf_simple_example + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + ios-app + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1.0 + LSRequiresIPhoneOS + + UILaunchStoryboardName + RunModelViewController + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.h b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.h new file mode 100644 index 0000000000..a4b358b4eb --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.h @@ -0,0 +1,24 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +@interface RunModelViewController : UIViewController + +- (IBAction)getUrl:(id)sender; + +@property(weak, nonatomic) IBOutlet UITextView *urlContentTextView; +@property(weak, nonatomic) IBOutlet UITextField *urlTextField; + +@end diff --git a/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.mm b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.mm new file mode 100644 index 0000000000..0dafb1f61e --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.mm @@ -0,0 +1,221 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "RunModelViewController.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" +#include "tensorflow/contrib/lite/string_util.h" +#include "tensorflow/contrib/lite/tools/mutable_op_resolver.h" + +#include "ios_image_load.h" + +#define LOG(x) std::cerr +#define CHECK(x) \ + if (!(x)) { \ + LOG(ERROR) << #x << "failed"; \ + exit(1); \ + } + +NSString* RunInferenceOnImage(); + +@interface RunModelViewController () +@end + +@implementation RunModelViewController { +} + +- (IBAction)getUrl:(id)sender { + NSString* inference_result = RunInferenceOnImage(); + self.urlContentTextView.text = inference_result; +} + +@end + +// Returns the top N confidence values over threshold in the provided vector, +// sorted by confidence in descending order. +static void GetTopN(const float* prediction, const int prediction_size, const int num_results, + const float threshold, std::vector >* top_results) { + // Will contain top N results in ascending order. + std::priority_queue, std::vector >, + std::greater > > + top_result_pq; + + const long count = prediction_size; + for (int i = 0; i < count; ++i) { + const float value = prediction[i]; + + // Only add it if it beats the threshold and has a chance at being in + // the top N. + if (value < threshold) { + continue; + } + + top_result_pq.push(std::pair(value, i)); + + // If at capacity, kick the smallest value out. + if (top_result_pq.size() > num_results) { + top_result_pq.pop(); + } + } + + // Copy to output vector and reverse into descending order. + while (!top_result_pq.empty()) { + top_results->push_back(top_result_pq.top()); + top_result_pq.pop(); + } + std::reverse(top_results->begin(), top_results->end()); +} + +NSString* FilePathForResourceName(NSString* name, NSString* extension) { + NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension]; + if (file_path == NULL) { + LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." << [extension UTF8String] + << "' in bundle."; + } + return file_path; +} + +NSString* RunInferenceOnImage() { + std::string graph; + const int num_threads = 1; + std::string input_layer_type = "float"; + std::vector sizes = {1, 224, 224, 3}; + + NSString* graph_path = FilePathForResourceName(@"mobilenet_v1_1.0_224", @"tflite"); + + std::unique_ptr model( + tflite::FlatBufferModel::BuildFromFile([graph_path UTF8String])); + if (!model) { + LOG(FATAL) << "Failed to mmap model " << graph; + } + LOG(INFO) << "Loaded model " << graph; + model->error_reporter(); + LOG(INFO) << "resolved reporter"; + +#ifdef TFLITE_CUSTOM_OPS_HEADER + tflite::MutableOpResolver resolver; + RegisterSelectedOps(&resolver); +#else + tflite::ops::builtin::BuiltinOpResolver resolver; +#endif + + std::unique_ptr interpreter; + tflite::InterpreterBuilder(*model, resolver)(&interpreter); + if (!interpreter) { + LOG(FATAL) << "Failed to construct interpreter"; + } + + if (num_threads != -1) { + interpreter->SetNumThreads(num_threads); + } + + int input = interpreter->inputs()[0]; + + if (input_layer_type != "string") { + interpreter->ResizeInputTensor(input, sizes); + } + + if (interpreter->AllocateTensors() != kTfLiteOk) { + LOG(FATAL) << "Failed to allocate tensors!"; + } + + // Read the label list + NSString* labels_path = FilePathForResourceName(@"labels", @"txt"); + std::vector label_strings; + std::ifstream t; + t.open([labels_path UTF8String]); + std::string line; + while (t) { + std::getline(t, line); + label_strings.push_back(line); + } + t.close(); + + // Read the Grace Hopper image. + NSString* image_path = FilePathForResourceName(@"grace_hopper", @"jpg"); + int image_width; + int image_height; + int image_channels; + std::vector image_data = + LoadImageFromFile([image_path UTF8String], &image_width, &image_height, &image_channels); + const int wanted_width = 224; + const int wanted_height = 224; + const int wanted_channels = 3; + const float input_mean = 127.5f; + const float input_std = 127.5f; + assert(image_channels >= wanted_channels); + uint8_t* in = image_data.data(); + float* out = interpreter->typed_tensor(input); + for (int y = 0; y < wanted_height; ++y) { + const int in_y = (y * image_height) / wanted_height; + uint8_t* in_row = in + (in_y * image_width * image_channels); + float* out_row = out + (y * wanted_width * wanted_channels); + for (int x = 0; x < wanted_width; ++x) { + const int in_x = (x * image_width) / wanted_width; + uint8_t* in_pixel = in_row + (in_x * image_channels); + float* out_pixel = out_row + (x * wanted_channels); + for (int c = 0; c < wanted_channels; ++c) { + out_pixel[c] = (in_pixel[c] - input_mean) / input_std; + } + } + } + + if (interpreter->Invoke() != kTfLiteOk) { + LOG(FATAL) << "Failed to invoke!"; + } + + float* output = interpreter->typed_output_tensor(0); + const int output_size = 1000; + const int kNumResults = 5; + const float kThreshold = 0.1f; + std::vector > top_results; + GetTopN(output, output_size, kNumResults, kThreshold, &top_results); + + std::stringstream ss; + ss.precision(3); + for (const auto& result : top_results) { + const float confidence = result.first; + const int index = result.second; + + ss << index << " " << confidence << " "; + + // Write out the result as a string + if (index < label_strings.size()) { + // just for safety: theoretically, the output is under 1000 unless there + // is some numerical issues leading to a wrong prediction. + ss << label_strings[index]; + } else { + ss << "Prediction: " << index; + } + + ss << "\n"; + } + + LOG(INFO) << "Predictions: " << ss.str(); + + std::string predictions = ss.str(); + NSString* result = @""; + result = [NSString stringWithFormat:@"%@ - %s", result, predictions.c_str()]; + + return result; +} diff --git a/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.xib b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.xib new file mode 100644 index 0000000000..93f334b985 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/RunModelViewController.xib @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tensorflow/contrib/lite/examples/ios/simple/data/grace_hopper.jpg b/tensorflow/contrib/lite/examples/ios/simple/data/grace_hopper.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d2a427810f679db537236c5430873a81a62ef412 GIT binary patch literal 73746 zcmex=zKMNGjWx(E@lO+TPE zg%l;`6{n>ZA=^~I!0;a{V15Cz7pv5NO5{K;V&PzOFU?Fz1$mW00PJQ4CI<#EW&npH ziyUugkdH!sS!z)cR3Rt^A#ze|!8w`95Z#q zjL?)KBJPt|nOdaa>7wACnwMIXSdw3);FMSlN^l_mFoMnC5%o(gR|wC{OUW-URtU)` z%}vTn%*5YF1;xxwsC9t_RE|9ZCl4PK`pH0#{JGoxAelq;`~InJWYyx znUcK2YPN#xIvMNi!?XW0xR%_H+x5>rGEH{sg+dw1>e^NJL;tm2R>}Ux&SAS) zSwk&F(9%F|<3#x%cdolkl%TTdesx5;x!AU~7|9tq@ z+^JLk?Y{c<)sfvCbjg}){UX?mJcV zpP~NpuC3YPN6aNzFRs(r`ux4-z27yv+jD=}f03JX{#AXL z@vnN1^{bah`Imb=$6h@Tt#&{1U*gM5*Y1`&ezBPE zS!8s#43Ep!H@ekflk2wqP03}KDAk$o(7SBIU$e%Szp6dHUEO^9rSTWcz%?YybM4$V@g^RhThdK<2L9-&0#QANm)k_3FG_ zR7&J@{s0cFZvts z%4A1)Xy`oYWVuthng5pkXZW@1o7$WU>JfT2TMk^RZ+~0yu5tG4=~w5yS<`K#vu!&2 zzU#Y=3&!2|znfToA?u}URhF0-yW8$%wW;f6_Fr8So_#ana@@Z zD%um3TV+{u#^a=W)m8Ro^8~+%9NU_9BmV!z=T+~o=iJ+P>Enmb*{cn=UO670y5@HJ zE!Mu;X@O~$t1qYS_%}^={)w-m=hxRq6jUAl`10V^HxlQ2<|JIus@*p``*&>l_PY~b znC?&foBUt6nJ3nLF{)NBfpZyJAn>X24%y`Mo z1iei?)qU0sbFb~z{BnKwuYJqDo_qZG{^SX#3|Ez2zm>G>hU$_34C^lyeL2^CA>(C4 z$aV#uo;y`hwqYL*=RW#=-DB>$FZ2J2-an@`=jMtdam$LP|1+ODf7)Urk4uZ}pC3H< z+dlO8`s=%H8*6W{2<(sD78)EElWn)z@V@VU?-&0j|1-=eshfSZAnx@$ld36OuSaNa zJKpcZ`Q`1(cPAEKt-p|McKQ8(2Cpr#-J$uvZF8#&zx8?;@0@=wIu!gbJ#N3+bj`2XWgxO&03Jkrl~ho$M>KE|sTBWrUD ze}(K%&0F;9-_-p3zw|%f*d?=m&5oa@QXjNW*7@>FZQAkWO-A{P_h0{-7KMsziqWKG;!y?=`a4?{?G6_>&7~#jk!AOzceilz9Stwdp2M3>KPmF-z>Y9`=5cS zU8+Xx`eC0wp_IHQ*{AhNl=Yg?y9T}umQTCwYL`06Qt)|ZC;n)-I}_5Td4-|C%Z$}z4|rzz~R#w6LMCr$)0^euiEYX zoAov?Z?1cJ^Xro$zmuyxXa}ZtQlKS^enoa#<%Ne!VEAL*r*?l#;*5?a%X7&Ny;#=jBrDE5vZMq-Z ze*fM5H{rML^qAi{;~tb%daicb{hd$V@s?j$WoFuQ%Zs!TDS03dECS3A_Mts_AN%cJ=J$_w>ov} z{G#?3j~+gLnsOlby7=C=IdAlCe_4I^-^_Ra8Ek{rzkbJ;eAMS~>9zfC+Y=h)=X|PN zwJ$#CtgO2Bs;8GOow^^nUP4$ed2MoR^1S=G_8+Y4?O!hM@HnT?C#{hEXj^2Gq9$*K z{Pe?%SLyD0Q&XN2`RYkk!mSSs)8$I?H~%?up?blWJAeF^_ZS(>-MjebF?gX~chq|Cb+BrHbCa zW>8opZ5ll_G~D2&!XCpnZOmi*0XDU&a6GVfiL2?_b16X}-F;`O4AV3TapET_TS2^z?++=ij;iLp$8K^xBiyG}m|c zOW&7H+w;Bp-1psoo+}y{2~-9Aj@Wcpitk$1T7I$Yqakx{z)8BmU*O+f+ zYvxRsiu|IPzG>rv{LeZ4OSgZXDm1|%xnaG><-AoV&pbQ%`OL%{RptL)U#agtV{N=^ zT^%dG$la-z-=><)UvzxaotmyWfB9d0zx+OWu7kPaw5W(zGLOZA-pU+j&)qg#>iwcu zYhtEfwOjLU;@5s=^ZMn%dAXH*Z(rP~)jqs`!`su>>UQsc^^N=0=~tG%M|Z1#$zKun zIyNKy;*Ma?>|1~8zx-#IAO9lfKLg_?n;&OZ1^#UM7j2SQIQ_KV-tL2slM<5+ub+-` zR(;PNUwHFBLtV6eY<9lquL)C8Czn!IAiZjnhZe^7Il6o&VrK;>dL*D*hkDf=JtqJ+hptYUzl(kgmDZ^-_qqgq;{V>DNEE6Spzic>aqrj9@_ToRo|10bN z8G5_Bud#kvyRdP-?0WtFHybPOtb6?@CHmJs*7*Io>-%4A?ETMhadp>`CGpc;`3jPA zxAqjDyv0!59P`}zbJG!K6rBP7dLbF%?Y;|YUW=3 zHT7a?z59#$2lF3XmD~7FPkXV@>0M#2+wMl(|5kn2_AFD}wL1snZcE)S-`VA6RZ+p_ z_~6AZy}9kz>h{zouUj*x-R^Z1>rK^7w=&k$<@(+__MhS4=0B?Mzlz)Nbjn^|RNvLN zzxVRpD}_ee4Cl=}^EH8e+BLH~$L&Ag4lav~t~ahdSrU-_H}^k-*p2#dE7iIF-)67d z+P`ksi(21|jpxfQ9oxD7&h@Rc-~DG;tEYIb`cB-ZnD0W*EsuGsoSySse#vva<$n@t zQ%}E^jefI!>vCz~u*<>Q4YyfT&-z{~t-o;oqV0cW_wCJ|re><9C}83kkvGpSbI!vx z1=|biR+Y;1R23CTyvjVCeYcY@?RnSSmrwGN@9x^~doKO*udMRF0eg}xf2=c$yI-;D zc-V!^>|FCZ^1EMN`}Hb)`9I++UN7Rd-e0W0(N0+3xB2kFqw;0%+kdWqE?0Tf)_lLb z(weR_k>0nH`Db%fhpyQ-@rt3L_SwBXBGHK}`}ni_9+(SmTwWcjaM@~!3a>!j!oGqj zoIMA7+9vh13jAmIfA!gahGT5O`L85=*B<=R6CbplkAKDY-?RVDny&vbY}e$|n;g`f zC8XFJRL?j5-uL`nsDl_? zm+x+Vxlohi$jltO)r*BS8D;N0aJ24TGS`2m#*Ld|kMEu7`u1z;4%^nVd<@MymnZ*S zVK+1T&h=x@CS7K8X_gZC<=1&jpuxk;MDa@bHN6e5^LJm**sS4tPBcutf6?7}=NJF% zvb2w|I_5p+^(N--d%1;QUI~3STNU#@H~Ej8sr@sZpNy{Oe<~;oXfUu{jne;KRLJ@9 zbWf;v$45T-X-j4+TkHL54UgQn_u$8^i|;VcxSp3C^?t$EICm|*!Xj4%h8qmqisw8@ zwUj*ZUQRKOW=W+&W?ou8Xp|!qG&Tbo za^h!jWJqDiXGj7MQw1<&Fyu3oFyu27gINU(3=F~qR46c*G3YTEfDDA#&Bo&u;Oedr zSegj35@Z`Y0~2VJlo9*r5C>7Is!vHWpT9Rz^W421RBTLq{RjKqa=s!bT(E zi3^?BHy#X1DpD5taIwi)^x?-zi<~zd3QkrLyHwn4^61m##T+iGYKKEoricqNFflT* zGP1I;v9qy(ox;c@$gF7SD8v$&SlFnob>#zM$<-(UXB>1t-+k%Toy7&%4r4KCrf;ldE};b^Lwg7 z@@8|7@~H{|CbyO@_2lqUV&XhA*Xy0&uGM>ELvFl$QQ?g-QO~w6{#&-nCZNJ#2o}!d@4qmYy1yt6BWr`ToAgzY<( znYz2@Fw8nL0xnf0UFQsXpy=myF}LXUWU|%-}5#x@&pLW5FE*u9qFp zOqK`FnEb8UBV5#RmHvI5@7=Z2ei!W9p>feg(Q?`HxEYC{dw`q zy3%UjNzFSm>q-wC?#M1YW+^=H`_8K}>omjzgWK$0oWK9w>i5yF#~#Q0OFe!1!jtnY zau1~rJX|uZ$BEgvza>2D)OL^8+t&6yxii;9J#hkyXQ7~`&4NZ7|LYQ8zG@zGmuYy{ zJ~eAu?Tl8zO_HL<)+r0O-FX+XGB31OJ6&+ir0yFVU)-C$>E&r{tMrVe9;+Fy8EVe# zH(0Kzr*%E`mUZ2mJhORA?`<7sG@+5uneOh}zW841S_H!c_p39if^Y-Jd^%<8lSvL6Y{rIfb z&*agab(^~WCNDnv&!k%Ty9ocRzuu)=-xpTal+@%o1+x+l-U<%XKI-LX zYkGaJt#jRtRZ+HUzuq`oqQ#=|?V-iz;FZ_5tq;4k;bLHavD^_x8NMRfjrYF%-EFqM zGiaIi*Sp~pzP~IrGT?C1f2)yq@44BxJBlkVhVF{COHrOu*d8Jk8Mudum0i+BjPXCi z>P4^Iz5ixa@v~=5{A0N7pGa-6X8V!d*Y~`&&AS^b9T8kMZJHtDQPK2=r)6AQ*hM%D zl_KPRyxp4{9PSxfJgZM9Z(C1SPc>u2!zaawEuCkoT?+m)+Wa z`(ApmqQhYwqsNk_QgPfKiVre%EqBNW91~RxeV46ia;136)-|k0Q%#s>>D^{OeQxEw zS+>^pmo6z}u8wo9=gR7Lcz3(&^Qm@A^+kH|%FVs9XOEn@D*Wb>?MD4`cGFgdPO0j; zxqL!Zc;jL*A(OpK#fBI2f0keP9xI<6H}!7V)!M^nT@w@-8rT+g*Ls;<7m3d=^$A&0 zn90RcB>P;>W!=%M;V(=sZ3whkyyZ*V@4eTS{*w0EB_bSpRZOMEI<@Gum#uhz)pzaG)pz}7X9g|1({RW_?D_tnd*AmSc4V7tE*7k-pphA>Q8(ko zhn?YiMKT#vVzT`%>#JP~E6B)mCD%J`M7*~k=@n{jg`L?|&_I^iD_6bxBjmB*qyJ`51p#B-yN!UCnx+$)}N3z?1#$F}?}Ta|}6_Z+W1=V6f0CWEqQy z9XG2?6th^?%!V2BPAGORa}MO#@$kS?K3z#>&6SV1IKLfG61u1$vs6S;z}eql`z{7=&;0UtW6n z?kqj89fxOK%{rN?JkRJHSHls{^lt$^)e9Cf=sC~UxxT~s&rVe8LIKQzrm}))ZbXsA%fon(V7oLE>3Bp&{~a3I+uzop~*0lg4)|B@*_2c6Ouy7Mdxf7O)UuBj}n z+@Hh1DUn=D6 zIdZ1qT@%-|`yKbvOwF18g)(cle(yM+7?tqMP~rSNp6r)r52m#4ulB+E8n2U=9nF|k-*qd_7G|}} z@mj`ek$ku(MSF&~#KIGzYZW8*oesDggWqf({S4o;I9`a*?&Z0tO) z7}B+SR)o}h{i73|3Iin`OkZ&H9AkyXw8F_DybQixuN*{#EmdYq99ZqCvU1M3EHUk! zfBadb_>-8N7}DF*B0qFa5fFaMI=}sA65FJL2TrZuM0%Q@vL0x-B-6|0;CkWo5tmRV zi}{*%$#>Z0-f!jSUg9q)dPK+Y1k;q@nNvEZSnx*q{G4;`jMgG%8ztV$#j1agFZosV zuiy7es`jgOJgHJvJNt};KK9!Cz7L;$b!zRc4-3}^zFY8MmBgO4e!3pd@1|b$*?rsW zZP&?@ZgUNO_ZqWod6`_;UGBJTuH6j&Wp8uKHa(l)di3&#b2i)8+`4@rs9xl~tNxN# zAEav9b~(=4Rd1G`v+kX4xpn=OsBN29UaYQ+xxiJ-o-A6o$=dE==*ZClI^TlJ6?w8_R-f`3@dByWZ#Akh)#{cjeTQspdWmJIc4T_FdmlbLeI7&7Rb~ zw=?%N*oqff$Z-VOY`G*@nsvsa#rTMuvnT61mUgCHjm|gPtc4yq<({bh!734(O2wmc=9(7f zI~T3foV8Snf1hPg|IEXSv-uAFDtT{OZzR7b_~q=6evw6BD=HI-( zOGBrse6^SqRBZUN_wd?Zrn4jW?7N$LGdHr`*Vgf?R$0)Zd*%Nb+HB76iM??rZ+dvO zuho=X*?@g_!Y{8~RdcZSX-@9LyFa71t2eLu&+sa=iZ5`HUPho4*CK@p$A2eTU0u7v zIJ9a_hULx!X^dJsRRUMVhO86Yn$^!c$@0!B$AioD4@*j?UEi6vop>0BG`^Aw zd;68kkbTAbO~+nk=$9Y)aaTXE>y+GOor=J0sXa*8>ch% z7`86tKeE_!zSyzOyWH2UBt!pNm(>OZu3N?#cJ{E>Za0~E%X;HkvbAr<xx7V|K&hD%mBmK#p8kW^FdFB8zsd%IpDHa8$7OM7N>%f9nH zUfyXQS{aQ?85ScBhk&Y4r*w@B-%>nsW1$G~9a+}&_x zM%fxBuIZdT{~1yYl{_yw8Z^lKUMP_Aa`_aIhy8DtN$^~7IVPbH`S2#cu0kL4Lu2JV zoKHDSURl&~ziI9|v5rAMhrv*N#RL`3pmv!DN*-rVE6;E$YHwaLML28ql){$QClgJK z<8K9U^h$ZDaWC_X<+6#MgpkX^*GkeYJ^g!Bm>QVaZyt@kw}B^1QQ_H=Nj|@= z6}F1(kbJ@@QKw_nyi~wslG99$ha4+pmU=DXNxf0{p@D6I>luD|Aw|9E8b<7lD$@)) ze6Bo8=(X~ilg4!Zm_+L-ktGLOdS{7fl~$B<6v`Y73U=X2zt6bY;icli+MgZ=R~%vB z<7Y^5v#2=0uAmaqqtbau$Wr7<^}?jZFP%FZm;_B`W=)!Sl84uq=fq#J54H-IO~(F zoYp%n)xM-Ozs9pnAd-Jk!<19<;Xhe+&-4;lY*4buVae6RG6g}K>Z0b@1Hmk!Gqehu z|1+FF*c~i3(Q4LYBb6qh1h-nlWfNIUvRGb4J~osPnHpBadAKDlxKQMTw&}kU>$^Yv z=yOqF2y*FZoG!B{D(hEqkT|17=K*m=t+#HD{F{4H4A*_}-pJkIvFZMiBcEJ+jaeBq zk2;=GbJ*J}k+ib&lwu-7>*+3rnJy2HTnS-TYi^pv^618s-6u9#cN%Qj=CPvM>AA$& zO9u+3G!?|1IJs(<$|GUZ_Yp_4TH{n6PwxpW{?WaoO331@pT+Wg$-gV+F4?(CuH-?3 zUSN;F?rDyxcCsI>JsDmK9jLq1o!i+{m*{zAW|G618IC7?I-8X)>J&tnI;i<-6mTp# z9QxuCm-#*~VSR~~5BWQi8zQmI?=EUE;N}XxQge_jvL7`W^onmTdpeP-XX0*52!%&3}fs`>QP8JzV~^ zTq;J6H+*qlg2g$-WU;*2DLXo9e-`fDAyu;bYQ-h?sWAc04jLRYIhLHhDSNw8@98_s z&u72)79PFP`q}@!X)nB_doLf>+%$c~ zcOfR8u)-r%hgW2m>U}QW6;krG$lPwk(+JL1;1=!>;66JP!7?c1?q=jyiYAsM;qGyg@``r7ka z^enku@6EDdT`hab{S*79-+rq(ZxR39_ov-g|Lk9!?|Wk@S4pq>;<~%x#kJEalcV?j zS#y6^(x3PDb@XB#-(}_=yYqd+9+MrmXBP)Ou-uwy?_3(VW95f>!+&m$dupD(n!E2! z+`JiU(>J!WT{i!+I&fu(@UySpwO0AMpN!|}l*wEDO4~bC`|PoE{~7X@hfLeFHoM%a zd5(B+W_Z%Jh1uyX;k)!jRbH=kTUeOvb=^7j)9vEdzQ@iN%Z&kz4zJM-)?=)^?j?B(aW>Nxi2?u{3`WpI_DA- z!+T-HpZ@N1uj0?VpP6s6GUj$v%;haRmT5=Fh9@jLwMN4++q*X{D|@T?)n#XW+v304 zUEF%}PM-PumHCOefmgKe+4Va5uFuRiyu9+Psq9O$#Zs^0mOp)URcQ6aZrSsvLblzX zbXPcg`=;9k+g8?BE~(nSer?$7?#ogoFKv64UHi|l>|Xw!??=^F)p@^@4$jJxHI7^M zJmYGuH`ju?6{ZU;_jTVg33vWCf7z+|$yd9>1OB|Z@m5#Mdb#fEcUPxwxml2LWqHP} z{!iPMy0=}q`d!+8mipq)@~d9l4%vUFO6})1hSgd@x?jIYZjYVaxyJsPozhc|=c<>* zJe{7EoOi3dJ9WO_hV^cXU&fg1Jh!65xl!5q)(-i;*^BOPc{l$`#ItJ|OG|xyy^>je z`~FsbYo=2h82|eo z%{RUIoq2nFbl#!vJn8(fxz1VNdQF{Huf21f`E=l{*{A)d|5%(aKiS%U)z$h$-`8Bv z_}RAVt>6-!e<*l5hwr5>mg#yIWpnomUW)bJ7M-`-aAmsRr^(uZD;m5Q9sAOhZZtYdhmaS0lB$e*&B_*DX)Tr+9L97G!97DNK@J5w&0l z@l{FGP!Vvs>JTTXqq(iJMM36aM}jfSlstt(%k&oE6%*tFRl@oHJg}&2I<2$0p|?n5 zW{9HEqYXR-MtR#kQZD{3Iuumt5h8T__Pf@l>x8t552UOTW6PM?Yc(;XbA1UDpMp1M z-_55t89B}}&Qs=GBG91ZbJD1(cisdJhNeS~arS=<7_OU&C}d{3ynSad$MB=Ws?~=* zTF)3c-LLE1+@~}+ka+vLudY7dy)@z~94sQ*YCYepf z&*GC*449@XT0RmtWER(CSTkkG`4pwc&7D?Giyqh{G&LSvBDp}by`*0vlwm>|V^D{i zyo;);)DkV0_>2-m8BdD^6Wawu3)NC4e|44!n4V~NSaap0#LF5Btdbm7`U~=eYFN(V z^yxXyQ6$7zB({pZaGIpkvBhrA>*8KHP5N}A?a5}Y^zOvQC$9om2p(7vw9_TU$hAT6 zAcv9dgr~|gorGHVaK?!1b9iZ{@LrjH&Y2ez*6Kene(&OuV zQqG>1lQuaOx=gW8+xj=|Wv0VV`Ow~~iOq95?S6Fal+Z9b{;Fs0gtm9vN~NErR;5h- zag5EXJT(1*qqzQzT7jyR3wyR^&)#4z}0zN1mWvqKu9CodRIV2~_xU%x$f(&?2} zg}+N5o0_eai&I$AC(-e#?e*5Ux3`zZPFt%S^s4)=_oZy_IUl$T-{#MrxvOU5yrX}= z-BFztqN>^(xH9Q==F!67rWxn*CLB$gC|&iJfB((gspj3c3-g_3TwVH9{!OoTO$)=~ z?}gnz&IVqV5}c$^ZhSFdj-$^Hy}}vSwp`>``e0J0!i=-&5+2^qcCKLXGP|AHd+lkV z-=&lDmbx=NYhgdF6l{}py2!h_LNJXX$RYHBe~lN<1uJ8H< z*Y>3=xxB6mJovWzt!Dhk%GolBEz1>RHFXM%jei7Lt2^#m|97+Z6<=L;=i>}sS-%f6 z%hmrqvHzmue+DxAi!2QFkK*LWB)Wd-f=S;eTws_v zvv<0lrMc!F@r{x~7qzZ0ROfh5|Ig{3#k<|CJ>8iBJJv|e1qck=QVcV1@42j1G7wdG22*izpG=5Ndv@8bv*I*mt0W5}55CVXDYjhq zpW(`NvENee`I{G~Z;3Wry42vop@BbQRa&{&@VfPjOr6_r?>tbsT=HOf z(=Wb`mv_G^?pf})>}K9=_uwnP9=$z!X2m{#{|i#HzkhXiK7G57mAh0kw9WF1X|3X= z{+Dv|mu`v)U%q2wf34TnT0sw?@3UrY5f2SFnQ-6yW>0swRG8wqTecB>#j&Vd#Ms~+4IYN zyI4bibFsUsn_{Bx2i^VNyZGvA53{9WcdjkF@V%<#>axhdE8gdyURis7`Qsj&6@hQN z7thVOSGcbJ;or%wt4>}wn}2xmzt1hDSNCps_4nP>sXMnVIlX*K0IyqnChv+5XQZ@3 zqUIkLeJ&E4u=7V|fui-kFS);#uadq}{q@_A)A^UUr*AR4AGCulSmdka347Cf^KG)u z-c^_N&)U-9F{5l*OZa+$7cXxAP>#72sy+S4@AGbd`tQuIy%@i21%q|z-Rv~=8+Eg{ z{by+ZYGvPK<=E%_pW*ReFZE^HqjD5%cXt-gULlgbb_LsfS-rk(i}=p92LG7Zy5nti z-D~GBw_7*sX{N}DZ27y)`pog#XA;l$x_-L%?&SKNH@x3#B>i@u_}YZ?lg&%fdhY$^ z{~3&|HnyZ|?7I4U+T27PE1Rz6dBzdzh0mU2?UK19cPQe)SJriBTbBynFO#~soLT3x z=tJ4ftE|ip*Dv~YW!CN36a1%lXV{$l5SHSE7J=T3&YNRo4B|zuOu|l zVe(T&)ujhM?cB5OKZCN>ojRYq<^DR;+O)ULF|1fD`&q-o_=uc*n^fcCX|95tA)zs0 z+_M@v=CLkwO=P;Marma?Ia85d)dilxZ0o%5OyG2_Pf}XGjUk}mnXsZ?ok3%eqvRs3 z0}4V?0WXyej|SUs=2K`5T$UP?u<%FXt_c%XcxZ@6{CMlsP&J0I$r@*F=H%<=uMFO};CI)V701Ocvn=2{7P#}}y@~r? zWpB7<5x9l(Rmh|XcIPwuT}%4?b8m~z*8O(cEO^3KEsJ8|nGrWnJXztpwEKuM+tm{D z+xaWZb|;7K%G%m{MfQfe1as0&zNf*}tJ8h2pUaycvL(yo?$Tg&h5VG{>&r5OiyN*i zik*1=*p7T5sT51rHewz!wU*FdGYf3wYxqlcfBuKn?<>$+&@ z6*-0_X%6a=M;#aLin-2Y?f>?stQ^zQZLg!YxqbeucZfmed8R}X|9*=X@tQ9;Ef2e- zmsxeG@*S7T#PiK=>k3Vsc$VImRe$nY=H1$5f%dAWESQ{J-UUC6d8a6&GC$X+ezrBk zEvw~Ur`(9n-eI!SVZw^bXKY)nGY}5Rfd|BRbSLk2=En(ZYbJpg(kGgqt zy7|L@YO9myf0=)B?Xj)VW@AF+bMGvJteILtk`rTe9pDjEsVDp~c zh0C*juf{Oy@vz=1nCU37^~IHiu~%O6>Sro!m-(Qq@g+TYd&1B9%Ujv*=K5|Im411F zh2gpFAy>nGtJsj*=;>i=Lf;n_KmTZzE#~%m$M&F38|f=StwP7e7czIMw>+79(NyTs z2Pv-gGh9?wRqs#f)_07!bk}OyrAr(5mOc3EZ8Nzga;J02ysjldtG!n*T)2Y6_(&38 z#9^jJ6Bmc4iq@)P_h-ztE93pQRjf((9m86O2Nt%AZ;EO%HTv{rcC_#Hb<2L6#A|Y^ zN=QN9hS6fq4)^4r`GNbteQqe*m;Ng8N6dJ=WId*%!t<BiSmVA?~mIb z`TL+MQ}=XRwshZwm0{BZBdj{5zAj`bnh@of_3hZKg{(Y>Gw zEqk-Yb#`vn6F*+M@#g$@x9*fBnh^n6%5(1KtN6yu56QCnz2?Q2ZTEjw@n@Gy&&o5I z^U|!hKm>Fzt9D^-^dO3n?u_AOe% zY}&0nbM;IwS6{2BIoDpT`=+I!`fql<;r-`*kym0fom$16tN}2POuXdllHPv;ld+_pr9oKi1e|~LwWV-D;nT0;#ceI3J z&wP3=Hoe$7cyr#8XXVoZQ+|Bczj%MziA;`X{}}{LtIoD=J5zo>a^K$8N3Wli{W{%S zEg|N+JZ$CSs0E_0*{(e0DO>B~m2-Ld^bbel3-6WNK70PPdh1i~T}SGTR!?c0yy8WM z{Iu1py`!ZLl?u7f#SGzX6{%UIX!e`^xccA1Gh$9-#68(&(x&h zRcU5uTD$T4qZhU(9d1eF-90Z`KHK=*)6-k0%+2&)EIfbf+faomKc)s~^>IqxPQANz z@6)YEzi!yOX<^{hJYAVb=R?;Rr>|)Dj#>G!PU5<`v!hDi(T9ist~imL^0A>nK{LQ| zvqQ4;(ZGibS?&hg8eXyW-@f_jTJGK5m1!Wb?=I=wp!CgWPVpUY%reARp)X4 z%Yi$uE^WKJF3YNQ?bFunE4N-xOnwowaru=3b-!tQ`?9l4&-`cbGWn~xujKVF=j61< zwduZJay7&@Jw1~){o?Wk`+c46>)*=BTd#ZbN=e?E{|qZ`U%jpWR@QO;lKSV`d%i}7 zSiZS*@#&=#k5{h~3hK&T?iH@GMZfUiv0klhPyQ`BBP?MUdE-s_-o`bz*R4DEe!Hn; zc5$F&xo1;Up3RnZQb*+H-1po5XZx{Sy=6D@zFfY&-P(F}*V^KjL8(XW5)Lt+*DT($ z{n69DMLWgC^=2+M-gbFO&DE>Y+H>=kMS7m)SKXJtX>pKRRn111n^x2I&X0H}Yq%ro z&aB^a??$iPw==`+(Y3jIb@j9F)T~?S`)AG7GcV3Bz9jPE{eOnymf8EsdOwuc?(!==%sOR@&EwC!)9k;uy>z>FYwqgI&`aM8n66ti zd=LHYd;L{q(u!XqbE2NLWodZGv^j5>wr0`o&$=tG{obiQ^{d1;Ue#5yUh@sEFr9zY zHL-k#h^m9sj@rQO{ePshV^`ahh@S79W;f||^tt}ryV~AXZzETHdi|fFOyOPp=06^PZq)ur7riZ5GsE}N?c3^8Qdi#o z+~s~oZ;tiU{;$<)YyWnHZ%Gd7m_5s4-~4&=KOXy@_$u1=+gyi(FYaAr{%W@5m9F}m zJw>7U_m)eRX6@})k>qjH;_Nlssih~LYqfrLNMO<7&(X6Mzc_EZ+Jm*tgG1usQvo5@ zfCEt8RzqPtHbfkDVG^Wh^>C$2# z+@Tt-vmjHC*??)4pNfp5mfBUd)XE(Pj_z^k)P8s@fMI9q@s}UkCJ5|uwchaExg$eC z^1R?ond*g{Q>QsBY42f@T03V7t5$V|bK4{jl@`Vrk12}A2A*8u8?6l*TrPWe6`rb5 z>`|C(pn9X3MO*KO)+L`4nMYO$-V>7YnJ`t6(@%CnSUjhSh$P=l2ZiapULI;L%Vphp z6ohZ3%(F9SSMHlUX*%l>;Ug?9X^b(&no|}?IC?%lA|m&56;9Px;$l{w(B#n!j=rd6j7&YS=0Xc>$A4ACR}dxYW-i_C5>7Fe=H zc>T@J>&iqq{uSs!l)s&kzv)%C>9T(-M2r$1}Ma9GxX) zov%rU9W8b{_iK0NtOqZz1O`j2{4;Y0?@L>Mt!q{b-aRusVfCM(rtCk%Rr9Tfx8A-` zxYV`aiBybxbxZQroMY-cP6cePYZDR~BW*vc0r4;Ym?icUpdx0@98wl6UM`-i*jve~zO=hvYeUJrWT z^?T>}Eq!rydE~+~1vAQX7R}xF{@P#LO;>mGGOm%zxF{4np=ZLAn69t|lXEYYiPvm- zt(nrNf7P1p{^e55n`RI0#0#vqXgs7ZugiURb6Bc_tzqE5*@qH#Sbx}M8T$3@xu=y; zzuw+j9bQ>|X9ttN#&5&Rbzv60``(z|y8lYtYRZz5rUg9n#Eh%Hr2c2K}E7hF^46_!r{65Q( z9PDwTAwd74Y@W>Ow9Efu>{BJyuZrEV-z)Cpd0(YuR+Vp0lGQVIF3j8!edo@(d%bSE zR{foEY~4q{S(BJnY01U}pFL&}^T2D;I%XT&ll+q{E*`f%x%`fHg~>0gd7UdiA77lK zY1X<|`}MCaFSoE?UUaLZyV-$PZ*AaIspV78eOcU5svh)C)_+InzCD$EJBzOJ1}rt% zxX0GYmp57ZTe8om<-u&z_A_vvw0_vITd@B^xWwU2nr^EfMu^oMf3{3|^Xof)x2wwy z-<6(bzH!!P*|(i>A0F7My$stFHUE|MH0$f9-u}9`-!t;OVu&`qg2xv&_S9^f9b6)~IIrt*$X?s@vALD~_fXIM4d_Y3CAAzRWm#kh6i%p`nM`mzL|by|}gIdsv>Vh0a<%Ewh)uSJX=RE%AtkJ_Jq$rOEvyZuV1X8(-sdQWQIP3Np#e&TIZ=4aP(-TU3Er_7m^ zd)f7#;8Lx*Q3s`YFE6)R*tco>i|=oLmtD_vxL7*%t$t|hudi8)u5CP@artA_p8Ls< z`|_{q|J;1}YHqpi>C1oZZhgPHeb0u)9=A>3Wv`gEb;dQb1%Gzks<)aQ`K@x%iT@1q zVs%cRUba1V_ja4$N{qNQ1@WQ+6*7dD*T=d{YW{k7u zw96+quKKDuO>9=~B_~ng8NN|HpHAtS+9gE9Z7?o)DciSV+O91bQl?ygd{-QLZup-; zB;0KG+Etee%*$^V+{wKmlMxlR>e;31|K_iH_W4S(`00-W4fuS&k{U7oXLW7g!%T-Ph5 zdg@oM1|B{B;ppA@wq=VGUQIPy-B`E4Y%;gJ%lVo2Hnp3smH+hT;uX!cCZqY0Lpl5qeCQB#W>dO@~o6Q_0yHxIxbFI61Tu0u2hELr~ z%CqhL^EOMHo|d`Jz1xjdH`Cql%9X9ZW`DHEy80(@viGxemv_sPGp^nG>Kr2WdZml? zmFoWtm)6vqd)w*T1ZKv@h24`ZJ>csVdgt5LN!iR+ub-ZHnY-m}wcgLn#Q4h9FXQum zxBB}%ymp^=?pg&~`|jv3`)0q||HCAITl}Cx$&#lUfp(n-=E{bTT2Zd#-Dw$satdJ*O}}6mwlg`^ZU-mxSLj|t6sdW zn)X}u@!qHd%{~TV8|LqN5V_XmLW|E74|nX}lt|Ezpl+>g^6 z-tITjl`8u!{d}?S)?}X@3#_!Z?6%`uDs%m-*}{X1cAM^)wk3E1 zdRA`0D-TCw=NbMbChRO>3t}veZD0Of{#A6Y`IUmwEcZ*R-l$zX^@;sf{?Dy$=RG4C zO%8djw0rT|Evqmn(lmH3sHy+rfPkQ)C4-UE<`#xa9wA32Y&yhj()XzA>RJC50Y<9; zNe!XzGm-=~IJ9k+&rwv@4rJ3xQJ-~?gR|gGWI}a2gkA(NKFiAcuzOX_yq~H|KEYTyH6F?*6 zp-;nhXlew1o5RGRZL}~|CGt7Lfp(!(qv)f;5-gcAsZ*YI{cvee;7D7@G>5;ugi*l!~d&~MGvw}R9IB^E>BQ*=R_8kj8EtL zCE7M{EjVe-^;@aBMC7wjh@_yx%i}@X-}lKq;(r+19{kd7+0qR)JNFgKxL&>+xFuSt z)b?o4ujCm_&*e(>oz^M%?4Bi@r!~ud*;lcy<^_UF9j{IiIIO_5ra|IE{9V@B)iLw$ zP4)D=&djvp&l6+YIW|gZS-J|3d@dXRP87Jta%V!r(-jNeo>NQS86x7@`J*~Xpf2VF2` zE=D~Ij%9D`Ew{xUcHQ<)Ha*~vKiAbiPMY!Nzb4+^_Vce^bN|aV47n*rrRhobt&1#o zF+94ocWzUWVeC4Y)J)5(=Fo7=WySMaMaK8-FPtb z%5t~e@^SN5`YxHi;qe6J{qK%lJ6v|hY+rWNSBc|ltFli-%~*KKG<@p(;;5zN4cD(| zJupyHmk!#_&i37HYk!Dakzb#M(cev9)BRqjn&0LUWt_B~?~(Q9zSEayZ@XWf$GbLc zufZip!>8=5!bY>E9AqhtpLA^X_t}jrb~Pkzn>9(wpY^fC`-nil!pdN~OE>P`_1OBx zy6&^r+HCvKW}f8sn(M1;cw3eKGi;dMRe!{BiLmVYRlE9gUY@op{m!*$uGKo8nP-o; zgjG#W+3;#_+veOkw|ZBd4{CK2)wO8nSas`WnYm`@O7$L@BAW*bb~HCW?$SE$_cJzd z|MES{mwRtrw|-ZYp#R;1{2z;F-nqDA%lfcu55%r5@U`l-iK^Rn-0Nq+Bj2y$%a?FC z2_6wT*`NN~pz8D8m)q~B#J^Umo87J@d;ZR;`pDH)v;H1gsh20I?XuYKkkp|V8HQ&n zPc9!+uD!lR+I{7FiPpGVd6A2E7Ihs>U(GDCP%*fnH^zF(!<+LzE%f`l)TZ_K@9r;8 z-%AEQTl`gjHiM|c$F>G8U&mNa<(7qBS$#%NPs~%k&9JcC(rEfO*W7KdKBs6pp1l;@ zw?M>af_t4dM=RsPI}5~FIa(g=DQLVU{z9OC8QX-exyxG=1Xy=XczQsx%!wt--7{VD zufUeh2`@dDJZQZa&li!mXoq*h-{lNK;=*Sxg)p^Nlso0RH+aYND!j^?qT-S+lD#DD zanr#NEl$fxojr_kZI}KtlWR85lY~4|yi1FXYQtMg4hpfe#He`{Hpo0xnCqfdcSymU zt8xZ|Fayi+m(8y>9+|2u$@=z9o2Zf0b)}{yUcobd>|U@XSbC;#Ff9wX#`MbF{M63Y@6NLeLs*24xjfZ4>~Lw{>fN4pCmnNxj%%#& zJkm7zee(v9-9DGgnrc=nIm}dl(V%tNl=+~^oo8njF;3ckT3;u5+p{^RS2hJqou@Z_ zTg--oMHeik+Ib8cmv|SSTAuV`_P?dC=1zONEqbn3^>MFXq1(Po_3P}}Ubnq&(zjV# zLNC9_NX@r@=I*-f{x0X0Rvz*1&TlQd8`xL(aC=-x_FT7J^8ZAY07tX zmswe}O4UD46)HWm?W^C(4Y}J(=WcjE#hlyydilakhtTSk7i`XyM!ieUv^@Li#g!M= za{fB+D7kz4y_xs)sO!bPR;??(?OfUXa&PAR9gn4qo?GZGZd&rd=eI>cFBf;(&7YHB zIo>(8F~nH!yY}u=>()jstc_}!e#N*_YwyXklatu4>EumQVEJ6A-1tyXkGp5Grqe06 z8a~fkiEBiKmRboE6_w@`7Iy}P2+TP1VS0+1`9qHI>FxWNPADxCcw?Y3DdssZ$L5mM zbn}AsIh=M&d3`fZo++Bx@-tRkR7LS%s^xcA@g-jGeXltEGA*rUJf?p(L#x%;L8U8Q zH70rUq0$YzrQ;*dKh?iizUr@YsgRS$3Sl9JgOSQzZBrg}RU1YWH*kmu_VmTLENEP6 z>ciOi_I+OlYmV>sQD9iEt{CIL zDR_Pk69?xKEv=d9_xb8L0$5x;zl#d+O;z#qIUv-)&|v9sSn>Dg2Iq4s>p13bm#7FA z2=sP0VAfAzQri@Fr}1$^o8r<10-Fz|Z~4!VBETdruwzckg+k#b=M+|7(v$tvOU0b~M$!&wtdZ!64z>FgxH> zqWb(2v&Gz93`-seObRG-T)M>cq@mqmS&xT>`wp>8xl^h+LDR1n!dZkC^tx~|zK=pcH6@PtNuUg@ECi6g*Z;*lRNQ?)Eb-8h626g)P%dfi#9v-GRMYG0$O@22Zk zU*Em6Y}MCSQd7l^-qg*=V*0jofyFZRqYe?t4v*5((zqV&IDBupzocL7Yq2#G#b?YB zX32JHSWxh;$LdL1sDYxo-HFqx9CqP6VioH*8D@)2oZt{>`$&6li(7|JCp$x^_3z|O zf4Y|)>pk}JhUtT9>9!S*RUI9bI{a&*cU7-i^~Tn|HaSzfd%H-*@fjI<|HO`29tsz` zu_ik1(+QKg@y_X=y;s@4aYJnPjK-HJ3CeG4%e9Lj}94bKF+E=>*(dJ5(gbut#RGEJ-GJD?6u2&*8laoHF06k z_J0o-^G43tRI_$f^v;-uN7pTTRXdgJu0N80?{3c?+$PECXT^T%`LuVX2bTDn8!o=$ zuf1IHp76v@uubknZY1JQ?Rq(V_puwkCPg`VpvCU4G z4tCBJ7L0v@B|m4~m34R&yYg7GhoZ~WgDXXrCoUG?c;@-=Xp)V29KUG2u940DS9$-` zR8_;h=ldVFqEd04We2VC$FsVuN zoV?ovxSC}AA`dD2d!fM@WKpIkAeuI5&Yr|p7ADR^LI<1}GT%z7s4zLEPtQ>hP-cGD zvdO{V{60ow&Gr;c>kXWT887*yzCTv*AnMVB3w!rYI3w`*_ruE!Q>S15JK?~FlAP_& zmTO&H(~@S=v41MVvxUZT2VOk4)e2sbb4#H473-TX_v%)>bP!s~F`>-x>I18r%RTK2 zyQl2=&+zcqF~yKL36F9wv8js1^JLPG$vMqoTEC&zz*%%h;w<5Lft!|P@$#rlv|~BM z(xlQgceBu}!>0t;=A5cfd8o)ct3{HJwYZHz_?gSm!WnPcGD}YRC}}XcsT{GGu;7ov z=}pXTib39Y-%eq03lUl%;u_fguAN6f=97%SAfJvC1E-OvtA+CxGk43%P60-lCkv*$ zi924)#K_0cG;@`o{8GLtyweRNW`2{bkaoQn;g@YEee>E=Ce?ErA)&Pzl1rd3_Q-yPm6qL<-71y(IQnx z(V6-G^~a$dGbQ}*a?M+CSkr^&$ZwYgSMw%>b52_P==-6Bu&k5`EguE1{;p%;%;5E$ zaVa9FKv#Y6_a zA5E*e9&{>cyzPu|2za*AsNkfRTSTJew@H->MjmcjZk9`Y0|dA?1o-To$n6jpu)w#b z&|~H+hosLU%)2GFNqYEHGsG$$^I-a+W9`+nS#w_O1?eS89$F!NpIHRD)gGAi+?kgz zu`{KkMVqN9KtR^e;^3-xnWr3Wl64saih>_YlrHV```VRXqwJF5d7@loj=wY0!w$u; znk1%%a}{3oF}^OVnmXTi?fB0iaZ^)fg-}rZ!lDj2#-c@fGVBs_FL4?yHuB$Pc<-uH zs~>w$bcKh*B!?LfCUrkQEj5+xePmDBtkho9%%-%K#s3*v>(7b^bTJ6a<*9Gd->>6p zyyvQltkaAO?+#VY?)hjSLpaNqb-2HS{QW`qqZ)s0`OX|Gs(CF6yO!6|WcA zDzGr6ZvVT6U`4}*cP)mK)-fLwIkhXx<@uaUW9}^;6YXy*Xsui1$vmZ^!(D)9a({V> z#^al$-`fB5uKrx9 z=~i9KmQ8WCSXI7|=We`9#`dE-wQMGyTY3BHD$6KOCg+|{J=1ch=g(jEXvs;DiX~;M z^ZGVz;a#S=LEmgTqXD1cro|g>W)yERT`Qa&#isC7a=z)cPHE+X+g*2VD`?A^zSX4d zlH#haR(81;lYjAkYk26h@ZhOt%Ng%Ywq*;icJ@BNqAay!%7G7U+mn`kEsbLAVT|I^ z^p`yJ=gMhuzC-TJb>~Y2GS&G`&Xc>rf9u_hdua~>zNx?O`&f86ET8d#$;z1wN@{X~ z!oQq4lU`iav=ry4pFQi+l6v*jZ!!uut}!qiV%|Jqdj2Ny7q4fE?O1Z?w84ItGWRcw zxxd&=$WUQ$S$dn}%TxuC4NY9OYbVYZ^{}`2CG@9v-E#SVvtw9vP6#+X)~Gh?Zr7VO zRYK`(Vc!9(ux`e(Ngov^O^Ca3V)^Be!;L0~J(ayvE&VyVpLwga|J2qto_P7?>h^}W zMh^@kPUd^Y>|r`+{Nb_e#Jjd$Q*Qe%=yB&v7v`PRo!P!`vj0?Be^Hr4JJ+*pJH82> zc(ned^nse?O^o)85>t1j{hPdHn&?Rbk9YnXwKDcEw%jo7c`` z-Q!xcPJPDGV_U8ts{FU_Zn1-W)a=>Ig(iJJraUW3{mAhJ@4iV}y{XpP=$LQlu;aPN z-p^0}nrGF{;Hl$Fs#ufJZfK>sUfA>;*W$AK&6aiCI#0X)Dv2d`B_-Pz-pc!SYuVJY zL<0xS!$K?xVmnu_P<+rSb!GBVs~PH?u9v);EY}&u89Zxn)M~Z;JApy_@T(@~J!f+c zXjNN$bLp6^+w|bVeI0EP`#Hx%H3XkuQWtEFmG#(Sl4+MVQEAhh&&MqOGCT~M$?iSt z!L#W?916{rW$vjf?CpaR9xLAnILT77*Y;=HE|dQZXB#I^+t~eT);GZkD=rCa4VYxV ziAyAisiJJMBA=6k(}J@V3k^CHd{4YQp=P+_6ce{wQ|M{6HG33`CK*mRXnaoY+DV_5 zsdv1*gx&PC7ATzYsw_XC*uruseO9o4`okth_eEz!W=!(q%;e(U%@}L(;V}b)W8pK- zIR*kwD?5Ydzcpi$+faNZ-~{t6^|u1=;u#IPzaE;Q zFj=Y4V$w_<1~<>RAdXN+)klx-v#bm`Kdbh0j=+VN%{u)(mTR?CWQ_IHRh_yj4lwh4 zVo?(LRWD*{v!KQ2Qr($H4hy#I-aebrj-ew%rRjm%sz<^W3C7`C*=NN!cO7b1*{IRI z{69mdq67Q?3r~ONT`k@hFYRr%*4Juj6aO;Puez_It(J1Vy&N@Dcg1m!CRIc6`BQXq z{Z{L$Prs=0(DGqV#J;=|=QVTgExf&Qt<)0kTghFYu1@&RAd*}D@?>G9#b4#y5uxe9 zleR8V(70rMu-!yx!3lxQ4|bemF1=E))Te4|tl7qeFfPJiQ0$0HH7kMw_oL1 z##zW9I$z^WlSlCZ8;{_LhjiNyKXP|g_|(9s@K9%xqhVVsL$Rm(`<4gnQ*@p`NO5ty zo@G44h5eAuS{84Wr^k7OvZ{R&uZGTP&UCsZonYb)a z?#gnQxi0n~i@$%A_7sawaXks|kYrz`W>W{A4?VMjjU*mUIP+Yz@Spk}^NGrB(;8bE z*%WG~^jJK!IBCQue0V?KjgIY17QXd)H!c?@xC)r`OQy>*KdsyU%3NW?&eopiXV*#X zztzR$aguS`VvYSZYt?_PJ&^s6ZMEIykTZ3{GVu}dzyA3@{PigJ%YTMfYs>fFOyfUu z>EachOK;OJX3aae+9${>O_;O5W6z%pS?^0YvY35NGp#oKy{Nb4@=+028TL5Y{@)oj zNBvyyA7P&IW0}FPkocg*nU@o5!zbJdxnbEoud$ceHfrGv?am2D`fnf8yKBEa=vIc+ zw{({kZYR?-OpGSvG|sRPsxQf#={n`MkEQ}kUhKMv zzV3ruOer%GC4H>!NiR)tU{*>tnYU}xmsP(7+jLij7p{t7RFz`W{OabN(IOQU%wl}F zCDrJTc+r9^CB2%&w`tRrZ#jtuHo1rt>J(qv$<;hd!ueLqdFSB!Z+qG};a?96AM`L4-)CN4iq?hgO?6lxe z?8avQ5}t)?IWBoM{|l5bwVJ@ISAC#(L(0Sjg%PE_nyW=sKDp&an_n(EQS7sjgJ-G4 zR}GUbTOS@c!E^9q!?~qfWTwV(R=q6b5Cm$9`iMlLq9`?ngD`{<1$*qp&nTux{es+&v)%w18$&z_j*evx9 zovprhi~VS+<2gx%isHRz`?fEc;HUk5PHC-erP{AuGVh{~X>Ul=VA*jpWv27N^8p#2 z3_KstI5=@|>siRp5xa!L?WjbemurE?;P{pS%-`tF`Rfb zfrCXqb0VjPtJi`6i}OdEm;+cvbo%T!FFdL~sjR84gCS$`lqtbF1yeYEOg$pmMFYIb z9A~O}CJHI@-8@u~Ij7@L_RTpArl-16(iZcC>mmlzf-4~W5dt98dZ|cwVO(kp1!dKkC7H|DGszLaM)DG9;XWxU(?wym_#w>)s2hUhBGP@9L!GI~qr{79B{R1LFS^2Y5<<#!>gx}S?wk5E zdT;*IsqZCnayeOJ{~X_z#q2*l=&_XIiS3_~Ew^kqzLL}NTE`N}E&V1_B)zvOvW76L z$lmZxUiIhTEvx#)+g4fU-F3fuM^B(?$x0r1Njsa3r(UuKEOVAS>a#Iomw4!rxiY~A zLxS4wXouPMU8tC_cWsCEib%nh*!x1Vsgqe=^VpNGqaQ&P{{ zMMGk)I!*3b#6Icq(se2mI2}?|%WRZhPZx^lROq#uWBPOR_oMHhR;GWgI zMbGXD(#Y)7a5+`|==Y}`q5oDEJzLhR%f~M3<>9LS-d*|X{eSx>+`sdmVWxDgZT;%4 z_x>I;KdWYwf8|!Zy+_eH7Hj!OY#MzW)rNKwM_)h7erdfzY|iNnQ-9IIOY?p#wrE&q z%pj}X`2IhGN#KEyhadklyj}3DKCpb7+u>aMWvfna_P?@||8lxZE7STv;mzt*ak=b| zyIrTQ_4=9-SnKrdQsIhHed7z013o^}yZBGzTFKvuJMT&yGUc|4x)bDj{@I&n&knyk zdhy$pwX6nLzIFvIYfY&+cgb@48>hX6?*gN=rrm4`%w6Qo9p$&;!iASVS$^*OJgF-6 z)Up2zA`zKQ&zX4cs4izHO}+ooZQ8w!+gH4DjSli#_aX4$Iyu+l!4vmPOx~QfElW{z z3)|&9hKkx1CyR}9_Xy5*nYQuQteEGM9ON>SYCmeOEP8lCWcjCCvvyx~PUh>qtg-Ry z9Pat8W(G|wd{TnVMH$Mr2N%xFp2LyxHsg?X<)Y|ViBtV|UbI=R;B{@MGa`&!f`2 zl3p$Lo?jvtt8jDWJ4I!?b6cxodRMOP>d#&_C*WvM@41Ig3w6`xYRMcp%FuI=$yoE= zy{)GlGhC#mB|Nv#extr|Pl0}k9Jkb^;wc{BN4x9z)~qklFsEuJ-L)-&bU zl+yi|3RJdb?T}-va69^7=FV436kMkFm}>Q|O32>%*8A39Q>L}%Wx^7n>wF!2W_@}T z<0j>#UXr8dDCOO;>|1lk#CdEc)~zR&Jd$X=!DMjIqr>c#XJv;u*EUofRM66s|| z6VhUCdJ(E&mbqP@JzX}M7iFxf_0p@VWbU(gBv9DdxmLsOeCZ9a|W6HmzJH>Z)KPu%al@74&&175=k>+lw zZRFz|adp{s@#RY%gw&|=&3msC{H^!&`u7>#clRCsDza)i+cgun>C=9;8a+3Tn_fF# z$9-q%^uWk)>%8~dHeK6lbTMl7%4t{fcCO4c3VqDu)J{NILG969fZv&!9gzE?H{iy_M@Y7J>G_UD}y@-{hCw*RxB#RxA&hX|_ON@*#aQ z^Q5^;Ikl$jZ{=oMu>X`(Lruv%A-J|DCpMQPq^nS?&+Im5=YvihjJz zqPpXnKHE8qTc&#JX8O(M^_uE8D{-OUnwBc}r}2L8tl5sdvhFfknl9D2n1QSCoJDi~ zw(htMOLyGhHS=4&#^8)`piJ_m-);|`i?%Kg>veeclEp$MRpe8i*|(>=&v@M_$hS(! zU8fm+C+fw{gcTAGqGSyg99vv|H#1kvbp7k4)>S%slX#4KeqC8-Yacr2%V)u}|MG6V z+b+#&^(~Zh%c2LbS0%{0HYj<&nX%~B{W8PJi`H`P5zZFdXmLI0eR=e|=kbNRRI@EJ zw@hM=;JhULP->P_Li6f_4h%=8^FPfLVoC1ld^p9`<8#@A{ZZ~!ot~~K6MJ_(zEYrC z*>a98SWqy3W!PLZh2jps&l)op82{LCK;%Qm&ekx&?ieni$#FM&-zfcVOtUV!z0PcR z|K&R@PqS}TG$|Ke{wu%d{rQZAE|(X`o{d_dF;lE#*Sjq`(6Y{;sdtKP#xTn@vKIcwOwH99&r#%L~gRB7pzWUW_O zI}dCL{joj-c-`@EelzwmGN%1u+fC)HXd2t*xX z+WT(a8k4R2v&_EtFU^nL`f{h1f$6k2B^TOSW29ov#m4h3&-`6D)Ay>`@3fmQMOIDN z%E+ZT@hOi_ip_qWFmvr)c{?(aIr-VW7JSUVd(JqoXx_7ofWoc=xz|3N-8ECsZt=|J zuIhp--|781)zT39pP~F=j@X}r_wJm(91(HJDc}lkt-Da?v)}Pq+P9~2O%K(~SP>t2 z#N<-(ti5l3q-|{ZefCr<|I1~Mp7m<^ub6i5T|)N#rCaTFD`WoZUR`Axy_M57Sh8&9 zyV#xVvjg9_y?egas;m3H+v=m6Z>b%>ta!L({`4H#eUI1unzVSu+xT_+;-7J+P1{g# zJs>l=zVo8l!ELemhvq%sAG*5S>C)ufoRwGfiZ*q1<^E?lwIt@PZeM@mp7^)R=H1-B zbJ`Y7#;|v*D`TW;wv=p*I&Nz(HqEtYXO{4-MH|jecRKrf!o6pgD%ApB*X)1&D>pUv z*zC8rzGp^#cMKJ0vp7=!O+DCf;iA39ftMA+6(1DNxp3_Die2x1{o2gFF5+}>;FdcE zibs-vB&{%)J@=oXVBO(LuXD@VrLMlu`jvQWM!e_#+LKFr8&5r1a&=emR^gv!YmZuA zp7#9W=A5eV^*N5U{1FYI`=;hPCGyXhk$7*{?u<)!58b_bU&HO4W{&)GRwcCH?6e!*Mx_H*Tb)rlY zPd&2#sU$r0#Gitm=}8N^Al6 zhFtHgP03qc=oTF9&u~#Px-%zhsl??wnY}wNO;-(FyiE1g((cUA+%Taf3$)TUpEA1h z%tT4lB|!D9VMOXxt$9;d=q_=Xn6`9UvS*KVyWy>s`*yBe@{;jfq2GtcI!V({_C$N> zOxmoO_OH2m=bf9E?K)T8&C5{Tb!h4v86i#SwhyUtIK2OEe>VKwORJk#7>=iYF3EvUbeS^LAqCGm6zO) z$Pc+&T>bKGbcWWeK7}dY{Y1C>ed)XDxAfh!&xKBtJFbSDTfD<`!DqMACHLOt<(}BL zTSUXqBiSXaNDSrr><8@itrx${PlP1(xXq=uikpOcdgjk`sJ%8bLcO9_3}w= z$iI~Ao~4B=j=t?WH$x<5Gk0L|tov%ePX}Aw&)Bvqa{JAIrTz~S6IdFL9uCRP`1-cD z=yK5f1zxPnDx)^3%y`;&cZGSidD-n!DYLkJyQ-tNhAg?|ecJWC>Q*iOZU15eCweVk zck5EN{OV{E4~8W>TZA6XPfZGbbTm2qQF!`{B`>$?8dxo@^2s%i%Qfw0KEFDw)yQOz z{$~l9wYmPgKOWuI@BPI*;?1G%sLLhC)L+|AFFHP9*J~4{&-E{l3*6j(>y1+1q9~un z6YFFzZr^_A{n~Z0<=d{6t}R#`?KdgP-B?!iR=r-Xr`r7X4ZA1iKX=TXDs{BJa@ni7 zlgnP-J^kzlY{Zi{{Av*MScW-!CvcN&b! z(`FR@IP8*VE@i#$X=2o)4_12d_x8$_aG$yMtYS&dci&Z8 z&ss^(RZ&{NzuJc{+4oW2qr35`doC{9w0LLq)zr%xS5_T*bltEet86Lj*^sAxv3%Xz zPHzv(jVjff5;8l|V=7N}KwwH|{-GcG_qz*bMJ#pqbx-eksqya6*5l6Mm&~TU-E~Ks z>3LUXmW=16SfAHXsRyN3nD$Ovu{CtXb@3~!j*Av8+Op&WqtwsZbMf0(nO)J0x?Z?6 za`C0f4y;R68sGBHGi=;(J9yKMm3i0pdkbH#tSH+y<i5W8)DmouCFfLf3ZjkzrFL;F{ zuSW0=_3Ipl2NDECrYyhD!n~}a#j3c0t3X!V}*&sW}{P?tFYmfXhLHN#*1bX9>?DMa|W!55Fyt z^<(i_$-02o^C)|kqndk=x&6%q78apK5vJhnJq#vn^B7LwXV}ubRJ_Gz9>TBcd6%T9l?_V3xK z>1Cy>c7A!;HjQbinCJUGleBlj-#T9ZXHeO4dG(I#g${QWeyhry^SYatAF6R-*XboI zV=i9(es1^8Us;dtoiAJ^{o1oat#Hcgxy$P}R&HJEw|8%uz^{a+1y!Cdl+Em64w<+pJj;+qSSMED)b$0G`|Aot{qP%B5y4rcX>F!pQm-Y+l&9oxc z@bqoEvFgjWpK%8{7xAymi8WksR{dMDw+oAew#x?A%aJDvd+!I8`d$w22qs8#ss37(#lm#qBorbCN)g@#??d|_cBHbs}e zA2}8Fu9%nDIrETu^@EJAAAL)nu|-Bo^%RMj2pM&~j|;ss>vdMDYX{@OgTX;_6}}(4 zdN(#PCwI;2hF^9wt`#!rB-meIQnFJudUNGZM?Aid3R6b zQO+ytCdSzrTo7D%prKR0X0dv~#`W7{uWsIv6|(4y%C{4OHzl5~;F!{!j>8&t_DpTQ0vWVz6*U_iCoS8!LY%ddGc_T2MCWT0((B z(%!Xw<*S1~$#nfazvlXZf2*!ne;4}tTVwZ%h->u+ufIyU_Sx{)ezvuKKW0m>Yq|Ym zI$){m+8um3QSl<+FdBuyChlH)KDt)u;`UTZ9yu5ma zL2IARjJxN5RN#HI>)ERfdse!1|4w_c%42Snuk-qkm-Bxny-fC-n!T*5B}C;@6W>Hr zyMP(Tu9t1S=s5KjcddH+teG!u`m^`Fn=DkmRBY+0l|fr2j~-le{Zi2ExL2pnPMh}Y zWmm{e-=*s=U3zqFuKQb7J%e@nzU!{Mwo<(+AA024n$Y-wMLTwXyDWKo?ZmIVwW$Rn zhdm1%q}IKel5yAJYRVm#3zkQXJJgoEew5X;{xy!hbq<4<+i$FAI`oq^l7-&-k|Tk3P?azmXpK77wQH(0ahGG&HPKYFNi3JA?kei>+Fi?aMr{5PzU@m&HwdiQu_VS^DsO4PwVnJI4)Lw` znWpq0=A7Z=oZCWD&w^^JrZN_LYfiY7Zt$Gt+tH1a*YRu$nG!T5_`uW9s_Hz^InlYH zCvEi#84XS;FW-D=(Uq->+!mEGt#f{)yq%jnedDsV8<#CjOG|IKEKz*urY~@!1CcX=~=PGb$RWB~~&s=q7$v2a2E1$7zo@s80 zczJt6ShmTw8I^@RMghr*@mYLZ^<23YY&pBaOZ`^YQcLdxJ9ixG)|xf#?Uk32TnT%< z`xKN+vZm#=^RBybBgj;fl_z`7n^UXz?Ko2_ZZvu3(skG8syf}wnttfXXSd*9tsc1v zPjr1P7AT*7$TV?Qw%w(LaaXsz;+nQf%ylV$=B#7UT6gYR`6(@)vsKpDfcL`9*5Bva zLKX&oXPewvEi!>Gs=%{TVq@gnHJg*29V;)b==1eF{%)7mjvs1!glRl_~_p1S-fFab+4p6 z@cHn~?bW?|OSewtu4AbFyE6x_j-6)u06*93)P9Sx~6pgt*{WTa8o@$9$70Cz+v2;3_3Z)rO zzv)rkz!el6V)$9&5**KBDy6`}KR!RG|OiHe6JU8c;6 zDOtd(ZsKZLUK1$s$BBDJ-^8$wTpm-BLsX3>Nft)TTV#1;o3}y1k>?WcXJ~Ll_WWna zQ8M3@k!{#tJde3EMDje}jdQolYftB_I5jQwt}p-8lR}uG+OC(~jorO>>N|;RcTTRWuKwIxs4Ex1+%}2}_lEFl%X<)V&_tVy3g9a=nd_tj-m+;Z1if6RKg`tORxivwDphP>rG zq;Iby{Ohbk?9yiw9409Ts{7x|Dc&14f8vumeppL^_vfUDv)?Aqd7E$2A;X~N_b1?Q zUQS7-8;6~ zFWixB>7gY&{fhr)i{t5MUxw{kadbUfUdy*%v-8zn{AW1jeQ{gK{L-kkZtU!QX)l8( z2~5n`b?erJ=YgWOvKk%tLRiG+x(3fEc2oDbvG3l}G_jY8QrDR$a`#tsmj5kzVR8A& zsU@O{N3>2A_MUye%4%{;s+H8*bz9x9>Dm>cz5)J*zi$bd+`emTD)H%o&ZOkO;l#jJ9qW7D1DH?HDocUFDQTc)d(w~X5(AT9RzhNE6btaI*Zzxi@-x!03f z-#VWj4)rR2EN=djahBwVnZ0Jew{6?1b|;u6Yzu_8;mw6% zfzxjvKAMt}`)jWA{z3TkTW5A_UNhHap4t#Dlg5>H9hXafmdeHOu{>-)Dl}EIv2=%9+3kw0 zjhkOt7nkmRJ7r?Vj;rg6xR{xjORAnNWT}kdc{XF>ob=esOCy4JGuXaa@bMt`oD20g zd!kBgogSU)apP>il4{9g^13)QdHsumyG2iL>Ni=3*W5hu_4bv=H#cl9%3XJ&{aV z)!A<*k7(rrMkb~P{>1paB%WnUD?}%qVhGmReVM~pSZjt?0E1ZO(STYDCx)3x3p#zY zI*ej1o^vdEX#e&UgOK8!>G!I8_>w$-%+?W*dr|O|b*4~DO5x)}(gu$HM`kSINnqTe zrVo4+B$;PD=DcI$^s$Ah@uT4V78M32uXVzQj~p;zwOE(^)v2k? zePbY>Rl)mK2KoP2AGP_ZNX}oqPrh$uL&?d?f;kWO?Amyq?=pWD(}R6ag&)hxdh}&2 z;*^sRQV6NESy1fJ#5FJU?+1yc;>`9E{>3F%t^_khO|D?&E|{h*H2D*Eu)tZdtip3f zj0$b9EO=7d3K{WAhVYXzbwRbC3!)?ou@RaE^P{(ommvCUwr^o|=X_ zg^gOt7uZ5=7Hp9bs_{H7aeSpwOovO0pU=`8ixgKZunt*zy65l2&^I#_*BxMqJ0c*( z#N)}elfg#erQ_;@MQvLa9*W8;`;2SOC)X$34Rv=n4?nI$r>-eXRTUCfng^GDJ?nZ;_2+<^iA*w>qctbO}BHf(9W z5tbMRu$3~nybG(Uo_$G z^QCXgqkmbqK0fQ{ViCL~jnOJB__vbY>{6@T+b=I(;8i%Qm*$e--kt1S@mA!4obZ(!vvaBjBRQubt`sH`$9OE>w_j9=H8&dtHc>9lJ z?_Ggwt3yl|mNKj;HcVbyaI1Q+8WTeX@2bZJOH&_Q7Pu45oqcib3QrY1mQ;;(Pr{zN zuiV{Vw$gmwty(p;`z&g?A>xx#39c87dQX4aK2+Con+EZ@T}xqOq}yoIH!P2}AA z&q;`Ot$n*Y^V+5V3}5*sHKf#kI8-UvGkf;kE7Q(BT5&OI_1%~3mTM9YUuN=qkS@5Z z`_;vj9-FT$hXxTBDX%TM)gt)b==+f+mmpZ+CZ|{^hI%c_V>E(YM!Mw`4 z*JbMpt#x{n9h7&OxG!G2NuT@R&2w&Um~LV6NaWS8$DU3%ub0La#`{`juRZ!{diuRqb(cwJw)bpX>A6cw z_1;NOwzKaFX7Vt1T#GA+J8?WY^p@wdsx1sHS{kP}sqa*LlN)f~ckB5T$7W7AU}9L? zSCZT$#W}%(andrwmwUzTol|?8ZfT!BwQiHm^|g2JpSit6DDJ-T_uk8m67wYaZzegK zIxLHAzRwu)L|x%cpehGTP|994cPP ztlA7bMO!A-ojiW`%i)VfMg~_I1-z9zJ*twVc1->EnL)6Dk>#vo`|gcmK`y@%MOhj+ z-fB)^Ho72HWB9Rw)ga)3noU?hhw}7$1)fd}NjsOfsC!J3{d?hnXWyg+g6;dHR&nHK zS=%I@V@xrbvq7!mfc`{2+Z`$ogvBPR>sS31xWve^=)*^Lj_R$PXDrGTV*EK6gpJi^ zGOIeyGvMV|q@Z|q-qau#bI-h>v&J5i5>~M2X&bm-6+AK5)5hm1594Z~{Q_Du{R$Rd zzBFrRaHmHAO`iBV_PQ*3NMS$ z5IUNE)LL=r4LNq!L+RfS2hZ5zlVa@T*}iG_%U`|A{xf*Z&8nLkpeA?Lw=#LMQpELo z9_IpHt9_aiHbwTQxV${kVl=Nt@!bsJ8Qg}qk6SuNanE6zc`lXBa^8t`k5@^Z<>>QG zIOUL1qu8s`4W*Z^b+{m)+~|yl{m~jPqBsiBHvq?BonypIu*hH||>8)m(wA zUQ_0Trgq0EtIazcb!zgAU&|i$&5ouph-EjZ3 zYYP~;{9NWNnt0Xv>|V9dT%}9jUh!V+=-u{u=ceA>odvNfmc0S$a~7?>TwT1q`}3|C zze~;C8CzCn&;Qn8&&?7o8hT@+PD|Ofpxp1XWj9y-o4Dy#(5~#2A=R+~>(m2PeXMS; zU!#3pWN+CzKK7|0smFAx`>v+$xLw!wa@R|1b_G5e?&?P?{xeLw9X&_pR{tfDsg7+- zQa8W7+TG#hyGH1v&a}Jkd4KsUH9O|TB&Xf|H~Hn(T^&lv5)1E^EWLNt#X5iWtG3ou ztxZCQeZsTzu9qFz_~dfQ?U$kxWuI@8YU9t@s&pr_Yvr}XOOxBpuAHvDq(e+EqkeT79@uC0INb7QC6EDZb?TIA`qUZZte|G8r; zFW*YI{5orQ>ua4info?(Pp}mZFbc5XxGwwe+4O*zx&>CVw{D#`bE0knZ^Vwbv(@fy z$i%o$=AdU?;uE`Pi7N~D9eXiV_+WXj<(h($bF&ux z-jerr=YsqG-T@5Hw#RW)RG4>eB2f% z@5DM@kxtI*4t<-nRpsX7#}fK4Ut9SXEQvYgS8&z0i-BYRKC`fCVp}vedt8!wZf#WZ zKrNwEx2TU#_hL$K>CcQEI*iOx*GvUOGUQzzO9xz%l2kSq)rvl_ewu9Lfh((*Gd+sh zeCCK-rb*#j`AzPd9^U?0lO6DqHSYVf^xmg;e$MIM{QFYf(Iz>Y;|l4&b(lnJ)Z94# zGjwsXDC)+#2FnGJm(MB|I*JZx}KN_iBlXra0){h3UW=GTJ?k38M|kKbfG!qAk! z>Lf5{_X9?TAV)S%-r^{+)Rj$4MfUrI=g2Hb5MfB0?2)OKpmM%UfZ=0f5oe-JKTjBA znM2FzrgDxvh8+TXocehbeHhf3_t$5xJ`^K-Ohhj?g4er$FKA=k z_4{A@y>5S1JNHB?Ut^11^JhNBk9f z@<2rKm(u3=W#6*3PyJph<#0FLYR;Q!F_%9btF}3wb!XAlj6Hjwt-N_(b$RflceTd( zm!rM&^5tLMj}egeNf0U1pQahwsdqf6AxQUc_0B7M@7>Ld+p^Y)ZH2?*n|}=bOYQ{o z9I%uLaQEViS2CLY*<1Wk`cB{9K?f6D7FfpWu^Fi>Tz>k9Z#vke9Fn<$!7Fxk=K;FZi`p!G&OW&SSF+? z@$SH~iXJw}=TfhqhFodYY?#+uc&Wh1?%WBdEQ3RJ0wN}hQ>P1f{a(mC&BnjQ(SXq< zWWwX$>C6TOmgOJcwW}q!70BPJZqIluFyn&U8>xpg4{sG!R?ofl?tAvJ+mQx)rVGSO zVzfB?ZSS_nW$SK*L@!}lUm+xuF591YV@=s^tN7&!rv0DgKZ^Ez+A(Kk%tpPirQ53` zQc_EfyTAR-JJ(#SYrCJ!`H~CEb0T5{cZFJSZZ`^Pd1$+kF=Ac5Rj-y6SA(lh;yRmk zSC7jHnlf;mQm&Y?F~;J!)qjQ$%YUT4GmHAM_+F3oy!2Z;CImh(`Rit0&amt33sWnP zOU4lw@0c(A;DCMTK!*9r7M@u{u6TQ+QWe8XuE}4j~+1^ zJ+tx)dGDUAZMKh=_8xWL zyROvSV&j|h9uro2Hizw%eEB=D)MnAPkAEU$1%ml?SFvllzCUx!HLUHT>c4`a(uC9Wf9 z;hOfF-QHnW*>ta!SFW6qbjw}8j5{-nPhk>M>O;}KdH3XZ-w4=U&1&^lN`~dY<;LLI z*L2>>i|lo|#Pd{@{h0QwEMJ?`B9lWS)>?P0lel&@c}m%i0^?tc#a{hqxX2&a=Vlfz z;;36T;oA=0nu4d(TS9hqz5H@J@PX%9H$lt)43DO0JnwV8w(*0Bb??>Kd$0QL=RB%) ze_4DzAlvhq;RUnTTGh2{xkMyiUS4*IgF(}$;LK6`rQ6=>*1Zj#wd#(=uY^a}R3_S8 ziPM!S+0v%G%g+NFb;otoZUlef(%Zgp|0S@KA{KZWJ!4UtO=5-rXw`c{=17~P-1 zE-X~#v~_8I_{3YD9E-OpZOq=X@YIeU56xw{sxzg&Rta)uq_XS^16{b)ISI+xFp3j;~4KA#?c=P3riNCU(=4uMJ zOv(t}r&v)?+#GU6fki^WCh5ZOv@I`}J!d<6{;G-p5uSwW+ZVqwo6U5kFz)kB!6jQ& zmd!ThTf*U9WVJBFS%Jl9YxBIb>Shxhg?yB=gBr4#1NSt*`T5`4+2j7S0PkrX%K4{xWjq<>JyT;hD#@ubVn+*j~zxIJA_X=kG+l zlUJ%s->yj3lXx}dCkMBZ(url)3+H96W=@J;^|JS=N>*i>RX}F`Hp6xAY^A*YMJ`+N zS{^Z%?w(&Cbnn#U)z@^Fu3Yn}ZBos9n=MD&OFb^tW|hv<3gay(n|k3ALzwZ~jI$T* zY}e1MyOTP3PVj#QG1i*yRd;8ev+-Yqrj<@!hSiw^JfGgkTCU&8R&c%|pzJ>Q+K z)a;%4{h`ITxyL7{1c@-KvZ~(hJjUQMc-}PVdmGAu9g0K~{6wmxPn5lI^#Zl(;{f`pD zk$uJu%(n_olr7tWXSlh&V`1Ef3^V?t)0f8{68HNiv6$zOpw-+mm73=ft}os`PJu=0S)gn54+c(k>gpQ(P}Z@?oJ&XCi6$}$N;A(B%jOzJ$$ zC@^_ms>K@3MxBa}ZKo2_&}6H*>UHuv#DtfQ>89N z@Ua=~%KEkJ>GtURVP(s07al2i-*dYA?CzpkFOyZ!5+KbEAtn4N3imD z9c7smQ_;tH%-ZYG+! zUA5gqSz(S(&@QLRuk%B`G@laLxg&9o-lElU%RUy|I^}C5b?`&m%#}S8+y1#qe>#}A zJlA{56rFdG+nuAry#!Z0TI$F$DgDPHrOEG{3XaG;e0s^p>3hQdeadRi9!fPUQiOX- zQl|<|;aauMscZAh4^0h1s|w$oVpA<-dCcw+GKop8`IR9Hb7!j20``W~6)p+$m0Ak9 z4Ch>a+4M-}k-@jW1uL0fnXX(R&MGO=5&I*)&_e5!_G<52r}d5(|FwJCe&YDlDyFW^ zmXf#29!`xn{blt1`r;Qmcdfeg>0IW`W0qR^Wv^aUdH<<+ z6WDc9R<||NWLwN~AsPo-OomuLYn`!B?J|jb7(#Dm$j&AOFye-$3J$lu7jns6u zNw$wQYWLi6S){e(S9b@i$D4-|`u2DIKb*dkwQbME_im~O^-MgcTU`En?AuH)=cc;T zQql_~GNzr4;rw_|x$qTZ-g1$UcRhAf{xe)&ek&;$e>ik#zszOz0Oq>cEi>Tv@qkX@=Exfa8=hBFmmYWY2`5Xz|?&UgF?y>KZ zJriX3dK0EHp0#ZG?AG$?R@HVVJw5FWTi;I6in~xC8kzU4S7GbJchBqv5-vT~@6xy% zyg78!&QhH2GldVS;Ly=s#8OX@|{RE;dId+E%&5jDz&vU-!JP0(odOXntig+%GrZqQIERhPp#?NSEY8mFWq+~x79H6ZARt< zhibRNxL3T(ua|0T@fOxPpAVR?ZD(#!yS!3eOj_&cdA^)xM~$10nztq^Efrn%oSpsJ zI`2+qhJ&Z=ejY2mvQMZmHRy;@>t3}>yo^1UU$$#zWL#>O6P%^@GWvF}FXNlSqimj& z+*aKPV|q7t*`#kT>z2>8D&5X*z4gqK-rQ@)_vW)~`t~P!>t&uLS6w45&nY|=>XrWN z)q1Jl$LI9CHK|;!TPou!VWV;ThmQx_iOiun(7@?gKDE%$0!)#e$!;T)N7+JKOa^*xc*lEYtQ) zj=CVo7~GfQGR5kC{<;UJ&Rtsa%2=VG_u|XkuDkP=M(I@FTBo>xpGUEGX?6~;wYA&E z!?FrO?OT<0dT~41@2*h&$~5g^X4UziYpde|d}gvvZ!;9#ck-{>yKOJsKg;_pUeTGA z?Roh9t7mQh8T`^Vm)&BUn&!DxibG`rL*U0`>9X(PWZita5jyhv9|`8kd$`SP;HKhI#3d zM-S~@99r@0SD#LSu9oQOrj>`}mij)*{C-?8J!VSQWk$htWf9p;2bUf&T^{Ss%WGtF zOt02E+_h(p=+m_ssw}$;uOz!}yrbCsrLdcIqm>GXNC z^%t>OBKvlxEz_D*_fMX?U%BqS{y~Yp>U+1gb|`LIa(oik&O3KD&vH9-soSKr)l zaf@}l=~K+=;{RnF-Miw@e+It^j8mpcshi8xA7aV>@;myc*tH3H`G@tdS?;-g%52%! z=~FA7B`O}hq{XP+VWA?a#ivoDut0W2K}671m6f5*$1{y32=*TGPQcGeGH<)$Sx}c$1 z$H_~E@i6nuBYihGm?l^susp!T7^AkjO#! zRuu%==csKg3Y_>xKq_FxGoPT^xf4>vJPc$vFnDILOgv+q&ZB7AIceQGZoVm<@plpv zWJ)IO5d8X(fmfZuT5#to4v!Xwy%SQzf{inetY8un32wioJw2g~IbcyjP^Fqd=Tiwa zg{dc;ju!Y7pWY`nRfV5tO2q-gUCUMhJ`$y;(Iux<@_4bR0JU-rE?yMIxAc;Kbw z9&_VcbPDrdM^Du~w(MnY!LM@DxU8LLRm`#)w(l|zkM)Ya6tYc=W!vfrp+Yi?QiOuf zikhcfayhgvbDE6OJ=>-7Gk4Ym)t#DlXUCu2ahf4}!(X4Oes|;T`W3ndD>Hf5sfZ-@ zPtxF5EZMLkL?O+{?nH%(&%(Hm7Rhs-TNbdG=rr&$JA_Q1l9ZGuxNTM&=VMcrQ z!;BRjSEY=OO!c#!sC|3kt+<6;76Ody$E3e`vPJT6eCS;tY;jflfG?j*M~Hy`g-+6Ji(G2k2IW&E%N1IQ^J*t?~||{|wp7 zvO}z1vgpchPl-6|m6xD#4;D-Q zdvW8s{g;Po-gPSuzwDcnFP8bNUbI5Q{8IGc)k1IhSd*>x-QAv-Upn=xk>n!3B-6*u zv+v#(xZ+&#+?=8C*J7>v{~4+S53HN!Qyb`S6x4T}?Xu^Vi|hM2P8kJ-`YN^WE)jd0 z^El}6zg;&^FTFBxJwx)UnTLPaT#TyNxv_esZQI1dbvvFWBb$}N2l3F6=s<~-kG{H{f*|{mm5pVZu>8`jGg!B+0`Xa zGG{QZ<81RWj(8Pqdt2*5toF)YMV_X4_K*51_m`a2UA3WhX>m~Zl(tRJdSYu)8b?5)9bniq@buhKJ@J+@S> z;To4tG3U_-w*E=Sf?r!*JQ=%e-kylGg_ePb{3CqD%)VFX7xup5ViDW?X+{IXs{J>G zH@!RgYi~GLsX@*$nl)9T?_aoa z@t;$T4<>O2>|-kw&0D%<+0#YO3c`~%hONBZw?^#VtIcYj+$(o_uHX|C|GPiz=C4`L zudQ%WH*|WVGO;w&SiCZM<&|w)Lw8K@`QZ0s!dKxr$1W{kD0Eo+D@-fYZ1>uPU#1Mq zLdPerw44@bE5*MxV^!uA*Gtc(@^~`sD}|J9MNRg!F);spn|pohIAd{mA;)y9JqVr&u25D0M%xy5qp^%^ZjK)-TWbbmgnyQO`*_i%x`Y*_v9k zsB!TtuKTNY=5FR%$ZpVQl=0DL=c3w0ow212Uou&j9rblF5wGXCEMaCgf6c1jovRBO z&U$inOFO=5zqssH759WadrsI?GM@Iy?z;2gqJ*BebKUKu%QiEMTQYvp*73qE8QSlx1D!2}hK!>q!G7#RdYbWSNY>bIq^F!=8i5Y~0pJEPj8 zWYIEd-jzcP94;=aO8I&@LlXcOroC|D{KH#hpRk9Zs^3 zZ?kBfSf+VMD%H@S*v}x!OK&j>j7VphDqO!|>~ z>Lz=4;`t|^%xx!}OsmrO`SNyR_`5&;s^7zd%#|2T<4d@_gp~?gK5K+rnRkB4vEU{1 z=WH%n_9CZT?cxf1t!)Y)KlUy4TWY>jV&Nu*lL8<56n&2_31!N0+B{M4>Nel2+jp0$ z2j#8m;rPTByZ6FlE&CF$^(-EzI{8e?7MB3BQxJsk3*Yx zt}NeUb*p3_Ecq8((j43Wu7YpR?L?(oL=pptiROUQ{d%<1M_~YFWFkX``yx|*=grP zr%ibkwr8bkDp#(%(6f`KS8~cM3X2{mE_>7$`$F5(%S+c&WnSK1?$x)R{n`?F=HXfY zFy}{?_DD-^S?92=cKQJn`TOuhTKRD;I2#n%$e%ttR9p8nE zOH7ha)oGkuTj=kZ&2#3Yd{AL;>De1gF8b`so1Nz9tS^(K%RYN$cX4E|>B=oz*H-cc zbahF8`RvAf_JP%Th!|+WBK=CUrs{&JJsG)rwA6dzb*J~%h}jXD>K3^Hcq+4 z&Yl%_<7n;c*q662F}_h$7u@_HFw^_uQ^xY8$S zkxTi3m44j~rK$xA+A~63z3j}qlU8cas(8rhrt)3*tMl{?79Znp z-)CUx{Gi>R*ucGI%^jhl_T3D3+PJ%ZIEQrZs-0O|aDd}kQ<25O$8!`sojYfE3e4@~ zc*gPWT#RDD%s8Vnk9!s}^c!DmifCkGdUhDJ?T5p5Z)kVuqK`fmqLZwTBp+ zMHsoe9@NcX*vNTYr0^7Xzo63O2cGX*m-$+13C#($aAsU#DEzQfeG?~Bwq}wl3sbxW(t$oTBMD zk5t$kG>oP*ir!<JvtUU6MDpf0)ZCY_-j+^=v z83)!BKDm1t&H~qcgq)Ypb7No?JbqZhLC9^UOCe4SyPHUj+9S2(6)eq$GxJ_E6pQj z2cN)mt;CjvYZU`J1!}7IB)$lYw=ybHSeAW|&oJr5*^}3ZZ_Y#A4gwVd|dvZ=*_X|Y5U#WZm+!a?N+GD z3I!&mm4U~mE@tLZ_F9)Mc5mr|tjH_6M|d91j9ML^c2~8eWt#2A^Iw*&v|GA4e%JPU z)_L_SL#9hUw0zKHlzSt(G;-B~Gbe@j8I&!0o6z%Y)ugw*bJ^L#S1v!hWUhtehi|TX zUR|?q+1$5JpIPqO43Rl8j8Z)lp1c#BS#aEP%hiQk4NR7Pu@jrKS6o`?O!eFHhWv_e$RC>)6Jb{UkWK)a(>IRpINmt_SLL-xyriS`F?&mgW2o)RJR~C z*J)S2US1-#_;1v#S3B=Z+v;zY5xaJ6+rvVqbN6RGdbU{iC39nKI9u?uKYsoGrF-A5 z_bR!&?q-(IR4tz^TM{NR%{JLG&nR!X`?A)K!Z+Fl7n6Q%KIir2r}>rn@meo>S>3w3 z^i%V7y^8(cebx!iz8`igFDoiWQ%BqPPE_xwW${&dYb#2*@&oSrWIBkSx$xrR`|4cT zdso-4xl+NpWwp+&r^2%T8AOT`UGBXJyp|Wa=8{RM)x>>mZGYI$&79n57qc@tHFNi* zXIEGj9%VW!{<@3xZtK~#M~`kS%viQy%h{l!-}|d%vrDV<15AH)7af@}saL;WY_<4I z|7R+5Rpo`(Z+bU3Drw!m0)Ykr58vt}v%A}`u$~X+yv!oxbd>4x%2-EWiG`rM3(MNgs^A9r4OWpASDy80!(@!J(XP=4)W6}~LRUUU7fZC$&aL*#Yg(E|2E2KMi| zmp-qkTy}G3$*y$&=_V1`z3ROj`+S$qwptqdz;eZ<7%!p6DiWQwf@}8eOO)HTFHPq7 zVUTES_t#c?zF3i(tr&Uzk^-!1r4B)gckPPekq$lUhs zK2N#Yyh}C98;>dny}kGB)r!>Jo_tSSB_^LEv~rY=IpgKOS6~mx{|vkdfU6k_nR`k#Cc}FG=F2c`n-p{{&D|V zR-$Wz(i@WoHd=RPc0=-SFsI(}fOW z&IA4&tSLP0nFq=aFdWEIXKrj!V{_)hW}R{^{x2Ty7Y;~ z#)XRQ6AFbq6(h7bjTk02Sbm(6dzB;CziQIrmJJ=9<)W%d z;Y#qHx*=suE z(sMhnn2G0B8U9;5VSua{V6WY-QCg8PNl!<>3HiK%-6Dj;pPb?A5r#wQV$kIBsHsaGFJxw zU8f}RaK%E+>D!bdzt!-hPj2|w>UktIGiwru1DEKV#(+r+9{)~cnx!=HAY*9fX5qRL zzbJP#KDLAf-{vU3mGOG|_@*F}u&U#2q4g<0w0kpK?VM!w z#phRFF<phPFX=g4Drvr@Tzx6!!mI3USKOwrN|ms6tzTOIab@2VvwQn} zlS3axe{Z<&^mJu|;hETg%bTkH=qXhDl&pQU+E#i4*IZVoZ|+KW%4;{>y0+Bny3beB zNtzqAY$o_A=p=rbT~su=D}Tqjpq`w{C5zoQzcY{K^$*y=Ie|<4D(~O)m*whr;;-j! zovKi}_X$Ht;?WzPYr0b9w$-*R+!k&6-f60y0n3MFheh9>F1ur1J-K+t^{=b^{d_VW z95bJ~@%H@X)@v>IuFm#f+4*aVZr+XU+owBS-2L@#Kz8&SZ?CED*8esh)DoNjBj?ck z$DfZHzSGmIz8CXdw6aiTlF!M)&*g_bjgNCUD88SwRC>+Fw-J4}1HD&Y{l0l-{p7OW zcZ+<#=}gT0S)6fu-JX}H{N1;I%~>_s{?ye#{vXO)YF;e){&w2dS*mWU_B=6Vy7=Dk zq+QSk>t||b{D0N$J}@mrUiSU7*x7p9xZc!B1qK@}kTGgK@k9Du-5Jl1v3ke1?RzE4 z+qXM^o9(`b>sV&-W}k?OSOdD&eYKU_1TBfrOUfj@_nF?;ThCiJYsswz!cJSO{HGi@ zU#Q*!vw7qWO;QGe$3<~1R^R{i!m_C14BJY$u~RdyjkmHBA~ zGZT)pD@U)eJewFgk&7>VmihNyKK=#n+J|=3-ARu3_jWg87kkAe$ds99@lZIZ)_vP9 zDebcxr>*Hf`K@-wi_n=qmsdE4cQI@_sF3tqQryl#R9$S&VJJsq%@VNzrRz=PXuUYg~&eVAL_v4QB+qXvso=UmG zJ<-BVdTqDmmW9R-LYYfzV^=Xzi+5Tw96-ZnF4#A5*m6d2!mU$5*c|wokh~>sF2ibEnD#fu?4C zi91&pT{B&}fNe|B`Xk}?Id;dUFZbOwWm@w=M%RwlSHEUSDYH$m7F8CR_w07skE5{eGw|#eYjiT|V2KJe!W=VfrwDa?tpVZ7nQ`ArU&N}jN>2(7f)JMrkZe;Z#}^O~%PvwAeacyZCx zS!oUIHdQ_U#JB0))<4s`?Ms#QQRBpjhq24AEQs+d`l|W(V|MR`C%5KtUi>yQHL38p z(UX8lt%eg_LryptGu9m4oN=V?G}GLu@Q~=j+`y?XEwr;P6;yL_H_tw`;nST`S%(Qy zPlMYpickIC!zk72pYhGXsY1kcJ~N-u5eD~?Jqk(_P0Q0bIh<1FNL@Z0qQi88*?6M% z^j$*V%O)P;b~oZ&f0cnjXEU>ik5SW%mSaKAk2M^Nr7AoV50;#ZXg%-<2z! zhg%HRxeDJF7WI7OveC)qVb`KWCjWg5QZ0%lhg$Vye;i;a+9~RJ$oY_*o5akd8BSW3 z3?C0kNlIz3czkorTGILUG)EDGn^aK6svl0DeI(Ue7z|cOFt}AN`c{5OPeDy}QI@}) zz?X><%=2OtclO-cvAk#ogRqYi)9QPo;)g`^Qm?L9l{8Z+5!8dc`lyKI*=8RWu}AHa zqHpFg94gRNu~0hU$+IMpah_3##muMDrh?KPA;u^C^?o|b)VS42Bnt^GXJuAp_;*o^ry$M zU*hLxO;=Pob4JNMDBG{SqT{IL=~5H7)rDS~ug==NdT#Ub#FQyPRrAwoH@m*wUFjjW zDDl4d>RnYGMvltI)K+KRx+bbIt-Y$(VWY2kXjbl{qIkJCdg`r(@rw$r&Nv#ano(fb zRdavOy28YJW?i>a56>*PbjpdEnw2w^wX+d;rVL zJQa!L{M>`HyLV*dL~q>Ml&)9)X7#+xqrqFw9}4oZQG9TQXM@MJ-KC2ztX=lJ;7XI> z>v!d%A%>cpn+ug#(~2%N&tFxTyWBIsN+qtUc!gPS*V?SvOOsCZJx$fh)vuh+`0q=? zoj*s`yBDmk-EwQ|(o+(@*Ckz@)E2Cpar~{FruIiB?{dD@4VObQW-@i=mH|I{sy zw`Esd+bL7$zVhDA!gD6Ap$2E}=G_gy68*UI&iYp&c_lMkr7{mY+nUX|?6E+x;*dns zsuigg?na^EyYyNVIo`I2guARrn(|yd()0A*7KH*O$CL^cCAFxV-=FtOI4)`a(4rbs zdq_~j@no{^D|<%PXE$&TwR%D*1io-rnjYbEP>_Rnu-w?pdt* zE%MO6h`6fRrGGbOgvCg=o~df=b$)PSXP$`U3)2Njs~ir8%ctuU-|f2gd$Cp5_ryC= z%r}GmSBaRuu|K`${LTR0_~Mh#ugc2pb&R_2!11Vb!{jbCp+0|A)iQr~=gr4gZ)U%d zp>6tO6@x?GLd&8Lf?cm%-#?sHR$#cPkWW8@>j86j!DsnzF%y|JgIRV6pE;l+(!%n4 zL)Ycyjir)nR@9m+9$8T&!`<>YKx*CE1(Pl~nRHH^wxpZ&tX!;r=4Hi0mE|+8&JK_| z5PjxB%qIbXIY~E6J2}_#nr2nad7cs0u=$aY!y4&o|Cn~mg`6^*CiI+l-8GNV;lh$h zJO18_6Wp+U!L6S2jAv~<80Y+x*m$aQVu0_HY}t%&rUG+41QJ@dJiRM9?c15cWDgf( zrXr5^mp#`nvWOhg**RDOi z%KXYlh*OtgYG}`|HJ7dh&pNT&M#)R6)@`?1^O@EC2^}i~IiIBlIxb*dGFh!r#sAH- z-nFfJv%X5Y%~0B!uBtrQjc>*LBGWBe+h*5#o%G#uE6esAM{mB_irruG?nhtvx3uc4 z=xeqvla7C(f`7LCGdG{N#5(Nm$rV2~^|>C+fFh?D|vMjJRhZ`RX=X4 z|H)YRpW)qZ+1dV|)@EN<-nKU2{C|d}vLBxaxjClayeXId^J=Z%=XLKgI_v$nZr?U9 zPxtGc`F|_kxaNB5@|n!8o_*!{jE1mep%kVA>{D7i6%C$G@KnhRP-$0b`?ASanBdm)o-QEKPV!SX_4Z!NYX*|z=BUJos?e0@8p^NE+sWjI4-3; zcYEpW?NxQ_t`_<|bkNK8w>JE|@mAdVM>#25OFHL>imGfp7#VnF71tJ-SEdr;u|`o3 zr!9DPUSo4^ZSCHh45zR6p6k$Dw$bb0{<0nMpI`MBJUzP0H@Dt#UQOCQ_TW{ot_1I4 zNGohwBGgjm|6AE}^Vhhud+TDQ_AlRU|8Le#-WB!CQ_rrLUuNFVxSqE_VabogjtPcK zO?86T7tTGpMry(3Gc5@_PVeJl30JffQcRww=DfqnH@G}A<4uE8ae<0=i@`kCW5Fj* zskjJDS}#!0Cb06RLXD4MFK3@n$m5$VK3c*G%a5@!>=aPfPyE}X#5qqxV4fyJd(w{P z4&^R~^GXZ!B854Y&GhigvKA7ayYxh;fVDpd8$;--r^*a&44MW!*F#QD`Ww`$?$a`9 z!KvC7$3jnQqooHs|JNQ(Qsh&4b;VzpLAd3&K-uBf z7i;&r2dv^KRf>2}7|0``-t}DSn#lssz>7Tv3TMtqacLe&KPL0+(VVoRH*GroZ&XEN zt_WV;7pS;xpR;aJ{3@%d25WWwZakf@(f{PW!jBEdza6t$s%ak7e4y^`$F_LkID5xc z@;uc&?w5)urZG;*$z5T3Ad7Q3-yJ1((~<^}c1~^kIQWKC_2^Fz20nA^|ZSugtm$ndkFW*mKQrUZwXPb+*B+jvf^mc-Rtru zTJP4^UB9eXE3LG4ZJpb!t<&Xw_rJLG`~I1lV*AdRrH1G43r}r%^5m{o#`agmn-gy} z1(mMKe>0nF_g<%_R)aGkJ667VddN_|Flx(d+oLHB3no~)D9o3!eyAm?utG>MCHI)6 zvhKZjrT51Rw_dbT+~MEH@R;vMIE$w85gFSFmYd!=T6lM`25U^5`%Lq9(}Hic0-P$X zDl>`%9vQeNcKo~He}R+d)11C$Z3w_pCyQp8@v~=R>W3A@Tp8M{5bZy>$hHT%wHJruGPT30zmtk z?&xmwf~^TGRlN)XT&um#zvz2foaW@ z#(NFi*|CCKbx%fxvg{L^Yg%{Kk z$7!3zW=GmBk4%``8j{*Eq3cD&SJn@V(#^*-RTzqOtS)f&18btm7J-k$w*WkhDFb=lurDOFqdAC!&jRm}HX!^ptoWb#-0^0nJ?x82S)?bx9G z%KFUrnfn%>zg4|+aZR-9EFR;&cBykGu7?#_hVM(+P@&T?=h@b!Uu7hHn4a%A!PcX#?wjvDT~>3ywm59x(t~}4Uv9Ol8Beoy{w!Z&ZSI=)J+~s&{i4|yu7%el zekIzs?6a9M$w8%KlICrxUy}3Ul;=EE(3!wv!IAaI^HA)*6S0>#4V#we#S0nCS)iaU zdAebKi-MP&&t5_ z?0T{|?n+yWz>)@$$(m^v5-JMrz6A%>J6kSsF)a7b+34wbsne5bt3b1z#zw`4CXdjN zIp;t8I5?4a$_E7xzWd#08B9!GURsy){AtL+3o)Emyq(iFNvL@~W?=RXTzEF-9LtYt z<wiy7*z`JY)}D#G{~C9Gx@&NG=I@7@ z=RY1-7MeCET;x=xAi40+wDsw@4kJf`(AR@xvg4nU8D4~ zkM~V_QGB`~#BqA!ISUP?nYmtmU+&%4brZVIw9JbwcH-U*uj3Zp{?c6fYnSizr6tT) z_ijy%{?D*e{P1-4mc@Q|xl)g2OPpFB$9si|g}ujARMasxO+4&i>Loc`59=f-* z)jc`TH!HqlN9Kx)e++Y%MztQ6jJvluJE(wlYUU2x7Ma7#HZC&mnVNK2bBa2DC%@C) z1q>FC54fbcGgUwFjF=c8Y_ybf63>%1sV6Q*S*_ZKoK#%$7FfFPIm_8H>8#IzCa0Na z-DlRnM=!e8xEQ=3xuC^{Jbzy5q~<+k44E3{u`@T{_RV>H^i zQ)zAUm-aOGOpjOp8BD)zpH*UR|Ff{Vw)&Y@ z#uMAa-(8nKYY*I=`tsY`Z(B{`-YvZ{dzoME?)o_&ii&q;^dAp?b4+Y?;o_i@u2k#w zbxARK(?!eXwmrO3wkzw@b(3?`V*~7b>^#I|x|m00`FH2e%1qXut@-754>9tb^;1mr zHZ%~>Gj3j@B$8gK>Yvi1t}$ta#vD}!Mz5*IZ!$az(3m2x@z}C)(jlo*nczJK7-Bvj zQTTSqhr#0EV{SQ(c_$p0X8BxUJeJMCtT5TL%(*;77k z-ufy~H9Jc(*h2vK2}yK&{_O{GT@eV7_n_oY=iyVslKnODEE4&5><>w1+@ zuclCphg8%>TfZ{ij+3jFY)<*I_`;RE?P~2y^Zh2Sua3Mockgb-D=}ZQo%p+~=AVAo zV$Hj7_kP~hyY95j`s#9+f#a6R55wj^7uiprpI$DeXX|`Lm;augqwbQI|7P~jUuN{R z?)Kvqo4O;I-IzIUCmj_kvYfLoG@BzOdD{dA1`Zj;tq}}9e6=@^C32`rnER*hHNO+8 zxaH;6^VNNfYgv!qmAcrp)k6M!+;Y=(YpcTUp7gu(Z`n`d6`fx5KbY)1W^qbMWM1vX z9k1qIcwhbP>8{<}tKJujTV+LEvR0L95-4K{5y;|wdBNkk=YmOH4Jis5!cET>Xn35y zJ;kC|S3xeQ`$l);+cuX(p-|Q$zk4?nyEwUZE(P?Qm?(GoA&0L4=S%~JXIplzV4BLc z<5^|xgm1E@76KkzY7?$%8mu&aYY@SxBG8tWt&t>mh24AE{QZI~4GXNN9(6pTbUv_c zi_)UZBQqB{&$+&J39FyN%ta~bLXixmJLY&wh^QqB$ykcbTe$R6A)`xIOGI*KaktY0 zk6BMmrYPLuIl&b!bK(r208eE&Uy=vQV$TWcO$Xk@{JSti{yxX59Xbb9*I70me%0c0 zK#MJ@K89I{FX7F^VmnU#kIpAmk2y3e%<$e7AkpwjMKe-O!a2ol(gVJ!;gUi#RTz{~ zygnDa{n*s&uBj#F;H~pONMM=p(J(=$rw5muS^4c01Eag(>M2?i0u~($ySp>b+BL9r z+ns%P4=bKIepc|y-BZu%d6HYMtUnqsxxZY3)ns)?_67!DM+@)v9E+1jujw^qt?CWm z^GeqKy4mcdVK?)7bLW0!>?!)^>pUU5-#EtqSH59(vA1}DYgu-GPl37e;R!-7eWsk> zyv-~+KH%ML+rX$x58q|le9^W&x8&oDVn6xPkDKlrdv_nLO{<=h^09g0%`Um4NB(kh zoDFVcSSQ%JId9tSdw1Onw=u3=;jMKkz}Kpm<5ix|Kl?*3&dgFvtUW0`Ei~fFiOYWd zjo|Mn&qqx}C1n@icQyO-b2)6MqTyHu?s`P}zwqB-ldZeGosen~Im zO7Il9V7VE^rK|n7Je1k}EZrxBb@$b&w!R7f8CLjjZWHvEdK`4)Q>C=t;gagW(i4lu7SEYY)(aXOG%WlziiEBzdK`GB z(wZn>IPrwe2f@Su4wbDQ&(8QfzRl&(GOOn^^N|Aarl*QEZW8Ah*afM9srN9_bUyIQ+N|39tou3OAe*n`k<{=I=^gPsJ$@T0NI^ zt(arL)1e`_$BmmsvA{mlKTmyQ;;t!LuO0~Z*9drJhHyE)^0rS8_-1^<~?BFa2}SC%EFlvm(*|3_MY9v$Rcii+1d3 z*STfH6R_*|qPe>o_vf7X&+xrFZK;<6gWgK>zsiR;*I%mtH~skO+7_pdbx)KncAO5_ zT2{7fInVXWHwxaa`nmOn+=Hnin^wzk$NFqPbG*4Rduy^ZchiNln#Lk#UyMN*BL!Gt~1$lUuFHvwg1`sJXfxJ`-)rjSMp=Bf))1N zyUZi7CA8w|f`4M}W--~Ts*_et+a+6iW~0W+lb3cbos}xIwwLSrl-FXO4W_eF?8G+6 z>~vcpzU|5lSNFZg=5p~B2`bP3IY;KkOZFw-)Yx8b-CLQdx<0dLos2Tq=fs<9Pv8A3 zT)g{=;H|hfEfZyK%zkw5&EYNLW>@PjFHdyYp1SY8-;7-2z(cczIueWTMs2)uZ40k| z_0Gyjv40&eych5I&v4h)-`V%JKyDeo;=ShJpI-x`e>Jb3I{(FghG*8(7VfCI{PNwL zJ%u-|au1gzUJP6MHbsYd>RZOFy<556<~RG_>sxw`!>a!AtW!>_~x${pF=(Jh}(zi#SN44A+) zMTmKu_-s9`#0KVY#;2~rhDwYk788~qw4HEQL!?-}|4QN=$5UywH=G?*KZ`B!?BP1Y zBjBjdYPby9}n#s&;{N(NSv;|vRuI9M!iG_=;=YtUl+DAug5E_htdKrM8Ymk(!Cp!GFI2akY9 z)hCSE0vd`v7|sWI6uO=eZChY`!b7-4Y}%ynI~tX`49gD*tNxDXF;LX-P`A6mF!7v9 zQ&_r9q6yoM7OQ8SHpUa4bzJ$m`SdZazR&ks)7g||JXF;lALYNuTh+F`Q}~c~pFo_N zvll}zM{BW`TL)i}Z=eJB?Z@0}t6NUjpT2wc+V7>Yo0hD&e>uu)mv5C~%KWVPE7$)> zUb;fhsij8wri|99Ps{G^h>pJ;7<6m3R?B(MaFc&(pN^Mi-wa%{*5g&G+tx34)VO1q zXF1j0cvuay(=vi*ANS{9luj-#(OhM8 z?pCnBr;yd^mA9;#Q?#c|Z=A8r(?4&<5#}vk4(UQ6T5mXYB_G}wJd@<(rPUc8B%Zp< zS7*|#_O`7aPOB$8=AG;%Y>?ugX3*VoK+CNl${~1?ryAFz<$JO^MSE3OxhZAr@h@_o z7UlI=^EAsLK|XzVKbJ>>#qoPKT%GLnAVg({)**%!F(Lvpp0}{F3c0WtGl^+*?U?$$ zWzw$%fgMv%Fz`1tFkIo&5?LS++rXltP_lp{sNjI?-Y4}rssZi%YYvt4Qj+=i@{ygfPygEAZWq9z;yEC@DTQ7Uo?dBu5*(^t^ z+y40eixn@jkN$eOuRi#Bc-+-tS&_aL0wa-?H1I?ggrF^&WgX(d5;J)mcGB#fBHxOQo({($ioZFA+77%Xbq; zWA%d-tE#R)J%1*4_quJL8n^A%pO_lYUHCE?Sw%=KEaqHcm<=a-RTNqI} z+ChWmO1U;e++{hNwLxpErfOfaS{+)w zDx~Oqbk}Z)*ZXa)W=`L|biZ%(6~3j3Kd(3VNUf8fa$NoHmH!MM+!gd9Ie#adkn^nQ zN|uyUIKW$c)ga}h&yJmnLJiC_KTct1ys@K?{g^;|(T*n~PlT8R^-g{~ZlM_D>g2<- z`+>*9rUbu$1Fj1utXRVO(ZJbBC`92GUsYmq&!GgPhJX_Zd<&}mB?Q`;56lbfQS00u zrtnCuXpR`Wz|IGLs{RR0iYyEVkMUF)|N=PkDwtY&e_H&fuuvl@;L6!RiqZdd$7$ zpfeBKG=V-H#bv%~+$&$Iws`pKI{$E6%rLF}g^&^7w|NJTad$TySs1`oFirCiV=u4P zq>!LM)&6Z#`#CgL8hLddzVNd{W663MyA{1h1iJ0)cL{I?Og!Xb&UYs9z{hXp&MHfG z)>u4cbLG?fd!eDy=|hWv1t3!ldc+&;^gCVgKSNukr~S{ol)?}5Ip-VzU5>o&fn<5*7MoK~ zn;zYftV^|j#`bM*-+zX;$JWWbEYCjWHBV#tM5)6ccFpCU?WZW*v(R~hTI;0bSyJ-}{r^k5CU(jx8NQxZu<4MI@q6Pr$>HoV4ea|YuO4`L*;u@|=z^}##4wBW z;2X1fuPwYJb*k&x)__NaefKMvXD{(hVSnG}|5$-F?w?s6>gcIIUF~zk zqKPZc70p`C%%`X8;K*g!=urE)G zgvA1z<}j%&Kegik^Bk$?yM1mg_f!?pbZywPt8j+qL=RQ36_FnDt2yM-8Cr{uYb0^1 zFgOXRt(ft##blA_oHnM)L(?XhdVH|=(G+rIS+e^?3zN!91;*q1nA7+fO(YM_aBAQG zx*=p6&e*XrMWaK-oMSk*6YG5v%LMXMa(7+ z0|!3a&n*Rf2RKE~)+p*s@ZbL3fl;#cDU)MRx&>Qyqr$QYtKYV?tPV)7vg?uPQIp(pusP8?>D4>32j=vnwc-DIh%-3x|WIo7Sd>&v(7S>mBv zk-X*Ebpek4gg4@E@0h)=UiK%~y-p$U-f?#KZYGhi6{_O=lE=Lleh&Jb`l8F4?`G)z zt9oDA@0v`=T`?(h$CmoPM`P`$|7YO;x&On%%XhrrT;9&~^mFmE&1-t=CcmtDS#A}* z?xDXF^P}wxKGc6_%UvD(mtWQX+`n+fPupfYTw?NACHa`~@)gg}aB&Nvn5D&S9t>S> z3{Q_S@szJv5hdynclf5=3AKfBiDy<69Zg9{YJDQGVy2(ai?XH9f>Ic{HZX8EFbXIf zOJk%|G@YpCR+S z>N(^83_6Y-Gu|)>PhF>5;bf?y__#$>N{Cm&~t%tNu%kR z!zGWR-`I#-l<_LG#@}$dQpLd((|y!U!cnWhPxjJd##1jRaXxNi682>GP%WOGV#u=4 z#;9RZkVoN3+tmgWx|V6NEbF#=Q6#YBnVMq5thX#OZYl}sw-i<=`&{T`Y8O4I$Y9d2 zbIP0s<{hins5PH*Idy`^(@rLFfy{&F)?tdPIt*9xJx};BLKShT6iL2Oj=d zxOC~JU)G8p3MNhKjMx=#-k!sklA(S^xTo{u<$qr|CzbR#D1UeQbi#l0wu%Fc#j`k^ zI_Jq)vzRo*ersn6b3fvwC)HVS_~F5jbu;}q3d{K1Qvx*``BxN3YI<>OMrX;ND)?mnb(6MgHN#lPfTo0Z&s=Ao*-i9=WBBck5T*Y$R zCV6Cg9^N1)XTakXae=Rbd7|w+0kPEOXB%cYcP-F9ZE&fPztD4uN#pYAvtM>>5PF#< z>B;f-QL=vCp4Hp#-F_$b=c>-VQx7}3q$+Pc{uXmD;t02i#iH$^Z$Gp?oi*vAW?A#` zT{0<`eKTG)u3D8*{{Cuo-Sl7WEB`ZizN}kv^*_Vp@`zz-5+AB=2>~9k({Z~_bIsMG9z0dTg{*JEU|9^RxP^rvvjkXof?6;?+ z%;e-L?ulBKzAIbiNy!=QO(immC)XTP-Cou!G2z{X8Sde$&vW`zt!`mTa`=uk?n7GD|pt* z-e-4$g@svFVx6QT%e;H{IXE~S57s0+xWq80>i|=>cXt%a2gcM}8#Wzj@-v=ZB6(t- z#SE6p2?<4QOaU>QCP}VjCRA$>1%)!nxw#3x=ZaXTCczo^YBY&dA~E$Lds` z&M}4iVT+Ezteyl0$;KHHItpu5FUwbP`aEe+G~o7X+T%3y@l763YEWe)<@|Iu4pq^yX5F=HvudX-JU(xBBVZx>B^ZrUpIXJ_fB z{|sGPzr(H<@5w8>9lPQA&bp=#S1+-fy8mZ*8O&F%Q#E_ZHt%XLlNHgyU)HXT&WPxYSyafHVV&bm$*1j zmpm;oq1mALjQ|e^r=~{s6k%1CMa$)GlpJJOFrD-F!gVv5&)#BDT$Px2CP;;O%hr9N1-w~1dhT5Cu5bNy>v~PVL%&6zn!GeFacUGBTx|B3 zE-iB+reMuS6J`Z@H3l?zEoj@Z<$+km^S0t985K_9iN=#2a|ce^xt4DNQ%X;J zvX|Vk19j8(G@dv&x{IkOA<5UxbUdY zl5>X4>J<%)S`G#b97lTm6lx3nCS7ote$=3uMIre<=hIsgTFO)%Cp?-U(=b6Oh^_eB zp;rdYMh1LSSgit17J9fWDE2qVQ**1GaI=Xc^v#=x96W7zujX!zc9^AgWqHWoxgoQI zX4wA9wi0n_bZ}fAKKoIt@XvKq{>tuOyCb@AeQsUv^8Z(M?{1u*l5};RL#ia7RJ*!ZuZ5i6A>O1y817hco-;uJ?g4=hA@7`75o^Mun<65F{ zr=I4Kb5bjw`Y7JBy&&Dw`RnfohKUYujpbc5&o@0}ygX4{b3T25L zOZwa;Q+Q-G+6DF*z4_R@(uwcr-yQ~L8#UV#8v-mWeV$KhVicGdGBbtiLni}ocHE;& zoS#~+blVg(aCqp2aB`gUoDiR5w`^{n$&`(Ly%h^3e3xhU)7Bi|XjWJ? z??*C&hB)I*K6i$7>>d+re>feFH1BPDwEa^XUa$~GrBolKbAdrD6 z!#aJ6I)h-Pljf{)pW}`%_?lPm3R}MCT}j8L1isDNtAt}7-RjT$6p%i1{zuvG3MaC3 zALRav{Iay#!Mu8x-qP~2W0$utd3ffOD&HnM@zAG#OlIBRF4uHhH##@xn34edMJe6> z6q7r*zIIKWXqd(9;W#Dy_LVuamWnkBEOcsUlud6l*zr{K?e)T~x8LTKOTU^ockNp9 z@E!GR!L09;-+O!8UH+MzxbyDTP9K3)3w|%|`t$TvzJKoam%G#drT!Eb|K%BWVOdzO zdHwTHsr_G)JSK&NTFU+26Z8J}-g{pw=52i&9PV|pS3@&MNNBN#>BO~+EVDY58a=ft zj;!VI2zdO>E!$K>v1rkh>^6x+o1_q>yM+>WgM%E60u}8HN>*^B-7k34t|Y|i#3Xq8 z5T}b)hzH-r8$N1EW-Hhdw3;6&{)+2dveMH+&49y=iRo3yJ2@r>-{2rd+Z|349*PGm zo^zN6)&$8&`%IAYHR$rn(lTK1KFcs&PAjDG5F@kaRppk8t9hL@|J^hAwo2&AoQCsT z6{b4&3)ttq$Y9%&w4?0sl1UQDxBu>);5!he`Fqn8;l>NIykbj;g`;KoeZUUE{Oj~5x-SS&)MsxQvmlqaXXxf^-M~yvD-RALy?N?j(PCZ-H z5VSaKnMG!H&*O7BpPHN|i12;vI=0T}$IRJNK6JfJP;z+q`(S(4QDYasKMyR_e-c?j zRph=lUEv6;c*Sxm{Wb%a$E>Dp8NvJ#g|n9DG@Oc2-tZ;wZbF&vtlqC{SEiP|UK{o7 zTr|7VzTmw#(+c=gh=Yqx4wd!?;D{VFCf%lzS2 z`yhwnKRr)B@6x<##lAR>DKh5rHT9QwcQ7y<>Dn@%)9S^cxhjPVCSUD1$)d2jiP=VZ z{eOo44Eis<`u^;mdu*bH!&eR4MX_6E6?MpeDm%2$JNDgHn=e2F|mo3=OU8Sj9c#9v_pFVLp9= zFTlg8^Lu~!&5jQr=KM~Wxllz(;}C&3@&xKI7Dyzpmy@(F-PVPkxtm{g2_xx$B+fn3@a% zn+jzoK5flpNm|G?)pc6oUAwri+}mUOZiH6dji|qPEpqY#X3yszFa2k*mS@>D&3WOi zoPZ$N%{?bBzcu%*H8ivn+#b-iQM^wjYOzQ{{5z4inL<2=o^$;!Naaf9uk4GyYU}rE zeJOXj&S|gmROXN~Q=Caf(l_@EKMSL(ExH|Lto_uYA0x1T-rzIc~r(92)V zg}TcwE}Lrjt=23vC;!*s(lwEOhavO|00(+nE{FO@LcKE*{Dk0&_n zn5vrIV=`fvzt*CU4mOG6pQcToc;3Qcszu@%gR3q3zNi_t`ycILWmx3h^!&CE(-kMq z0#2m{>yUE=-JZ&)90OWD9GQCf-4O@I=vhK(OsR|_k3!}!Jmy?$aYAe6l*1YY8gB%= zB0dN(cv+k%m~mF2cB=8W=S;B&SmosJa!inVY{DS$j(v-|iB-eS5bHA*oz6bzc}|{H z&W&F7{dU;}bFsW7`Wi;BZU@gk@3!#I@}KS6lR4Cve&5dPx9EsPd)P9L{|x3S(p&WV zg(sRuL{FL4tL(A1xYPEDg7AkX%c$Z?Bh?gk&Tq$_|GRRpL;b?CYTKynF9&4}_f0(Q z*t)R(>T*^2?oC`vyg$jT|0VoQs>mV3hd<}!+d0Zh-yQKeGr!91@xSZ$of;iibu_KD z{}Zs@D!T4=*|vQ1ZH=>YqmEtO`q#~(>X!Ewv5mKGtL>lt?(j><8ci|1<;Hzy14GT_%z2V) z+8H@yglA}moVWSm>>044v(rO0CZr|QD8`3Lpk!97l+9wBMBb=U1`nB4lUPsy{|df%?p+0g3{wRwJhi3yYA64~IOFcE8? z%NxTy@Aqgh{J*w+(u!80z@AS_M1*v3` zTgIR3vV8udKGvnGGp0P=TytjUm#V}sjdRZ))rw5Gy6R0mOW_tL=Lu7^xH@ho2ARG( zQ?oo$^rP-Ak;2$v54;vir6!%Sbk6jmPn@cZy_2n-~eU>eM^5b#Km$vUNXRItv+_fio+q!Q%!dKlt^)0rx zw2oy)`v#TV2z#Y<@ZIK3l?m&fUuC|$ZDFoQ z)fQdTp!CnBzv4wB7Dis)u{7VU_oir`)U~en8NAu6r%OWe8pN{a23dta~O~ z=T%I|o8lWaHS4B<-Hzo!585Zb3S06lB>PU>&7ePP?Yiy6KiL^)=C4iGxLnJ;cHPof zp;y*bXY9EuWV2_hR%ms-u-zG6501I74bLx+o0#f$;?j%)wM6%a9@(!A1=9}*PHFs5 zVt4P+WWC(1?ZGdjHqEuly5N1(z;8L*l(g-0qe_d+CqL+%Uv)U`+vWNNA3ZLAiMSN; z=oQm!H~Z6z3tnB_Yi|Ac$@3DOfPib$H}tx+o8JgEGF<*Dci!<8+pgb?vtYR7E8W&t z70OY-m*wyCpTTJUak1#Vhvs^1eKjkqW!uEUpl6m}v-Wz=dVeDGOY{qK=PyM@%V#M? zJC@xKeJi55ZLU@otJFf7XI*`|3741q>g)QSUHR-O5s_W#p?G8pO!f)R(m3yVH=1{vTZ?cx@ zo{fjr>uTqIpDq^8krjQVl&iHhdS%O&g&*1+R|#r(?y6G_Z{lE0Ka{xm`0s?4%)F(F zjE_0f3Zyy|=KWcCp<~M{V+COYKMx@e;oq(zW#=_i7gqW-F}2M6aiFk(S?}JJLkz+J zLc*r(`&2Siez$Cl$Yu_gR5{*;aG*%PUmRUH3eEbZy(Uxt^{U zuYB#hSoF-YL1@OrrjC1e;xm?1Zu<6HsrS)gJ(iI>HsvZddOPyaW2 z>94Z+zpCOE+dm4)UbXu_gXV;(sU1(BcAhln-Zi`RY4)ij?00rwVe~SPS5@%;Vh>Rz0a$ z-=n9nI9RB5=K+?p;+{TfhMg;-ukBc&5s;`KIj7~%<@Zf1LpL5WnlmM7cD|P2)DYe? zlYEw{bi5K?a9ZifhjUibPrsjD9rmAL=fBD2qS*mr@7FnNNhR!DzwCVbY>RDQx1ZYA zdt=+~tly=*{*hl#Yl+8PE7}sc>dW)$8EOYV90^%H>CBZM;h{eYmwKElIsCEIR5sf8 z%FW9qt9beHjwT)7c#zxM@$n&T*R|DA{XAFHmFhY+MZTS4)xFZ}SM<@lUoUg7j9%+K z*{aI>tX+*M~*Mb^S71SpUoLS6Abeqg%sG=e#}cT~hhi zLqB5f&ZA!QN@iKt&3$z(@p#!b&wVy2Prj5)oBzy|cL~#UJLfm+gcuI;F1xs5vQmo0 zvD2wH%DV3@xMB8c|H_rlg=;>>p4+uzU9v{#ijtR?m(QIte|3m%bNWql+0$9ucchkc z@4Av1n*Zumuh*=(O9Y($PTQq3o$u4^Hr^H1GhcKo|9kwnW_xMWT=Nx~bEeGt+stAj zkml9vGI2`ouT{-+Kh@oRv+7jN@;$$%N_}VB`8MjyUaLgW_g9|n_pZN`>i)-Y@|Kcw z7fU~Lm)*$I46VDHomn2$yZ@E8saD`hhv2epdqS+DLqA=W`g^Q*@vp+$scE6BQ+?0u zTj;j*sQa(dt%mD$injdB)2>`3Q*FF^SK%@dzx7*AzqVVP_a^yB;H1e@)aFf@lC?kQ z$>t!@V0IqEV~kfgC792IHHm-%G1lt=)I+)$9B#v3r6xtdWk|9M&4ew?gvNe}=98XN9)bnX4?Ayn3yT;PRO-YUb&- z*F9PASSqZ4<>>~MOolA(DG&Q5J-n~+Ha760tn!u{vv=oi_g;D5s59!j@RCoK{?ERB z+GVNtpCR2n|K4|N>9n0&bo2FeUu-=dn5`M2%Xm)1UvFN9PJHX{fW0hN(sxZgf8yJm z)7w&WckGplyYbHSfspzGi$j-B`AvQMV25Ut#7<3N$G%Dlt0=*<^H@2$yV@q$2`%Pn zlQ`Vq;CYzSky71`*DuP0th$ zsyLi;<+%OH~Yv3jyZz|%Y{8}&Y!L(t#}ws;CG@3-u6cMz>Ok$gzRVRY)A#pF?5;Vt<7@dVvjgpQ^X@jf zxxTADzjf(GkE~86C+|&GVpkO=p5NblYf(l?_@X1H+SxM?EO^M!#B!=fe3RG`4lX999}Z`Y{BKJnE+{OU8tZD&vi=t1oTW^Qw7WXXorcEO?evi>Z~lMb%N2`Jh3A80*XZM(?=0SLHrRnJyMOYpTDo zQdGUc+Sy)vw`!TsF4bOix#G6endPt9&O4vA_x6r&bq_Aqdhwqj_`HZx!{VpY-#2WQ zIW?2N#HzpA%I)^GySgcN&8GQp4!QR10x!$MGv7ki2j2KoxH&zhj&D%BN~Lal-pXyMyo2bGN^?n?2FGRn3IO=~;h4+4_>4++FjNcWS#A zEmoRbk>SVnq>sU;=i_7h6ej7OIeX{tlb(?G##YUxXG&x4>WgJ{O)YOwrhKc z9_OuRJx@FPjTtUZkf~4Y=@KuWb|Uv()k> zfAZ|t4*Oo7ZepDKw(Fa6US>bP-}?UyVP>_tSL}Mdwyfp1Wy|8Xw3uuEG^dDnbJ4C_ z<=VNIF5aJ?^D(fU*Wu)w`a9L!vtDevc4u~NanSR?FRNvJ*;2NAe=K^p#Cy-hw0@@Z zm#<6v#*OhR(4&k8xxoLgE*=F!%{BAp4ly*e+30ZhxmR~tP8ESO0qp3FiXa5 z=>sDt#rApLM@3F=W=Le}oiOD9d*|DxM|;kF>r^T@n3SsA+@Px=Xu!b9_rgd|BJ&LY zI~C^u^RP)C2V^Xg+Eg?=RFAYyR5{I~V$S*+-QtOASzQJ#0TV8ovND~R ztl`Jm>8sKtnzDE9NAVd4O4eKpp220iVYc$BNKMbCMQjUNeg^XVPM7A+-TJjTD}1Vr z#1Vz5k#bUtjAk{hei`&yrh582?XpF;e+hQyx`eH`w)a*<%{j(%ng$&Q=grufyJg$& zvbKf~mtIcvTB+W0n9s2IWww}E+l8J>9Lqkat%%>P?Em4CV96ct*6tgez9mwIGfY?Z zC{I53)qd)Ut;YlIem2{6{k`q-;>Ke!m7%#)zVEoYW~J^7(S41t<>uZp_nK z>}l_cU!j5Cw;o*RQRlc@`kx{8%eMF1qNCq#@7uNN*3);XR-q}E-Bz1w&tRYV(d*Tz z^Q(6T6{v+xJ6oVKv#M(Lzbly$-!-`cdYn$Guf8j-esW5>Mqt97+8MIma?|B}-c&|9|zAM|F`}NeydF_#R zbpCyHcauMJ%G}iadPr{ArhE5J-CKIsd{#eSs@K)bp!#L!^_K=7j$Zk5^U%$&7*_wDcd=I^{?w)fPn)~%OUTVB=CR!D!jI(TBX z<<3{eQ}>*0e{*%+x4EY?cHA{Pd+gMuE4Mc-JsK@_cG;op=5dP`rk{D|8@T-}=3HMi{pn%5!tS|;j_2Oav}{_M z^msq<=&-+gS`D>TS{483c_x9MMxyyD7Yql=+{P^!oOkQ}5 z$cDek@mufziZ}eux8SerQ{PQnx4z{v7kQg~g@28!R8wEug}5LaL1KY z#r)E(_k(KhnqA&p8(r0V?UO{Lzo*R6Uo)4O>B{T29+1t||$CsY|x+MDK^s`T+ zXNP1S(%vP|6S#BNtG!zy;sSRpEjo2P->mJ$17)LR57xZIm!w=2V+LA^OFYNhh|q{7%<6wRE@9 z?X17%izBWs-yIo$dD*R$$$oRK&a81ie0tTKeP0qTbFML2sZ-MN_!#%CM-LaSDm6X8 zu%wsa^iAQdrMGrUZ#<`Kvc>#!#l@#L=F7dA)pJc(uXXd9?YDD}C+%OoJ#gomQ{`b- z=LT;3${Zmlm}Pl*V%%N%K1=bSyBF?Q^_Sk!UAc41dcD=xmiw)l!|E=s$UZx0deL!@ zgv%HI#(WB>3e%e=f42U8pKr{>u!XJ)>sDNT`R#Fj*V^OS<*K6pOgT3nsk*u}eY&<` zX-NAUmz)dBfF)Tx!CHif8>^=;+k&VtUC!mzMhf7q;wNG?iEVjJxD9TPKH!!MBCN^{Wr3IZpV{ zk-gNGWloCnT<7)EWc?(sO+9-&Sox?=Ud zyim`l=B1r;E=L}|DfLq{C?LVE%7$bMk( zQt4FC_&&qIN2v3-{jC578|yPdGZ^MhIN%u5qB-Y$4a4Ob@wa~`UYdAW$JB{2T;xn2 z6N8i^gDKa)7oL4Whj$4^3Wcd2>B-H9OBIqXc`B}2F)8?6gTOZtRUw(^Q=G=r=C6}-s%#WtZl31C7bje3 zFv;bKil4EK?Sz=iD<(B}v9ueur6f*#dy0{t7ZIj&W4Vj zNxj9Mx9%L<$<($dja@Ivc>kIhbE(|Wz`oj7CC7FKrFzP4a^zP$puk}`f0xoj$*@(s zq;x*Lzj^5Dajrd?E3>!!Y~LFhyggOA+MU~M>z30qLsq1}>JF8=DSth*XLbLqTF=PV zsk&>LnO9zUn(7*rpBlFEYn`(HEX^*KFD;>r!Mhw4vs7c7WL?tlFb0d>dHC_2m$_b4 z&vL22FSjTBXRtoGy58{Wt{>ZX&Ypd5<=uR(uIoIDuD@97({PAwmXFBH&Ldm*TWf19cWk`<^?6A2jf=NeURk0Mv^eN$>z7&ai&ids{<;2TaQ?%#Qu)|Y`=Bs!>E5i8 z6H8xaZcl5Q#-7di=E{c!M{9O81v7*&aV>JZ;$iVpwyP^jsi<^m$yytq&aJWQrXHMq zsy6+~m$e#^W^ecPar{gvNV=`K!;n`*Xc_j*o~S>LTT-*}~0ytV#x@y=+8=>DLuY%k+Z?R|TqGYr^t7l%DcKcM6oVDs+EB#oBV8Rvvbdd0Z<`TjWW+Hx;Dw!kc0 z)^-2gu(hR8Pxfui>RTE5D~hrBWZ_fhX?Z=xd-7Udnl69pbfI%j;uI^-N2Z<}-wzyN z3fCz-r1)v=-z%9h?}9wFKYONj&ior+l6Xa1QH#;B$;tZKiFfi??p$!_7df{r zCD^mq!>XUDz@u`4(Q)CHzZVXOpW>XyB-PcmBvdpju;|G<@7j}=Cm)7)3)-a2kSO$# zvTT_3{GUpz5d%xxGq+%aM=5H%Jf;gCOyw12a%t)|nQ}&jA?pf<%BN<-A5(bh-Y79m zHV{ilWpZ7ydPTC$?nMa-^In>7<6F}>jn#0{L5Eq5Tuaoio3zA4UE8@tGPzN914Fo- z-eVQkM5n-~Z=Bvt2h9y!+_LG}y{&9_HXgjyV02>MiK}-)3s)VRZ&+YF&B8T$iTStN z7B}w}UoM}rt1xd};QKwrVROyI8Ppaq%RG7Ap7~;@O?lemki!<6&zG+=)z4ccx$|n4 z#dD+2#|$q#f820*m(26&mR|*m#YJs6HReBRley0GY$xNCl4;&MKEF4=$9Y0U@6Od= z4n1a-(=v(rJd7uc8#*Q~_lRNnyFsSn>0yrIg3yV^t^qR-6vP`%)-zz}h!b)--(xZ1 zQOqOt8DS6Sn6q3F-SOe@_G3KDKPr7^oMOem8RIplaiaEpj^yL}w*`cFsujqDaR|F> z$Z6DdFu1ueElIUI|E6U@p^O+q@hr!ju9cS$-)3nNJi4@7_3s4+KdwTH`oe}n2lWOC z$19dwYm~g_n7KDSVOn)sf`Q|K-q&A*r& zA){@2?yF1;H@9G8FE8(vTfOCLUs=aZf3opwMkDMzcLuNx2SLPp5w66=%!c21jkt}yMA^Qc+Rmn zGR;YN#-8Ur=ULl|7TOp+o_?LLxOqzpE7z+zKb$!frbzU!afwUj7W6PuNalBFS+r%% z-?ALLB&%-z_=RcfZD(d8MC5@RkEi>O1@u=FU8L&M%J9O~5q4rN845kGRdw z%PJEXt`tf|Nl95ZG$hTQ)yg4}BEGpt@)cjky!e}M8dqg8$=&cfDbC7Nz~kX4v&Tt2 z|JFHiEwR&GiSJwFSZ*sAA1j#iXl0YAa4MWt3-59YYL?%W z$b4#9%BNOVrvD5jirju24neJjk*VDN+H)2#bai*yhnVaXt+7#0BuD zl$9)}Sp)0^8d7^U2kR8Hu2=beSo7f=3yXEjS})ID)_wfl0hWA;4`+nuxEnuZwN?tB zmT}TiCAqy_qQlCw>+x?1C)u-4L+3Fwa_(K|wqP-j;_7>xN0x|YbuwFi`}cxHHbKU* z(3M?hI)~Rmi3%s%l=fkA+rnEYqFcaHjd|VV{!) zPM#`LT#e`7ZC?4H=?%|?3Co)f`zJN1wWPB;)UWs(^;I@PuRc@Jx%kTBKIPpU0U`YV zFFl`i?~>f%rKc7i()(>6aQo`bbs2G%S(~PM?*1@i(H_oKJeLni7KdD37_ni(mODJ3 zw7(oTGrcSKrFve9vUPCrZ>^P6!z!hiR`JGID=Mv;w4!k(m$TnYHM<{P9%{uwg>y>I z^9ia8xOo)*TBxbZGN*&(%qc;?B!f`bzaIn{r|jUr*T%@nc;LxOrM(mQg}?dYz z)pF{);b5@flne81mL&%*HT^Da=sJhqF>!D;z4Hf5k@he{1oT_X* zE;?h1dxN&&lxv0JGuS+qZ*?!xRardu^eTs$JYp=@_l7)ek(>fLZ`yK!2ZPMQun2{3 zjvUe7rv*(Ej;U}u>NRN(dyyk!z`Ov9G{*xzAKREXm(S4lpH|#9)g_df=b^QT(Q&yS z#w_a#nOhiGCaG*_a`BM}d?L?1DIB@#qoiuKGtFlw&k|&d} zKR-v3poSpVLqn#Dhx%DgDhhl@)m%D`#VC3#bMJWDz*MtphYbHdfxxF0=`E~U?T33b z`&URX6c%r3J+LBZ=7AXuJw8t@9r>$0oPrw;%+OrH*deU>NU=fGkyZ7hnm}XOk_lx@ zG1WE-XYYu&u`(Svm(don=66Lh1BZ~BVG!3hCzXT{Mk5~P#vsNA)(fpoCrQrv(7+-i zBCPt*kSnFei(_T_mtIh1yWMKS;QSpgyke6G)aRvVp z0|CC-E!}CWIaV@APFTsY>U_(x2{9(SJ&$OFwlFvpxy(GrX(XX@WaISjhaHU1_ehFN z;#}px^n3XpFF`H?wHR?j&dt*Tx&->>GfoR&;5;&?SpF{inTiELb6AwP^n{KizUy{( z&Jp{4?TuDmOYrTSFT0+epBxkS;F#g@H(AB8fpr|Mw^!X!wX87=eXpK(d#5VLvzzO0 zS$VJA)!4S}^p-pK0@?l7ok|G`?5+)0wT&+iNr=y_?zs{H@oYz|&mq*Q>;U!{K z%JQFqr)qiTTEBOfTKC;9cG|GwzTfO9?%BR)GAkDSit2xP?{(2#lfPy!FP{HkWb!vY zefz!nPFr=w@}CsvN%^veRAwgcH+d;C>%!C3+2_x{`k86%ZvLM^>#Fqj=5Mo=4+c#; zd!^#>-7B9pbUj%!=A1dBAkfN`lq|D2bAPVht?lB0PPatGma>NO#_JndPk;Y8^Rm>2 zOHUOq=LvYK3f{eJzz{BZ=7FCrA{@aHfJ9mh3wba+> zIy)#VJTt{8V8bzGT?|}G#z2|B@r|;N#bb-mMGW99Woa~CmUCG>mGg3ajiUZR?SIH z;7X{AQtgI0FLgTKHVB-o7XI!eEHFWxRraukE9Z;}$D5WgePWxeAj-ln;BVlRCMYJw zAmDQ$W#^ebONSZHRF4HIvpODC6h5oDjG2Sa@pgxC%K?36hly8pn$#;wOc-ABR4^?0 zFhM}(X<&oPH%Atklt#z5r#T#i7G73VefprwZU&>FgyJ0bf};%F%7;{z2d-Ta6Q#w( zT{16n!c2!2QyJ;TNtT?EBD0*GgcUYVkV$-3Ja27*Qv++LOUNOqc?SyqPLgKmjpPY+ zm}sXaCGMf&B+7DnPuW2Q4}}n?7NsBJj#iGGmOeiI#~7HVel?W|sXNUjGFfQC0_Oi$ zp0n^oaSFx#XSm@uzkJ2A+3Z4Z?Oxp8fBN<^XS@3nm);${zcQmhb3vxMNW{^!`G(;O zU)v`67Q9oLy}GNmYv-&#Oq2U$5|}#s3*R?2#z^_sD6k8@nUiZXn`L*3NaJyh;7Kbi zmKF3FY5(?36urdU)H?CJMKWhmF^?PHuS6Sx2Wl%+UJ7iEV3s?u`j&$Ho}F5cZ!)l) z=P|ZC;NGN1orp85(JiZ&X8ELQJ_6hF-4R|PELS7U-AHxM)N)QT|A#P++rt) z8pUyQJPQ6che0SnuxH8gyL=2jT1qD#^5*krPLXe2`k5<0IZhxvthrHbaiKP7vrPj|+T76y}_73Y$4_o~+XuH9a|eaVBVCbJG~ z*?Vl(_8FPVQ+u=j`o_$gy=cW#zZ*AlO7&jG{hBtVx8(AxiL#9U#`5fce+){ zl{Md9{bvY28GL@?M)UlIeouF&#_oI88uUE(+?|Zee|^({j}wX(U-Rxf%y z_sf3<{cB$}FF!dYwoU6~sMSmN8-0r=mupQ5y6u1Io5Xzsja~aLZ+}7{$9^YpXRUVl2cN;T@i zZDE(otXe)#tXqU8D;7@p^>8y=vE|{NJF7ztr!4lI+8t;ttj=;ywn10x>Eup^lg?c- zb2S`O=7mT+@i!I9yAf-k5QK!Clj(Ad{=x zsNhJMqlKu`DybG8H#P1l44d;DKDJGDJR+Z?*s$fyPLIdG7rc}di8F3_>UU%cyUyhAF)5x? zeA*0cQxDqCJXEx5s`F8C8%0^G;@>O39`@3|%5Z3pNW)J5yNt$i0_~N|7Rr*z3> + +std::vector LoadImageFromFile(const char* file_name, int* out_width, + int* out_height, int* out_channels); + +#endif // TENSORFLOW_EXAMPLES_IOS_IOS_IMAGE_LOAD_H_ diff --git a/tensorflow/contrib/lite/examples/ios/simple/ios_image_load.mm b/tensorflow/contrib/lite/examples/ios/simple/ios_image_load.mm new file mode 100644 index 0000000000..cb19377d7e --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/ios_image_load.mm @@ -0,0 +1,80 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ios_image_load.h" + +#include +#include +#include +#include + +#import +#import + +std::vector LoadImageFromFile(const char* file_name, int* out_width, int* out_height, + int* out_channels) { + FILE* file_handle = fopen(file_name, "rb"); + fseek(file_handle, 0, SEEK_END); + const size_t bytes_in_file = ftell(file_handle); + fseek(file_handle, 0, SEEK_SET); + std::vector file_data(bytes_in_file); + fread(file_data.data(), 1, bytes_in_file, file_handle); + fclose(file_handle); + CFDataRef file_data_ref = + CFDataCreateWithBytesNoCopy(NULL, file_data.data(), bytes_in_file, kCFAllocatorNull); + CGDataProviderRef image_provider = CGDataProviderCreateWithCFData(file_data_ref); + + const char* suffix = strrchr(file_name, '.'); + if (!suffix || suffix == file_name) { + suffix = ""; + } + CGImageRef image; + if (strcasecmp(suffix, ".png") == 0) { + image = CGImageCreateWithPNGDataProvider(image_provider, NULL, true, kCGRenderingIntentDefault); + } else if ((strcasecmp(suffix, ".jpg") == 0) || (strcasecmp(suffix, ".jpeg") == 0)) { + image = + CGImageCreateWithJPEGDataProvider(image_provider, NULL, true, kCGRenderingIntentDefault); + } else { + CFRelease(image_provider); + CFRelease(file_data_ref); + fprintf(stderr, "Unknown suffix for file '%s'\n", file_name); + *out_width = 0; + *out_height = 0; + *out_channels = 0; + return std::vector(); + } + + const int width = (int)CGImageGetWidth(image); + const int height = (int)CGImageGetHeight(image); + const int channels = 4; + CGColorSpaceRef color_space = CGColorSpaceCreateDeviceRGB(); + const int bytes_per_row = (width * channels); + const int bytes_in_image = (bytes_per_row * height); + std::vector result(bytes_in_image); + const int bits_per_component = 8; + CGContextRef context = + CGBitmapContextCreate(result.data(), width, height, bits_per_component, bytes_per_row, + color_space, kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); + CGColorSpaceRelease(color_space); + CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); + CGContextRelease(context); + CFRelease(image); + CFRelease(image_provider); + CFRelease(file_data_ref); + + *out_width = width; + *out_height = height; + *out_channels = channels; + return result; +} diff --git a/tensorflow/contrib/lite/examples/ios/simple/main.mm b/tensorflow/contrib/lite/examples/ios/simple/main.mm new file mode 100644 index 0000000000..05cb55ddd7 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/main.mm @@ -0,0 +1,22 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +int main(int argc, char *argv[]) { + @autoreleasepool { + NSString *delegateClassName = @"AppDelegate"; + return UIApplicationMain(argc, argv, nil, delegateClassName); + } +} diff --git a/tensorflow/contrib/lite/examples/ios/simple/simple.xcodeproj/project.pbxproj b/tensorflow/contrib/lite/examples/ios/simple/simple.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..9277c230b8 --- /dev/null +++ b/tensorflow/contrib/lite/examples/ios/simple/simple.xcodeproj/project.pbxproj @@ -0,0 +1,359 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 1C0D734B1ECCC460008C1DAB /* CoreGraphics.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1C0D734A1ECCC460008C1DAB /* CoreGraphics.framework */; }; + 1CA45FFF1ECCC356002FA6A4 /* UIKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1CA45FFE1ECCC356002FA6A4 /* UIKit.framework */; }; + 594C14AE1FB8F9B500EE8BFE /* libtensorflow-lite.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 594C14AD1FB8F9B500EE8BFE /* libtensorflow-lite.a */; }; + 594C14B11FB9037100EE8BFE /* labels.txt in Resources */ = {isa = PBXBuildFile; fileRef = 594C14AF1FB9037100EE8BFE /* labels.txt */; }; + 594C14B21FB9037100EE8BFE /* mobilenet_v1_1.0_224.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 594C14B01FB9037100EE8BFE /* mobilenet_v1_1.0_224.tflite */; }; + 59A3D0011CF4E68100C4259F /* AppDelegate.mm in Sources */ = {isa = PBXBuildFile; fileRef = 59A3CFF21CF4E68100C4259F /* AppDelegate.mm */; }; + 59A3D0031CF4E68100C4259F /* grace_hopper.jpg in Resources */ = {isa = PBXBuildFile; fileRef = 59A3CFF51CF4E68100C4259F /* grace_hopper.jpg */; }; + 59A3D0081CF4E68100C4259F /* ios_image_load.mm in Sources */ = {isa = PBXBuildFile; fileRef = 59A3CFFB1CF4E68100C4259F /* ios_image_load.mm */; }; + 59A3D0091CF4E68100C4259F /* main.mm in Sources */ = {isa = PBXBuildFile; fileRef = 59A3CFFC1CF4E68100C4259F /* main.mm */; }; + 59A3D00B1CF4E68100C4259F /* RunModelViewController.mm in Sources */ = {isa = PBXBuildFile; fileRef = 59A3CFFF1CF4E68100C4259F /* RunModelViewController.mm */; }; + 59A3D00C1CF4E68100C4259F /* RunModelViewController.xib in Resources */ = {isa = PBXBuildFile; fileRef = 59A3D0001CF4E68100C4259F /* RunModelViewController.xib */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 1C0D73481ECCC41B008C1DAB /* CoreImage.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreImage.framework; path = System/Library/Frameworks/CoreImage.framework; sourceTree = SDKROOT; }; + 1C0D734A1ECCC460008C1DAB /* CoreGraphics.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreGraphics.framework; path = System/Library/Frameworks/CoreGraphics.framework; sourceTree = SDKROOT; }; + 1CA45FFE1ECCC356002FA6A4 /* UIKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = UIKit.framework; path = System/Library/Frameworks/UIKit.framework; sourceTree = SDKROOT; }; + 5911579B1CF4011C00C31E3A /* tf_simple_example.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = tf_simple_example.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 594C14AD1FB8F9B500EE8BFE /* libtensorflow-lite.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "libtensorflow-lite.a"; path = "../../../gen/lib/libtensorflow-lite.a"; sourceTree = ""; }; + 594C14AF1FB9037100EE8BFE /* labels.txt */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = labels.txt; sourceTree = ""; }; + 594C14B01FB9037100EE8BFE /* mobilenet_v1_1.0_224.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = mobilenet_v1_1.0_224.tflite; sourceTree = ""; }; + 59A3CFF11CF4E68100C4259F /* AppDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = ""; }; + 59A3CFF21CF4E68100C4259F /* AppDelegate.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = AppDelegate.mm; sourceTree = ""; }; + 59A3CFF51CF4E68100C4259F /* grace_hopper.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = grace_hopper.jpg; sourceTree = ""; }; + 59A3CFFA1CF4E68100C4259F /* ios_image_load.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ios_image_load.h; sourceTree = ""; }; + 59A3CFFB1CF4E68100C4259F /* ios_image_load.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_image_load.mm; sourceTree = ""; }; + 59A3CFFC1CF4E68100C4259F /* main.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = main.mm; sourceTree = ""; }; + 59A3CFFD1CF4E68100C4259F /* RunModel-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "RunModel-Info.plist"; sourceTree = ""; }; + 59A3CFFE1CF4E68100C4259F /* RunModelViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RunModelViewController.h; sourceTree = ""; }; + 59A3CFFF1CF4E68100C4259F /* RunModelViewController.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = RunModelViewController.mm; sourceTree = ""; }; + 59A3D0001CF4E68100C4259F /* RunModelViewController.xib */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.xib; path = RunModelViewController.xib; sourceTree = ""; }; + 73DBC33C5DD9A526EE6D1EF2 /* libPods-tf_simple_example.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-tf_simple_example.a"; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 591157981CF4011C00C31E3A /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 594C14AE1FB8F9B500EE8BFE /* libtensorflow-lite.a in Frameworks */, + 1C0D734B1ECCC460008C1DAB /* CoreGraphics.framework in Frameworks */, + 1CA45FFF1ECCC356002FA6A4 /* UIKit.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 24D7686C331131624F4454A0 /* Frameworks */ = { + isa = PBXGroup; + children = ( + 594C14AD1FB8F9B500EE8BFE /* libtensorflow-lite.a */, + 1C0D734A1ECCC460008C1DAB /* CoreGraphics.framework */, + 1C0D73481ECCC41B008C1DAB /* CoreImage.framework */, + 1CA45FFE1ECCC356002FA6A4 /* UIKit.framework */, + 73DBC33C5DD9A526EE6D1EF2 /* libPods-tf_simple_example.a */, + ); + name = Frameworks; + sourceTree = ""; + }; + 591157921CF4011C00C31E3A = { + isa = PBXGroup; + children = ( + 59A3CFF11CF4E68100C4259F /* AppDelegate.h */, + 59A3CFF21CF4E68100C4259F /* AppDelegate.mm */, + 59A3CFF31CF4E68100C4259F /* data */, + 59A3CFFA1CF4E68100C4259F /* ios_image_load.h */, + 59A3CFFB1CF4E68100C4259F /* ios_image_load.mm */, + 59A3CFFC1CF4E68100C4259F /* main.mm */, + 59A3CFFD1CF4E68100C4259F /* RunModel-Info.plist */, + 59A3CFFE1CF4E68100C4259F /* RunModelViewController.h */, + 59A3CFFF1CF4E68100C4259F /* RunModelViewController.mm */, + 59A3D0001CF4E68100C4259F /* RunModelViewController.xib */, + 5911579C1CF4011C00C31E3A /* Products */, + 24D7686C331131624F4454A0 /* Frameworks */, + ); + sourceTree = ""; + }; + 5911579C1CF4011C00C31E3A /* Products */ = { + isa = PBXGroup; + children = ( + 5911579B1CF4011C00C31E3A /* tf_simple_example.app */, + ); + name = Products; + sourceTree = ""; + }; + 59A3CFF31CF4E68100C4259F /* data */ = { + isa = PBXGroup; + children = ( + 59A3CFF51CF4E68100C4259F /* grace_hopper.jpg */, + 594C14AF1FB9037100EE8BFE /* labels.txt */, + 594C14B01FB9037100EE8BFE /* mobilenet_v1_1.0_224.tflite */, + ); + path = data; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 5911579A1CF4011C00C31E3A /* tf_simple_example */ = { + isa = PBXNativeTarget; + buildConfigurationList = 591157B21CF4011D00C31E3A /* Build configuration list for PBXNativeTarget "tf_simple_example" */; + buildPhases = ( + 591157971CF4011C00C31E3A /* Sources */, + 591157981CF4011C00C31E3A /* Frameworks */, + 591157991CF4011C00C31E3A /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = tf_simple_example; + productName = tf_ios_makefile_example; + productReference = 5911579B1CF4011C00C31E3A /* tf_simple_example.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 591157931CF4011C00C31E3A /* Project object */ = { + isa = PBXProject; + attributes = { + LastUpgradeCheck = 0830; + ORGANIZATIONNAME = Google; + TargetAttributes = { + 5911579A1CF4011C00C31E3A = { + CreatedOnToolsVersion = 7.2; + DevelopmentTeam = EQHXZ8M8AV; + ProvisioningStyle = Manual; + }; + }; + }; + buildConfigurationList = 591157961CF4011C00C31E3A /* Build configuration list for PBXProject "simple" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = English; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 591157921CF4011C00C31E3A; + productRefGroup = 5911579C1CF4011C00C31E3A /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 5911579A1CF4011C00C31E3A /* tf_simple_example */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 591157991CF4011C00C31E3A /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 59A3D00C1CF4E68100C4259F /* RunModelViewController.xib in Resources */, + 594C14B11FB9037100EE8BFE /* labels.txt in Resources */, + 59A3D0031CF4E68100C4259F /* grace_hopper.jpg in Resources */, + 594C14B21FB9037100EE8BFE /* mobilenet_v1_1.0_224.tflite in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 591157971CF4011C00C31E3A /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 59A3D0091CF4E68100C4259F /* main.mm in Sources */, + 59A3D0011CF4E68100C4259F /* AppDelegate.mm in Sources */, + 59A3D00B1CF4E68100C4259F /* RunModelViewController.mm in Sources */, + 59A3D0081CF4E68100C4259F /* ios_image_load.mm in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 591157B01CF4011D00C31E3A /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MTL_ENABLE_DEBUG_INFO = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 591157B11CF4011D00C31E3A /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 8.0; + MTL_ENABLE_DEBUG_INFO = NO; + SDKROOT = iphoneos; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 591157B31CF4011D00C31E3A /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_DEBUG_INFORMATION_LEVEL = default; + CODE_SIGN_IDENTITY = "iPhone Developer"; + DEVELOPMENT_TEAM = EQHXZ8M8AV; + ENABLE_BITCODE = NO; + GCC_ENABLE_CPP_EXCEPTIONS = YES; + GCC_ENABLE_CPP_RTTI = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + ../../../../../../, + ../../../downloads/flatbuffers/include/, + ../../../downloads/eigen/, + ../../../downloads/, + ); + INFOPLIST_FILE = "$(SRCROOT)/RunModel-Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 9.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LIBRARY_SEARCH_PATHS = ../../../gen/lib/; + OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; + OTHER_LDFLAGS = "$(inherited)"; + PRODUCT_BUNDLE_IDENTIFIER = "com.google.tflite-simple-example"; + PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE = "1072bd47-ff19-4e5f-8107-d912748f83f1"; + PROVISIONING_PROFILE_SPECIFIER = "Google Development"; + SEPARATE_STRIP = NO; + }; + name = Debug; + }; + 591157B41CF4011D00C31E3A /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_DEBUG_INFORMATION_LEVEL = default; + CODE_SIGN_IDENTITY = "iPhone Developer"; + DEVELOPMENT_TEAM = ""; + ENABLE_BITCODE = NO; + GCC_ENABLE_CPP_EXCEPTIONS = YES; + GCC_ENABLE_CPP_RTTI = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + ../../../../../../, + ../../../downloads/flatbuffers/include/, + ../../../downloads/eigen/, + ../../../downloads/, + ); + INFOPLIST_FILE = "$(SRCROOT)/RunModel-Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 9.2; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks"; + LIBRARY_SEARCH_PATHS = ../../../gen/lib/; + ONLY_ACTIVE_ARCH = YES; + OTHER_CPLUSPLUSFLAGS = "$(OTHER_CFLAGS)"; + OTHER_LDFLAGS = "$(inherited)"; + PRODUCT_BUNDLE_IDENTIFIER = "com.google.tflite-simple-example"; + PRODUCT_NAME = "$(TARGET_NAME)"; + PROVISIONING_PROFILE_SPECIFIER = ""; + SEPARATE_STRIP = NO; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 591157961CF4011C00C31E3A /* Build configuration list for PBXProject "simple" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 591157B01CF4011D00C31E3A /* Debug */, + 591157B11CF4011D00C31E3A /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 591157B21CF4011D00C31E3A /* Build configuration list for PBXNativeTarget "tf_simple_example" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 591157B31CF4011D00C31E3A /* Debug */, + 591157B41CF4011D00C31E3A /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 591157931CF4011C00C31E3A /* Project object */; +} diff --git a/tensorflow/contrib/lite/g3doc/apis.md b/tensorflow/contrib/lite/g3doc/apis.md index e8f5566f11..fe208e47d1 100644 --- a/tensorflow/contrib/lite/g3doc/apis.md +++ b/tensorflow/contrib/lite/g3doc/apis.md @@ -267,7 +267,7 @@ try (Interpreter interpreter = new Interpreter(file_of_a_tensorflowlite_model)) The `Interpreter.java` class drives model inference with TensorFlow Lite. In most of the cases, this is the only class an app developer will need. -#### Initializing an `Interpreter` Mith a Model Mile +#### Initializing an `Interpreter` With a Model File The `Interpreter` can be initialized with a model file using the constructor: diff --git a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md index 121c4c2c95..9ade04eb8c 100644 --- a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md +++ b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md @@ -54,7 +54,7 @@ counterparts: * [tf.sigmoid](https://www.tensorflow.org/api_docs/python/tf/sigmoid) * [tf.space_to_depth](https://www.tensorflow.org/api_docs/python/tf/space_to_depth) -## Straighforward Conversions, Constant-Folding and Fusing +## Straightforward Conversions, Constant-Folding and Fusing A number of TensorFlow operations can be processed by TensorFlow Lite even though they have no direct equivalent. This is the case for operations that can diff --git a/tensorflow/contrib/lite/ios_makefile.inc b/tensorflow/contrib/lite/ios_makefile.inc new file mode 100644 index 0000000000..345ed26212 --- /dev/null +++ b/tensorflow/contrib/lite/ios_makefile.inc @@ -0,0 +1,31 @@ +#Settings for iOS. +ifeq($(TARGET), IOS) BUILD_FOR_IOS_SIMULATOR + : = false ifeq($(IOS_ARCH), x86_64) BUILD_FOR_IOS_SIMULATOR + : = true endif ifeq($(IOS_ARCH), i386) BUILD_FOR_IOS_SIMULATOR + : = true endif ifeq($(BUILD_FOR_IOS_SIMULATOR), true) IPHONEOS_PLATFORM + : = $(shell xcrun-- sdk iphonesimulator-- show - sdk - platform - + path) IPHONEOS_SYSROOT + : = $(shell xcrun-- sdk iphonesimulator-- show - sdk - + path) else IPHONEOS_PLATFORM + : = $(shell xcrun-- sdk iphoneos-- show - sdk - platform - + path) IPHONEOS_SYSROOT + : = $(shell xcrun-- sdk iphoneos-- show - sdk - path) endif IOS_SDK_VERSION + : = $(shell xcrun-- sdk iphoneos-- show - sdk - version) MIN_SDK_VERSION + : = 9.0 +#Override IOS_ARCH with armv7, armv7s, arm64, i386, or x86_64. + IOS_ARCH + : = x86_64 CXXFLAGS + += -miphoneos - version + - min = $(MIN_SDK_VERSION) - DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + - fembed - bitcode - Wno - c++ 11 - narrowing - mno - thumb + - fno - exceptions + - isysroot ${IPHONEOS_SYSROOT} - arch $(IOS_ARCH) - O3 CCFLAGS + += -miphoneos - version + - min = $(MIN_SDK_VERSION) - fembed - bitcode - mno - thumb + - isysroot ${IPHONEOS_SYSROOT} - arch $(IOS_ARCH) - + O3 LDFLAGS + : = -fembed - bitcode - miphoneos - version + - min = ${MIN_SDK_VERSION} - arch $(IOS_ARCH) OBJDIR + : = $(OBJDIR) ios_$(IOS_ARCH) / LIBDIR + : = $(LIBDIR) ios_$(IOS_ARCH) / BINDIR + : = $(BINDIR) ios_$(IOS_ARCH) / DEPDIR : = $(DEPDIR) ios_$(IOS_ARCH) / endif diff --git a/tensorflow/contrib/lite/java/demo/app/build.gradle b/tensorflow/contrib/lite/java/demo/app/build.gradle index e1470fe717..b76eaad8bb 100644 --- a/tensorflow/contrib/lite/java/demo/app/build.gradle +++ b/tensorflow/contrib/lite/java/demo/app/build.gradle @@ -36,8 +36,8 @@ android { } repositories { - flatDir { - dirs 'libs' + maven { + url 'https://google.bintray.com/tensorflow' } } diff --git a/tensorflow/contrib/lite/models/testdata/g3doc/README.md b/tensorflow/contrib/lite/models/testdata/g3doc/README.md index 83760e420f..46b24248f0 100644 --- a/tensorflow/contrib/lite/models/testdata/g3doc/README.md +++ b/tensorflow/contrib/lite/models/testdata/g3doc/README.md @@ -86,25 +86,34 @@ same input. ### Models: -[Speech hotword model (Svdf rank=1)] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/testdata/speech_hotword_model_rank1.tflite) +[Speech hotword model (Svdf +rank=1)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank1_2017_11_14.tflite) -[Speech hotword model (Svdf rank=2)] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/testdata/speech_hotword_model_rank2.tflite) +[Speech hotword model (Svdf +rank=2)](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_hotword_model_rank2_2017_11_14.tflite) -[Speaker-id model] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/testdata/speech_speakerid_model.tflite) +[Speaker-id +model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_speakerid_model_2017_11_14.tflite) -[TTS model] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/testdata/speech_tts_model.tflite) +[TTS +model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_tts_model_2017_11_14.tflite) -[ASR AM model] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/testdata/speech_terse_am_model.tflite) +[ASR AM +model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/speech_terse_am_model_2017_11_14.tflite) ### Test benches -[Speech hotword model test] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_hotword_model_test.cc) +[Speech hotword model +test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_hotword_model_test.cc) -[Speaker-id model test] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_speakerid_model_test.cc) +[Speaker-id model +test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_speakerid_model_test.cc) -[TTS model test] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_tts_model_test.cc) +[TTS model +test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_tts_model_test.cc) -[ASR AM model test] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_terse_am_model_test.cc) +[ASR AM model +test](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/models/speech_terse_am_model_test.cc) ## Android Support The models have been tested on Android phones, using the following tests: @@ -112,5 +121,3 @@ The models have been tested on Android phones, using the following tests: [Hotword] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/android/BUILD?rcl=172930882&l=25) [Speaker-id] (https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/android/BUILD?rcl=172930882&l=36) - - diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h index 5d06165772..bdb5e01538 100644 --- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h +++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h @@ -1454,9 +1454,9 @@ inline int ANeuralNetworksModel_finish(ANeuralNetworksModel* model) { * {@link ANeuralNetworksExecution_setOutputFromMemory} and * {@link ANeuralNetworksExecution_setOperandValue}. * - * To build a model that can accomodate inputs of various sizes, as you may want - * to do for a CNN, set the size of the dimensions that will vary at run time to - * 0. If you do so, provide the full dimensions when calling + * To build a model that can accommodate inputs of various sizes, as you may + * want to do for a CNN, set the size of the dimensions that will vary at run + * time to 0. If you do so, provide the full dimensions when calling * {@link ANeuralNetworksExecution_setInput} or {@link * ANeuralNetworksExecution_setInputFromMemory}. * diff --git a/tensorflow/contrib/lite/schema/upgrade_schema_test.py b/tensorflow/contrib/lite/schema/upgrade_schema_test.py index 754400e888..b5002e6f75 100644 --- a/tensorflow/contrib/lite/schema/upgrade_schema_test.py +++ b/tensorflow/contrib/lite/schema/upgrade_schema_test.py @@ -252,7 +252,7 @@ def JsonDumpAndFlush(data, fp): class TestSchemaUpgrade(test_util.TensorFlowTestCase): - def testNonExistantFile(self): + def testNonExistentFile(self): converter = upgrade_schema_lib.Converter() non_existent = tempfile.mktemp(suffix=".json") with self.assertRaisesRegexp(IOError, "No such file or directory"): diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD index 5e40a13d3c..ecddb4b807 100644 --- a/tensorflow/contrib/lite/testing/BUILD +++ b/tensorflow/contrib/lite/testing/BUILD @@ -187,6 +187,7 @@ tf_cc_test( srcs = ["generated_examples_zip_test.cc"], data = [":optest"], shard_count = 10, + tags = ["no_oss"], deps = [ ":parse_testdata_lib", "//tensorflow/contrib/lite:builtin_op_data", diff --git a/tensorflow/contrib/lite/testing/parse_testdata.cc b/tensorflow/contrib/lite/testing/parse_testdata.cc index 2b67052cad..d745ed2715 100644 --- a/tensorflow/contrib/lite/testing/parse_testdata.cc +++ b/tensorflow/contrib/lite/testing/parse_testdata.cc @@ -232,7 +232,7 @@ TfLiteStatus CheckOutputs(tflite::Interpreter* interpreter, // invoke { // id: xyz // input: 1,2,1,1,1,2,3,4 -// ouput: 4,5,6 +// output: 4,5,6 // } class Invoke : public Message { public: diff --git a/tensorflow/contrib/lite/testing/test_runner.h b/tensorflow/contrib/lite/testing/test_runner.h index 04ee4d9f7d..f4b26949b5 100644 --- a/tensorflow/contrib/lite/testing/test_runner.h +++ b/tensorflow/contrib/lite/testing/test_runner.h @@ -63,7 +63,7 @@ class TestRunner { // Run the model. virtual void Invoke() = 0; - // Verify that the contents of all ouputs conform to the existing + // Verify that the contents of all outputs conform to the existing // expectations. Return true if there are no expectations or they are all // satisfied. virtual bool CheckResults() = 0; diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h index f2fce2b249..04b0813523 100644 --- a/tensorflow/contrib/lite/toco/model.h +++ b/tensorflow/contrib/lite/toco/model.h @@ -129,7 +129,7 @@ enum class AxesOrder { // The type of the scalars in an array. // Note that that does not by itself tell whether the values in the array are // real (are literally interpreted as real numbers) or quantized (only acquire -// a meaning as real numbers in conjuction with QuantizationParams). +// a meaning as real numbers in conjunction with QuantizationParams). // // In practice though: // float values are always real diff --git a/tensorflow/contrib/lite/tools/benchmark_model.cc b/tensorflow/contrib/lite/tools/benchmark_model.cc new file mode 100644 index 0000000000..ef43f64131 --- /dev/null +++ b/tensorflow/contrib/lite/tools/benchmark_model.cc @@ -0,0 +1,95 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/lite/kernels/register.h" +#include "tensorflow/contrib/lite/model.h" +#include "tensorflow/contrib/lite/string_util.h" +#include "tensorflow/contrib/lite/tools/mutable_op_resolver.h" + +#ifdef TFLITE_CUSTOM_OPS_HEADER +void RegisterSelectedOps(::tflite::MutableOpResolver* resolver); +#endif + +#define LOG(x) std::cerr +#define CHECK(x) \ + if (!(x)) { \ + LOG(ERROR) << #x << "failed"; \ + exit(1); \ + } + +namespace tensorflow { +namespace benchmark_tflite_model { + +std::unique_ptr model; +std::unique_ptr interpreter; + +void InitImpl(const std::string& graph, const std::vector& sizes, + const std::string& input_layer_type, int num_threads) { + CHECK(graph.c_str()); + + model = tflite::FlatBufferModel::BuildFromFile(graph.c_str()); + if (!model) { + LOG(FATAL) << "Failed to mmap model " << graph; + } + LOG(INFO) << "Loaded model " << graph; + model->error_reporter(); + LOG(INFO) << "resolved reporter"; + +#ifdef TFLITE_CUSTOM_OPS_HEADER + tflite::MutableOpResolver resolver; + RegisterSelectedOps(&resolver); +#else + tflite::ops::builtin::BuiltinOpResolver resolver; +#endif + + tflite::InterpreterBuilder(*model, resolver)(&interpreter); + if (!interpreter) { + LOG(FATAL) << "Failed to construct interpreter"; + } + + if (num_threads != -1) { + interpreter->SetNumThreads(num_threads); + } + + int input = interpreter->inputs()[0]; + + if (input_layer_type != "string") { + interpreter->ResizeInputTensor(input, sizes); + } + + if (interpreter->AllocateTensors() != kTfLiteOk) { + LOG(FATAL) << "Failed to allocate tensors!"; + } +} + +int Main(int argc, char** argv) { + InitImpl("", {}, "", 1); + return 0; +} + +} // namespace benchmark_tflite_model +} // namespace tensorflow + +int main(int argc, char** argv) { + return tensorflow::benchmark_tflite_model::Main(argc, argv); +} diff --git a/tensorflow/contrib/lite/tools/mutable_op_resolver.h b/tensorflow/contrib/lite/tools/mutable_op_resolver.h index cc1a8e27e6..be60cf476d 100644 --- a/tensorflow/contrib/lite/tools/mutable_op_resolver.h +++ b/tensorflow/contrib/lite/tools/mutable_op_resolver.h @@ -19,6 +19,16 @@ limitations under the License. #include "tensorflow/contrib/lite/context.h" #include "tensorflow/contrib/lite/model.h" +// Needed to resolve unordered_set hash on older compilers. +namespace std { +template <> +struct hash { + size_t operator()(const tflite::BuiltinOperator& op) const { + return std::hash()(op); + } +}; +} // namespace std + namespace tflite { // An OpResolver that is mutable, also used as the op in gen_op_registration. diff --git a/tensorflow/contrib/mpi/BUILD b/tensorflow/contrib/mpi/BUILD index 20ceef5004..d9d55faf50 100644 --- a/tensorflow/contrib/mpi/BUILD +++ b/tensorflow/contrib/mpi/BUILD @@ -72,6 +72,7 @@ cc_library( "//tensorflow/core:worker_proto_cc", "//tensorflow/core/distributed_runtime:base_rendezvous_mgr", "//tensorflow/core/distributed_runtime:session_mgr", + "//tensorflow/core/distributed_runtime:tensor_coding", "//tensorflow/core/distributed_runtime:worker_env", "//third_party/mpi", ], diff --git a/tensorflow/contrib/nn/python/ops/cross_entropy.py b/tensorflow/contrib/nn/python/ops/cross_entropy.py index 61c1d1c6d9..5045f2c957 100644 --- a/tensorflow/contrib/nn/python/ops/cross_entropy.py +++ b/tensorflow/contrib/nn/python/ops/cross_entropy.py @@ -116,7 +116,7 @@ def deprecated_flipped_sparse_softmax_cross_entropy_with_logits(logits, Raises: ValueError: If logits are scalars (need to have rank >= 1) or if the rank - of the labels is not equal to the rank of the labels minus one. + of the labels is not equal to the rank of the logits minus one. """ return nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits, name=name) diff --git a/tensorflow/contrib/nn/python/ops/sampling_ops.py b/tensorflow/contrib/nn/python/ops/sampling_ops.py index 2ae529e015..98749cff7e 100644 --- a/tensorflow/contrib/nn/python/ops/sampling_ops.py +++ b/tensorflow/contrib/nn/python/ops/sampling_ops.py @@ -34,7 +34,7 @@ def _rank_resample(weights, biases, inputs, sampled_values, num_resampled, log(sum_j exp((w_i * x_j + b_i) / resampling_temperature)) - where w_i, b_i are the weight and bias of the i-th class, repsectively, + where w_i, b_i are the weight and bias of the i-th class, respectively, and j ranges over the rows of `inputs`. For efficiency, we rearrange the computation to diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py index 289359e5ec..9685b58392 100644 --- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py +++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py @@ -114,7 +114,6 @@ class CoupledInputForgetGateLSTMCell(rnn_cell_impl.RNNCell): The class uses optional peep-hole connections, and an optional projection layer. - Layer normalization implementation is based on: https://arxiv.org/abs/1607.06450. diff --git a/tensorflow/contrib/slim/README.md b/tensorflow/contrib/slim/README.md index f7a85557ca..dc92ae0c85 100644 --- a/tensorflow/contrib/slim/README.md +++ b/tensorflow/contrib/slim/README.md @@ -441,7 +441,8 @@ module. Consider the simple case where we want to train the VGG network: ```python import tensorflow as tf -vgg = tf.contrib.slim.nets.vgg +import tensorflow.contrib.slim.nets as nets +vgg = nets.vgg # Load the images and labels. images, labels = ... @@ -559,9 +560,10 @@ examine the following sample of training the VGG network: ```python import tensorflow as tf +import tensorflow.contrib.slim.nets as nets slim = tf.contrib.slim -vgg = tf.contrib.slim.nets.vgg +vgg = nets.vgg ... @@ -809,9 +811,10 @@ Putting it all together: ```python import tensorflow as tf +import tensorflow.contrib.slim.nets as nets slim = tf.contrib.slim -vgg = tf.contrib.slim.nets.vgg +vgg = nets.vgg # Load the data diff --git a/tensorflow/contrib/slim/python/slim/evaluation.py b/tensorflow/contrib/slim/python/slim/evaluation.py index cdb720b36b..3caf4e02da 100644 --- a/tensorflow/contrib/slim/python/slim/evaluation.py +++ b/tensorflow/contrib/slim/python/slim/evaluation.py @@ -34,7 +34,7 @@ the metrics and finally call the `evaluation` method: "mse": slim.metrics.mean_squared_error(predictions, labels), }) - inital_op = tf.group( + initial_op = tf.group( tf.global_variables_initializer(), tf.local_variables_initializer()) @@ -42,7 +42,7 @@ the metrics and finally call the `evaluation` method: metric_values = slim.evaluation( sess, num_evals=1, - inital_op=initial_op, + initial_op=initial_op, eval_op=names_to_updates.values(), final_op=name_to_values.values()) diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index 45d6454526..f34291c203 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -25,7 +25,6 @@ py_test( srcs_version = "PY2AND3", deps = [ ":summary_ops", - ":summary_test_internal", ":summary_test_util", "//tensorflow/python:array_ops", "//tensorflow/python:errors", @@ -46,7 +45,6 @@ py_test( srcs_version = "PY2AND3", deps = [ ":summary_ops", - ":summary_test_internal", ":summary_test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -119,15 +117,3 @@ py_library( "//tensorflow/python:platform", ], ) - -py_library( - name = "summary_test_internal", - testonly = 1, - srcs = ["summary_test_internal.py"], - srcs_version = "PY2AND3", - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/python:lib", - "//tensorflow/python:platform", - ], -) diff --git a/tensorflow/contrib/summary/summary_ops_graph_test.py b/tensorflow/contrib/summary/summary_ops_graph_test.py index fe55bf93e2..703adb7b46 100644 --- a/tensorflow/contrib/summary/summary_ops_graph_test.py +++ b/tensorflow/contrib/summary/summary_ops_graph_test.py @@ -21,7 +21,6 @@ import tempfile import six from tensorflow.contrib.summary import summary_ops -from tensorflow.contrib.summary import summary_test_internal from tensorflow.contrib.summary import summary_test_util from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 @@ -33,10 +32,10 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.platform import test from tensorflow.python.training import training_util -get_all = summary_test_internal.get_all +get_all = summary_test_util.get_all -class DbTest(summary_test_internal.SummaryDbTest): +class DbTest(summary_test_util.SummaryDbTest): def testGraphPassedToGraph_isForbiddenForThineOwnSafety(self): with self.assertRaises(TypeError): diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 3fe421a7e9..54433deb28 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -21,7 +21,6 @@ import tempfile import six from tensorflow.contrib.summary import summary_ops -from tensorflow.contrib.summary import summary_test_internal from tensorflow.contrib.summary import summary_test_util from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 @@ -35,8 +34,8 @@ from tensorflow.python.ops import state_ops from tensorflow.python.platform import gfile from tensorflow.python.training import training_util -get_all = summary_test_internal.get_all -get_one = summary_test_internal.get_one +get_all = summary_test_util.get_all +get_one = summary_test_util.get_one class TargetTest(test_util.TensorFlowTestCase): @@ -137,7 +136,7 @@ class TargetTest(test_util.TensorFlowTestCase): self.assertEqual(3, get_total()) -class DbTest(summary_test_internal.SummaryDbTest): +class DbTest(summary_test_util.SummaryDbTest): def testIntegerSummaries(self): step = training_util.create_global_step() diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py index 794c5b8bab..915820e05b 100644 --- a/tensorflow/contrib/summary/summary_test_util.py +++ b/tensorflow/contrib/summary/summary_test_util.py @@ -19,13 +19,38 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import os +import sqlite3 +from tensorflow.contrib.summary import summary_ops from tensorflow.core.util import event_pb2 +from tensorflow.python.framework import test_util from tensorflow.python.lib.io import tf_record from tensorflow.python.platform import gfile +class SummaryDbTest(test_util.TensorFlowTestCase): + """Helper for summary database testing.""" + + def setUp(self): + super(SummaryDbTest, self).setUp() + self.db_path = os.path.join(self.get_temp_dir(), 'DbTest.sqlite') + if os.path.exists(self.db_path): + os.unlink(self.db_path) + self.db = sqlite3.connect(self.db_path) + self.create_summary_db_writer = functools.partial( + summary_ops.create_summary_db_writer, + db_uri=self.db_path, + experiment_name='experiment', + run_name='run', + user_name='user') + + def tearDown(self): + self.db.close() + super(SummaryDbTest, self).tearDown() + + def events_from_file(filepath): """Returns all events in a single event file. @@ -58,5 +83,17 @@ def events_from_logdir(logdir): """ assert gfile.Exists(logdir) files = gfile.ListDirectory(logdir) - assert len(files) == 1, "Found not exactly one file in logdir: %s" % files + assert len(files) == 1, 'Found not exactly one file in logdir: %s' % files return events_from_file(os.path.join(logdir, files[0])) + + +def get_one(db, q, *p): + return db.execute(q, p).fetchone()[0] + + +def get_all(db, q, *p): + return unroll(db.execute(q, p).fetchall()) + + +def unroll(list_of_tuples): + return sum(list_of_tuples, ()) diff --git a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py b/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py index cccf444db8..a56beeeb2c 100644 --- a/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py +++ b/tensorflow/contrib/tensor_forest/hybrid/python/models/decisions_to_data_then_nn_test.py @@ -80,7 +80,7 @@ class DecisionsToDataThenNNTest(test_util.TensorFlowTestCase): isinstance(self.params.num_trees, tensor_forest.ForestHParams)) with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testContructionPollution"): + "DecisionsToDataThenNNTest_testConstructionPollution"): graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( self.params) @@ -95,7 +95,7 @@ class DecisionsToDataThenNNTest(test_util.TensorFlowTestCase): for _ in range(100)]) with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testInferenceContruction"): + "DecisionsToDataThenNNTest_testInferenceConstruction"): graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( self.params) graph = graph_builder.inference_graph(data, None) @@ -111,7 +111,7 @@ class DecisionsToDataThenNNTest(test_util.TensorFlowTestCase): labels = [1 for _ in range(100)] with variable_scope.variable_scope( - "DecisionsToDataThenNNTest_testTrainingContruction"): + "DecisionsToDataThenNNTest_testTrainingConstruction"): graph_builder = decisions_to_data_then_nn.DecisionsToDataThenNN( self.params) graph = graph_builder.training_graph(data, labels, None) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index a1d61a7932..fce0663aa5 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -455,6 +455,7 @@ tf_cuda_library( "util/mirror_pad_mode.h", "util/padding.h", "util/port.h", + "util/ptr_util.h", "util/reffed_status_callback.h", "util/saved_tensor_slice_util.h", "util/sparse/group_iterator.h", @@ -493,6 +494,11 @@ cc_library( ], ) +cc_library( + name = "ptr_util", + hdrs = ["util/ptr_util.h"], +) + cc_library( name = "reader_base", srcs = ["framework/reader_base.cc"], diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h index 223dd12f8f..b620127d90 100644 --- a/tensorflow/core/graph/graph.h +++ b/tensorflow/core/graph/graph.h @@ -455,7 +455,6 @@ class Graph { // the corresponding NodeDef to reflect the change. // REQUIRES: The control edge must exist. void RemoveControlEdge(const Edge* e); - // Updates the input to a node. The existing edge to `dst` is removed and an // edge from `new_src` to `dst` is created. The NodeDef associated with `dst` // is also updated. diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 6861a51795..efe8ac05a3 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -1068,7 +1068,7 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() { if (simplified_node != nullptr) { nodes_to_simplify.PushBack(simplified_node); } - // When `node` is simplifed to another node rather than in-place, the + // When `node` is simplified to another node rather than in-place, the // consumers of `node` are already redirected to `simplified_tensor`. // Re-push the consumers into `nodes_to_simplify` for further // optimizations. diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 9279514e6b..dcffb28513 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2583,8 +2583,13 @@ tf_kernel_library( tf_kernel_library( name = "batch_matmul_op", + srcs = [] + if_mkl([ + "mkl_batch_matmul_op.cc", + ]), prefix = "batch_matmul_op", - deps = MATH_DEPS, + deps = MATH_DEPS + if_mkl([ + "//third_party/mkl:intel_binary_blob", + ]), ) tf_kernel_library( @@ -6325,11 +6330,11 @@ cc_library( srcs = ["summary_interface.cc"], hdrs = ["summary_interface.h"], deps = [ - "//tensorflow/compiler/xla:util", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", + "//tensorflow/core:ptr_util", ], ) diff --git a/tensorflow/core/kernels/batch_matmul_op_complex.cc b/tensorflow/core/kernels/batch_matmul_op_complex.cc index a58ec02726..96216764fd 100644 --- a/tensorflow/core/kernels/batch_matmul_op_complex.cc +++ b/tensorflow/core/kernels/batch_matmul_op_complex.cc @@ -17,8 +17,10 @@ limitations under the License. namespace tensorflow { +#if !defined(INTEL_MKL) TF_CALL_complex64(REGISTER_BATCH_MATMUL_CPU); TF_CALL_complex128(REGISTER_BATCH_MATMUL_CPU); +#endif #if GOOGLE_CUDA TF_CALL_complex64(REGISTER_BATCH_MATMUL_GPU); diff --git a/tensorflow/core/kernels/batch_matmul_op_real.cc b/tensorflow/core/kernels/batch_matmul_op_real.cc index 1900ed8e31..8d155ca62b 100644 --- a/tensorflow/core/kernels/batch_matmul_op_real.cc +++ b/tensorflow/core/kernels/batch_matmul_op_real.cc @@ -17,8 +17,10 @@ limitations under the License. namespace tensorflow { +#if !defined(INTEL_MKL) TF_CALL_float(REGISTER_BATCH_MATMUL_CPU); TF_CALL_double(REGISTER_BATCH_MATMUL_CPU); +#endif TF_CALL_half(REGISTER_BATCH_MATMUL_CPU); TF_CALL_int32(REGISTER_BATCH_MATMUL_CPU); diff --git a/tensorflow/core/kernels/cwise_op_asinh.cc b/tensorflow/core/kernels/cwise_op_asinh.cc index a7673afd0b..822d72e068 100644 --- a/tensorflow/core/kernels/cwise_op_asinh.cc +++ b/tensorflow/core/kernels/cwise_op_asinh.cc @@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tensorflow/core/kernels/decode_bmp_op.cc b/tensorflow/core/kernels/decode_bmp_op.cc index 086369a9f1..6d9fdfcf33 100644 --- a/tensorflow/core/kernels/decode_bmp_op.cc +++ b/tensorflow/core/kernels/decode_bmp_op.cc @@ -34,8 +34,10 @@ class DecodeBmpOp : public OpKernel { explicit DecodeBmpOp(OpKernelConstruction* context) : OpKernel(context) { OP_REQUIRES_OK(context, context->GetAttr("channels", &channels_)); OP_REQUIRES( - context, channels_ == 0 || channels_ == 3 || channels_ == 4, - errors::InvalidArgument("channels must be 0, 3 or 4, got ", channels_)); + context, + channels_ == 0 || channels_ == 1 || channels_ == 3 || channels_ == 4, + errors::InvalidArgument("channels must be 0, 1, 3 or 4, got ", + channels_)); } void Compute(OpKernelContext* context) override { @@ -66,11 +68,11 @@ class DecodeBmpOp : public OpKernel { channels_ = bpp / 8; } - // Current implementation only supports 3 or 4 channel + // Current implementation only supports 1, 3 or 4 channel // bitmaps. - OP_REQUIRES(context, (channels_ == 3 || channels_ == 4), + OP_REQUIRES(context, (channels_ == 1 || channels_ == 3 || channels_ == 4), errors::InvalidArgument( - "Number of channels must be 3 or 4, was ", channels_)); + "Number of channels must be 1, 3 or 4, was ", channels_)); // if height is negative, data layout is top down // otherwise, it's bottom up @@ -117,6 +119,9 @@ uint8* DecodeBmpOp::Decode(const uint8* input, uint8* const output, dst_pos = (i * width + j) * channels; switch (channels) { + case 1: + output[dst_pos] = input[src_pos]; + break; case 3: // BGR -> RGB output[dst_pos] = input[src_pos + 2]; diff --git a/tensorflow/core/kernels/dynamic_partition_op_test.cc b/tensorflow/core/kernels/dynamic_partition_op_test.cc index 0e8fbc0a67..9a7ed0af21 100644 --- a/tensorflow/core/kernels/dynamic_partition_op_test.cc +++ b/tensorflow/core/kernels/dynamic_partition_op_test.cc @@ -16,6 +16,7 @@ limitations under the License. #include #include +#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/fake_input.h" #include "tensorflow/core/framework/node_def_builder.h" @@ -23,10 +24,14 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/graph/testlib.h" #include "tensorflow/core/kernels/ops_testutil.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/random/simple_philox.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" namespace tensorflow { namespace { @@ -153,5 +158,58 @@ TEST_F(DynamicPartitionOpTest, Error_IndexOutOfRange) { << s; } +Node* DynamicPartitionNode(Graph* g, Node* in0, Node* in1, int num_partitions) { + Node* ret; + TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DynamicPartition") + .Input(in0) + .Input(in1) + .Attr("num_partitions", num_partitions) + .Finalize(g, &ret)); + return ret; +} + +template +static Graph* DynamicPartition(int num_partitions, int dim) { + Graph* g = new Graph(OpRegistry::Global()); + // Always use a 128MB buffer. + const int kRows = ((128 << 20) / sizeof(T)) / dim; + Tensor data(DataTypeToEnum::value, TensorShape({kRows, dim})); + data.flat().setRandom(); + + random::PhiloxRandom philox(301, 17); + random::SimplePhilox rnd(&philox); + Tensor partitions(DT_INT32, TensorShape({kRows})); + for (int i = 0; i < kRows; i++) { + partitions.flat()(i) = rnd.Uniform(num_partitions); + } + DynamicPartitionNode(g, test::graph::Constant(g, data), + test::graph::Constant(g, partitions), num_partitions); + return g; +} + +#define BM_DYNAMIC_PARTITION(DEVICE, T, num) \ + static void BM_##DEVICE##_dynpart_##T##_##num(int iters, int dim) { \ + const int64 items = ((128 << 20) / sizeof(T)); \ + const int64 tot = static_cast(iters) * items; \ + testing::ItemsProcessed(tot); \ + testing::UseRealTime(); \ + test::Benchmark(#DEVICE, DynamicPartition(num, dim)).Run(iters); \ + } \ + BENCHMARK(BM_##DEVICE##_dynpart_##T##_##num)->Arg(1)->Arg(256) + +BM_DYNAMIC_PARTITION(cpu, float, 2); +BM_DYNAMIC_PARTITION(cpu, float, 100); +BM_DYNAMIC_PARTITION(cpu, double, 2); +BM_DYNAMIC_PARTITION(cpu, double, 100); +BM_DYNAMIC_PARTITION(cpu, complex64, 2); +BM_DYNAMIC_PARTITION(cpu, complex64, 100); + +BM_DYNAMIC_PARTITION(gpu, float, 2); +BM_DYNAMIC_PARTITION(gpu, float, 100); +BM_DYNAMIC_PARTITION(gpu, double, 2); +BM_DYNAMIC_PARTITION(gpu, double, 100); +BM_DYNAMIC_PARTITION(gpu, complex64, 2); +BM_DYNAMIC_PARTITION(gpu, complex64, 100); + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/mkl_batch_matmul_op.cc b/tensorflow/core/kernels/mkl_batch_matmul_op.cc new file mode 100644 index 0000000000..d9713075be --- /dev/null +++ b/tensorflow/core/kernels/mkl_batch_matmul_op.cc @@ -0,0 +1,238 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/math_ops.cc. + +// This file uses MKL CBLAS batched xGEMM for acceleration of TF Batch +// Matrix-Matrix Multiplication (MatMul) operations. +// We currently register this kernel only for MKL supported data +// types (float, double, complex64, complex128). The macro INTEL_MKL is defined +// by the build system only when MKL is chosen as an option at configure stage +// and when it is undefined at build time, this file becomes an empty +// compilation unit + +#define EIGEN_USE_THREADS + +#if defined(INTEL_MKL) +#include +#include "mkl_cblas.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/numeric_types.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/type_traits.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +#define MKL_Complex8 tensorflow::complex64 +#define MKL_Complex16 tensorflow::complex128 + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; + +template +class BatchMatMulMkl : public OpKernel { + public: + explicit BatchMatMulMkl(OpKernelConstruction *context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("adj_x", &adj_x_)); + OP_REQUIRES_OK(context, context->GetAttr("adj_y", &adj_y_)); + } + + virtual ~BatchMatMulMkl() {} + + void Compute(OpKernelContext *ctx) override { + const Tensor &lhs = ctx->input(0); + const Tensor &rhs = ctx->input(1); + OP_REQUIRES(ctx, lhs.dims() == rhs.dims(), + errors::InvalidArgument("lhs and rhs has different ndims: ", + lhs.shape().DebugString(), " vs. ", + rhs.shape().DebugString())); + const int ndims = lhs.dims(); + OP_REQUIRES( + ctx, ndims >= 2, + errors::InvalidArgument("lhs and rhs ndims must be >= 2: ", ndims)); + TensorShape out_shape; + for (int i = 0; i < ndims - 2; ++i) { + OP_REQUIRES(ctx, lhs.dim_size(i) == rhs.dim_size(i), + errors::InvalidArgument( + "lhs.dim(", i, ") and rhs.dim(", i, + ") must be the same: ", lhs.shape().DebugString(), " vs ", + rhs.shape().DebugString())); + out_shape.AddDim(lhs.dim_size(i)); + } + auto batch_size = (ndims == 2) ? 1 : out_shape.num_elements(); + auto lhs_rows = lhs.dim_size(ndims - 2); + auto lhs_cols = lhs.dim_size(ndims - 1); + auto rhs_rows = rhs.dim_size(ndims - 2); + auto rhs_cols = rhs.dim_size(ndims - 1); + if (adj_x_) std::swap(lhs_rows, lhs_cols); + if (adj_y_) std::swap(rhs_rows, rhs_cols); + OP_REQUIRES(ctx, lhs_cols == rhs_rows, + errors::InvalidArgument( + "lhs mismatch rhs shape: ", lhs_cols, " vs. ", rhs_rows, + ": ", lhs.shape().DebugString(), " ", + rhs.shape().DebugString(), " ", adj_x_, " ", adj_y_)); + out_shape.AddDim(lhs_rows); + out_shape.AddDim(rhs_cols); + Tensor *out = nullptr; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, out_shape, &out)); + if (out->NumElements() == 0) { + return; + } + if (lhs.NumElements() == 0 || rhs.NumElements() == 0) { + functor::SetZeroFunctor f; + f(ctx->eigen_device(), out->flat()); + return; + } + + auto rhs_reshaped = rhs.template flat_inner_dims(); + auto lhs_reshaped = lhs.template flat_inner_dims(); + auto out_reshaped = out->template flat_inner_dims(); + const uint64 M = lhs_reshaped.dimension(adj_x_ ? 2 : 1); + const uint64 K = lhs_reshaped.dimension(adj_x_ ? 1 : 2); + const uint64 N = rhs_reshaped.dimension(adj_y_ ? 1 : 2); + + std::vector m_array(batch_size, M); + std::vector n_array(batch_size, N); + std::vector k_array(batch_size, K); + std::vector lda_array(batch_size, adj_x_ ? M : K); + std::vector ldb_array(batch_size, adj_y_ ? K : N); + std::vector ldc_array(batch_size, N); + std::vector group_size(1, batch_size); + std::vector a_array; + std::vector b_array; + std::vector c_array; + a_array.reserve(batch_size); + b_array.reserve(batch_size); + c_array.reserve(batch_size); + for (int64 i = 0; i < batch_size; i++) { + a_array.push_back(&lhs_reshaped(i, 0, 0)); + b_array.push_back(&rhs_reshaped(i, 0, 0)); + c_array.push_back(&out_reshaped(i, 0, 0)); + } + + MklCblasGemmBatch(CblasRowMajor, adj_x_, adj_y_, &m_array[0], &n_array[0], + &k_array[0], &a_array[0], &lda_array[0], &b_array[0], + &ldb_array[0], &c_array[0], &ldc_array[0], 1, + &group_size[0]); + } + + private: + bool adj_x_; + bool adj_y_; + + void MklCblasGemmBatch(const CBLAS_LAYOUT Layout, const bool TransA, + const bool TransB, const MKL_INT *M_Array, + const MKL_INT *N_Array, const MKL_INT *K_Array, + const float **A_Array, const MKL_INT *lda_Array, + const float **B_Array, const MKL_INT *ldb_Array, + float **C_Array, const MKL_INT *ldc_Array, + const MKL_INT group_count, const MKL_INT *group_size) { + std::vector TransA_Array( + group_size[0], TransA ? CblasTrans : CblasNoTrans); + std::vector TransB_Array( + group_size[0], TransB ? CblasTrans : CblasNoTrans); + std::vector alpha_Array(group_size[0], 1.0); + std::vector beta_Array(group_size[0], 0.0); + cblas_sgemm_batch(Layout, &TransA_Array[0], &TransB_Array[0], M_Array, + N_Array, K_Array, &alpha_Array[0], A_Array, lda_Array, + B_Array, ldb_Array, &beta_Array[0], C_Array, ldc_Array, + group_count, group_size); + } + + void MklCblasGemmBatch(const CBLAS_LAYOUT Layout, const bool TransA, + const bool TransB, const MKL_INT *M_Array, + const MKL_INT *N_Array, const MKL_INT *K_Array, + const double **A_Array, const MKL_INT *lda_Array, + const double **B_Array, const MKL_INT *ldb_Array, + double **C_Array, const MKL_INT *ldc_Array, + const MKL_INT group_count, const MKL_INT *group_size) { + std::vector TransA_array( + group_size[0], TransA ? CblasTrans : CblasNoTrans); + std::vector TransB_array( + group_size[0], TransB ? CblasTrans : CblasNoTrans); + std::vector alpha_Array(group_size[0], 1.0); + std::vector beta_Array(group_size[0], 0.0); + cblas_dgemm_batch(Layout, &TransA_array[0], &TransB_array[0], M_Array, + N_Array, K_Array, &alpha_Array[0], A_Array, lda_Array, + B_Array, ldb_Array, &beta_Array[0], C_Array, ldc_Array, + group_count, group_size); + } + + void MklCblasGemmBatch(const CBLAS_LAYOUT Layout, const bool TransA, + const bool TransB, const MKL_INT *M_Array, + const MKL_INT *N_Array, const MKL_INT *K_Array, + const MKL_Complex8 **A_Array, const MKL_INT *lda_Array, + const MKL_Complex8 **B_Array, const MKL_INT *ldb_Array, + MKL_Complex8 **C_Array, const MKL_INT *ldc_Array, + const MKL_INT group_count, const MKL_INT *group_size) { + std::vector TransA_array( + group_size[0], TransA ? CblasConjTrans : CblasNoTrans); + std::vector TransB_array( + group_size[0], TransB ? CblasConjTrans : CblasNoTrans); + std::vector alpha_Array(group_size[0], {1.0f, 0.0f}); + std::vector beta_Array(group_size[0], {0.0f, 0.0f}); + cblas_cgemm_batch( + Layout, &TransA_array[0], &TransB_array[0], M_Array, N_Array, K_Array, + static_cast(&alpha_Array[0]), + reinterpret_cast(A_Array), lda_Array, + reinterpret_cast(B_Array), ldb_Array, + static_cast(&beta_Array[0]), + reinterpret_cast(C_Array), ldc_Array, group_count, group_size); + } + + void MklCblasGemmBatch(const CBLAS_LAYOUT Layout, const bool TransA, + const bool TransB, const MKL_INT *M_Array, + const MKL_INT *N_Array, const MKL_INT *K_Array, + const MKL_Complex16 **A_Array, + const MKL_INT *lda_Array, + const MKL_Complex16 **B_Array, + const MKL_INT *ldb_Array, MKL_Complex16 **C_Array, + const MKL_INT *ldc_Array, const MKL_INT group_count, + const MKL_INT *group_size) { + std::vector TransA_array( + group_size[0], TransA ? CblasConjTrans : CblasNoTrans); + std::vector TransB_array( + group_size[0], TransB ? CblasConjTrans : CblasNoTrans); + std::vector alpha_Array(group_size[0], {1.0f, 0.0f}); + std::vector beta_Array(group_size[0], {0.0f, 0.0f}); + cblas_zgemm_batch( + Layout, &TransA_array[0], &TransB_array[0], M_Array, N_Array, K_Array, + static_cast(&alpha_Array[0]), + reinterpret_cast(A_Array), lda_Array, + reinterpret_cast(B_Array), ldb_Array, + static_cast(&beta_Array[0]), + reinterpret_cast(C_Array), ldc_Array, group_count, group_size); + } +}; + +#define REGISTER_BATCH_MATMUL_MKL(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("BatchMatMul").Device(DEVICE_CPU).TypeConstraint("T"), \ + BatchMatMulMkl) + +TF_CALL_float(REGISTER_BATCH_MATMUL_MKL); +TF_CALL_double(REGISTER_BATCH_MATMUL_MKL); +TF_CALL_complex64(REGISTER_BATCH_MATMUL_MKL); +TF_CALL_complex128(REGISTER_BATCH_MATMUL_MKL); + +} // end namespace tensorflow +#endif diff --git a/tensorflow/core/kernels/prefetch_dataset_op.cc b/tensorflow/core/kernels/prefetch_dataset_op.cc index 1a6b7e078e..b02269f525 100644 --- a/tensorflow/core/kernels/prefetch_dataset_op.cc +++ b/tensorflow/core/kernels/prefetch_dataset_op.cc @@ -37,6 +37,8 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel { int64 buffer_size; OP_REQUIRES_OK( ctx, ParseScalarArgument(ctx, "buffer_size", &buffer_size)); + OP_REQUIRES(ctx, buffer_size > 0, + errors::InvalidArgument("buffer_size must be > 0")); *output = new Dataset(ctx, input, buffer_size); } diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc index ad28d77ffd..97c0c2c099 100644 --- a/tensorflow/core/kernels/summary_interface.cc +++ b/tensorflow/core/kernels/summary_interface.cc @@ -16,7 +16,6 @@ limitations under the License. #include -#include "tensorflow/compiler/xla/ptr_util.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/resource_mgr.h" @@ -28,6 +27,7 @@ limitations under the License. #include "tensorflow/core/lib/png/png_io.h" #include "tensorflow/core/lib/wav/wav_io.h" #include "tensorflow/core/util/events_writer.h" +#include "tensorflow/core/util/ptr_util.h" namespace tensorflow { namespace { @@ -229,7 +229,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { } mutex_lock ml(mu_); events_writer_ = - xla::MakeUnique(io::JoinPath(logdir, "events")); + tensorflow::MakeUnique(io::JoinPath(logdir, "events")); if (!events_writer_->InitWithSuffix(filename_suffix)) { return errors::Unknown("Could not initialize events writer."); } diff --git a/tensorflow/core/lib/io/path.cc b/tensorflow/core/lib/io/path.cc index d93dd0296e..83f15e134d 100644 --- a/tensorflow/core/lib/io/path.cc +++ b/tensorflow/core/lib/io/path.cc @@ -14,8 +14,22 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/lib/io/path.h" + +#include +#include +#include +#include +#include +#if !defined(PLATFORM_WINDOWS) +#include +#endif + +#include + #include "tensorflow/core/lib/strings/scanner.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/env.h" namespace tensorflow { namespace io { @@ -60,8 +74,7 @@ std::pair SplitPath(StringPiece uri) { auto pos = path.rfind('/'); #ifdef PLATFORM_WINDOWS - if (pos == StringPiece::npos) - pos = path.rfind('\\'); + if (pos == StringPiece::npos) pos = path.rfind('\\'); #endif // Handle the case with no '/' in 'path'. if (pos == StringPiece::npos) @@ -112,7 +125,7 @@ StringPiece Extension(StringPiece path) { string CleanPath(StringPiece unclean_path) { string path = unclean_path.ToString(); - const char *src = path.c_str(); + const char* src = path.c_str(); string::iterator dst = path.begin(); // Check for absolute path and determine initial backtrack limit. @@ -229,5 +242,52 @@ string CreateURI(StringPiece scheme, StringPiece host, StringPiece path) { return strings::StrCat(scheme, "://", host, path); } +// Returns a unique number every time it is called. +int64 UniqueId() { + static mutex mu(LINKER_INITIALIZED); + static int64 id = 0; + mutex_lock l(mu); + return ++id; +} + +string GetTempFilename(const string& extension) { +#if defined(PLATFORM_WINDOWS) || defined(__ANDROID__) + LOG(FATAL) << "GetTempFilename is not implemented in this platform."; +#else + for (const char* dir : std::vector( + {getenv("TEST_TMPDIR"), getenv("TMPDIR"), getenv("TMP"), "/tmp"})) { + if (!dir || !dir[0]) { + continue; + } + struct stat statbuf; + if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) { + // UniqueId is added here because mkstemps is not as thread safe as it + // looks. https://github.com/tensorflow/tensorflow/issues/5804 shows + // the problem. + string tmp_filepath; + int fd; + if (extension.length()) { + tmp_filepath = io::JoinPath( + dir, strings::StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX.", + extension)); + fd = mkstemps(&tmp_filepath[0], extension.length() + 1); + } else { + tmp_filepath = io::JoinPath( + dir, + strings::StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX")); + fd = mkstemp(&tmp_filepath[0]); + } + if (fd < 0) { + LOG(FATAL) << "Failed to create temp file."; + } else { + close(fd); + return tmp_filepath; + } + } + } + LOG(FATAL) << "No temp directory found."; +#endif +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/path.h b/tensorflow/core/lib/io/path.h index 8d02baa5bb..47bb2b998d 100644 --- a/tensorflow/core/lib/io/path.h +++ b/tensorflow/core/lib/io/path.h @@ -89,6 +89,9 @@ void ParseURI(tensorflow::StringPiece uri, tensorflow::StringPiece* scheme, string CreateURI(tensorflow::StringPiece scheme, tensorflow::StringPiece host, tensorflow::StringPiece path); +// Creates a temporary file name with an extension. +string GetTempFilename(const string& extension); + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index d7afd02df6..ceda11663a 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -2333,11 +2333,25 @@ REGISTER_OP("Cross") .Input("b: T") .Output("product: T") .Attr("T: realnumbertype") - // TODO(cwhipkey): implement these shape inference constraints here: - // * Both inputs have the same shape. - // * Input rank >= 1. - // * input_shape[-1] == 3. - .SetShapeFn(shape_inference::UnchangedShape) + .SetShapeFn([](InferenceContext* c) { + ShapeHandle a_shape; + ShapeHandle b_shape; + // * Input rank >= 1. + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &a_shape)); + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &b_shape)); + + // * Both inputs have the same shape. + TF_RETURN_IF_ERROR(c->Merge(a_shape, b_shape, &a_shape)); + + // * input_shape[-1] == 3. + if (c->RankKnown(a_shape)) { + int rank = c->Rank(a_shape); + auto dim = c->Dim(a_shape, rank - 1); + TF_RETURN_IF_ERROR(c->WithValue(dim, 3, &dim)); + } + c->set_output(0, a_shape); + return Status::OK(); + }) .Doc(R"doc( Compute the pairwise cross product. diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc index 28f9969de5..3dfa776d26 100644 --- a/tensorflow/core/ops/math_ops_test.cc +++ b/tensorflow/core/ops/math_ops_test.cc @@ -515,4 +515,15 @@ TEST(MathOpstest, RequantizationRange_ShapeFn) { INFER_ERROR("must be rank 0", op, "?;?;[2]"); } +TEST(MathOpsTest, Cross_ShapeFn) { + ShapeInferenceTestOp op("Cross"); + + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]"); + INFER_ERROR("Dimension 0 in both shapes must be equal, but", op, "[3];[5]"); + INFER_ERROR("Dimension must be 3 but", op, "[3,5];[3,5]"); + + INFER_OK(op, "?;?", "?"); + INFER_OK(op, "[?];[?]", "in0"); + INFER_OK(op, "[1,?,3];[?,?,?]", "in0"); +} } // end namespace tensorflow diff --git a/tensorflow/core/platform/default/build_config_root.bzl b/tensorflow/core/platform/default/build_config_root.bzl index c63fb28ff9..6e98f12114 100644 --- a/tensorflow/core/platform/default/build_config_root.bzl +++ b/tensorflow/core/platform/default/build_config_root.bzl @@ -10,7 +10,9 @@ def tf_sycl_tests_tags(): def tf_additional_plugin_deps(): return select({ - "//tensorflow:with_xla_support": ["//tensorflow/compiler/jit"], + str(Label("//tensorflow:with_xla_support")): [ + str(Label("//tensorflow/compiler/jit")) + ], "//conditions:default": [], }) @@ -19,37 +21,37 @@ def tf_additional_xla_deps_py(): def tf_additional_license_deps(): return select({ - "//tensorflow:with_xla_support": ["@llvm//:LICENSE.TXT"], + str(Label("//tensorflow:with_xla_support")): ["@llvm//:LICENSE.TXT"], "//conditions:default": [], }) def tf_additional_verbs_deps(): return select({ - "//tensorflow:with_verbs_support": [ - "//tensorflow/contrib/verbs:verbs_server_lib", - "//tensorflow/contrib/verbs:grpc_verbs_client", + str(Label("//tensorflow:with_verbs_support")): [ + str(Label("//tensorflow/contrib/verbs:verbs_server_lib")), + str(Label("//tensorflow/contrib/verbs:grpc_verbs_client")), ], "//conditions:default": [], }) def tf_additional_mpi_deps(): return select({ - "//tensorflow:with_mpi_support": [ - "//tensorflow/contrib/mpi:mpi_server_lib", + str(Label("//tensorflow:with_mpi_support")): [ + str(Label("//tensorflow/contrib/mpi:mpi_server_lib")), ], "//conditions:default": [], }) def tf_additional_gdr_deps(): return select({ - "//tensorflow:with_gdr_support": [ - "//tensorflow/contrib/gdr:gdr_server_lib", + str(Label("//tensorflow:with_gdr_support")): [ + str(Label("//tensorflow/contrib/gdr:gdr_server_lib")), ], "//conditions:default": [], }) def if_static(extra_deps, otherwise=[]): return select({ - "//tensorflow:framework_shared_object": otherwise, + str(Label("//tensorflow:framework_shared_object")): otherwise, "//conditions:default": extra_deps, }) diff --git a/tensorflow/core/util/ptr_util.h b/tensorflow/core/util/ptr_util.h new file mode 100644 index 0000000000..f902b3ffa1 --- /dev/null +++ b/tensorflow/core/util/ptr_util.h @@ -0,0 +1,80 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_UTIL_PTR_UTIL_H_ +#define TENSORFLOW_CORE_UTIL_PTR_UTIL_H_ + +// Utility functions for pointers. + +#include + +#include +#include +#include + +namespace tensorflow { + +namespace helper { + +// Trait to select overloads and return types for MakeUnique. +template +struct MakeUniqueResult { + using scalar = std::unique_ptr; +}; +template +struct MakeUniqueResult { + using array = std::unique_ptr; +}; +template +struct MakeUniqueResult { + using invalid = void; +}; + +} // namespace helper + +// Transfers ownership of a raw pointer to a std::unique_ptr of deduced type. +// Example: +// X* NewX(int, int); +// auto x = WrapUnique(NewX(1, 2)); // 'x' is std::unique_ptr. +// +// WrapUnique is useful for capturing the output of a raw pointer factory. +// However, prefer 'MakeUnique(args...) over 'WrapUnique(new T(args...))'. +// auto x = WrapUnique(new X(1, 2)); // works, but nonideal. +// auto x = MakeUnique(1, 2); // safer, standard, avoids raw 'new'. +// +// Note: Cannot wrap pointers to array of unknown bound (i.e. U(*)[]). +template +std::unique_ptr WrapUnique(T* ptr) { + static_assert(!std::is_array::value || std::extent::value != 0, + "types T[0] or T[] are unsupported"); + return std::unique_ptr(ptr); +} + +template +typename helper::MakeUniqueResult::scalar MakeUnique(Args&&... args) { + return std::unique_ptr(new T(std::forward(args)...)); +} + +// Overload for array of unknown bound. +// The allocation of arrays needs to use the array form of new, +// and cannot take element constructor arguments. +template +typename helper::MakeUniqueResult::array MakeUnique(size_t n) { + return std::unique_ptr(new typename std::remove_extent::type[n]()); +} + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_UTIL_PTR_UTIL_H_ diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md index 15d6d77f5e..c52279b212 100644 --- a/tensorflow/docs_src/extend/adding_an_op.md +++ b/tensorflow/docs_src/extend/adding_an_op.md @@ -341,9 +341,9 @@ Assuming you have `g++` installed, here is the sequence of commands you can use to compile your op into a dynamic library. ```bash -TF_INC=$(python -c 'import tensorflow as tf; print(tf.sysconfig.get_include())') -TF_LIB=$(python -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') -g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC -I$TF_INC -I$TF_INC/external/nsync/public -L$TF_LIB -ltensorflow_framework -O2 +TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') ) +TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') ) +g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]} -O2 ``` On Mac OS X, the additional flag "-undefined dynamic_lookup" is required when @@ -451,17 +451,17 @@ Now that you know how to build a basic (and somewhat restricted) op and implementation, we'll look at some of the more complicated things you will typically need to build into your op. This includes: -* [Conditional checks and validation](#conditional_checks_and_validation) -* [Op registration](#op_registration) +* [Conditional checks and validation](#conditional-checks-and-validation) +* [Op registration](#op-registration) * [Attrs](#attrs) - * [Attr types](#attr_types) + * [Attr types](#attr-types) * [Polymorphism](#polymorphism) - * [Inputs and outputs](#inputs_and_outputs) - * [Backwards compatibility](#backwards_compatibility) -* [GPU support](#gpu_support) - * [Compiling the kernel for the GPU device](#compiling_the_kernel_for_the_gpu_device) -* [Implement the gradient in Python](#implement_the_gradient_in_python) -* [Shape functions in C++](#shape_functions_in_c) + * [Inputs and outputs](#inputs-and-outputs) + * [Backwards compatibility](#backwards-compatibility) +* [GPU support](#gpu-support) + * [Compiling the kernel for the GPU device](#compiling-the-kernel-for-the-gpu-device) +* [Implement the gradient in Python](#implement-the-gradient-in-python) +* [Shape functions in C++](#shape-functions-in-c) ### Conditional checks and validation @@ -1228,10 +1228,10 @@ into a single dynamically loadable library: ```bash nvcc -std=c++11 -c -o cuda_op_kernel.cu.o cuda_op_kernel.cu.cc \ --I $TF_INC -I$TF_INC/external/nsync/public -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC + ${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC g++ -std=c++11 -shared -o cuda_op_kernel.so cuda_op_kernel.cc \ -cuda_op_kernel.cu.o -I $TF_INC -I$TF_INC/external/nsync/public -fPIC -lcudart -L$TF_LIB -ltensorflow_framework + cuda_op_kernel.cu.o ${TF_CFLAGS[@]} -fPIC -lcudart ${TF_LFLAGS[@]} ``` `cuda_op_kernel.so` produced above can be loaded as usual in Python, using the diff --git a/tensorflow/docs_src/get_started/input_fn.md b/tensorflow/docs_src/get_started/input_fn.md index 0db5c6143a..f0dcdc47ff 100644 --- a/tensorflow/docs_src/get_started/input_fn.md +++ b/tensorflow/docs_src/get_started/input_fn.md @@ -211,8 +211,8 @@ def get_input_fn_from_numpy(data_set, num_epochs=None, shuffle=True): ### A Neural Network Model for Boston House Values In the remainder of this tutorial, you'll write an input function for -preprocessing a subset of Boston housing data pulled from the [UCI Housing Data -Set](https://archive.ics.uci.edu/ml/datasets/Housing) and use it to feed data to +preprocessing a subset of Boston housing data pulled from the UCI Housing Data +Set and use it to feed data to a neural network regressor for predicting median house values. The [Boston CSV data sets](#setup) you'll use to train your neural network diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index 63742828b0..8d0eb7966f 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -84,7 +84,7 @@ install it now: * [Python 3.5.x 64-bit from python.org](https://www.python.org/downloads/release/python-352/) * [Python 3.6.x 64-bit from python.org](https://www.python.org/downloads/release/python-362/) --TensorFlow supports Python 3.5.x and 3.6.x on Windows. +TensorFlow supports Python 3.5.x and 3.6.x on Windows. Note that Python 3 comes with the pip3 package manager, which is the program you'll use to install TensorFlow. @@ -98,7 +98,6 @@ To install the GPU version of TensorFlow, enter the following command: 
    C:\> pip3 install --upgrade tensorflow-gpu
    - ## Installing with Anaconda **The Anaconda installation is community supported, not officially supported.** @@ -219,6 +218,11 @@ ImportError: cannot import name 'descriptor' + +     38896424 + + 
    Could not find a version that satisfies the requirement tensorflow
    + + - diff --git a/tensorflow/docs_src/mobile/ios_build.md b/tensorflow/docs_src/mobile/ios_build.md index a04655052f..4c84a1214a 100644 --- a/tensorflow/docs_src/mobile/ios_build.md +++ b/tensorflow/docs_src/mobile/ios_build.md @@ -98,7 +98,7 @@ There are three demo applications for iOS, all defined in Xcode projects inside ## Building the TensorFlow iOS libraries from source -While Cocapods is the quickest and easiest way of getting started, you sometimes +While Cocoapods is the quickest and easiest way of getting started, you sometimes need more flexibility to determine which parts of TensorFlow your app should be shipped with. For such cases, you can build the iOS libraries from the sources. [This diff --git a/tensorflow/docs_src/mobile/mobile_intro.md b/tensorflow/docs_src/mobile/mobile_intro.md index 3a002c4da2..17dbf1c3e6 100644 --- a/tensorflow/docs_src/mobile/mobile_intro.md +++ b/tensorflow/docs_src/mobile/mobile_intro.md @@ -156,7 +156,7 @@ easy cases on device. Doing on-device computation can also signal when it's time to switch to working on the cloud. A good example of this is hotword detection in speech. Since devices are able to constantly listen out for the keywords, this then triggers a -lot of traffic to cloud-based speech recognition once one is recognised. Without +lot of traffic to cloud-based speech recognition once one is recognized. Without the on-device component, the whole application wouldn’t be feasible, and this pattern exists across several other applications as well. Recognizing that some sensor input is interesting enough for further processing makes a lot of diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md index d9e8875c38..44cacff5db 100644 --- a/tensorflow/docs_src/mobile/optimizing.md +++ b/tensorflow/docs_src/mobile/optimizing.md @@ -115,7 +115,7 @@ If you look at the resulting file size, you should see that it’s about a quart of the original at 23MB. Another transform is `round_weights`, which doesn't make the file smaller, but it -makes the file compressable to about the same size as when `quantize_weights` is +makes the file compressible to about the same size as when `quantize_weights` is used. This is particularly useful for mobile development, taking advantage of the fact that app bundles are compressed before they’re downloaded by consumers. diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 4333f94486..a49973d550 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -776,7 +776,7 @@ The output type is a tuple of three ComputationDataHandles: | `batch_var` | `ComputationDataHandle` | 1 dimensional array (\\(\sigma^2\\)) | The `batch_mean` and `batch_var` are moments calculated across the batch and -spatial dimensions using the formulars above. +spatial dimensions using the formulas above. ## BatchNormInference diff --git a/tensorflow/docs_src/tutorials/image_recognition.md b/tensorflow/docs_src/tutorials/image_recognition.md index df13eabead..32257f87d6 100644 --- a/tensorflow/docs_src/tutorials/image_recognition.md +++ b/tensorflow/docs_src/tutorials/image_recognition.md @@ -5,7 +5,7 @@ tell apart a lion and a jaguar, read a sign, or recognize a human's face. But these are actually hard problems to solve with a computer: they only seem easy because our brains are incredibly good at understanding images. -In the last few years the field of machine learning has made tremendous +In the last few years, the field of machine learning has made tremendous progress on addressing these difficult problems. In particular, we've found that a kind of model called a deep [convolutional neural network](https://colah.github.io/posts/2014-07-Conv-Nets-Modular/) diff --git a/tensorflow/examples/how_tos/reading_data/convert_to_records.py b/tensorflow/examples/how_tos/reading_data/convert_to_records.py index d14c1f7c86..c89e839563 100644 --- a/tensorflow/examples/how_tos/reading_data/convert_to_records.py +++ b/tensorflow/examples/how_tos/reading_data/convert_to_records.py @@ -52,17 +52,19 @@ def convert_to(data_set, name): filename = os.path.join(FLAGS.directory, name + '.tfrecords') print('Writing', filename) - writer = tf.python_io.TFRecordWriter(filename) - for index in range(num_examples): - image_raw = images[index].tostring() - example = tf.train.Example(features=tf.train.Features(feature={ - 'height': _int64_feature(rows), - 'width': _int64_feature(cols), - 'depth': _int64_feature(depth), - 'label': _int64_feature(int(labels[index])), - 'image_raw': _bytes_feature(image_raw)})) - writer.write(example.SerializeToString()) - writer.close() + with tf.python_io.TFRecordWriter(filename) as writer: + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example( + features=tf.train.Features( + feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw) + })) + writer.write(example.SerializeToString()) def main(unused_argv): diff --git a/tensorflow/examples/speech_commands/input_data.py b/tensorflow/examples/speech_commands/input_data.py index 6d75fbb92b..751652b330 100644 --- a/tensorflow/examples/speech_commands/input_data.py +++ b/tensorflow/examples/speech_commands/input_data.py @@ -240,7 +240,8 @@ class AudioProcessor(object): # Look through all the subfolders to find audio samples search_path = os.path.join(self.data_dir, '*', '*.wav') for wav_path in gfile.Glob(search_path): - word = re.search('.*/([^/]+)/.*.wav', wav_path).group(1).lower() + _, word = os.path.split(os.path.dirname(wav_path)) + word = word.lower() # Treat the '_background_noise_' folder as a special case, since we expect # it to contain long audio samples we mix in to improve training. if word == BACKGROUND_NOISE_DIR_NAME: diff --git a/tensorflow/examples/speech_commands/train.py b/tensorflow/examples/speech_commands/train.py index a54bcbdb32..f5bf04305a 100644 --- a/tensorflow/examples/speech_commands/train.py +++ b/tensorflow/examples/speech_commands/train.py @@ -156,7 +156,8 @@ def main(_): predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) - confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices) + confusion_matrix = tf.confusion_matrix( + expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', evaluation_step) diff --git a/tensorflow/examples/udacity/1_notmnist.ipynb b/tensorflow/examples/udacity/1_notmnist.ipynb index 39674e1aa4..dffe5d37c6 100644 --- a/tensorflow/examples/udacity/1_notmnist.ipynb +++ b/tensorflow/examples/udacity/1_notmnist.ipynb @@ -46,13 +46,13 @@ "# These are all the modules we'll be using later. Make sure you can import them\n", "# before proceeding further.\n", "from __future__ import print_function\n", + "import imageio\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "import sys\n", "import tarfile\n", "from IPython.display import display, Image\n", - "from scipy import ndimage\n", "from sklearn.linear_model import LogisticRegression\n", "from six.moves.urllib.request import urlretrieve\n", "from six.moves import cPickle as pickle\n", @@ -325,13 +325,13 @@ " for image in image_files:\n", " image_file = os.path.join(folder, image)\n", " try:\n", - " image_data = (ndimage.imread(image_file).astype(float) - \n", + " image_data = (imageio.imread(image_file).astype(float) - \n", " pixel_depth / 2) / pixel_depth\n", " if image_data.shape != (image_size, image_size):\n", " raise Exception('Unexpected image shape: %s' % str(image_data.shape))\n", " dataset[num_images, :, :] = image_data\n", " num_images = num_images + 1\n", - " except IOError as e:\n", + " except (IOError, ValueError) as e:\n", " print('Could not read:', image_file, ':', e, '- it\\'s ok, skipping.')\n", " \n", " dataset = dataset[0:num_images, :, :]\n", diff --git a/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java b/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java index 9a1b7592b3..a24150484e 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java +++ b/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java @@ -265,6 +265,36 @@ public final class OperationBuilder { return this; } + public OperationBuilder setAttr(String name, Shape[] value) { + int[] numDimensions = new int[value.length]; + int totalNumDimensions = 0; + for (int idx = 0; idx < value.length; ++idx) { + int n = value[idx].numDimensions(); + numDimensions[idx] = n; + if (n > 0) { + totalNumDimensions += n; + } + } + // Flatten the shapes into a single array to avoid too much overhead in the + // native part + long[] shapes = new long[totalNumDimensions]; + int shapeIdx = 0; + for (Shape shape : value) { + if (shape.numDimensions() > 0) { + for (long dim : shape.asArray()) { + shapes[shapeIdx++] = dim; + } + } + } + Graph.Reference r = graph.ref(); + try { + setAttrShapeList(unsafeNativeHandle, name, shapes, numDimensions); + } finally { + r.close(); + } + return this; + } + public OperationBuilder setAttr(String name, String[] value) { Charset utf8 = Charset.forName("UTF-8"); Object[] objects = new Object[value.length]; @@ -297,8 +327,6 @@ public final class OperationBuilder { // The names of all the setAttr* family functions below correspond to the C library types, not the // Java library types. Roughly, setAttrFoo calls the TensorFlow C library function: TF_SetAttrFoo. - // TODO(ashankar): - // - setAttrShapeList: Which would take in a long[][] private static native void setAttrString(long handle, String name, byte[] value); @@ -324,5 +352,8 @@ public final class OperationBuilder { private static native void setAttrShape(long handle, String name, long[] shape, int numDims); + private static native void setAttrShapeList( + long handle, String name, long[] shapes, int[] numDims); + private static native void setAttrStringList(long handle, String name, Object[] value); } diff --git a/tensorflow/java/src/main/native/operation_builder_jni.cc b/tensorflow/java/src/main/native/operation_builder_jni.cc index e03be7b110..55d214a7c4 100644 --- a/tensorflow/java/src/main/native/operation_builder_jni.cc +++ b/tensorflow/java/src/main/native/operation_builder_jni.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/java/src/main/native/operation_builder_jni.h" +#include #include #include "tensorflow/c/c_api.h" #include "tensorflow/java/src/main/native/exception_jni.h" @@ -262,6 +263,41 @@ JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrShape( env->ReleaseStringUTFChars(name, cname); } +JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrShapeList( + JNIEnv* env, jclass clazz, jlong handle, jstring name, jlongArray shapes, + jintArray num_dims) { + TF_OperationDescription* d = requireHandle(env, handle); + if (d == nullptr) return; + std::unique_ptr cshapes; + std::unique_ptr cdims; + std::unique_ptr cnum_dims; + const int num_dims_length = env->GetArrayLength(num_dims); + if (num_dims_length > 0) { + const int shapes_length = env->GetArrayLength(shapes); + cshapes.reset(new int64_t[shapes_length]); + cdims.reset(new int64_t*[num_dims_length]); + cnum_dims.reset(new int[num_dims_length]); + jlong* shapes_elems = + static_cast(env->GetPrimitiveArrayCritical(shapes, nullptr)); + std::memcpy(cshapes.get(), shapes_elems, shapes_length << 3); + env->ReleasePrimitiveArrayCritical(shapes, shapes_elems, JNI_ABORT); + int64_t* cshapes_ptr = cshapes.get(); + jint* num_dims_elems = + static_cast(env->GetPrimitiveArrayCritical(num_dims, nullptr)); + for (int i = 0; i < num_dims_length; ++i) { + cnum_dims[i] = static_cast(num_dims_elems[i]); + cdims[i] = cshapes_ptr; + if (cnum_dims[i] > 0) { + cshapes_ptr += cnum_dims[i]; + } + } + env->ReleasePrimitiveArrayCritical(num_dims, num_dims_elems, JNI_ABORT); + } + const char* cname = env->GetStringUTFChars(name, nullptr); + TF_SetAttrShapeList(d, cname, cdims.get(), cnum_dims.get(), num_dims_length); + env->ReleaseStringUTFChars(name, cname); +} + JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrStringList( JNIEnv* env, jclass object, jlong handle, jstring name, jobjectArray values) { diff --git a/tensorflow/java/src/main/native/operation_builder_jni.h b/tensorflow/java/src/main/native/operation_builder_jni.h index 2e72bd68da..cf0abe4829 100644 --- a/tensorflow/java/src/main/native/operation_builder_jni.h +++ b/tensorflow/java/src/main/native/operation_builder_jni.h @@ -169,6 +169,14 @@ JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrTensorList( JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrShape( JNIEnv *, jclass, jlong, jstring, jlongArray, jint); +/* + * Class: org_tensorflow_OperationBuilder + * Method: setAttrShapeList + * Signature: (JLjava/lang/String;[J[I)V + */ +JNIEXPORT void JNICALL Java_org_tensorflow_OperationBuilder_setAttrShapeList( + JNIEnv *, jclass, jlong, jstring, jlongArray, jintArray); + /* * Class: org_tensorflow_OperationBuilder * Method: setAttrStringList diff --git a/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java b/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java index 6dc233987b..0a4a8cf4e3 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java @@ -148,6 +148,19 @@ public class OperationBuilderTest { } } + @Test + public void setAttrShapeList() { + // Those shapes match tensors ones, so no exception is thrown + testSetAttrShapeList(new Shape[] {Shape.make(2, 2), Shape.make(2, 2, 2)}); + try { + // Those shapes do not match tensors ones, exception is thrown + testSetAttrShapeList(new Shape[] {Shape.make(2, 2), Shape.make(2, 2, 2, 2)}); + fail("Shapes are incompatible and an exception was expected"); + } catch (IllegalArgumentException e) { + // expected + } + } + @Test public void addControlInput() { try (Graph g = new Graph(); @@ -175,6 +188,30 @@ public class OperationBuilderTest { } } + private static void testSetAttrShapeList(Shape[] shapes) { + try (Graph g = new Graph(); + Session s = new Session(g)) { + int[][] matrix = new int[][] {{0, 0}, {0, 0}}; + Output queue = + g.opBuilder("FIFOQueue", "queue") + .setAttr("component_types", new DataType[] {DataType.INT32, DataType.INT32}) + .setAttr("shapes", shapes) + .build() + .output(0); + assertTrue(hasNode(g, "queue")); + Output c1 = TestUtil.constant(g, "const1", matrix); + Output c2 = TestUtil.constant(g, "const2", new int[][][] {matrix, matrix}); + Operation enqueue = + g.opBuilder("QueueEnqueue", "enqueue") + .addInput(queue) + .addInputList(new Output[] {c1, c2}) + .build(); + assertTrue(hasNode(g, "enqueue")); + + s.runner().addTarget(enqueue).run(); + } + } + private static boolean hasNode(Graph g, String name) { return g.operation(name) != null; } diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 5e7a6c0b59..12d81c4383 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3341,6 +3341,7 @@ py_test( tags = [ "no_gpu", "no_oss", + "no_pip", "no_pip_gpu", "notap", ], @@ -3387,6 +3388,7 @@ py_test( srcs_version = "PY2AND3", tags = [ "no_gpu", + "no_windows", ], deps = [ ":array_ops", diff --git a/tensorflow/python/data/util/nest.py b/tensorflow/python/data/util/nest.py index 2f89c006d2..2455395635 100644 --- a/tensorflow/python/data/util/nest.py +++ b/tensorflow/python/data/util/nest.py @@ -376,6 +376,16 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True): "structure has length %s, while shallow structure has length %s." % (len(input_tree), len(shallow_tree))) + if check_types and isinstance(shallow_tree, dict): + if set(input_tree) != set(shallow_tree): + raise ValueError( + "The two structures don't have the same keys. Input " + "structure has keys %s, while shallow structure has keys %s." % + (list(_six.iterkeys(input_tree)), + list(_six.iterkeys(shallow_tree)))) + input_tree = list(_six.iteritems(input_tree)) + shallow_tree = list(_six.iteritems(shallow_tree)) + for shallow_branch, input_branch in zip(shallow_tree, input_tree): assert_shallow_structure(shallow_branch, input_branch, check_types=check_types) diff --git a/tensorflow/python/data/util/nest_test.py b/tensorflow/python/data/util/nest_test.py index 0bd0a5f443..90dd7dfe77 100644 --- a/tensorflow/python/data/util/nest_test.py +++ b/tensorflow/python/data/util/nest_test.py @@ -268,6 +268,15 @@ class NestTest(test.TestCase): nest.assert_shallow_structure(inp_ab2, inp_ab1) nest.assert_shallow_structure(inp_ab2, inp_ab1, check_types=False) + inp_ab1 = {"a": (1, 1), "b": {"c": (2, 2)}} + inp_ab2 = {"a": (1, 1), "b": {"d": (2, 2)}} + expected_message = ( + r"The two structures don't have the same keys. Input " + r"structure has keys \['c'\], while shallow structure has " + r"keys \['d'\].") + with self.assertRaisesRegexp(ValueError, expected_message): + nest.assert_shallow_structure(inp_ab2, inp_ab1) + def testFlattenUpTo(self): input_tree = (((2, 2), (3, 3)), ((4, 9), (5, 5))) shallow_tree = ((True, True), (False, True)) diff --git a/tensorflow/python/estimator/export/export.py b/tensorflow/python/estimator/export/export.py index 31e9933c6f..3b295a7e35 100644 --- a/tensorflow/python/estimator/export/export.py +++ b/tensorflow/python/estimator/export/export.py @@ -57,7 +57,7 @@ class ServingInputReceiver(collections.namedtuple( groups of receiver tensors, each of which may be a `Tensor` or a dict of string to `Tensor`. These named receiver tensor alternatives generate additional serving signatures, which may be used to feed inputs at - different points within the input reciever subgraph. A typical usage is + different points within the input receiver subgraph. A typical usage is to allow feeding raw feature `Tensor`s *downstream* of the tf.parse_example() op. Defaults to None. """ diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py index 1862e325e2..17d018aa88 100644 --- a/tensorflow/python/estimator/training_test.py +++ b/tensorflow/python/estimator/training_test.py @@ -1016,7 +1016,7 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase): is_the_final_export): del export_path, checkpoint_path, eval_result estimator.times_export_was_called += 1 - # final_export is happend at the end. + # final_export is happened at the end. self.assertEqual(0, estimator.times_final_export_was_called) if is_the_final_export: estimator.times_final_export_was_called += 1 @@ -1361,7 +1361,7 @@ class TrainingExecutorRunLocalTest(test.TestCase): is_the_final_export): del export_path, checkpoint_path, eval_result estimator.times_export_was_called += 1 - # final_export is happend at the end. + # final_export is happened at the end. self.assertEqual(0, estimator.times_final_export_was_called) if is_the_final_export: estimator.times_final_export_was_called += 1 diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index e4992afbca..d9391dd6c5 100644 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -556,6 +556,7 @@ py_test( srcs = ["_impl/keras/utils/data_utils_test.py"], srcs_version = "PY2AND3", tags = [ + "no_windows", "noasan", # times out "notsan", ], diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py index b029e5161f..ec7a5dcffd 100644 --- a/tensorflow/python/keras/_impl/keras/backend.py +++ b/tensorflow/python/keras/_impl/keras/backend.py @@ -2487,7 +2487,7 @@ class Function(object): """Runs a computation graph. It's possible to pass arguments to `tf.Session.run()` via `session_kwargs`. - In particular additonal operations via `fetches` argument and additional + In particular additional operations via `fetches` argument and additional tensor substitutions via `feed_dict` arguments. Note that given substitutions are merged with substitutions from `inputs`. Even though `feed_dict` is passed once in the constructor (called in `model.compile()`) diff --git a/tensorflow/python/keras/_impl/keras/callbacks.py b/tensorflow/python/keras/_impl/keras/callbacks.py index 40a996a03f..16109b52b3 100644 --- a/tensorflow/python/keras/_impl/keras/callbacks.py +++ b/tensorflow/python/keras/_impl/keras/callbacks.py @@ -768,7 +768,7 @@ class TensorBoard(Callback): self.writer.add_summary(summary, epoch) self.writer.flush() - def on_train_end(self, _): + def on_train_end(self, logs=None): self.writer.close() diff --git a/tensorflow/python/keras/_impl/keras/callbacks_test.py b/tensorflow/python/keras/_impl/keras/callbacks_test.py index 97a650a992..79dfcd1bb6 100644 --- a/tensorflow/python/keras/_impl/keras/callbacks_test.py +++ b/tensorflow/python/keras/_impl/keras/callbacks_test.py @@ -19,16 +19,18 @@ from __future__ import division from __future__ import print_function import csv -import multiprocessing import os import re import shutil +import threading +import unittest import numpy as np from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.platform import test +from tensorflow.python.summary.writer import writer_cache try: import h5py # pylint:disable=g-import-not-at-top @@ -498,7 +500,10 @@ class KerasCallbacksTest(test.TestCase): values = [] with open(fp) as f: for x in csv.reader(f): - values.append(x) + # In windows, due to \r\n line ends we may end up reading empty lines + # after each line. Skip empty lines. + if x: + values.append(x) assert 'nan' in values[-1], 'The last epoch was not logged.' def test_TerminateOnNaN(self): @@ -678,23 +683,41 @@ class KerasCallbacksTest(test.TestCase): batch_size=5)] # fit w/o validation data should raise ValueError if histogram_freq > 0 + cbs = callbacks_factory(histogram_freq=1) with self.assertRaises(ValueError): - model.fit(x_train, y_train, batch_size=BATCH_SIZE, - callbacks=callbacks_factory(histogram_freq=1), epochs=3) + model.fit( + x_train, y_train, batch_size=BATCH_SIZE, callbacks=cbs, epochs=3) + + for cb in cbs: + cb.on_train_end() # fit generator without validation data should raise ValueError if # histogram_freq > 0 + cbs = callbacks_factory(histogram_freq=1) with self.assertRaises(ValueError): - model.fit_generator(data_generator(True), len(x_train), epochs=2, - callbacks=callbacks_factory(histogram_freq=1)) + model.fit_generator( + data_generator(True), len(x_train), epochs=2, callbacks=cbs) + + for cb in cbs: + cb.on_train_end() # fit generator with validation data generator should raise ValueError if # histogram_freq > 0 + cbs = callbacks_factory(histogram_freq=1) with self.assertRaises(ValueError): - model.fit_generator(data_generator(True), len(x_train), epochs=2, - validation_data=data_generator(False), - validation_steps=1, - callbacks=callbacks_factory(histogram_freq=1)) + model.fit_generator( + data_generator(True), + len(x_train), + epochs=2, + validation_data=data_generator(False), + validation_steps=1, + callbacks=cbs) + + for cb in cbs: + cb.on_train_end() + + # Make sure file writer cache is clear to avoid failures during cleanup. + writer_cache.FileWriterCache.clear() def test_TensorBoard_multi_input_output(self): np.random.seed(1337) @@ -767,6 +790,9 @@ class KerasCallbacksTest(test.TestCase): callbacks=callbacks_factory(histogram_freq=1)) assert os.path.isdir(filepath) + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') def test_LambdaCallback(self): with self.test_session(): np.random.seed(1337) @@ -789,14 +815,15 @@ class KerasCallbacksTest(test.TestCase): # Start an arbitrary process that should run during model # training and be terminated after training has completed. + e = threading.Event() + def target(): - while True: - pass + e.wait() - p = multiprocessing.Process(target=target) - p.start() + t = threading.Thread(target=target) + t.start() cleanup_callback = keras.callbacks.LambdaCallback( - on_train_end=lambda logs: p.terminate()) + on_train_end=lambda logs: e.set()) cbks = [cleanup_callback] model.fit( @@ -807,8 +834,8 @@ class KerasCallbacksTest(test.TestCase): callbacks=cbks, epochs=5, verbose=0) - p.join() - assert not p.is_alive() + t.join() + assert not t.is_alive() def test_TensorBoard_with_ReduceLROnPlateau(self): with self.test_session(): diff --git a/tensorflow/python/keras/_impl/keras/engine/training_test.py b/tensorflow/python/keras/_impl/keras/engine/training_test.py index e2a06e8e77..17a26f978e 100644 --- a/tensorflow/python/keras/_impl/keras/engine/training_test.py +++ b/tensorflow/python/keras/_impl/keras/engine/training_test.py @@ -18,6 +18,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os +import unittest + import numpy as np from tensorflow.python.keras._impl import keras @@ -783,6 +786,9 @@ class TestDynamicTrainability(test.TestCase): class TestGeneratorMethods(test.TestCase): + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') def test_generator_methods(self): arr_data = np.random.random((50, 2)) arr_labels = np.random.random((50,)) diff --git a/tensorflow/python/keras/_impl/keras/estimator_test.py b/tensorflow/python/keras/_impl/keras/estimator_test.py index 1144aa3152..a7ea3b48a3 100644 --- a/tensorflow/python/keras/_impl/keras/estimator_test.py +++ b/tensorflow/python/keras/_impl/keras/estimator_test.py @@ -33,6 +33,7 @@ from tensorflow.python.keras._impl.keras import testing_utils from tensorflow.python.keras._impl.keras.applications import mobilenet from tensorflow.python.platform import gfile from tensorflow.python.platform import test +from tensorflow.python.summary.writer import writer_cache try: @@ -132,6 +133,8 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir) def tearDown(self): + # Make sure nothing is stuck in limbo. + writer_cache.FileWriterCache.clear() if os.path.isdir(self._base_dir): gfile.DeleteRecursively(self._base_dir) @@ -153,6 +156,8 @@ class TestKerasEstimator(test_util.TensorFlowTestCase): est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16) after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1) self.assertLess(after_eval_results['loss'], before_eval_results['loss']) + + writer_cache.FileWriterCache.clear() gfile.DeleteRecursively(self._config.model_dir) def test_evaluate(self): diff --git a/tensorflow/python/keras/_impl/keras/models_test.py b/tensorflow/python/keras/_impl/keras/models_test.py index 86acac4604..61938066b9 100644 --- a/tensorflow/python/keras/_impl/keras/models_test.py +++ b/tensorflow/python/keras/_impl/keras/models_test.py @@ -54,10 +54,11 @@ class TestModelSaving(test.TestCase): model.train_on_batch(x, y) out = model.predict(x) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) new_model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) out2 = new_model.predict(x) @@ -95,13 +96,14 @@ class TestModelSaving(test.TestCase): model.train_on_batch(x, y) out = model.predict(x) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model( fname, custom_objects={'CustomOp': CustomOp, 'custom_loss': custom_loss}) + os.close(fd) os.remove(fname) out2 = model.predict(x) @@ -125,10 +127,11 @@ class TestModelSaving(test.TestCase): model.train_on_batch(x, y) out = model.predict(x) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) out2 = model.predict(x) @@ -144,9 +147,10 @@ class TestModelSaving(test.TestCase): model.add(keras.layers.Dense(3)) model.compile(loss='mse', optimizer='sgd', metrics=['acc']) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) def test_saving_with_tf_optimizer(self): @@ -161,9 +165,10 @@ class TestModelSaving(test.TestCase): optimizer=training_module.AdadeltaOptimizer(0.1), metrics=['acc']) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) def test_saving_right_after_compilation(self): @@ -177,9 +182,10 @@ class TestModelSaving(test.TestCase): model.compile(loss='mse', optimizer='sgd', metrics=['acc']) model.model._make_train_function() - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) def test_saving_lambda_numpy_array_arguments(self): @@ -194,10 +200,11 @@ class TestModelSaving(test.TestCase): model = keras.models.Model(inputs, output) model.compile(loss='mse', optimizer='sgd', metrics=['acc']) - _, fname = tempfile.mkstemp('.h5') + fd, fname = tempfile.mkstemp('.h5') keras.models.save_model(model, fname) model = keras.models.load_model(fname) + os.close(fd) os.remove(fname) self.assertAllClose(mean, model.layers[1].arguments['mu']) diff --git a/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py b/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py index 14b2f08442..47c5b4cff0 100644 --- a/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py +++ b/tensorflow/python/keras/_impl/keras/utils/data_utils_test.py @@ -22,6 +22,7 @@ from itertools import cycle import os import tarfile import threading +import unittest import zipfile import numpy as np @@ -164,6 +165,9 @@ class TestEnqueuers(test.TestCase): self.assertEqual(len(set(acc) - set(range(100))), 0) enqueuer.stop() + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') def test_generator_enqueuer_processes(self): enqueuer = keras.utils.data_utils.GeneratorEnqueuer( create_generator_from_sequence_pcs(TestSequence([3, 200, 200, 3])), @@ -185,6 +189,9 @@ class TestEnqueuers(test.TestCase): with self.assertRaises(StopIteration): next(gen_output) + @unittest.skipIf( + os.name == 'nt', + 'use_multiprocessing=True does not work on windows properly.') def test_generator_enqueuer_fail_processes(self): enqueuer = keras.utils.data_utils.GeneratorEnqueuer( create_generator_from_sequence_pcs(FaultSequence()), diff --git a/tensorflow/python/keras/_impl/keras/utils/io_utils.py b/tensorflow/python/keras/_impl/keras/utils/io_utils.py index 1c8299c27d..a8fc18c17a 100644 --- a/tensorflow/python/keras/_impl/keras/utils/io_utils.py +++ b/tensorflow/python/keras/_impl/keras/utils/io_utils.py @@ -63,11 +63,11 @@ class HDF5Matrix(object): 'HDF5 and h5py installed.') if datapath not in list(self.refs.keys()): - f = h5py.File(datapath) - self.refs[datapath] = f + self._f = h5py.File(datapath) + self.refs[datapath] = self._f else: - f = self.refs[datapath] - self.data = f[dataset] + self._f = self.refs[datapath] + self.data = self._f[dataset] self.start = start if end is None: self.end = self.data.shape[0] @@ -78,6 +78,9 @@ class HDF5Matrix(object): def __len__(self): return self.end - self.start + def __del__(self): + self._f.close() + def __getitem__(self, key): if isinstance(key, slice): start, stop = key.start, key.stop diff --git a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py index ce2faf2d96..d56c4484ce 100644 --- a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py +++ b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py @@ -120,7 +120,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'): layer_id = str(id(layer)) for i, node in enumerate(layer._inbound_nodes): # pylint: disable=protected-access node_key = layer.name + '_ib-' + str(i) - if node_key in model.container_nodes: + if node_key in model._network_nodes: # pylint: disable=protected-access for inbound_layer in node.inbound_layers: inbound_layer_id = str(id(inbound_layer)) layer_id = str(id(layer)) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index f15b3baabe..f6721de32a 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2362,7 +2362,7 @@ cuda_py_test( cuda_py_test( name = "slice_op_test", - size = "medium", + size = "large", srcs = ["slice_op_test.py"], additional_deps = [ "//third_party/py/numpy", @@ -2942,6 +2942,20 @@ tf_py_test( ], ) +tf_py_test( + name = "prefetch_dataset_op_test", + size = "small", + srcs = ["prefetch_dataset_op_test.py"], + additional_deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + tf_py_test( name = "range_dataset_op_test", size = "small", diff --git a/tensorflow/python/kernel_tests/decode_bmp_op_test.py b/tensorflow/python/kernel_tests/decode_bmp_op_test.py index 783492a6f2..35f8f76991 100644 --- a/tensorflow/python/kernel_tests/decode_bmp_op_test.py +++ b/tensorflow/python/kernel_tests/decode_bmp_op_test.py @@ -64,6 +64,81 @@ class DecodeBmpOpTest(test.TestCase): decoded = decode.eval() self.assertAllEqual(decoded, img_bytes) + def testGrayscale(self): + img_bytes = [[[255], [0]], [[255], [0]]] + encoded_bytes = [ + 0x42, + 0x40, + 0x3d, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x36, + 0, + 0, + 0, + 0x28, + 0, + 0, + 0, + 0x2, + 0, + 0, + 0, + 0x2, + 0, + 0, + 0, + 0x1, + 0, + 0x8, + 0, + 0, + 0, + 0, + 0, + 0x10, + 0, + 0, + 0, + 0x13, + 0xb, + 0, + 0, + 0x13, + 0xb, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0xff, + 0, + 0, + 0, + 0xff, + 0, + 0, + 0, + ] + + byte_string = bytes(bytearray(encoded_bytes)) + img_in = constant_op.constant(byte_string, dtype=dtypes.string) + decode = image_ops.decode_bmp(img_in) + + with self.test_session(): + decoded = decode.eval() + self.assertAllEqual(decoded, img_bytes) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/prefetch_dataset_op_test.py b/tensorflow/python/kernel_tests/prefetch_dataset_op_test.py new file mode 100644 index 0000000000..646324cb95 --- /dev/null +++ b/tensorflow/python/kernel_tests/prefetch_dataset_op_test.py @@ -0,0 +1,59 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test PrefetchDataset.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class PrefetchDatasetTest(test.TestCase): + + def testBufferSize(self): + buffer_size = array_ops.placeholder(dtypes.int64, shape=[]) + iterator = dataset_ops.Dataset.range(10).prefetch( + buffer_size=buffer_size).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op, feed_dict={buffer_size: 5}) + for m in range(10): + self.assertEqual(m, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testInvalidBufferSize(self): + buffer_size = array_ops.placeholder(dtypes.int64, shape=[]) + iterator = dataset_ops.Dataset.range(10).prefetch( + buffer_size=buffer_size).make_initializable_iterator() + init_op = iterator.initializer + + with self.assertRaisesRegexp(errors.InvalidArgumentError, "buffer_size"): + with self.test_session() as sess: + sess.run(init_op, feed_dict={buffer_size: 0}) + + with self.assertRaisesRegexp(errors.InvalidArgumentError, "buffer_size"): + with self.test_session() as sess: + sess.run(init_op, feed_dict={buffer_size: -5}) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 8c327d7e27..fbb13bb72c 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -920,6 +920,7 @@ class SeparableConv2D(Conv2D): trainable=trainable, name=name, **kwargs) + self.data_format = data_format self.depth_multiplier = depth_multiplier self.depthwise_initializer = depthwise_initializer self.pointwise_initializer = pointwise_initializer @@ -1231,9 +1232,8 @@ class Conv2DTranspose(Conv2D): def build(self, input_shape): if len(input_shape) != 4: - raise ValueError('Inputs should have rank ' + - str(4) + - 'Received input shape:', str(input_shape)) + raise ValueError('Inputs should have rank 4. Received input shape: ' + + str(input_shape)) if self.data_format == 'channels_first': channel_axis = 1 else: diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py index 81a7cf28bb..bd26ff6696 100644 --- a/tensorflow/python/ops/math_ops_test.py +++ b/tensorflow/python/ops/math_ops_test.py @@ -61,7 +61,7 @@ class ReduceTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testReduceInvalidAxis(self): if context.in_eager_mode(): - # The shape check is in run a graph contruction time. In eager mode, + # The shape check is in run a graph construction time. In eager mode, # it misses the check, magically return result given wrong shape. return x = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index ae2d46a2b7..3643861a16 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -1697,7 +1697,7 @@ class variable_scope(object): # pylint: disable=invalid-name v1 = foo() # Creates v. v2 = foo() # Gets the same, existing v. assert v1 == v2 - + ``` Basic example of sharing a variable with reuse=True: diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index a1e4305de1..e0748d87e2 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -200,7 +200,7 @@ class Variable(object): @compatibility(eager) `tf.Variable` is not compatible with eager execution. Use - `tfe.Variable` instead which is compatable with both eager execution + `tfe.Variable` instead which is compatible with both eager execution and graph construction. See [the TensorFlow Eager Execution guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) for details on how variables work in eager execution. @@ -1064,7 +1064,7 @@ class PartitionedVariable(object): """A container for partitioned `Variable` objects. @compatibility(eager) `tf.PartitionedVariable` is not compatible with - eager execution. Use `tfe.Variable` instead which is compatable + eager execution. Use `tfe.Variable` instead which is compatible with both eager execution and graph construction. See [the TensorFlow Eager Execution guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index c39d0fa5b1..ccfb9aac53 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -65,7 +65,7 @@ class PrintModelAnalysisTest(test.TestCase): ' ScalarW (1, 1/1 params)\n', f.read()) - def testSelectEverthingDetail(self): + def testSelectEverythingDetail(self): ops.reset_default_graph() dev = '/device:GPU:0' if test.is_gpu_available() else '/device:CPU:0' outfile = os.path.join(test.get_temp_dir(), 'dump') diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index 98ac197204..74ee1e5fa8 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -726,6 +726,8 @@ class SaverTest(test.TestCase): class SaveRestoreShardedTest(test.TestCase): + _WRITE_VERSION = saver_pb2.SaverDef.V1 + def _get_test_dir(self, dirname): test_dir = os.path.join(self.get_temp_dir(), dirname) gfile.MakeDirs(test_dir) @@ -751,6 +753,7 @@ class SaveRestoreShardedTest(test.TestCase): "t0": t0.saveable, "t1": t1.saveable }, + write_version=self._WRITE_VERSION, sharded=True) variables.global_variables_initializer().run() t0.insert("k1", 30.0).run() @@ -771,7 +774,13 @@ class SaveRestoreShardedTest(test.TestCase): with sess.graph.device("/cpu:0"): v0 = variables.Variable(111, name="v0") t0 = saver_test_utils.CheckpointedOp(name="t0") - save = saver_module.Saver({"v0": v0, "t0": t0.saveable}, sharded=True) + save = saver_module.Saver( + { + "v0": v0, + "t0": t0.saveable + }, + write_version=self._WRITE_VERSION, + sharded=True) variables.global_variables_initializer().run() t0.insert("k11", 33.0).run() self.assertEqual(111, v0.eval()) @@ -789,7 +798,13 @@ class SaveRestoreShardedTest(test.TestCase): with sess.graph.device("/cpu:0"): v1 = variables.Variable(222) t1 = saver_test_utils.CheckpointedOp(name="t1") - save = saver_module.Saver({"v1": v1, "t1": t1.saveable}, sharded=True) + save = saver_module.Saver( + { + "v1": v1, + "t1": t1.saveable + }, + write_version=self._WRITE_VERSION, + sharded=True) variables.global_variables_initializer().run() t1.insert("k22", 44.0).run() self.assertEqual(222, v1.eval()) @@ -817,6 +832,7 @@ class SaveRestoreShardedTest(test.TestCase): "t0": t0.saveable, "t1": t1.saveable }, + write_version=self._WRITE_VERSION, sharded=True) variables.global_variables_initializer().run() t0.insert("k11", 33.0).run() @@ -982,6 +998,10 @@ class SaveRestoreShardedTest(test.TestCase): self._testPartitionedVariables(use_resource=True) +class SaveRestoreShardedTestV2(SaveRestoreShardedTest): + _WRITE_VERSION = saver_pb2.SaverDef.V2 + + class MaxToKeepTest(test.TestCase): def _get_test_dir(self, dirname): diff --git a/tensorflow/python/training/sync_replicas_optimizer.py b/tensorflow/python/training/sync_replicas_optimizer.py index 2a97d45daa..b52d101a21 100644 --- a/tensorflow/python/training/sync_replicas_optimizer.py +++ b/tensorflow/python/training/sync_replicas_optimizer.py @@ -99,7 +99,7 @@ class SyncReplicasOptimizer(optimizer.Optimizer): # Note that if you want to have 2 backup replicas, you can change # total_num_replicas=52 and make sure this number matches how many physical # replicas you started in your job. - opt = tf.SyncReplicasOptimizer(opt, replicas_to_aggregate=50, + opt = tf.train.SyncReplicasOptimizer(opt, replicas_to_aggregate=50, total_num_replicas=50) # Some models have startup_delays to help stabilize the model but when using diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py index 75f482e5a8..5c066e2bef 100644 --- a/tensorflow/python/util/nest.py +++ b/tensorflow/python/util/nest.py @@ -116,7 +116,7 @@ def flatten(nest): used instead. The same convention is followed in `pack_sequence_as`. This correctly repacks dicts and `OrderedDict`s after they have been flattened, and also allows flattening an `OrderedDict` and then repacking it back using - a correponding plain dict, or vice-versa. + a corresponding plain dict, or vice-versa. Dictionaries with non-sortable keys cannot be flattened. Users must not modify any collections used in `nest` while this function is @@ -296,7 +296,7 @@ def pack_sequence_as(structure, flat_sequence): keys is used instead. The same convention is followed in `flatten`. This correctly repacks dicts and `OrderedDict`s after they have been flattened, and also allows flattening an `OrderedDict` and then repacking it - back using a correponding plain dict, or vice-versa. + back using a corresponding plain dict, or vice-versa. Dictionaries with non-sortable keys cannot be flattened. Args: @@ -452,6 +452,17 @@ def assert_shallow_structure(shallow_tree, input_tree, check_types=True): "structure has length %s, while shallow structure has length %s." % (len(input_tree), len(shallow_tree))) + if check_types and isinstance(shallow_tree, dict): + if set(input_tree) != set(shallow_tree): + raise ValueError( + "The two structures don't have the same keys. Input " + "structure has keys %s, while shallow structure has keys %s." % + (list(_six.iterkeys(input_tree)), + list(_six.iterkeys(shallow_tree)))) + + input_tree = list(_six.iteritems(input_tree)) + shallow_tree = list(_six.iteritems(shallow_tree)) + for shallow_branch, input_branch in zip(shallow_tree, input_tree): assert_shallow_structure(shallow_branch, input_branch, check_types=check_types) diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py index c4020f4f3c..3d9e9f9684 100644 --- a/tensorflow/python/util/nest_test.py +++ b/tensorflow/python/util/nest_test.py @@ -385,6 +385,16 @@ class NestTest(test.TestCase): nest.assert_shallow_structure(inp_ab2, inp_ab1) nest.assert_shallow_structure(inp_ab2, inp_ab1, check_types=False) + inp_ab1 = {"a": (1, 1), "b": {"c": (2, 2)}} + inp_ab2 = {"a": (1, 1), "b": {"d": (2, 2)}} + expected_message = ( + r"The two structures don't have the same keys. Input " + r"structure has keys \['c'\], while shallow structure has " + r"keys \['d'\].") + + with self.assertRaisesRegexp(ValueError, expected_message): + nest.assert_shallow_structure(inp_ab2, inp_ab1) + def testFlattenUpTo(self): # Shallow tree ends at scalar. input_tree = [[[2, 2], [3, 3]], [[4, 9], [5, 5]]] @@ -429,8 +439,7 @@ class NestTest(test.TestCase): input_tree_flattened_as_shallow_tree = nest.flatten_up_to(shallow_tree, input_tree) self.assertEqual(input_tree_flattened_as_shallow_tree, [0, 1, 2, 3, 4]) - shallow_tree = collections.OrderedDict([("a", 0), - ("b", {"d": 3, "e": 1})]) + shallow_tree = collections.OrderedDict([("a", 0), ("c", {"d": 3, "e": 1})]) input_tree_flattened_as_shallow_tree = nest.flatten_up_to(shallow_tree, input_tree) self.assertEqual(input_tree_flattened_as_shallow_tree, diff --git a/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt index 6620a9d308..7de4008c45 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.callbacks.-tensor-board.pbtxt @@ -29,7 +29,7 @@ tf_class { } member_method { name: "on_train_end" - argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'logs\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "set_model" diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc index 2d59299da4..9809ad52de 100644 --- a/tensorflow/tools/benchmark/benchmark_model.cc +++ b/tensorflow/tools/benchmark/benchmark_model.cc @@ -622,7 +622,7 @@ int Main(int argc, char** argv) { RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference", warmup_runs, warmup_time_us / 1000000.0); - // Time from starting to intialize TF to getting the first result back. + // Time from starting to initialize TF to getting the first result back. // This also assumes that only one warmup run is performed. RecordBenchmarkEntry( output_prefix, benchmark_name, "meta-init-plus-first-inference", 1, diff --git a/tensorflow/tools/ci_build/builds/test_user_ops.sh b/tensorflow/tools/ci_build/builds/test_user_ops.sh index 4f1c61b8e9..358f82ac5d 100755 --- a/tensorflow/tools/ci_build/builds/test_user_ops.sh +++ b/tensorflow/tools/ci_build/builds/test_user_ops.sh @@ -76,17 +76,17 @@ echo "PYTHON_BIN_PATH: ${PYTHON_BIN_PATH}" pushd "${TMP_DIR}" -# Obtain paths include and lib paths to the TensorFlow installation -TF_INC=$("${PYTHON_BIN_PATH}" \ - -c 'import tensorflow as tf; print(tf.sysconfig.get_include())') -TF_LIB=$("${PYTHON_BIN_PATH}" \ - -c 'import tensorflow as tf; print(tf.sysconfig.get_lib())') - -if [[ -z "${TF_INC}" ]]; then - die "FAILED to determine TensorFlow include path" +# Obtain compilation and linking flags +TF_CFLAGS=( $("${PYTHON_BIN_PATH}" \ + -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') ) +TF_LFLAGS=( $("${PYTHON_BIN_PATH}" \ + -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') ) + +if [[ -z "${TF_CFLAGS}" || -z "${TF_LFLAGS}" ]]; then + die "FAILED to determine TensorFlow compilation or linking flags" else - echo "TensorFlow include path: ${TF_INC}" - TF_INCLUDE_PATH="-I${TF_INC} -I${TF_INC}/external/nsync/public" + echo "TensorFlow compile flags: ${TF_CFLAGS[@]}" + echo "TensorFlow link flags: ${TF_LFLAGS[@]}" fi # Check g++ availability @@ -145,7 +145,7 @@ if [[ ${IS_GPU} == "0" ]]; then "${GPP_BIN}" -std=c++11 ${EXTRA_GPP_FLAGS} \ -shared "${SRC_FILE}" -o "${USER_OP_SO}" \ - -fPIC ${TF_INCLUDE_PATH} -L "${TF_LIB}" -ltensorflow_framework || \ + -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]} || \ die "g++ compilation of ${SRC_FILE} FAILED" else @@ -184,7 +184,7 @@ else OP_KERNEL_O=$(echo "${OP_KERNEL_CC}" | sed -e 's/\.cc/\.o/') "${NVCC_BIN}" -std=c++11 \ -c -o "${OP_KERNEL_O}" "${OP_KERNEL_CU}" \ - ${TF_INCLUDE_PATH} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC || \ + ${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC || \ die "nvcc compilation of ${OP_KERNEL_CC} FAILED" CUDA_LIB_DIR="/usr/local/cuda/lib64" @@ -203,8 +203,8 @@ else USER_OP_SO="add_one.so" "${GPP_BIN}" -std=c++11 ${EXTRA_GPP_FLAGS} \ -shared -o "${USER_OP_SO}" "${OP_KERNEL_CC}" \ - "${OP_KERNEL_O}" ${TF_INCLUDE_PATH} -L "${CUDA_LIB_DIR}" -L "${TF_LIB}" \ - -fPIC -lcudart -ltensorflow_framework || \ + "${OP_KERNEL_O}" ${TF_CFLAGS[@]} -L "${CUDA_LIB_DIR}" ${TF_LFLAGS[@]} \ + -fPIC -lcudart || \ die "g++ compilation of ${OP_KERNEL_CC}" FAILED fi diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index c27f4953e3..2217b110e3 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -546,8 +546,8 @@ echo "" TMP_DIR="" DOCKERFILE_FLAG="" -if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ] || - ["${TF_BUILD_PYTHON_VERSION}" == "python3.6" ]]; then +if [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.5" ]] || + [[ "${TF_BUILD_PYTHON_VERSION}" == "python3.6" ]]; then # Modify Dockerfile for Python3.5 | Python3.6 build TMP_DIR=$(mktemp -d) echo "Docker build will occur in temporary directory: ${TMP_DIR}" diff --git a/tensorflow/tools/ci_build/windows/cpu/cmake/run_build.bat b/tensorflow/tools/ci_build/windows/cpu/cmake/run_build.bat index 6e600e2dcf..56bff07774 100644 --- a/tensorflow/tools/ci_build/windows/cpu/cmake/run_build.bat +++ b/tensorflow/tools/ci_build/windows/cpu/cmake/run_build.bat @@ -37,4 +37,4 @@ SET MSBUILD_EXE="C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe" %CMAKE_EXE% %CMAKE_DIR% -A x64 -DSWIG_EXECUTABLE=%SWIG_EXE% -DPYTHON_EXECUTABLE=%PY_EXE% -DCMAKE_BUILD_TYPE=Release -DPYTHON_LIBRARIES=%PY_LIB% -Dtensorflow_BUILD_PYTHON_TESTS=%BUILD_PYTHON_TESTS% -Dtensorflow_BUILD_CC_TESTS=%BUILD_CC_TESTS% -Dtensorflow_TF_NIGHTLY=%TF_NIGHTLY% :: Run msbuild in the resulting VS project files to build a pip package. -%MSBUILD_EXE% /p:Configuration=Release /maxcpucount:32 /verbosity:minimal tf_python_build_pip_package.vcxproj \ No newline at end of file +%MSBUILD_EXE% /p:Configuration=Release /maxcpucount:32 tf_python_build_pip_package.vcxproj diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh index f6e3d2e6c7..8520ca898f 100644 --- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh @@ -64,7 +64,7 @@ reinstall_tensorflow_pip ${PIP_NAME} # https://github.com/tensorflow/tensorflow/issues/12844 is fixed. bazel test -c opt $BUILD_OPTS -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ - --test_tag_filters=-no_pip,-no_windows \ - --build_tag_filters=-no_pip,-no_windows --build_tests_only \ + --test_tag_filters=-no_pip,-no_windows,-no_oss \ + --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \ --test_env=TF_SAVER_LENIENT_NAMES=True \ //${PY_TEST_DIR}/tensorflow/python/... diff --git a/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat b/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat index 44d8252a7a..832943ad6c 100644 --- a/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat +++ b/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat @@ -38,4 +38,4 @@ SET MSBUILD_EXE="C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe" %CMAKE_EXE% %CMAKE_DIR% -A x64 -DSWIG_EXECUTABLE=%SWIG_EXE% -DPYTHON_EXECUTABLE=%PY_EXE% -DCMAKE_BUILD_TYPE=Release -DPYTHON_LIBRARIES=%PY_LIB% -Dtensorflow_BUILD_PYTHON_TESTS=%BUILD_PYTHON_TESTS% -Dtensorflow_BUILD_CC_TESTS=%BUILD_CC_TESTS% -Dtensorflow_ENABLE_GPU=ON -DCUDNN_HOME=%CUDNN_HOME% -Dtensorflow_TF_NIGHTLY=%TF_NIGHTLY% :: Run msbuild in the resulting VS project files to build a pip package. -%MSBUILD_EXE% /p:Configuration=Release /maxcpucount:32 /verbosity:minimal tf_python_build_pip_package.vcxproj +%MSBUILD_EXE% /p:Configuration=Release /maxcpucount:32 tf_python_build_pip_package.vcxproj diff --git a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh index 25d327c818..47ca42d642 100644 --- a/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh +++ b/tensorflow/tools/ci_build/windows/gpu/pip/build_tf_windows.sh @@ -65,7 +65,7 @@ reinstall_tensorflow_pip ${PIP_NAME} # https://github.com/tensorflow/tensorflow/issues/12844 is fixed. bazel test -c opt $BUILD_OPTS -k --test_output=errors \ --define=no_tensorflow_py_deps=true --test_lang_filters=py \ - --test_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu \ - --build_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu \ + --test_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,no_oss \ + --build_tag_filters=-no_pip,-no_windows,-no_windows_gpu,-no_gpu,-no_pip_gpu,no_oss \ --test_env=TF_SAVER_LENIENT_NAMES=True \ --local_test_jobs=1 --build_tests_only //${PY_TEST_DIR}/tensorflow/python/... diff --git a/tensorflow/tools/dist_test/python/census_widendeep.py b/tensorflow/tools/dist_test/python/census_widendeep.py index 3a55781496..8feb5386e9 100644 --- a/tensorflow/tools/dist_test/python/census_widendeep.py +++ b/tensorflow/tools/dist_test/python/census_widendeep.py @@ -263,8 +263,7 @@ if __name__ == "__main__": "--data_dir", type=str, default="/tmp/census-data", - help="Directory for storing the cesnsus data" - ) + help="Directory for storing the census data") parser.add_argument( "--model_dir", type=str, diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel index 1a0145b078..3525c7524f 100644 --- a/tensorflow/tools/docker/Dockerfile.devel +++ b/tensorflow/tools/docker/Dockerfile.devel @@ -101,4 +101,3 @@ EXPOSE 6006 EXPOSE 8888 WORKDIR /root -CMD ["/bin/bash"] diff --git a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl new file mode 100644 index 0000000000..8180e5e7fb --- /dev/null +++ b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl @@ -0,0 +1,85 @@ +FROM tensorflow/tensorflow:latest-devel + +LABEL maintainer="Clayne Robison" + +# These arguments are parameterized. Use --build-args to override. +ARG TF_BRANCH=r1.4 +ARG WHL_DIR=/whl + +RUN apt-get update && apt-get install -y --no-install-recommends \ + golang \ + vim \ + emacs \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN pip --no-cache-dir install --upgrade \ + pip setuptools + +RUN pip --no-cache-dir install wheel + +# Download and build TensorFlow. +WORKDIR / +RUN rm -rf tensorflow && \ + git clone https://github.com/tensorflow/tensorflow.git && \ + cd tensorflow && \ + git checkout ${TF_BRANCH} +WORKDIR /tensorflow + +# Configure the build for CPU with MKL by accepting default build options and +# setting library locations +ENV CI_BUILD_PYTHON=python \ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} \ + PYTHON_BIN_PATH=/usr/bin/python \ + PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \ + CC_OPT_FLAGS='-march=native' \ + TF_NEED_JEMALLOC=0 \ + TF_NEED_GCP=0 \ + TF_NEED_CUDA=0 \ + TF_NEED_HDFS=0 \ + TF_NEED_S3=0 \ + TF_NEED_OPENCL=0 \ + TF_NEED_GDR=0 \ + TF_ENABLE_XLA=0 \ + TF_NEED_VERBS=0 \ + TF_NEED_MPI=0 +RUN ./configure + +# Build and Install TensorFlow. +# The 'mkl' option builds with Intel(R) Math Kernel Library (MKL), which detects +# the platform it is currently running on and takes appropriately optimized +# paths. The -march=native option is for code that is not in MKL, and assumes +# this container will be run on the same architecture on which it is built. +RUN LD_LIBRARY_PATH=${LD_LIBRARY_PATH} \ + bazel build --config=mkl \ + --config="opt" \ + --copt="-march=native" \ + --copt="-O3" \ + //tensorflow/tools/pip_package:build_pip_package && \ + mkdir ${WHL_DIR} && \ + bazel-bin/tensorflow/tools/pip_package/build_pip_package ${WHL_DIR} + +# Clean up Bazel cache when done, but leave the whl. +# This will upgrade the default Tensorflow version with the Intel MKL version +RUN pip --no-cache-dir install --upgrade ${WHL_DIR}/tensorflow-*.whl && \ + rm -rf /root/.cache + +WORKDIR /root + +#add welcome message with instructions + +RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/issue && cat /etc/motd' \ + >> /etc/bash.bashrc \ + ; echo "\ +||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||\n\ +| \n\ +| Docker container running Ubuntu \n\ +| with TensorFlow ${TF_BRANCH} optimized for CPU \n\ +| with Intel(R) MKL \n\ +| \n\ +||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||\n\ +\n "\ + > /etc/motd + +CMD ["/bin/bash"] diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu index 21a44ee404..041f45971b 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu @@ -102,5 +102,3 @@ WORKDIR /root EXPOSE 6006 # IPython EXPOSE 8888 - -RUN ["/bin/bash"] diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 index 9bcc3925a8..3bedc8cf34 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 @@ -113,5 +113,3 @@ WORKDIR /root EXPOSE 6006 # IPython EXPOSE 8888 - -RUN ["/bin/bash"] diff --git a/tensorflow/tools/docker/notebooks/2_getting_started.ipynb b/tensorflow/tools/docker/notebooks/2_getting_started.ipynb index e171b439fe..b0963ebc3f 100644 --- a/tensorflow/tools/docker/notebooks/2_getting_started.ipynb +++ b/tensorflow/tools/docker/notebooks/2_getting_started.ipynb @@ -159,7 +159,7 @@ "X = np.array([np.linspace(-2, 4, num_examples), np.linspace(-6, 6, num_examples)])\n", "X += np.random.randn(2, num_examples)\n", "x, y = X\n", - "x_with_bias = np.array([(1., a) for a in x]).astype(np.float32)\n", + "bias_with_x = np.array([(1., a) for a in x]).astype(np.float32)\n", "\n", "losses = []\n", "training_steps = 50\n", @@ -167,7 +167,7 @@ "\n", "with tf.Session() as sess:\n", " # Set up all the tensors, variables, and operations.\n", - " input = tf.constant(x_with_bias)\n", + " input = tf.constant(bias_with_x)\n", " target = tf.constant(np.transpose([y]).astype(np.float32))\n", " weights = tf.Variable(tf.random_normal([2, 1], 0, 0.1))\n", "\n", @@ -583,7 +583,7 @@ "# Split into x and y\n", "x, y = X\n", "# Add the bias node which always has a value of 1\n", - "x_with_bias = np.array([(1., a) for a in x]).astype(np.float32)\n", + "bias_with_x = np.array([(1., a) for a in x]).astype(np.float32)\n", "\n", "# Keep track of the loss at each iteration so we can chart it later\n", "losses = []\n", @@ -598,7 +598,7 @@ "with tf.Session() as sess:\n", " # Set up all the tensors.\n", " # Our input layer is the x value and the bias node.\n", - " input = tf.constant(x_with_bias)\n", + " input = tf.constant(bias_with_x)\n", " # Our target is the y values. They need to be massaged to the right shape.\n", " target = tf.constant(np.transpose([y]).astype(np.float32))\n", " # Weights are a variable. They change every time through the loop.\n", @@ -621,7 +621,7 @@ " loss = tf.nn.l2_loss(yerror)\n", "\n", " # Perform gradient descent. \n", - " # This essentially just updates weights, like weights += grads * learning_rate\n", + " # This essentially just updates weights, like weights -= grads * learning_rate\n", " # using the partial derivative of the loss with respect to the\n", " # weights. It's the direction we want to go to move toward lower error.\n", " update_weights = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n", @@ -743,7 +743,7 @@ "with tf.Session() as sess:\n", " # Set up all the tensors.\n", " # The input is the x values with the bias appended on to each x.\n", - " input = tf.constant(x_with_bias)\n", + " input = tf.constant(bias_with_x)\n", " # We're trying to find the best fit for the target y values.\n", " target = tf.constant(np.transpose([y]).astype(np.float32))\n", " # Let's set up the weights randomly\n", diff --git a/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb b/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb index 614a19c178..5585ebdcd3 100644 --- a/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb +++ b/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb @@ -135,6 +135,8 @@ "from six.moves.urllib.request import urlretrieve\n", "\n", "SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/'\n", + "#SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'\n", + "# for those who have no access to google storage, use lecun's repo please\n", "WORK_DIRECTORY = \"/tmp/mnist-data\"\n", "\n", "def maybe_download(filename):\n", diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py index 3677aaa886..cc46dd5162 100644 --- a/tensorflow/tools/pip_package/pip_smoke_test.py +++ b/tensorflow/tools/pip_package/pip_smoke_test.py @@ -66,9 +66,6 @@ BLACKLIST = [ "//tensorflow/contrib/timeseries/examples:data/period_trend.csv", # pylint:disable=line-too-long "//tensorflow/contrib/timeseries/python/timeseries:test_utils", "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:test_utils", # pylint:disable=line-too-long - - # TODO(yifeif): Remove when py_library(testonly=1) is ignored. - "//tensorflow/contrib/summary:summary_test_internal", ] diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index a493c6f2aa..3852b251d9 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -33,11 +33,16 @@ _VERSION = '1.4.0' REQUIRED_PACKAGES = [ 'absl-py', - 'enum34 >= 1.1.6', + # weakref.finalize introduced in Python 3.4 + 'backports.weakref >= 1.0rc1; python_version < "3.4"', + # enum module introduced in Python 3.4 + 'enum34 >= 1.1.6; python_version < "3.4"', + # Needed for unittest.mock in Python 2 + 'mock >= 2.0.0; python_version < "3.0"', 'numpy >= 1.12.1', 'six >= 1.10.0', 'protobuf >= 3.4.0', - 'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0', + 'tensorflow-tensorboard', ] project_name = 'tensorflow' @@ -52,20 +57,14 @@ if sys.version_info.major == 3: REQUIRED_PACKAGES.append('wheel >= 0.26') else: REQUIRED_PACKAGES.append('wheel') - # mock comes with unittest.mock for python3, need to install for python2 - REQUIRED_PACKAGES.append('mock >= 2.0.0') -# remove tensorboard from tf-nightly packages +# tf-nightly should depend on tb-nightly if 'tf_nightly' in project_name: - for package in REQUIRED_PACKAGES: - if 'tensorflow-tensorboard' in package: - REQUIRED_PACKAGES.remove(package) + for i, pkg in enumerate(REQUIRED_PACKAGES): + if 'tensorboard' in pkg: + REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.5.0a0, < 1.6.0a0' break -# weakref.finalize was introduced in Python 3.4 -if sys.version_info < (3, 4): - REQUIRED_PACKAGES.append('backports.weakref >= 1.0rc1') - # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ 'freeze_graph = tensorflow.python.tools.freeze_graph:main', @@ -76,13 +75,13 @@ CONSOLE_SCRIPTS = [ # is now declared by the tensorboard pip package. If we remove the # TensorBoard command, pip will inappropriately remove it during install, # even though the command is not removed, just moved to a different wheel. - 'tensorboard = tensorboard.main:main', + 'tensorboard = tensorboard.main:run_main', ] # pylint: enable=line-too-long # remove the tensorboard console script if building tf_nightly if 'tf_nightly' in project_name: - CONSOLE_SCRIPTS.remove('tensorboard = tensorboard.main:main') + CONSOLE_SCRIPTS.remove('tensorboard = tensorboard.main:run_main') TEST_PACKAGES = [ 'scipy >= 0.15.1', diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 68d663acfc..b61012f71e 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -80,7 +80,7 @@ def _apply_patch(repo_ctx, patch_file): bazel_sh = _get_env_var(repo_ctx, "BAZEL_SH") if not bazel_sh: fail("BAZEL_SH environment variable is not set") - cmd = [bazel_sh, "-c", " ".join(cmd)] + cmd = [bazel_sh, "-l", "-c", " ".join(cmd)] _execute_and_check_ret_code(repo_ctx, cmd) # Download the repository and apply a patch to its root diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD index e1563103c8..0a76adcf91 100644 --- a/third_party/flatbuffers/flatbuffers.BUILD +++ b/third_party/flatbuffers/flatbuffers.BUILD @@ -6,8 +6,11 @@ licenses(["notice"]) # Apache 2.0 FLATBUFFERS_COPTS = [ "-fexceptions", - "-Wno-implicit-fallthrough", -] +] + select({ + "@bazel_tools//src:windows": [], + "@bazel_tools//src:windows_msvc": [], + "//conditions:default": ["-Wno-implicit-fallthrough"], +}) # Public flatc library to compile flatbuffer files at runtime. cc_library( diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index f637873f14..8b73ddabdd 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -20,7 +20,7 @@ def if_mkl(if_true, if_false = []): """ return select({ - "//third_party/mkl:using_mkl": if_true, + str(Label("//third_party/mkl:using_mkl")): if_true, "//conditions:default": if_false }) diff --git a/third_party/nccl.BUILD b/third_party/nccl.BUILD index 3a2a3afe46..b2b8e18824 100644 --- a/third_party/nccl.BUILD +++ b/third_party/nccl.BUILD @@ -55,7 +55,7 @@ cc_library( ], "@org_tensorflow//tensorflow:ios": [], "@org_tensorflow//tensorflow:windows": [ - "ws2_32.lib", + "-DEFAULTLIB:ws2_32.lib", ], "//conditions:default": [ "-lrt", diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index bbc07905fc..c16eb3a12a 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -1,11 +1,8 @@ -# -*- Python -*- """Repository rule for Python autoconfiguration. `python_configure` depends on the following environment variables: - * `NUMPY_INCLUDE_PATH`: Location of Numpy libraries. * `PYTHON_BIN_PATH`: location of python binary. - * `PYTHON_INCLUDE_PATH`: Location of python binaries. * `PYTHON_LIB_PATH`: Location of python libraries. """ @@ -23,32 +20,13 @@ def _tpl(repository_ctx, tpl, substitutions={}, out=None): substitutions) -def _python_configure_warning(msg): - """Output warning message during auto configuration.""" - yellow = "\033[1;33m" - no_color = "\033[0m" - print("%sPython Configuration Warning:%s %s" % (yellow, no_color, msg)) - - -def _python_configure_fail(msg): +def _fail(msg): """Output failure message when auto configuration fails.""" red = "\033[0;31m" no_color = "\033[0m" fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) -def _get_env_var(repository_ctx, name, default = None, enable_warning = True): - """Find an environment variable in system path.""" - if name in repository_ctx.os.environ: - return repository_ctx.os.environ[name] - if default != None: - if enable_warning: - _python_configure_warning( - "'%s' environment variable is not set, using '%s' as default" % (name, default)) - return default - _python_configure_fail("'%s' environment variable is not set" % name) - - def _is_windows(repository_ctx): """Returns true if the host operating system is windows.""" os_name = repository_ctx.os.name.lower() @@ -73,11 +51,10 @@ def _execute(repository_ctx, cmdline, error_msg=None, error_details=None, """ result = repository_ctx.execute(cmdline) if result.stderr or not (empty_stdout_fine or result.stdout): - _python_configure_fail( - "\n".join([ - error_msg.strip() if error_msg else "Repository command failed", - result.stderr.strip(), - error_details if error_details else ""])) + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else ""])) return result @@ -163,21 +140,23 @@ def _symlink_genrule_for_dir(repository_ctx, src_dir, dest_dir, genrule_name, def _get_python_bin(repository_ctx): """Gets the python bin path.""" - python_bin = _get_env_var(repository_ctx, _PYTHON_BIN_PATH, - None, False) + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) if python_bin != None: return python_bin python_bin_path = repository_ctx.which("python") if python_bin_path != None: return str(python_bin_path) - path = _get_env_var(repository_ctx, "PATH") - _python_configure_fail("Cannot find python in PATH, please make sure " + - "python is installed and add its directory in PATH, or set the " + - "environment variable PYTHON_BIN_PATH.\nPATH=%s" % (path)) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % ( + _PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) def _get_python_lib(repository_ctx, python_bin): """Gets the python lib path.""" + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib != None: + return python_lib print_lib = ("< Date: Thu, 30 Nov 2017 17:37:21 -0800 Subject: [PATCH 0171/1924] Support compressed TensorProto format in constant folding for types iny16, int8, uint8, and bool, in addition to float ,double, int32, and int64, which were already supported. Add unit test for all types. PiperOrigin-RevId: 177533200 --- .../grappler/optimizers/constant_folding.cc | 24 +++++-- .../optimizers/constant_folding_test.cc | 63 ++++++++++++++++++- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index cf913d6f48..e0f39c2931 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -657,9 +657,9 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const { namespace { -#define SET_TENSOR_VAL_CASE(DTYPE, TYPE) \ +#define SET_TENSOR_VAL_CASE(DTYPE, TYPE, NAME) \ case DTYPE: \ - t->add_##TYPE##_val(static_cast(value)); \ + t->add_##NAME##_val(static_cast(value)); \ break; Status CreateConstantTensorAttrValue(DataType type, double value, @@ -668,10 +668,14 @@ Status CreateConstantTensorAttrValue(DataType type, double value, TensorProto* t = attr_tensor->mutable_tensor(); *t->mutable_tensor_shape() = shape; switch (type) { - SET_TENSOR_VAL_CASE(DT_FLOAT, float); - SET_TENSOR_VAL_CASE(DT_DOUBLE, double); - SET_TENSOR_VAL_CASE(DT_INT64, int64); - SET_TENSOR_VAL_CASE(DT_INT32, int); + SET_TENSOR_VAL_CASE(DT_FLOAT, float, float); + SET_TENSOR_VAL_CASE(DT_DOUBLE, double, double); + SET_TENSOR_VAL_CASE(DT_INT64, int64, int64); + SET_TENSOR_VAL_CASE(DT_INT32, int32, int); + SET_TENSOR_VAL_CASE(DT_INT16, int32, int); + SET_TENSOR_VAL_CASE(DT_INT8, int32, int); + SET_TENSOR_VAL_CASE(DT_UINT8, int32, int); + SET_TENSOR_VAL_CASE(DT_BOOL, bool, bool); default: return errors::InvalidArgument("Unsupported type: ", type); } @@ -721,6 +725,14 @@ NodeDef ConstantFolding::CreateNodeDef(const string& name, POPULATE_TENSOR_PROTO(tensor, t, int64, int64) } else if (tensor->dtype() == DT_INT32) { POPULATE_TENSOR_PROTO(tensor, t, int32, int) + } else if (tensor->dtype() == DT_INT16) { + POPULATE_TENSOR_PROTO(tensor, t, int16, int) + } else if (tensor->dtype() == DT_INT8) { + POPULATE_TENSOR_PROTO(tensor, t, int8, int) + } else if (tensor->dtype() == DT_UINT8) { + POPULATE_TENSOR_PROTO(tensor, t, uint8, int) + } else if (tensor->dtype() == DT_BOOL) { + POPULATE_TENSOR_PROTO(tensor, t, bool, bool) } } if (optimized) { diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index c72ed96520..32a691d3ee 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -173,11 +173,70 @@ TEST_F(ConstantFoldingTest, NeutralElement) { } } +TEST_F(ConstantFoldingTest, CreateConstNodes) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + +#define MAKE_TEST_GRAPH(TYPE) \ + Output TYPE##_const = \ + ops::Const(s.WithOpName(#TYPE "_const"), static_cast(10), {5}); \ + Output TYPE##_mul = \ + ops::Mul(s.WithOpName(#TYPE "_mul"), TYPE##_const, TYPE##_const); \ + Output TYPE##_id = ops::Identity(s.WithOpName(#TYPE "_id"), TYPE##_mul) + + MAKE_TEST_GRAPH(float); + MAKE_TEST_GRAPH(double); + MAKE_TEST_GRAPH(int64); + MAKE_TEST_GRAPH(int32); + MAKE_TEST_GRAPH(int16); + MAKE_TEST_GRAPH(int8); + MAKE_TEST_GRAPH(uint8); +#undef MAKE_TEST_GRAPH + + Output bool_const = ops::Const(s.WithOpName("bool_const"), true, {5}); + Output bool_and = + ops::LogicalAnd(s.WithOpName("bool_and"), bool_const, bool_const); + Output bool_id = ops::Identity(s.WithOpName("bool_id"), bool_and); + + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + ConstantFolding fold(nullptr /* cpu_device */); + GraphDef output; + Status status = fold.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(24, output.node_size()); + for (const NodeDef& node : output.node()) { +#define CHECK_RESULT(TYPE, FIELD) \ + if (node.name() == #TYPE "_mul") { \ + EXPECT_EQ(5, \ + node.attr().at("value").tensor().tensor_shape().dim(0).size()); \ + EXPECT_EQ(1, node.attr().at("value").tensor().FIELD##_val_size()); \ + EXPECT_EQ(10 * 10, node.attr().at("value").tensor().FIELD##_val(0)); \ + } + + CHECK_RESULT(float, float); + CHECK_RESULT(double, double); + CHECK_RESULT(int64, int64); + CHECK_RESULT(int32, int); + CHECK_RESULT(int16, int); + CHECK_RESULT(int8, int); + CHECK_RESULT(uint8, int); +#undef CHECK_RESULT + + if (node.name() == "bool_and") { + EXPECT_EQ(5, + node.attr().at("value").tensor().tensor_shape().dim(0).size()); + EXPECT_EQ(1, node.attr().at("value").tensor().bool_val_size()); + EXPECT_EQ(true && true, node.attr().at("value").tensor().bool_val(0)); + } + } +} + TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) { // Build a simple graph with a few trivially prunable ops. tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output a = ops::Const(s.WithOpName("a"), 10, {3}); + Output a = ops::Const(s.WithOpName("a"), 10, {5}); auto b = ops::Unique(s.WithOpName("b"), {a}); Output c = ops::Identity(s.WithOpName("c"), {b.y}); Output d = ops::Identity(s.WithOpName("d"), {b.idx}); @@ -1059,3 +1118,5 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices) { } // namespace } // namespace grappler } // namespace tensorflow + +// LocalWords: NewRootScope -- GitLab From 1a89cf58c021ef176c624b4070ee8422303e29a2 Mon Sep 17 00:00:00 2001 From: Max Galkin Date: Thu, 30 Nov 2017 18:01:03 -0800 Subject: [PATCH 0172/1924] Output unknown dimension root nodes with --vmodule=graph_properties=2 PiperOrigin-RevId: 177535370 --- .../core/grappler/costs/graph_properties.cc | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index fbc52e9bd1..ec44d11bdd 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -265,6 +265,79 @@ bool IsEnterWithQueue(const Node& node) { return false; } +bool HasAnyUnknownDimensions(const TensorShapeProto& proto) { + if (proto.unknown_rank()) { + return true; + } + for (const auto& dim : proto.dim()) { + if (dim.size() < 0) { + return true; + } + } + return false; +} + +void VerboseLogUnknownDimensionSources( + const Graph& graph, + const std::map>& + input_properties_map, + const std::map>& + output_properties_map) { + if (!VLOG_IS_ON(2)) { + return; + } + + VLOG(2) << "Nodes with known inputs, but with unknown output dimensions:"; + + // Find all nodes in the graph for which we + // do not have any unknown dimensions in their inputs, but + // we have some unknown dimensions in their outputs. + for (const Node* const node : graph.nodes()) { + if (node->num_outputs() == 0) { + continue; + } + + const auto& input_properties = input_properties_map.at(node->name()); + const auto& output_properties = output_properties_map.at(node->name()); + + bool has_unknown_inputs = false; + for (int i = 0; i < node->num_inputs(); ++i) { + if (HasAnyUnknownDimensions(input_properties[i].shape())) { + has_unknown_inputs = true; + break; + } + } + + if (has_unknown_inputs) { + continue; + } + + for (int i = 0; i < node->num_outputs(); ++i) { + if (HasAnyUnknownDimensions(output_properties[i].shape())) { + string inputs = "input_shapes=["; + for (int i = 0; i < node->num_inputs(); ++i) { + inputs += + PartialTensorShape::DebugString(input_properties[i].shape()); + } + inputs += "]"; + + string outputs = "output_shapes=["; + for (int i = 0; i < node->num_outputs(); ++i) { + outputs += + PartialTensorShape::DebugString(output_properties[i].shape()); + } + outputs += "]"; + + VLOG(2) << "Node: " << node->name() << ", Op: " << node->def().op() + << ", " << inputs << ", " << outputs; + + // don't log again for this node + break; + } + } + } +} + } // namespace // Queue of nodes to process. Nodes can be enqueued in any order, but will be @@ -1000,6 +1073,10 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) { } } + // Help trace the unknown dimensions to their origins. + VerboseLogUnknownDimensionSources(graph, input_properties_, + output_properties_); + return Status::OK(); } -- GitLab From 6e16af86658cd27b466c7c3ba270338b8f95f184 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 30 Nov 2017 18:24:27 -0800 Subject: [PATCH 0173/1924] Register more ops with bfloat16 types. PiperOrigin-RevId: 177537667 --- .../compiler/tf2xla/kernels/matmul_op.cc | 5 +- .../compiler/tf2xla/kernels/scan_ops.cc | 13 +- .../contrib/tpu/ops/cross_replica_ops.cc | 2 +- tensorflow/core/framework/numeric_types.h | 6 +- .../core/framework/op_def_builder_test.cc | 15 ++- tensorflow/core/framework/types.cc | 24 ++-- tensorflow/core/ops/array_ops.cc | 32 +++-- tensorflow/core/ops/math_ops.cc | 119 +++++++++--------- tensorflow/core/ops/nn_ops.cc | 66 +++++----- tensorflow/core/ops/random_ops.cc | 8 +- 10 files changed, 150 insertions(+), 140 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc index a62d233526..644abd5905 100644 --- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc @@ -85,10 +85,7 @@ class SparseMatMulOp : public MatMulOp { ~SparseMatMulOp() override = default; }; -REGISTER_XLA_OP(Name("SparseMatMul") - .TypeConstraint("Ta", kFloatTypes) - .TypeConstraint("Tb", kFloatTypes), - SparseMatMulOp); +REGISTER_XLA_OP(Name("SparseMatMul"), SparseMatMulOp); } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc index 3cc9d14411..650f8c7dc8 100644 --- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc @@ -35,6 +35,11 @@ limitations under the License. namespace tensorflow { namespace { +// TODO(phawkins): implement double-sized windowed reductions in XLA and remove +// the type constraint. +constexpr std::array kScanOpTypes = { + {DT_HALF, DT_BFLOAT16, DT_FLOAT}}; + class ScanOp : public XlaOpKernel { public: ScanOp(OpKernelConstruction* ctx, bool sum) : XlaOpKernel(ctx), sum_(sum) { @@ -124,17 +129,13 @@ class CumsumOp : public ScanOp { public: explicit CumsumOp(OpKernelConstruction* ctx) : ScanOp(ctx, /*sum=*/true) {} }; -// TODO(phawkins): implement non-float windowed reductions in XLA and remove the -// type constraint. -REGISTER_XLA_OP(Name("Cumsum").TypeConstraint("T", DT_FLOAT), CumsumOp); +REGISTER_XLA_OP(Name("Cumsum").TypeConstraint("T", kScanOpTypes), CumsumOp); class CumprodOp : public ScanOp { public: explicit CumprodOp(OpKernelConstruction* ctx) : ScanOp(ctx, /*sum=*/false) {} }; -// TODO(phawkins): implement non-float windowed reductions in XLA and remove the -// type constraint. -REGISTER_XLA_OP(Name("Cumprod").TypeConstraint("T", DT_FLOAT), CumprodOp); +REGISTER_XLA_OP(Name("Cumprod").TypeConstraint("T", kScanOpTypes), CumprodOp); } // anonymous namespace } // namespace tensorflow diff --git a/tensorflow/contrib/tpu/ops/cross_replica_ops.cc b/tensorflow/contrib/tpu/ops/cross_replica_ops.cc index cbbd19800e..d389050e67 100644 --- a/tensorflow/contrib/tpu/ops/cross_replica_ops.cc +++ b/tensorflow/contrib/tpu/ops/cross_replica_ops.cc @@ -22,7 +22,7 @@ namespace tensorflow { REGISTER_OP("CrossReplicaSum") .Input("input: T") .Output("output: T") - .Attr("T: {float}") + .Attr("T: {bfloat16, float}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( An Op to sum inputs across replicated TPU instances. Each diff --git a/tensorflow/core/framework/numeric_types.h b/tensorflow/core/framework/numeric_types.h index 29cac26244..bdd5af064b 100644 --- a/tensorflow/core/framework/numeric_types.h +++ b/tensorflow/core/framework/numeric_types.h @@ -58,7 +58,7 @@ struct bfloat16 { explicit EIGEN_DEVICE_FUNC bfloat16(const T& val) : bfloat16(static_cast(val)) {} - EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(float) const { + EIGEN_DEVICE_FUNC explicit operator float() const { float result; uint16_t* q = reinterpret_cast(&result); @@ -89,6 +89,10 @@ struct bfloat16 { return static_cast(float(*this)); } + EIGEN_DEVICE_FUNC explicit operator long() const { + return static_cast(float(*this)); + } + EIGEN_DEVICE_FUNC explicit operator char() const { return static_cast(float(*this)); } diff --git a/tensorflow/core/framework/op_def_builder_test.cc b/tensorflow/core/framework/op_def_builder_test.cc index c1511ebe34..9b24e3aa00 100644 --- a/tensorflow/core/framework/op_def_builder_test.cc +++ b/tensorflow/core/framework/op_def_builder_test.cc @@ -124,22 +124,23 @@ TEST_F(OpDefBuilderTest, AttrWithRestrictions) { "attr: { name: 'a' type: 'type' allowed_values { list { type: " "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, " "DT_UINT16, DT_INT8, DT_COMPLEX64, DT_COMPLEX128, DT_QINT8, DT_QUINT8, " - "DT_QINT32, DT_UINT32, DT_UINT64] } } }"); + "DT_QINT32, DT_UINT32, DT_UINT64, DT_BFLOAT16] } } }"); ExpectSuccess( b().Attr("a:{numbertype, variant}"), "attr: { name: 'a' type: 'type' allowed_values { list { type: " "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, " "DT_UINT16, DT_INT8, DT_COMPLEX64, DT_COMPLEX128, DT_QINT8, DT_QUINT8, " - "DT_QINT32, DT_UINT32, DT_UINT64, DT_VARIANT] } } }"); + "DT_QINT32, DT_UINT32, DT_UINT64, DT_BFLOAT16, DT_VARIANT] } } }"); ExpectSuccess(b().Attr("a:realnumbertype"), "attr: { name: 'a' type: 'type' allowed_values { list { type: " "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, " - "DT_INT16, DT_UINT16, DT_INT8, DT_UINT32, DT_UINT64] } } }"); + "DT_INT16, DT_UINT16, DT_INT8, DT_UINT32, DT_UINT64, " + "DT_BFLOAT16] } } }"); ExpectSuccess(b().Attr("a:{realnumbertype, variant , string, }"), "attr: { name: 'a' type: 'type' allowed_values { list { type: " "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, " "DT_INT16, DT_UINT16, DT_INT8, DT_UINT32, DT_UINT64, " - "DT_VARIANT, DT_STRING] } } }"); + "DT_BFLOAT16, DT_VARIANT, DT_STRING] } } }"); ExpectSuccess(b().Attr("a:quantizedtype"), "attr: { name: 'a' type: 'type' allowed_values { list { type: " "[DT_QINT8, DT_QUINT8, DT_QINT32, DT_QINT16, DT_QUINT16]} } }"); @@ -216,12 +217,14 @@ TEST_F(OpDefBuilderTest, AttrListOfRestricted) { b().Attr("a:list(realnumbertype)"), "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: " "[DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, " - "DT_UINT16, DT_INT8, DT_HALF, DT_UINT32, DT_UINT64] } } }"); + "DT_UINT16, DT_INT8, DT_HALF, DT_BFLOAT16, DT_UINT32, DT_UINT64" + "] } } }"); ExpectSuccess( b().Attr("a:list({realnumbertype, variant})"), "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: " "[DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, " - "DT_UINT16, DT_INT8, DT_HALF, DT_UINT32, DT_UINT64, DT_VARIANT] } } }"); + "DT_UINT16, DT_INT8, DT_HALF, DT_BFLOAT16, DT_UINT32, DT_UINT64, " + "DT_VARIANT] } } }"); ExpectSuccess( b().Attr("a:list(quantizedtype)"), "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: " diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc index faae19585d..48849f9dda 100644 --- a/tensorflow/core/framework/types.cc +++ b/tensorflow/core/framework/types.cc @@ -206,18 +206,18 @@ string DataTypeSliceString(const DataTypeSlice types) { } DataTypeVector AllTypes() { - return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_UINT8, DT_INT16, - DT_UINT16, DT_INT8, DT_STRING, DT_COMPLEX64, DT_COMPLEX128, - DT_INT64, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, - DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE, DT_VARIANT, - DT_UINT32, DT_UINT64}; + return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_UINT8, DT_INT16, + DT_UINT16, DT_INT8, DT_STRING, DT_COMPLEX64, DT_COMPLEX128, + DT_INT64, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, + DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE, DT_VARIANT, + DT_UINT32, DT_UINT64, DT_BFLOAT16}; } #if !defined(IS_MOBILE_PLATFORM) || defined(SUPPORT_SELECTIVE_REGISTRATION) DataTypeVector RealNumberTypes() { - return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_INT64, DT_UINT8, DT_INT16, - DT_INT8, DT_UINT16, DT_HALF, DT_UINT32, DT_UINT64}; + return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_INT64, DT_UINT8, DT_INT16, + DT_INT8, DT_UINT16, DT_HALF, DT_UINT32, DT_UINT64, DT_BFLOAT16}; } DataTypeVector QuantizedTypes() { @@ -227,14 +227,14 @@ DataTypeVector QuantizedTypes() { DataTypeVector RealAndQuantizedTypes() { return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_INT64, DT_UINT8, DT_UINT16, DT_UINT16, DT_INT8, DT_QINT8, DT_QUINT8, - DT_QINT16, DT_QUINT16, DT_QINT32, DT_HALF}; + DT_QINT16, DT_QUINT16, DT_QINT32, DT_HALF, DT_BFLOAT16}; } DataTypeVector NumberTypes() { - return {DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, - DT_UINT8, DT_UINT16, DT_INT16, DT_INT8, - DT_COMPLEX64, DT_COMPLEX128, DT_QINT8, DT_QUINT8, - DT_QINT32, DT_HALF, DT_UINT32, DT_UINT64}; + return {DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, + DT_UINT16, DT_INT16, DT_INT8, DT_COMPLEX64, DT_COMPLEX128, + DT_QINT8, DT_QUINT8, DT_QINT32, DT_HALF, DT_UINT32, + DT_UINT64, DT_BFLOAT16}; } #elif defined(__ANDROID_TYPES_FULL__) diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 9fa6423d59..6f4ea09206 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -724,8 +724,8 @@ REGISTER_OP("OnesLike") .Input("x: T") .Output("y: T") .Attr( - "T: {float, double, int8, uint8, int16, uint16, int32, int64, " - "complex64, complex128, bool}") + "T: {bfloat16, float, double, int8, uint8, int16, uint16, int32, " + "int64, complex64, complex128, bool}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns a tensor of ones with the same shape and type as x. @@ -738,7 +738,7 @@ y: a tensor of the same shape and type as x but filled with ones. REGISTER_OP("Diag") .Input("diagonal: T") .Output("output: T") - .Attr("T: {float, double, int32, int64, complex64, complex128}") + .Attr("T: {bfloat16, float, double, int32, int64, complex64, complex128}") .SetShapeFn([](InferenceContext* c) { ShapeHandle in = c->input(0); TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in)); @@ -776,7 +776,7 @@ diagonal: Rank k tensor where k is at most 1. REGISTER_OP("DiagPart") .Input("input: T") .Output("diagonal: T") - .Attr("T: {float, double, int32, int64, complex64, complex128}") + .Attr("T: {bfloat16, float, double, int32, int64, complex64, complex128}") .SetShapeFn([](InferenceContext* c) { ShapeHandle in = c->input(0); if (!c->RankKnown(in)) { @@ -1059,9 +1059,8 @@ REGISTER_OP("Reverse") .Input("dims: bool") .Output("output: T") .Attr( - "T: {uint8, int8, uint16, int16, int32, int64, bool, half, float, " - "double, complex64, " - "complex128, string}") + "T: {uint8, int8, uint16, int16, int32, int64, bool, half, " + "float, double, complex64, complex128, string}") .SetShapeFn([](InferenceContext* c) { ShapeHandle input = c->input(0); ShapeHandle dims; @@ -1137,9 +1136,8 @@ REGISTER_OP("ReverseV2") .Output("output: T") .Attr("Tidx: {int32, int64} = DT_INT32") .Attr( - "T: {uint8, int8, uint16, int16, int32, int64, bool, half, float, " - "double, complex64, " - "complex128, string}") + "T: {uint8, int8, uint16, int16, int32, int64, bool, half, bfloat16, " + "float, double, complex64, complex128, string}") .SetShapeFn([](InferenceContext* c) { ShapeHandle input = c->input(0); ShapeHandle axis; @@ -1834,7 +1832,7 @@ this operation. REGISTER_OP("CheckNumerics") .Input("tensor: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("message: string") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( @@ -4565,12 +4563,12 @@ REGISTER_OP("Bitcast") .Output("output: type") // All supported dtypes are listed here to include qint16 and quint16. .Attr( - "T: {float, double, int64, int32, uint8, uint16, int8, int16," + "T: {bfloat16, float, double, int64, int32, uint8, uint16, int8, int16," " complex64, complex128, qint8, quint8, qint16, quint16, qint32," " half}") .Attr( - "type: {float, double, int64, int32, uint8, uint16, int8, int16," - " complex64, complex128, qint8, quint8, qint16, quint16, qint32," + "type: {bfloat16, float, double, int64, int32, uint8, uint16, int8, " + "int16, complex64, complex128, qint8, quint8, qint16, quint16, qint32," " half}") .SetShapeFn([](InferenceContext* c) { ShapeHandle input = c->input(0); @@ -4782,7 +4780,7 @@ REGISTER_OP("QuantizeAndDequantize") .Attr("input_min: float = 0") .Attr("input_max: float = 0") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Deprecated(22, "Replaced by QuantizeAndDequantizeV2") .Doc(R"doc( @@ -4798,7 +4796,7 @@ REGISTER_OP("QuantizeAndDequantizeV2") .Attr("num_bits: int = 8") .Attr("range_given: bool = false") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle unused; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); @@ -4877,7 +4875,7 @@ REGISTER_OP("QuantizeAndDequantizeV3") .Attr("signed_input: bool = true") .Attr("range_given: bool = true") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle unused; TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused)); diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index ceda11663a..45ebfa203b 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -85,7 +85,7 @@ REGISTER_OP("BatchMatMul") .Input("x: T") .Input("y: T") .Output("output: T") - .Attr("T: {half, float, double, int32, complex64, complex128}") + .Attr("T: {half, bfloat16, float, double, int32, complex64, complex128}") .Attr("adj_x: bool = false") .Attr("adj_y: bool = false") .SetShapeFn([](InferenceContext* c) { @@ -184,7 +184,7 @@ _HostCast requires its input and produces its output in host memory. REGISTER_OP("Abs") .Input("x: T") .Output("y: T") - .Attr("T: {half, float, double, int32, int64}") + .Attr("T: {half, bfloat16, float, double, int32, int64}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Computes the absolute value of a tensor. @@ -210,29 +210,31 @@ value is computed as \\( \sqrt{a^2 + b^2}\\). )doc"); // Declares cwise unary operations signature: 't -> 't -#define UNARY() \ - Input("x: T") \ - .Output("y: T") \ - .Attr("T: {half, float, double, int32, int64, complex64, complex128}") \ +#define UNARY() \ + Input("x: T") \ + .Output("y: T") \ + .Attr( \ + "T: {half, bfloat16, float, double, int32, int64, complex64, " \ + "complex128}") \ .SetShapeFn(shape_inference::UnchangedShape) -#define UNARY_REAL() \ - Input("x: T") \ - .Output("y: T") \ - .Attr("T: {half, float, double}") \ +#define UNARY_REAL() \ + Input("x: T") \ + .Output("y: T") \ + .Attr("T: {half, bfloat16, float, double}") \ .SetShapeFn(shape_inference::UnchangedShape) -#define UNARY_COMPLEX() \ - Input("x: T") \ - .Output("y: T") \ - .Attr("T: {half, float, double, complex64, complex128}") \ +#define UNARY_COMPLEX() \ + Input("x: T") \ + .Output("y: T") \ + .Attr("T: {half, bfloat16, float, double, complex64, complex128}") \ .SetShapeFn(shape_inference::UnchangedShape) -#define UNARY_GRADIENT_COMPLEX() \ - Input("y: T") \ - .Input("dy: T") \ - .Output("z: T") \ - .Attr("T: {half, float, double, complex64, complex128}") \ +#define UNARY_GRADIENT_COMPLEX() \ + Input("y: T") \ + .Input("dy: T") \ + .Output("z: T") \ + .Attr("T: {half, bfloat16, float, double, complex64, complex128}") \ .SetShapeFn(shape_inference::UnchangedShape) REGISTER_OP("Neg") @@ -481,7 +483,7 @@ Computes atan of x element-wise. REGISTER_OP("IsNan") .Input("x: T") .Output("y: bool") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns which elements of x are NaN. @@ -494,7 +496,7 @@ Equivalent to np.isnan REGISTER_OP("IsInf") .Input("x: T") .Output("y: bool") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns which elements of x are Inf. @@ -507,7 +509,7 @@ Equivalent to np.isinf REGISTER_OP("IsFinite") .Input("x: T") .Output("y: bool") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns which elements of x are finite. @@ -520,7 +522,9 @@ Equivalent to np.isfinite REGISTER_OP("Sign") .Input("x: T") .Output("y: T") - .Attr("T: {half, float, double, int32, int64, complex64, complex128}") + .Attr( + "T: {half, bfloat16, float, double, int32, int64, complex64, " + "complex128}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns an element-wise indication of the sign of a number. @@ -533,7 +537,7 @@ For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`. REGISTER_OP("Floor") .Input("x: T") .Output("y: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns element-wise largest integer not greater than x. @@ -542,7 +546,7 @@ Returns element-wise largest integer not greater than x. REGISTER_OP("Ceil") .Input("x: T") .Output("y: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns element-wise smallest integer in not less than x. @@ -551,7 +555,7 @@ Returns element-wise smallest integer in not less than x. REGISTER_OP("Rint") .Input("x: T") .Output("y: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Returns element-wise integer closest to x. @@ -569,22 +573,23 @@ rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.] // Declares cwise binary operations signature: 't, 't -> 't. -#define BINARY_MORE() \ - Input("x: T").Input("y: T").Output("z: T").Attr( \ - "T: {half, float, double, uint8, int8, uint16, int16, int32, int64, " \ - "complex64, complex128}") +#define BINARY_MORE() \ + Input("x: T").Input("y: T").Output("z: T").Attr( \ + "T: {half, bfloat16, float, double, uint8, int8, uint16, int16, int32, " \ + "int64, complex64, complex128}") -#define BINARY_FEWER() \ - Input("x: T").Input("y: T").Output("z: T").Attr( \ - "T: {half, float, double, int32, int64, complex64, complex128}") +#define BINARY_FEWER() \ + Input("x: T").Input("y: T").Output("z: T").Attr( \ + "T: {half, bfloat16, float, double, int32, int64, complex64, " \ + "complex128}") REGISTER_OP("Add") .Input("x: T") .Input("y: T") .Output("z: T") .Attr( - "T: {half, float, double, uint8, int8, int16, int32, int64, complex64, " - "complex128, string}") + "T: {half, bfloat16, float, double, uint8, int8, int16, int32, int64, " + "complex64, complex128, string}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Returns x + y element-wise. @@ -600,8 +605,8 @@ REGISTER_OP("AddV2") .Input("y: T") .Output("z: T") .Attr( - "T: {half, float, double, uint8, int8, int16, int32, int64, complex64, " - "complex128}") + "T: {half, bfloat16, float, double, uint8, int8, int16, int32, int64, " + "complex64, complex128}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .SetIsAggregate() .SetIsCommutative() @@ -757,7 +762,7 @@ REGISTER_OP("Maximum") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {half, float, double, int32, int64}") + .Attr("T: {half, bfloat16, float, double, int32, int64}") .SetIsCommutative() .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( @@ -788,7 +793,7 @@ REGISTER_OP("Minimum") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {half, float, double, int32, int64}") + .Attr("T: {half, bfloat16, float, double, int32, int64}") .SetIsCommutative() .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( @@ -802,7 +807,7 @@ REGISTER_OP("Mod") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {int32, int64, float, double}") + .Attr("T: {int32, int64, bfloat16, float, double}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Returns element-wise remainder of division. This emulates C semantics in that @@ -817,7 +822,7 @@ REGISTER_OP("FloorMod") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {int32, int64, float, double}") + .Attr("T: {int32, int64, bfloat16, float, double}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Returns element-wise remainder of division. When `x < 0` xor `y < 0` is @@ -832,7 +837,7 @@ REGISTER_OP("TruncateMod") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {int32, int64, float, double}") + .Attr("T: {int32, int64, bfloat16, float, double}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Returns element-wise remainder of division. This emulates C semantics in that @@ -847,7 +852,9 @@ REGISTER_OP("Pow") .Input("x: T") .Input("y: T") .Output("z: T") - .Attr("T: {half, float, double, int32, int64, complex64, complex128}") + .Attr( + "T: {half, bfloat16, float, double, int32, int64, complex64, " + "complex128}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Computes the power of one value to another. @@ -946,7 +953,7 @@ REGISTER_OP("Atan2") .Input("y: T") .Input("x: T") .Output("z: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) .Doc(R"doc( Computes arctangent of `y/x` element-wise, respecting signs of the arguments. @@ -1064,15 +1071,15 @@ Returns the truth value of (x >= y) element-wise. // -------------------------------------------------------------------------- -#define EQUALITY_COMPARISON() \ - Input("x: T") \ - .Input("y: T") \ - .Output("z: bool") \ - .SetIsCommutative() \ - .Attr( \ - "T: {half, float, double, uint8, int8, int16, int32, int64, " \ - "complex64, " \ - "quint8, qint8, qint32, string, bool, complex128}") \ +#define EQUALITY_COMPARISON() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: bool") \ + .SetIsCommutative() \ + .Attr( \ + "T: {half, bfloat16, float, double, uint8, int8, int16, int32, " \ + "int64, complex64, quint8, qint8, qint32, string, bool, " \ + "complex128}") \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) REGISTER_OP("Equal") @@ -1291,7 +1298,7 @@ REGISTER_OP("MatMul") .Output("product: T") .Attr("transpose_a: bool = false") .Attr("transpose_b: bool = false") - .Attr("T: {half, float, double, int32, complex64, complex128}") + .Attr("T: {half, bfloat16, float, double, int32, complex64, complex128}") .SetShapeFn(shape_inference::MatMulShape) .Doc(R"doc( Multiply the matrix "a" by the matrix "b". @@ -2105,7 +2112,7 @@ REGISTER_OP("Range") .Input("limit: Tidx") .Input("delta: Tidx") .Output("output: Tidx") - .Attr("Tidx: {float, double, int32, int64} = DT_INT32") + .Attr("Tidx: {bfloat16, float, double, int32, int64} = DT_INT32") .SetShapeFn([](InferenceContext* c) { ShapeHandle unused; TF_RETURN_WITH_CONTEXT_IF_ERROR(c->WithRank(c->input(0), 0, &unused), @@ -2160,7 +2167,7 @@ REGISTER_OP("LinSpace") .Input("stop: T") .Input("num: Tidx") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .Attr("Tidx: {int32, int64} = DT_INT32") .SetShapeFn([](InferenceContext* c) { ShapeHandle unused; diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 59c4642e4d..102de94787 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -73,7 +73,7 @@ REGISTER_OP("AvgPool") .Attr("strides: list(int) >= 4") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::AvgPoolShape) .Doc(R"doc( Performs average pooling on the input. @@ -101,7 +101,7 @@ REGISTER_OP("AvgPoolGrad") .Attr("strides: list(int) >= 4") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -300,7 +300,7 @@ REGISTER_OP("FusedBatchNormV2") .Output("batch_variance: U") .Output("reserve_space_1: U") .Output("reserve_space_2: U") - .Attr("T: {half, float}") + .Attr("T: {half, bfloat16, float}") .Attr("U: {float}") .Attr("epsilon: float = 0.0001") .Attr("data_format: string = 'NHWC'") @@ -393,7 +393,7 @@ REGISTER_OP("FusedBatchNormGradV2") .Output("offset_backprop: U") .Output("reserve_space_3: U") .Output("reserve_space_4: U") - .Attr("T: {half, float}") + .Attr("T: {half, bfloat16, float}") .Attr("U: {float}") .Attr("epsilon: float = 0.0001") .Attr("data_format: string = 'NHWC'") @@ -508,7 +508,7 @@ REGISTER_OP("Conv2D") .Input("input: T") .Input("filter: T") .Output("output: T") - .Attr("T: {half, float}") + .Attr("T: {half, bfloat16, float}") .Attr("strides: list(int)") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) @@ -567,7 +567,7 @@ REGISTER_OP("Conv2DBackpropInput") .Input("filter: T") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {half, float}") + .Attr("T: {half, bfloat16, float}") .Attr("strides: list(int)") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) @@ -615,7 +615,7 @@ REGISTER_OP("Conv2DBackpropFilter") .Input("filter_sizes: int32") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {half, float}") + .Attr("T: {half, bfloat16, float}") .Attr("strides: list(int)") .Attr("use_cudnn_on_gpu: bool = true") .Attr(GetPaddingAttrString()) @@ -837,7 +837,7 @@ REGISTER_OP("DepthwiseConv2dNative") .Input("input: T") .Input("filter: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) @@ -884,7 +884,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropInput") .Input("filter: T") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) @@ -932,7 +932,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropFilter") .Input("filter_sizes: int32") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .Attr("strides: list(int)") .Attr(GetPaddingAttrString()) .Attr(GetConvnetDataFormatAttrString()) @@ -980,7 +980,7 @@ REGISTER_OP("Conv3D") .Input("input: T") .Input("filter: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) @@ -1073,7 +1073,7 @@ REGISTER_OP("Conv3DBackpropInputV2") .Input("filter: T") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) @@ -1116,7 +1116,7 @@ REGISTER_OP("Conv3DBackpropFilterV2") .Input("filter_sizes: int32") .Input("out_backprop: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) @@ -1163,7 +1163,7 @@ REGISTER_OP("AvgPool3D") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn(shape_inference::Pool3DShape) .Doc(R"doc( Performs 3D average pooling on the input. @@ -1190,7 +1190,7 @@ REGISTER_OP("AvgPool3DGrad") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {float, double}") + .Attr("T: {bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s)); @@ -1225,7 +1225,7 @@ REGISTER_OP("MaxPool3D") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {float}") + .Attr("T: {bfloat16, float}") .SetShapeFn(shape_inference::Pool3DShape) .Doc(R"doc( Performs 3D max pooling on the input. @@ -1253,8 +1253,8 @@ REGISTER_OP("MaxPool3DGrad") .Attr("strides: list(int) >= 5") .Attr(GetPaddingAttrString()) .Attr(GetConvnet3dDataFormatAttrString()) - .Attr("T: {float} = DT_FLOAT") - .Attr("TInput: {float} = DT_FLOAT") + .Attr("T: {bfloat16, float} = DT_FLOAT") + .Attr("TInput: {bfloat16, float} = DT_FLOAT") .SetShapeFn([](InferenceContext* c) { return UnchangedShapeWithRank(c, 5); }) @@ -1319,7 +1319,7 @@ data_format: The data format of the input and output data. With the REGISTER_OP("L2Loss") .Input("t: T") .Output("output: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::ScalarShape) .Doc(R"doc( L2 Loss. @@ -1341,7 +1341,7 @@ REGISTER_OP("LRN") .Attr("bias: float = 1.0") .Attr("alpha: float = 1.0") .Attr("beta: float = 0.5") - .Attr("T: {float, half} = DT_FLOAT") + .Attr("T: {half, bfloat16, float} = DT_FLOAT") .SetShapeFn([](InferenceContext* c) { return UnchangedShapeWithRank(c, 4); }) @@ -1376,7 +1376,7 @@ REGISTER_OP("LRNGrad") .Attr("bias: float = 1.0") .Attr("alpha: float = 1.0") .Attr("beta: float = 0.5") - .Attr("T: {float, half} = DT_FLOAT") + .Attr("T: {half, bfloat16, float} = DT_FLOAT") .SetShapeFn([](InferenceContext* c) { ShapeHandle s; TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &s)); // input_grads @@ -1402,8 +1402,8 @@ output: The gradients for LRN. REGISTER_OP("MaxPool") .Attr( - "T: {float, double, int32, int64, uint8, int16, int8, uint16, " - "half, qint8} = DT_FLOAT") + "T: {half, bfloat16, float, double, int32, int64, uint8, int16, int8, " + "uint16, qint8} = DT_FLOAT") .Attr("ksize: list(int) >= 4") .Attr("strides: list(int) >= 4") .Attr(GetPaddingAttrString()) @@ -1429,8 +1429,8 @@ output: The max pooled output tensor. REGISTER_OP("MaxPoolV2") .Attr( - "T: {float, double, int32, int64, uint8, int16, int8, uint16, " - "half, qint8} = DT_FLOAT") + "T: {half, bfloat16, float, double, int32, int64, uint8, int16, int8, " + "uint16, qint8} = DT_FLOAT") .Attr(GetPaddingAttrString()) .Attr("data_format: {'NHWC', 'NCHW', 'NCHW_VECT_C'} = 'NHWC'") .Input("input: T") @@ -1913,7 +1913,7 @@ backprops: The gradients: REGISTER_OP("Elu") .Input("features: T") .Output("activations: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Computes exponential linear: `exp(features) - 1` if < 0, `features` otherwise. @@ -1926,7 +1926,7 @@ REGISTER_OP("EluGrad") .Input("gradients: T") .Input("outputs: T") .Output("backprops: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::MergeBothInputsShapeFn) .Doc(R"doc( Computes gradients for the exponential linear (Elu) operation. @@ -1940,7 +1940,7 @@ backprops: The gradients: `gradients * (outputs + 1)` if outputs < 0, REGISTER_OP("Selu") .Input("features: T") .Output("activations: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)` @@ -1953,7 +1953,7 @@ REGISTER_OP("SeluGrad") .Input("gradients: T") .Input("outputs: T") .Output("backprops: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn(shape_inference::MergeBothInputsShapeFn) .Doc(R"doc( Computes gradients for the scaled exponential linear (Selu) operation. @@ -2015,7 +2015,7 @@ backprops: The gradients: `gradients / (1 + abs(features)) ** 2`. REGISTER_OP("Softmax") .Input("logits: T") .Output("softmax: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { return shape_inference::UnchangedShapeWithRankAtLeast(c, 1); }) @@ -2035,7 +2035,7 @@ softmax: Same shape as `logits`. REGISTER_OP("LogSoftmax") .Input("logits: T") .Output("logsoftmax: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { return shape_inference::UnchangedShapeWithRankAtLeast(c, 1); }) @@ -2057,7 +2057,7 @@ REGISTER_OP("SoftmaxCrossEntropyWithLogits") .Input("labels: T") .Output("loss: T") .Output("backprop: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .SetShapeFn([](InferenceContext* c) { ShapeHandle input; TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input)); @@ -2086,7 +2086,7 @@ REGISTER_OP("SparseSoftmaxCrossEntropyWithLogits") .Input("labels: Tlabels") .Output("loss: T") .Output("backprop: T") - .Attr("T: {half, float, double}") + .Attr("T: {half, bfloat16, float, double}") .Attr("Tlabels: {int32, int64} = DT_INT64") .SetShapeFn([](InferenceContext* c) { ShapeHandle features; diff --git a/tensorflow/core/ops/random_ops.cc b/tensorflow/core/ops/random_ops.cc index 5a436fb93e..31d9c82e53 100644 --- a/tensorflow/core/ops/random_ops.cc +++ b/tensorflow/core/ops/random_ops.cc @@ -29,7 +29,7 @@ REGISTER_OP("RandomUniform") .Output("output: dtype") .Attr("seed: int = 0") .Attr("seed2: int = 0") - .Attr("dtype: {half,float,double}") + .Attr("dtype: {half,bfloat16,float,double}") .Attr("T: {int32, int64}") .SetShapeFn(shape_inference::RandomShape) .Doc(R"doc( @@ -87,7 +87,7 @@ REGISTER_OP("RandomStandardNormal") .Output("output: dtype") .Attr("seed: int = 0") .Attr("seed2: int = 0") - .Attr("dtype: {half,float,double}") + .Attr("dtype: {half,bfloat16,float,double}") .Attr("T: {int32, int64}") .SetShapeFn(shape_inference::RandomShape) .Doc(R"doc( @@ -115,7 +115,7 @@ REGISTER_OP("ParameterizedTruncatedNormal") .Output("output: dtype") .Attr("seed: int = 0") .Attr("seed2: int = 0") - .Attr("dtype: {half,float,double}") + .Attr("dtype: {half,bfloat16,float,double}") .Attr("T: {int32, int64}") .SetShapeFn(shape_inference::RandomShape) .Doc(R"doc( @@ -145,7 +145,7 @@ REGISTER_OP("TruncatedNormal") .Output("output: dtype") .Attr("seed: int = 0") .Attr("seed2: int = 0") - .Attr("dtype: {half,float,double}") + .Attr("dtype: {half,bfloat16,float,double}") .Attr("T: {int32, int64}") .SetShapeFn(shape_inference::RandomShape) .Doc(R"doc( -- GitLab From e361bf18a3c71a1ec9985a478c419c04852a61d3 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Fri, 1 Dec 2017 10:31:22 +0800 Subject: [PATCH 0174/1924] add link to decode_bmp --- tensorflow/docs_src/api_guides/python/image.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/docs_src/api_guides/python/image.md b/tensorflow/docs_src/api_guides/python/image.md index a2c8c3c3c9..051e4547ee 100644 --- a/tensorflow/docs_src/api_guides/python/image.md +++ b/tensorflow/docs_src/api_guides/python/image.md @@ -19,6 +19,7 @@ Note: The PNG encode and decode Ops support RGBA, but the conversions Ops presently only support RGB, HSV, and GrayScale. Presently, the alpha channel has to be stripped from the image and re-attached using slicing ops. +* @{tf.image.decode_bmp} * @{tf.image.decode_gif} * @{tf.image.decode_jpeg} * @{tf.image.encode_jpeg} -- GitLab From 87e2f20c8b4f2ece313584c7c3c5588ee6ae5ece Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 30 Nov 2017 18:56:57 -0800 Subject: [PATCH 0175/1924] Automated g4 rollback of changelist 177505909 PiperOrigin-RevId: 177540002 --- .../grappler/optimizers/layout_optimizer.cc | 88 ++++++++---- .../optimizers/layout_optimizer_test.cc | 128 ++++++++++++++++++ 2 files changed, 190 insertions(+), 26 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index ef4b015295..cb8411ba5e 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include #include "tensorflow/core/framework/attr_value.pb.h" @@ -761,24 +762,52 @@ class AgnosticNodeProcessor : public NodeProcessor { bool IsNodeAfterNCHWToNHWC() const { std::set ops_format_agnostic = GetOpsFormatAgnostic(); - auto node = node_map_->GetNode(node_->name()); - while (node->input_size() > 0) { - int data_input_pos = 0; - if (IsConcatV1(*node) || IsSplit(*node)) { - data_input_pos = 1; - } - node = node_map_->GetNode(node->input(data_input_pos)); - if (IsNodeNCHWToNHWC(node->name())) { + std::deque queue; + auto first_node_pos = DataInputPos(*node_); + for (const auto& pos : first_node_pos) { + auto input_node = node_map_->GetNode(node_->input(pos)); + queue.push_back(input_node); + } + // The code will exit this while loop in one iteration in most cases, as the + // graph is already topologically sorted. + while (!queue.empty()) { + NodeDef* current_node = queue.front(); + queue.pop_front(); + if (IsNodeNCHWToNHWC(current_node->name())) { return true; } - bool connected = - ops_format_agnostic.find(node->op()) != ops_format_agnostic.end(); - if (!connected) { - return false; + // We only continue searching if the path is connected through + // format-agnostic nodes. + if (ops_format_agnostic.find(current_node->op()) != + ops_format_agnostic.end()) { + auto current_node_pos = DataInputPos(*current_node); + for (const auto& pos : current_node_pos) { + auto input_node = node_map_->GetNode(current_node->input(pos)); + queue.push_back(input_node); + } } } return false; } + + private: + std::vector DataInputPos(const NodeDef& node) const { + std::vector pos; + if (IsSplit(node)) { + return {1}; + } + if (IsConcatV1(node)) { + return {1}; + } + if (IsAdd(node) || IsMul(node) || IsRealDiv(node) || + IsSquaredDifference(node) || IsSub(node)) { + return {0, 1}; + } + if (node.input_size() > 0 && !IsControlInput(node.input(0))) { + return {0}; + } + return {}; + } }; class AddNProcessor : public AgnosticNodeProcessor { @@ -801,42 +830,49 @@ class BinaryOpProcessor : public AgnosticNodeProcessor { public: explicit BinaryOpProcessor(const OptimizeContext& opt_cxt) : AgnosticNodeProcessor(opt_cxt) { - is_4d_with_vector_ = Is4DOperateWithVector(); + is_4d_with_vector_ = IsNDOperateWithMD(4, 1); } protected: bool ShouldProcess() const override { + // TODO(yaozhang): Support IsNDOperateWithMD(1, 4): first input is a vector + // and the second input is a 4D tensor; and update CustomizedProcessing() + // accordingly. return !MustPreserve() && IsDimsFour(*node_) && HasOutputs() && IsNodeAfterNCHWToNHWC() && - (Is4DOperateWithND(4) || Is4DOperateWithScalar() || - Is4DOperateWithVector()) && + (IsNDOperateWithMD(4, 0) || IsNDOperateWithMD(4, 1) || + IsNDOperateWithMD(4, 4) || IsNDOperateWithMD(0, 4)) && IsOnGPU(); } std::vector GetInputPos() const override { - std::vector input_pos = {0}; - if (Is4DOperateWithND(4)) { + std::vector input_pos; + auto input0 = node_map_->GetNode(node_->input(0)); + auto input1 = node_map_->GetNode(node_->input(1)); + if (IsDimsFour(*input0)) { + input_pos.push_back(0); + } + if (IsDimsFour(*input1)) { input_pos.push_back(1); } return input_pos; } - bool Is4DOperateWithND(int n) const { + bool IsDimsFour(const NodeDef& node) const { + return NodeProcessor::IsDimsFour(node) || IsNodeNCHWToNHWC(node.name()); + } + + bool IsNDOperateWithMD(int n, int m) const { auto input0 = node_map_->GetNode(node_->input(0)); auto input1 = node_map_->GetNode(node_->input(1)); if (input0 && input1) { - return (IsDimsFour(*input0) || IsNodeNCHWToNHWC(input0->name())) && - ((n == 4) - ? (IsDimsFour(*input1) || IsNodeNCHWToNHWC(input1->name())) - : IsDimsN(*input1, n)); + bool input0_is_n = (n == 4) ? IsDimsFour(*input0) : IsDimsN(*input0, n); + bool input1_is_m = (m == 4) ? IsDimsFour(*input1) : IsDimsN(*input1, m); + return input0_is_n && input1_is_m; } return false; } - bool Is4DOperateWithScalar() const { return Is4DOperateWithND(0); } - - bool Is4DOperateWithVector() const { return Is4DOperateWithND(1); } - NodeDef* AddNodeShapeConst(const string& name, int num_channels) { NodeDef* node = graph_->add_node(); node_map_->AddNode(name, node); diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc index e8f7b8ac3c..363b4c3fd8 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer_test.cc @@ -298,6 +298,39 @@ TEST_F(LayoutOptimizerTest, Connectivity) { EXPECT_EQ(node_i2_output->input(0), "i1"); } +TEST_F(LayoutOptimizerTest, ConnectivityBinaryOpWithInputScalarAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto i1 = ops::Identity(s.WithOpName("i1"), conv); + auto i2 = ops::Identity(s.WithOpName("i2"), i1); + auto scalar_sub = ops::Const(s.WithOpName("scalar_sub"), 3.0f, {}); + auto sub = ops::Sub(s.WithOpName("sub"), scalar_sub, i2); + auto i3 = ops::Identity(s.WithOpName("i3"), sub); + auto i4 = ops::Identity(s.WithOpName("i4"), i3); + auto i5 = ops::Identity(s.WithOpName("i5"), i4); + auto scalar_mul = ops::Const(s.WithOpName("scalar_mul"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), scalar_mul, i5); + auto i6 = ops::Identity(s.WithOpName("i6"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + // Make the graph not in topological order to test the handling of multi-hop + // connectivity (here we say two nodes are connected if all nodes in the + // middle are layout agnostic). If the graph is already in topological order, + // the problem is easier, where layout optimizer only needs to check + // single-hop connectivity. + NodeMap node_map_original(&item.graph); + auto node_i1 = node_map_original.GetNode("i1"); + auto node_mul = node_map_original.GetNode("mul"); + node_mul->Swap(node_i1); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map_output(&output); + auto mul_node = node_map_output.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "scalar_mul"); + EXPECT_EQ(mul_node->input(1), "i5"); +} + TEST_F(LayoutOptimizerTest, PreserveFetch) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); auto conv = SimpleConv2D(&s, 3, 2, "VALID"); @@ -571,6 +604,101 @@ TEST_F(LayoutOptimizerTest, Sum) { */ } +TEST_F(LayoutOptimizerTest, MulScalarAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), scalar, conv); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "scalar"); + EXPECT_EQ(mul_node->input(1), "Conv2D"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAndScalar) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto scalar = ops::Const(s.WithOpName("scalar"), 3.0f, {}); + auto mul = ops::Mul(s.WithOpName("mul"), conv, scalar); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "scalar"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto i = ops::Identity(s.WithOpName("i"), conv); + auto mul = ops::Mul(s.WithOpName("mul"), conv, i); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "i"); +} + +TEST_F(LayoutOptimizerTest, Mul4DAndVector) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); + auto mul = ops::Mul(s.WithOpName("mul"), conv, vector); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + EXPECT_EQ(mul_node->input(0), "Conv2D"); + EXPECT_EQ(mul_node->input(1), "LayoutOptimizerReshapeNHWCToNCHW-mul-vector"); + auto mul_const = node_map.GetNode("LayoutOptimizerReshapeConst-mul-vector"); + Tensor tensor; + EXPECT_TRUE( + tensor.FromProto(mul_const->mutable_attr()->at({"value"}).tensor())); + Tensor tensor_expected(DT_INT32, {4}); + test::FillValues(&tensor_expected, {1, 2, 1, 1}); + test::ExpectTensorEqual(tensor_expected, tensor); +} + +TEST_F(LayoutOptimizerTest, MulVectorAnd4D) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + auto conv = SimpleConv2D(&s, 3, 2, "VALID"); + auto vector = ops::Const(s.WithOpName("vector"), {3.0f, 7.0f}, {2}); + auto mul = ops::Mul(s.WithOpName("mul"), vector, conv); + auto o = ops::Identity(s.WithOpName("o"), mul); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + LayoutOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(virtual_cluster_.get(), item, &output); + NodeMap node_map(&output); + auto mul_node = node_map.GetNode("mul"); + // TODO(yaozhang): Support vector as the first input and 4d tensor as the + // second input for BinaryOpProcessor. + EXPECT_EQ(mul_node->input(0), "vector"); + EXPECT_EQ(mul_node->input(1), + "LayoutOptimizerTransposeNCHWToNHWC-Conv2D-mul-1"); +} + } // namespace } // namespace grappler } // namespace tensorflow -- GitLab From 79ad4a423f3e9031eb841a164372cc7476cc112a Mon Sep 17 00:00:00 2001 From: Olivia Nordquist Date: Thu, 30 Nov 2017 19:46:05 -0800 Subject: [PATCH 0176/1924] enabling Tensor._set_shape() to work with the C API PiperOrigin-RevId: 177543170 --- tensorflow/python/client/tf_session.i | 43 ++++++++++++++ tensorflow/python/client/tf_session_helper.cc | 19 +++++++ tensorflow/python/client/tf_session_helper.h | 14 +++++ tensorflow/python/framework/ops.py | 57 +++++++++++++------ 4 files changed, 117 insertions(+), 16 deletions(-) diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 5fa1a7e8fc..d471a39b69 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -532,6 +532,49 @@ def TF_Reset(target, containers=None, config=None): %unignore TF_GraphGetTensorShapeHelper; %ignore TF_GraphGetTensorShape; +// We use TF_GraphSetTensorShape_wrapper instead of +// TF_GraphSetTensorShape +%ignore TF_GraphSetTensorShape; +%unignore tensorflow; +%unignore TF_GraphSetTensorShape_wrapper; + +// $input is a Python list of ints to a vector for TF_GraphSetTensorShape_wrapper +%typemap(in) (const std::vector& dims) + (std::vector dims_local){ + if ($input != Py_None) { + if (!PyList_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, tensorflow::strings::Printf( + "$symname: expected list but got %s ", Py_TYPE($input)->tp_name).c_str()); + } + size_t size = PyList_Size($input); + for (int i = 0; i < size; ++i) { + PyObject* item = PyList_GetItem($input, i); + dims_local.push_back(PyInt_AsLong(item)); + } + $1 = &dims_local; + } else { + $1 = nullptr; + } +} + +// We use TF_GraphGetTensorShape_wrapper instead of +// TF_GraphGetTensorShape +%ignore TF_GraphGetTensorShape; +%unignore tensorflow; +%unignore TF_GraphGetTensorShape_wrapper; + +// Build a Python list of ints and return it. +%typemap(out) std::vector tensorflow::TF_GraphGetTensorShape_wrapper { + $result = PyList_New($1.size()); + if (!$result) { + SWIG_exception_fail(SWIG_MemoryError, "$symname: couldn't create list"); + } + + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, PyInt_FromLong($1[i])); + } +} + %include "tensorflow/python/client/tf_session_helper.h" %unignoreall diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc index ad982e5dd8..e4bf09a0ca 100644 --- a/tensorflow/python/client/tf_session_helper.cc +++ b/tensorflow/python/client/tf_session_helper.cc @@ -407,4 +407,23 @@ TF_Function* TF_GraphToFunction_wrapper( opts, description, out_status); } +void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, + const std::vector& dims, + bool unknown_shape, TF_Status* status) { + if (unknown_shape) { + TF_GraphSetTensorShape(graph, output, nullptr, -1, status); + return; + } + TF_GraphSetTensorShape(graph, output, dims.data(), dims.size(), status); +} + +std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, + TF_Output output, + int num_dims, + TF_Status* status) { + std::vector dims(num_dims); + TF_GraphGetTensorShape(graph, output, dims.data(), num_dims, status); + return dims; +} + } // namespace tensorflow diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h index 6ed08d3a58..bb7171db31 100644 --- a/tensorflow/python/client/tf_session_helper.h +++ b/tensorflow/python/client/tf_session_helper.h @@ -168,6 +168,20 @@ TF_Function* TF_GraphToFunction_wrapper( const std::vector& inputs, const std::vector& outputs, const NameVector& output_names, const TF_FunctionOptions* opts, const char* description, TF_Status* out_status); + +// Set the shape of output. If unknown is true, `num_dims` must be set to +// -1 and `dims` is set to nullptr. +void TF_GraphSetTensorShape_wrapper(TF_Graph* graph, TF_Output output, + const std::vector& dims, + bool unknown_shape, TF_Status* status); + +// Return the shape of output. `num_dims` should be the output of +// TF_GraphGetTensorNumDims. If `num_dims = -1`, this should not be called. +std::vector TF_GraphGetTensorShape_wrapper(TF_Graph* graph, + TF_Output output, + int num_dims, + TF_Status* status); + } // namespace tensorflow #endif // TENSORFLOW_PYTHON_CLIENT_TF_SESSION_HELPER_H_ diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 5f945ac133..13e6426447 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -374,6 +374,19 @@ class Tensor(_TensorLike): A `TensorShape` representing the shape of this tensor. """ + if _USE_C_API: + graph = self._op._graph._c_graph # pylint: disable=protected-access + with errors.raise_exception_on_not_ok_status() as status: + num_dims = c_api.TF_GraphGetTensorNumDims(graph, self._as_tf_output(), + status) + if num_dims == -1: + dim_list = None + else: + with errors.raise_exception_on_not_ok_status() as status: + dim_list = c_api.TF_GraphGetTensorShape_wrapper( + graph, self._as_tf_output(), num_dims, status) + dim_list = [None if i == -1 else i for i in dim_list] + return tensor_shape.TensorShape(dim_list) return self._shape def __iter__(self): @@ -393,8 +406,8 @@ class Tensor(_TensorLike): yield self[i] def _shape_as_list(self): - if self._shape.ndims is not None: - return [dim.value for dim in self._shape.dims] + if self.shape.ndims is not None: + return [dim.value for dim in self.shape.dims] else: return None @@ -410,7 +423,7 @@ class Tensor(_TensorLike): Returns: Integer rank or None """ - return self._shape.ndims + return self.shape.ndims def get_shape(self): """Alias of Tensor.shape.""" @@ -441,14 +454,35 @@ class Tensor(_TensorLike): ``` Args: - shape: A `TensorShape` representing the shape of this tensor. + shape: A `TensorShape` representing the shape of this tensor, a + `TensorShapeProto`, a list, a tuple, or None. Raises: ValueError: If `shape` is not compatible with the current shape of this tensor. """ - # TODO(skyewm): call C API - self._shape = self._shape.merge_with(shape) + if not _USE_C_API: + self._shape = self._shape.merge_with(shape) # pylint: disable=protected-access + return + if not isinstance(shape, tensor_shape.TensorShape): + shape = tensor_shape.TensorShape(shape) + dim_list = [] + if shape.dims is None: + unknown_shape = True + else: + unknown_shape = False + for dim in shape.dims: + if dim.value is None: + dim_list.append(-1) + else: + dim_list.append(dim.value) + with errors.raise_exception_on_not_ok_status() as status: + c_api.TF_GraphSetTensorShape_wrapper( + self._op._graph._c_graph, # pylint: disable=protected-access + self._as_tf_output(), + dim_list, + unknown_shape, + status) @property def value_index(self): @@ -4521,15 +4555,11 @@ def control_dependencies(control_inputs): See @{tf.Graph.control_dependencies} for more details. - When eager execution is enabled, any callable object in the `control_inputs` - list will be called. - Args: control_inputs: A list of `Operation` or `Tensor` objects which must be executed or computed before running the operations defined in the context. Can also be `None` to clear the control - dependencies. If eager execution is enabled, any callable object in the - `control_inputs` list will be called. + dependencies. Returns: A context manager that specifies control dependencies for all @@ -4538,11 +4568,6 @@ def control_dependencies(control_inputs): if context.in_graph_mode(): return get_default_graph().control_dependencies(control_inputs) else: - if control_inputs: - # Excute any pending callables. - for control in control_inputs: - if callable(control): - control() return _NullContextmanager() -- GitLab From 6968ff07225ad88928922bc83e5522d4515cf963 Mon Sep 17 00:00:00 2001 From: Yao Zhang Date: Thu, 30 Nov 2017 20:21:19 -0800 Subject: [PATCH 0177/1924] Disable tuning for now. Re-enable when measurement-based estimator is ready. PiperOrigin-RevId: 177545499 --- .../core/grappler/optimizers/layout_optimizer.cc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index cb8411ba5e..e9436638f0 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -1623,20 +1623,13 @@ Status LayoutOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item, } TuningConfig config; - config.no_gemm = false; + config.no_gemm = true; + // TODO(yaozhang): Enable tuning with various TuningConfig choices wtih + // the measurement-based estimator. status = Tune(item, graph_properties, config, output); - // This is based on an empirical observation that if the introduced Transpose - // nodes is more than 30, not using GEMM implementation would result in better - // performance. - if (status.ok() && GetNumTranspose(*output) > 30) { - config.no_gemm = true; - status = Tune(item, graph_properties, config, output); - } - if (!status.ok()) { *output = item.graph; } - return status; } -- GitLab From 1ec61fafe13e5edce6e45d5a67e960efb9df618a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 20:30:18 -0800 Subject: [PATCH 0178/1924] Use latest nsync in tensorflow. Latest nsync builds with bazel on FreeBSD. PiperOrigin-RevId: 177545934 --- tensorflow/contrib/cmake/external/nsync.cmake | 2 +- tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt | 9 +++++++++ tensorflow/workspace.bzl | 8 ++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tensorflow/contrib/cmake/external/nsync.cmake b/tensorflow/contrib/cmake/external/nsync.cmake index 155c91cb97..0508006047 100644 --- a/tensorflow/contrib/cmake/external/nsync.cmake +++ b/tensorflow/contrib/cmake/external/nsync.cmake @@ -16,7 +16,7 @@ include (ExternalProject) set(nsync_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/public) set(nsync_URL https://github.com/google/nsync) -set(nsync_TAG 93815892dddafe9146a5f7e7042281d59d0f4323) +set(nsync_TAG 8502189abfa44c249c01c2cad64e6ed660a9a668) set(nsync_BUILD ${CMAKE_CURRENT_BINARY_DIR}/nsync/src/nsync) set(nsync_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/nsync/install) diff --git a/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt b/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt index 594c2492d4..aaae18a313 100644 --- a/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt +++ b/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt @@ -158,12 +158,21 @@ if (NOT "${NSYNC_LANGUAGE}X" STREQUAL "c++11X") elseif ("${CMAKE_SYSTEM_NAME}X" STREQUAL "NetBSDX") include_directories ("${PROJECT_SOURCE_DIR}/platform/netbsd") set (NSYNC_POSIX ON) + set (NSYNC_OS_EXTRA_SRC + "platform/posix/src/nsync_semaphore_mutex.c" + ) elseif ("${CMAKE_SYSTEM_NAME}X" STREQUAL "FreeBSDX") include_directories ("${PROJECT_SOURCE_DIR}/platform/freebsd") set (NSYNC_POSIX ON) + set (NSYNC_OS_EXTRA_SRC + "platform/posix/src/nsync_semaphore_mutex.c" + ) elseif ("${CMAKE_SYSTEM_NAME}X" STREQUAL "OpenBSDX") include_directories ("${PROJECT_SOURCE_DIR}/platform/openbsd") set (NSYNC_POSIX ON) + set (NSYNC_OS_EXTRA_SRC + "platform/posix/src/nsync_semaphore_mutex.c" + ) endif () endif () diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b61012f71e..25e036e24c 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -416,11 +416,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "nsync", urls = [ - "https://mirror.bazel.build/github.com/google/nsync/archive/93815892dddafe9146a5f7e7042281d59d0f4323.tar.gz", - "https://github.com/google/nsync/archive/93815892dddafe9146a5f7e7042281d59d0f4323.tar.gz", + "https://mirror.bazel.build/github.com/google/nsync/archive/8502189abfa44c249c01c2cad64e6ed660a9a668.tar.gz", + "https://github.com/google/nsync/archive/8502189abfa44c249c01c2cad64e6ed660a9a668.tar.gz", ], - sha256 = "e3bd4555415ace511338fc27e595351738eea4e9006f1612b76c82914770716b", - strip_prefix = "nsync-93815892dddafe9146a5f7e7042281d59d0f4323", + sha256 = "51f81ff4202bbb820cdbedc061bd2eb6765f2b5c06489e7a8694bedac329e8f8", + strip_prefix = "nsync-8502189abfa44c249c01c2cad64e6ed660a9a668", ) native.http_archive( -- GitLab From 6eec9c2ea33f3b86012cb0ea2aeb9e49e65bc716 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 21:08:05 -0800 Subject: [PATCH 0179/1924] [XLA] Hlo parser: support rng and reduce-precision. Also simplify the lexer by regarding several things as identifier. PiperOrigin-RevId: 177548483 --- .../compiler/xla/service/hlo_instruction.cc | 30 +++++++ .../compiler/xla/service/hlo_instruction.h | 5 +- tensorflow/compiler/xla/tools/parser/BUILD | 2 +- .../compiler/xla/tools/parser/hlo_lexer.cc | 32 ++------ .../compiler/xla/tools/parser/hlo_lexer.h | 14 +--- .../compiler/xla/tools/parser/hlo_parser.cc | 81 +++++++++++++++++-- .../xla/tools/parser/hlo_parser_test.cc | 25 ++++++ .../compiler/xla/tools/parser/hlo_token.h | 6 +- 8 files changed, 149 insertions(+), 46 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index b4bac18bcd..45825c7c76 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2060,6 +2060,14 @@ std::vector HloInstruction::ExtraAttributesToString() const { extra.push_back( StrCat("outfeed_config=\"", CEscape(outfeed_config_), "\"")); } + if (opcode() == HloOpcode::kRng) { + extra.push_back( + StrCat("distribution=", RandomDistributionToString(distribution_))); + } + if (opcode() == HloOpcode::kReducePrecision) { + extra.push_back(StrCat("exponent_bits=", exponent_bits_)); + extra.push_back(StrCat("mantissa_bits=", mantissa_bits_)); + } return extra; } @@ -3029,6 +3037,28 @@ string OpMetadataToString(const OpMetadata& metadata) { return Join(result, " "); } +string RandomDistributionToString(const RandomDistribution& distribution) { + return tensorflow::str_util::Lowercase(RandomDistribution_Name(distribution)); +} + +StatusOr StringToRandomDistribution(const string& name) { + static std::unordered_map* map = [] { + static auto* map = new std::unordered_map; + for (int i = 0; i < RandomDistribution_ARRAYSIZE; i++) { + if (RandomDistribution_IsValid(i)) { + auto value = static_cast(i); + (*map)[RandomDistributionToString(value)] = value; + } + } + return map; + }(); + auto found = map->find(tensorflow::str_util::Lowercase(name)); + if (found == map->end()) { + return InvalidArgument("Unknown distribution"); + } + return found->second; +} + std::ostream& operator<<(std::ostream& os, HloInstruction::FusionKind kind) { return os << ToString(kind); } diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index 768c027a42..088902e2a7 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -1285,9 +1285,12 @@ string ToString(HloInstruction::FusionKind kind); StatusOr StringToFusionKind( const string& kind_name); -// Custom stringification functions for protos that live inside HloInstruction. +// Custom (de)stringification functions for protos that live inside +// HloInstruction. string PaddingConfigToString(const PaddingConfig& padding); string OpMetadataToString(const OpMetadata& metadata); +string RandomDistributionToString(const RandomDistribution& distribution); +StatusOr StringToRandomDistribution(const string& name); std::ostream& operator<<(std::ostream& os, HloInstruction::FusionKind kind); diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD index ce936af6c3..97aacf6b39 100644 --- a/tensorflow/compiler/xla/tools/parser/BUILD +++ b/tensorflow/compiler/xla/tools/parser/BUILD @@ -34,9 +34,9 @@ cc_library( deps = [ "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/service:hlo", "//tensorflow/core:lib", "//tensorflow/core:regexp_internal", ], diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc index 56744440db..04247594ed 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -17,7 +17,6 @@ limitations under the License. #include -#include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/util.h" @@ -153,15 +152,15 @@ TokKind HloLexer::LexToken() { } } -// Lex a shape, name, keyword, opcode, attribute name, or the dim labels -// pattern. +// Lex a shape, name, keyword, attribute name, the dim labels pattern, and +// other identifiers. // // shape ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})? // name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: // keyword ::= HloModule, ENTRY, ... -// opcode ::= add, greater-than, ... // attribute_name ::= condition, body, dimensions, ... // dim_labels_pattern ::= [0-9bf]{2,}_[0-9io]{2,}->[0-9bf]{2,} +// identifiers ::= other cases that match [a-zA-Z_][a-zA-Z0-9_.-]* TokKind HloLexer::LexIdentifier() { { auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); @@ -220,20 +219,6 @@ TokKind HloLexer::LexIdentifier() { #undef KEYWORD - // See if this is an opcode. - auto opcode = StringToHloOpcode(identifier.ToString()); - if (opcode.ok()) { - opcode_val_ = opcode.ValueOrDie(); - return TokKind::kOpcode; - } - - // See if this is an fusion kind. - auto kind = xla::StringToFusionKind(identifier.ToString()); - if (kind.ok()) { - fusion_kind_val_ = kind.ValueOrDie(); - return TokKind::kFusionKind; - } - { auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); static LazyRE2 dim_labels_pattern = { @@ -244,8 +229,9 @@ TokKind HloLexer::LexIdentifier() { return TokKind::kDimLabels; } } - current_ptr_ = token_start_ + 1; - return TokKind::kError; + + str_val_ = identifier.ToString(); + return TokKind::kIdent; } // Lex names after a % character. @@ -428,14 +414,12 @@ string TokKindToString(TokKind kind) { return "kDxD"; case TokKind::kPad: return "kPad"; + case TokKind::kIdent: + return "kIdent"; case TokKind::kString: return "kString"; case TokKind::kShape: return "kShape"; - case TokKind::kOpcode: - return "kOpcode"; - case TokKind::kFusionKind: - return "kFusionKind"; case TokKind::kInt: return "kInt"; case TokKind::kDecimal: diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h index 5c9d1bf391..9daf6a11d3 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h @@ -18,9 +18,8 @@ limitations under the License. #include -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/tools/parser/hlo_token.h" +#include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/logging.h" @@ -48,6 +47,7 @@ class HloLexer { case TokKind::kDxD: case TokKind::kPad: case TokKind::kString: + case TokKind::kIdent: return str_val_; default: LOG(FATAL) << "This token does not have string value"; @@ -57,14 +57,6 @@ class HloLexer { CHECK(GetKind() == TokKind::kShape); return shape_val_; } - HloOpcode GetOpcodeVal() const { - CHECK(GetKind() == TokKind::kOpcode); - return opcode_val_; - } - HloInstruction::FusionKind GetFusionKindVal() const { - CHECK(GetKind() == TokKind::kFusionKind); - return fusion_kind_val_; - } int64 GetInt64Val() const { CHECK(GetKind() == TokKind::kInt); return int64_val_; @@ -114,8 +106,6 @@ class HloLexer { TokKind current_kind_; string str_val_; Shape shape_val_; - HloOpcode opcode_val_; - HloInstruction::FusionKind fusion_kind_val_; int64 int64_val_; double decimal_val_; }; diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc index 47979ec6f3..ddc1e69951 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" #include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/core/lib/gtl/map_util.h" @@ -104,6 +105,7 @@ class HloParser { kPaddingConfig, kMetadata, kFusionKind, + kDistribution, }; struct AttrConfig { @@ -174,6 +176,7 @@ class HloParser { bool ParseShape(Shape* result); bool ParseOpcode(HloOpcode* result); bool ParseFusionKind(HloInstruction::FusionKind* result); + bool ParseRandomDistribution(RandomDistribution* result); bool ParseInt64(int64* result); bool ParseDouble(double* result); bool ParseBool(bool* result); @@ -816,10 +819,36 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder, shape, operands[0], config ? *config : "")); break; } + case HloOpcode::kRng: { + optional distribution; + attrs["distribution"] = {/*required=*/true, AttrTy::kDistribution, + &distribution}; + if (!ParseOperands(&operands) || !ParseAttributes(attrs)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateRng(shape, *distribution, operands)); + break; + } + case HloOpcode::kReducePrecision: { + optional exponent_bits; + optional mantissa_bits; + attrs["exponent_bits"] = {/*required=*/true, AttrTy::kInt64, + &exponent_bits}; + attrs["mantissa_bits"] = {/*required=*/true, AttrTy::kInt64, + &mantissa_bits}; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseAttributes(attrs)) { + return false; + } + instruction = + builder->AddInstruction(HloInstruction::CreateReducePrecision( + shape, operands[0], static_cast(*exponent_bits), + static_cast(*mantissa_bits))); + break; + } case HloOpcode::kConditional: case HloOpcode::kCustomCall: - case HloOpcode::kReducePrecision: - case HloOpcode::kRng: case HloOpcode::kTrace: return TokenError(StrCat("parsing not yet implemented for op: ", HloOpcodeString(opcode))); @@ -1548,6 +1577,15 @@ bool HloParser::ParseAttributeHelper( static_cast*>(attr_out_ptr)->emplace(result); return true; } + case AttrTy::kDistribution: { + RandomDistribution result; + if (!ParseRandomDistribution(&result)) { + return false; + } + static_cast*>(attr_out_ptr) + ->emplace(result); + return true; + } } }(); if (!success) { @@ -2024,20 +2062,51 @@ bool HloParser::ParseMetadata(OpMetadata* metadata) { bool HloParser::ParseOpcode(HloOpcode* result) { VLOG(1) << "ParseOpcode"; - if (lexer_.GetKind() != TokKind::kOpcode) { + if (lexer_.GetKind() != TokKind::kIdent) { return TokenError("expects opcode"); } - *result = lexer_.GetOpcodeVal(); + string val = lexer_.GetStrVal(); + auto status_or_result = StringToHloOpcode(val); + if (!status_or_result.ok()) { + return TokenError( + Printf("expects opcode but sees: %s, error: %s", val.c_str(), + status_or_result.status().error_message().c_str())); + } + *result = status_or_result.ValueOrDie(); lexer_.Lex(); return true; } bool HloParser::ParseFusionKind(HloInstruction::FusionKind* result) { VLOG(1) << "ParseFusionKind"; - if (lexer_.GetKind() != TokKind::kFusionKind) { + if (lexer_.GetKind() != TokKind::kIdent) { return TokenError("expects fusion kind"); } - *result = lexer_.GetFusionKindVal(); + string val = lexer_.GetStrVal(); + auto status_or_result = StringToFusionKind(val); + if (!status_or_result.ok()) { + return TokenError( + Printf("expects fusion kind but sees: %s, error: %s", val.c_str(), + status_or_result.status().error_message().c_str())); + } + *result = status_or_result.ValueOrDie(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseRandomDistribution(RandomDistribution* result) { + VLOG(1) << "ParseRandomDistribution"; + if (lexer_.GetKind() != TokKind::kIdent) { + return TokenError("expects random distribution"); + } + string val = lexer_.GetStrVal(); + auto status_or_result = StringToRandomDistribution(val); + if (!status_or_result.ok()) { + return TokenError( + Printf("expects random distribution but sees: %s, error: %s", + val.c_str(), status_or_result.status().error_message().c_str())); + } + *result = status_or_result.ValueOrDie(); lexer_.Lex(); return true; } diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc index 90cdb87a1e..69d48d65bc 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -654,6 +654,31 @@ ENTRY %InfeedToOutfeed () -> (u32[3], pred[]) { %outfeed.1 = () outfeed((u32[3]{0}, pred[]) %infeed.1) } +)" +}, +// Rng +{ +"Rng", +R"(HloModule rng_module: + +ENTRY %Rng () -> f32[8] { + %constant = f32[] constant(0) + %constant.1 = f32[] constant(1) + ROOT %rng = f32[8]{0} rng(f32[] %constant, f32[] %constant.1), distribution=rng_uniform +} + +)" +}, +// Reduce precision +{ +"ReducePrevison", +R"(HloModule reduce_precision: + +ENTRY %ReducePrecision () -> f32[1] { + %constant = f32[1]{0} constant({3.14159}) + ROOT %reduce-precision = f32[1]{0} reduce-precision(f32[1]{0} %constant), exponent_bits=8, mantissa_bits=10 +} + )" } }); diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h index 07e48804d0..7928bee5c2 100644 --- a/tensorflow/compiler/xla/tools/parser/hlo_token.h +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -18,6 +18,9 @@ limitations under the License. #include +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/platform/types.h" + namespace xla { namespace tools { @@ -60,10 +63,9 @@ enum class TokKind { kDimLabels, // [0-9bf]{2,}_[0-9io]{2,}->[0-9bf]{2,} kDxD, // [0-9]+(x[0-9]+)+ kPad, // [0-9]+_[0-9]+(_[0-9]+)?(x[0-9]+_[0-9]+(_[0-9]+)?)* + kIdent, // other identifiers kString, // "abcd\"\n" kShape, // f32[2,3]{1,0} - kOpcode, // add - kFusionKind, // kLoop, kOutput, ... kInt, // 42 kDecimal, // 4.2 }; -- GitLab From e747fc911f0dc6f1bf0b9c0ac0b57ad1a704c542 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 23:20:35 -0800 Subject: [PATCH 0180/1924] Add additional linkopts argument to tf_custom_op_library. PiperOrigin-RevId: 177555877 --- tensorflow/tensorflow.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 76ef59484f..709a2d46e1 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -1197,7 +1197,7 @@ check_deps = rule( # Helper to build a dynamic library (.so) from the sources containing # implementations of custom ops and kernels. -def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[]): +def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]): cuda_deps = [ clean_dep("//tensorflow/core:stream_executor_headers_lib"), "@local_config_cuda//cuda:cuda_headers", @@ -1226,7 +1226,7 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[]): deps=deps + if_cuda(cuda_deps), data=[name + "_check_deps"], copts=tf_copts(), - linkopts=select({ + linkopts=linkopts + select({ "//conditions:default": [ "-lm", ], -- GitLab From 370e521762f3cbd558a7e56992e3b062236b626f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 23:46:38 -0800 Subject: [PATCH 0181/1924] Adds a fisher block for fully connected recurrent layers. `FullyConnectedSeriesFB` uses an approximation to the Fisher information matrix designed for RNNs. This CL also adds support for dtypes other than `float32` to `fisher_factors`. PiperOrigin-RevId: 177558080 --- .../python/kernel_tests/fisher_blocks_test.py | 54 ++- .../kernel_tests/fisher_factors_test.py | 117 +++++- tensorflow/contrib/kfac/python/ops/BUILD | 1 + .../contrib/kfac/python/ops/fisher_blocks.py | 210 +++++++++- .../contrib/kfac/python/ops/fisher_factors.py | 383 ++++++++++++++++-- tensorflow/contrib/kfac/python/ops/utils.py | 31 +- 6 files changed, 751 insertions(+), 45 deletions(-) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index 5f2b5c6cac..bdc950a4e6 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -301,8 +301,7 @@ class FullyConnectedDiagonalFB(test.TestCase): multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( self.w, [self.inputs], [self.outputs], [self.output_grads]) multiply_result_small, multiply_inverse_result_small = ( - self.runFisherBlockOps(self.w, - np.split(self.inputs, 2), + self.runFisherBlockOps(self.w, np.split(self.inputs, 2), np.split(self.outputs, 2), np.split(self.output_grads, 2))) @@ -584,8 +583,7 @@ class ConvDiagonalFBTest(test.TestCase): multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( self.w, [self.inputs], [self.outputs], [self.output_grads]) multiply_result_small, multiply_inverse_result_small = ( - self.runFisherBlockOps(self.w, - np.split(self.inputs, 2), + self.runFisherBlockOps(self.w, np.split(self.inputs, 2), np.split(self.outputs, 2), np.split(self.output_grads, 2))) @@ -608,8 +606,9 @@ class ConvDiagonalFBTest(test.TestCase): self.kernel_size, self.kernel_size, self.input_channels + 1, self.output_channels ]) - expected_result = (expected_result[:, :, 0:-1, :], np.reshape( - expected_result[:, :, -1, :], [self.output_channels])) + expected_result = (expected_result[:, :, 0:-1, :], + np.reshape(expected_result[:, :, -1, :], + [self.output_channels])) self.assertEqual(len(result), 2) self.assertAllClose(expected_result[0], result[0]) @@ -692,8 +691,8 @@ class ConvKFCBasicFBTest(test.TestCase): sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) - vector = (np.arange(1, 15).reshape(7, 2).astype(np.float32), np.arange( - 2, 4).reshape(2, 1).astype(np.float32)) + vector = (np.arange(1, 15).reshape(7, 2).astype(np.float32), + np.arange(2, 4).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) @@ -776,11 +775,50 @@ class ConvKFCBasicFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +class FullyConnectedSeriesFBTest(test.TestCase): + + def testFullyConnectedSeriesFBInit(self): + with ops.Graph().as_default(): + random_seed.set_random_seed(200) + inputs = array_ops.constant([1., 2.]) + outputs = array_ops.constant([3., 4.]) + block = fb.FullyConnectedSeriesFB( + lc.LayerCollection(), inputs=[inputs], outputs=[outputs]) + self.assertAllEqual([outputs], block.tensors_to_compute_grads()) + + def testInstantiateFactorsHasBias(self): + with ops.Graph().as_default(): + random_seed.set_random_seed(200) + inputs = array_ops.constant([[1., 2.], [3., 4.]]) + outputs = array_ops.constant([[3., 4.], [5., 6.]]) + block = fb.FullyConnectedSeriesFB( + lc.LayerCollection(), + inputs=[inputs], + outputs=[outputs], + has_bias=True) + grads = outputs**2 + block.instantiate_factors(((grads,),), 0.5) + + def testInstantiateFactorsNoBias(self): + with ops.Graph().as_default(): + random_seed.set_random_seed(200) + inputs = array_ops.constant([[1., 2.], [3., 4.]]) + outputs = array_ops.constant([[3., 4.], [5., 6.]]) + block = fb.FullyConnectedSeriesFB( + lc.LayerCollection(), + inputs=[inputs], + outputs=[outputs], + has_bias=False) + grads = outputs**2 + block.instantiate_factors(((grads,),), 0.5) + + def as_tensors(tensor_or_tuple): """Converts a potentially nested tuple of np.array to Tensors.""" if isinstance(tensor_or_tuple, (tuple, list)): return tuple(as_tensors(t) for t in tensor_or_tuple) return ops.convert_to_tensor(tensor_or_tuple) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py index 5e2ce5a309..f4a017fc77 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py @@ -67,6 +67,10 @@ class FisherFactorTestingDummy(ff.FisherFactor): def _num_sources(self): return 1 + @property + def _dtype(self): + return dtypes.float32 + def _compute_new_cov(self): raise NotImplementedError @@ -94,6 +98,10 @@ class InverseProvidingFactorTestingDummy(ff.InverseProvidingFactor): def _num_sources(self): return 1 + @property + def _dtype(self): + return dtypes.float32 + def _compute_new_cov(self): raise NotImplementedError @@ -121,7 +129,7 @@ class NumericalUtilsTest(test.TestCase): normalizer = 10. x = npr.randn(100, 3) - cov = ff._compute_cov(array_ops.constant(x), normalizer) + cov = ff._compute_cov(array_ops.constant(x), normalizer=normalizer) np_cov = np.dot(x.T, x) / normalizer self.assertAllClose(sess.run(cov), np_cov) @@ -267,13 +275,13 @@ class InverseProvidingFactorTest(test.TestCase): for i in range(1, ff.EIGENVALUE_DECOMPOSITION_THRESHOLD + 1): factor.register_damped_inverse(1. / i) ops = factor.make_inverse_update_ops() - self.assertEqual(ff.EIGENVALUE_DECOMPOSITION_THRESHOLD, len(ops)) + self.assertEqual(1, len(ops)) sess.run(tf_variables.global_variables_initializer()) new_invs = [] + sess.run(ops) for i in range(1, ff.EIGENVALUE_DECOMPOSITION_THRESHOLD + 1): # The inverse op will assign the damped inverse of cov to the inv var. - sess.run(ops[i - 1]) new_invs.append(sess.run(factor._inverses_by_damping[1. / i])) # We want to see that the new invs are all different from each other. for i in range(len(new_invs)): @@ -331,6 +339,16 @@ class FullFactorTest(test.TestCase): factor = ff.FullFactor((tensor,), 32) self.assertEqual([6, 6], factor.get_cov().get_shape().as_list()) + def testFullFactorInitFloat64(self): + with tf_ops.Graph().as_default(): + dtype = dtypes.float64_ref + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c') + factor = ff.FullFactor((tensor,), 32) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual([6, 6], cov.get_shape().as_list()) + def testMakeCovarianceUpdateOp(self): with tf_ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) @@ -351,6 +369,16 @@ class NaiveDiagonalFactorTest(test.TestCase): factor = ff.NaiveDiagonalFactor((tensor,), 32) self.assertEqual([6, 1], factor.get_cov().get_shape().as_list()) + def testNaiveDiagonalFactorInitFloat64(self): + with tf_ops.Graph().as_default(): + dtype = dtypes.float64_ref + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c') + factor = ff.NaiveDiagonalFactor((tensor,), 32) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual([6, 1], cov.get_shape().as_list()) + def testMakeCovarianceUpdateOp(self): with tf_ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) @@ -364,18 +392,25 @@ class NaiveDiagonalFactorTest(test.TestCase): class FullyConnectedKroneckerFactorTest(test.TestCase): - def _testFullyConnectedKroneckerFactorInit(self, has_bias, final_shape): + def _testFullyConnectedKroneckerFactorInit(self, + has_bias, + final_shape, + dtype=dtypes.float32_ref): with tf_ops.Graph().as_default(): random_seed.set_random_seed(200) - tensor = array_ops.ones((2, 3), name='a/b/c') + tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c') factor = ff.FullyConnectedKroneckerFactor((tensor,), has_bias=has_bias) - self.assertEqual(final_shape, factor.get_cov().get_shape().as_list()) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual(final_shape, cov.get_shape().as_list()) def testFullyConnectedKroneckerFactorInitNoBias(self): - self._testFullyConnectedKroneckerFactorInit(False, [3, 3]) + for dtype in (dtypes.float32_ref, dtypes.float64_ref): + self._testFullyConnectedKroneckerFactorInit(False, [3, 3], dtype=dtype) def testFullyConnectedKroneckerFactorInitWithBias(self): - self._testFullyConnectedKroneckerFactorInit(True, [4, 4]) + for dtype in (dtypes.float32_ref, dtypes.float64_ref): + self._testFullyConnectedKroneckerFactorInit(True, [4, 4], dtype=dtype) def testMakeCovarianceUpdateOpWithBias(self): with tf_ops.Graph().as_default(), self.test_session() as sess: @@ -418,6 +453,18 @@ class ConvInputKroneckerFactorTest(test.TestCase): self.assertEqual([1 * 2 * 3 + 1, 1 * 2 * 3 + 1], factor.get_cov().get_shape().as_list()) + def testConvInputKroneckerFactorInitFloat64(self): + with tf_ops.Graph().as_default(): + dtype = dtypes.float64_ref + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c') + factor = ff.ConvInputKroneckerFactor( + tensor, (1, 2, 3, 4), 3, 2, has_bias=True) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual([1 * 2 * 3 + 1, 1 * 2 * 3 + 1], + cov.get_shape().as_list()) + def testMakeCovarianceUpdateOpWithBias(self): with tf_ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) @@ -453,6 +500,16 @@ class ConvOutputKroneckerFactorTest(test.TestCase): factor = ff.ConvOutputKroneckerFactor((tensor,)) self.assertEqual([5, 5], factor.get_cov().get_shape().as_list()) + def testConvOutputKroneckerFactorInitFloat64(self): + with tf_ops.Graph().as_default(): + dtype = dtypes.float64_ref + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3, 4, 5), dtype=dtype, name='a/b/c') + factor = ff.ConvOutputKroneckerFactor((tensor,)) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual([5, 5], cov.get_shape().as_list()) + def testConvOutputKroneckerFactorInitNotEnoughDims(self): with tf_ops.Graph().as_default(): random_seed.set_random_seed(200) @@ -471,5 +528,49 @@ class ConvOutputKroneckerFactorTest(test.TestCase): self.assertAllClose([[43, 46.5], [46.5, 51.5]], new_cov) +class FullyConnectedMultiKFTest(test.TestCase): + + def testFullyConnectedMultiKFInit(self): + with tf_ops.Graph().as_default(): + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3), name='a/b/c') + tensor_list = [tensor] + factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=False) + self.assertEqual([3, 3], factor.get_cov().get_shape().as_list()) + + def testFullyConnectedMultiKFInitFloat64(self): + with tf_ops.Graph().as_default(): + dtype = dtypes.float64_ref + random_seed.set_random_seed(200) + tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c') + tensor_list = [tensor] + factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=False) + cov = factor.get_cov() + self.assertEqual(cov.dtype, dtype) + self.assertEqual([3, 3], cov.get_shape().as_list()) + + def testMakeCovarianceUpdateOpWithBias(self): + with tf_ops.Graph().as_default(), self.test_session() as sess: + random_seed.set_random_seed(200) + tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c') + tensor_list = [tensor] + factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=True) + + sess.run(tf_variables.global_variables_initializer()) + new_cov = sess.run(factor.make_covariance_update_op(.5)) + self.assertAllClose([[3, 3.5, 1], [3.5, 5.5, 1.5], [1, 1.5, 1]], new_cov) + + def testMakeCovarianceUpdateOpNoBias(self): + with tf_ops.Graph().as_default(), self.test_session() as sess: + random_seed.set_random_seed(200) + tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c') + tensor_list = [tensor] + factor = ff.FullyConnectedMultiKF((tensor_list,)) + + sess.run(tf_variables.global_variables_initializer()) + new_cov = sess.run(factor.make_covariance_update_op(.5)) + self.assertAllClose([[3, 3.5], [3.5, 5.5]], new_cov) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD index b2272a4cee..3d731c7bc2 100644 --- a/tensorflow/contrib/kfac/python/ops/BUILD +++ b/tensorflow/contrib/kfac/python/ops/BUILD @@ -38,6 +38,7 @@ py_library( ":utils", "//tensorflow/python:array_ops", "//tensorflow/python:framework_ops", + "//tensorflow/python:init_ops", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:special_math_ops", diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index e822a1213a..cf734d56ad 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -38,6 +38,7 @@ from __future__ import division from __future__ import print_function import abc +import enum # pylint: disable=g-bad-import-order import six @@ -153,7 +154,7 @@ class FullFB(FisherBlock): self._factor.register_damped_inverse(damping) def multiply_inverse(self, vector): - inverse = self._factor.get_inverse(self._damping) + inverse = self._factor.get_damped_inverse(self._damping) out_flat = math_ops.matmul(inverse, utils.tensors_to_column(vector)) return utils.column_to_tensors(vector, out_flat) @@ -411,7 +412,7 @@ class ConvDiagonalFB(FisherBlock): (self._strides[1] * self._strides[2])) if NORMALIZE_DAMPING_POWER: - damping /= self._num_locations ** NORMALIZE_DAMPING_POWER + damping /= self._num_locations**NORMALIZE_DAMPING_POWER self._damping = damping self._factor = self._layer_collection.make_or_get_factor( @@ -487,8 +488,9 @@ class KroneckerProductFB(FisherBlock): return 1.0 def multiply_inverse(self, vector): - left_factor_inv = self._input_factor.get_inverse(self._input_damping) - right_factor_inv = self._output_factor.get_inverse(self._output_damping) + left_factor_inv = self._input_factor.get_damped_inverse(self._input_damping) + right_factor_inv = self._output_factor.get_damped_inverse( + self._output_damping) reshaped_vector = utils.layer_params_to_mat2d(vector) reshaped_out = math_ops.matmul(left_factor_inv, math_ops.matmul(reshaped_vector, @@ -720,3 +722,203 @@ def _concat_along_batch_dim(tensor_list): def _num_conv_locations(input_shape, strides): """Returns the number of locations a Conv kernel is applied to.""" return input_shape[1] * input_shape[2] // (strides[1] * strides[2]) + + +class SeriesFBApproximation(enum.IntEnum): + """See FullyConnectedSeriesFB.__init__ for description and usage.""" + option1 = 1 + option2 = 2 + + +class FullyConnectedSeriesFB(FisherBlock): + """FisherBlock for fully-connected RNN cells. + + See the following preprint for details: + https://openreview.net/pdf?id=HyMTkQZAb + + See the end of the appendix of the paper for a pseudo-code of the + algorithm being implemented by multiply_inverse here. Note that we are + using pre-computed versions of certain matrix-matrix products to speed + things up. This is explicitly explained wherever it is done. + """ + + def __init__(self, + layer_collection, + inputs, + outputs, + has_bias=False, + option=SeriesFBApproximation.option2): + """Constructs a new `FullyConnectedSeriesFB`. + + Args: + layer_collection: The collection of all layers in the K-FAC approximate + Fisher information matrix to which this FisherBlock belongs. + inputs: List of tensors of shape [batch_size, input_size]. + Inputs to the layer. + outputs: List of tensors of shape [batch_size, input_size]. + Outputs of the layer (before activations). + has_bias: Whether the layer includes a bias parameter. + option: A `SeriesFBApproximation` specifying the simplifying assumption + to be used in this block. `option1` approximates the cross-covariance + over time as a symmetric matrix, while `option2` makes + the assumption that training sequences are infinitely long. See section + 3.5 of the paper for more details. + """ + + assert len(inputs) == len(outputs) + # We need to make sure inputs and outputs are tuples and not lists so that + # they get hashed by layer_collection.make_or_get_factor properly. + self._inputs = tuple(inputs) + self._outputs = tuple(outputs) + self._has_bias = has_bias + self._num_timesteps = len(inputs) + self._option = option + + super(FullyConnectedSeriesFB, self).__init__(layer_collection) + + @property + def num_registered_minibatches(self): + # TODO(b/69411207): Add support for registering additional minibatches. + return 1 + + def instantiate_factors(self, grads_list, damping): + + self._input_factor = self._layer_collection.make_or_get_factor( + fisher_factors.FullyConnectedMultiKF, ((self._inputs,), self._has_bias)) + + self._output_factor = self._layer_collection.make_or_get_factor( + fisher_factors.FullyConnectedMultiKF, (grads_list,)) + + damping /= self._num_timesteps**NORMALIZE_DAMPING_POWER + + pi = utils.compute_pi(self._input_factor.get_cov(), + self._output_factor.get_cov()) + + self._damping_input = (damping**0.5) * pi + self._damping_output = (damping**0.5) / pi + + if self._option == SeriesFBApproximation.option1: + self._input_factor.register_option1quants(self._damping_input) + self._output_factor.register_option1quants(self._damping_output) + elif self._option == SeriesFBApproximation.option2: + self._input_factor.register_option2quants(self._damping_input) + self._output_factor.register_option2quants(self._damping_output) + else: + raise ValueError( + "Unrecognized FullyConnectedSeriesFB approximation: {}".format( + self._option)) + + def multiply_inverse(self, vector): + # pylint: disable=invalid-name + + Z = utils.layer_params_to_mat2d(vector) + + # Derivations were done for "batch_dim==1" case so we need to convert to + # that orientation: + Z = array_ops.transpose(Z) + + if self._option == SeriesFBApproximation.option1: + + # Note that L_A = A0^(-1/2) * U_A and L_G = G0^(-1/2) * U_G. + L_A, psi_A = self._input_factor.get_option1quants(self._damping_input) + L_G, psi_G = self._output_factor.get_option1quants(self._damping_output) + + def gamma(x): + # We are assuming that each case has the same number of time-steps. + # If this stops being the case one shouldn't simply replace this T + # with its average value. Instead, one needs to go back to the + # definition of the gamma function from the paper. + T = self._num_timesteps + return (1 - x)**2 / (T * (1 - x**2) - 2 * x * (1 - x**T)) + + # Y = gamma( psi_G*psi_A^T ) (computed element-wise) + # Even though Y is Z-independent we are recomputing it from the psi's + # each since Y depends on both A and G quantities, and it is relatively + # cheap to compute. + Y = gamma(array_ops.reshape(psi_G, [int(psi_G.shape[0]), -1]) * psi_A) + + # Z = L_G^T * Z * L_A + # This is equivalent to the following computation from the original + # pseudo-code: + # Z = G0^(-1/2) * Z * A0^(-1/2) + # Z = U_G^T * Z * U_A + Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A), transpose_a=True) + + # Z = Z .* Y + Z *= Y + + # Z = L_G * Z * L_A^T + # This is equivalent to the following computation from the original + # pseudo-code: + # Z = U_G * Z * U_A^T + # Z = G0^(-1/2) * Z * A0^(-1/2) + Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A, transpose_b=True)) + + elif self._option == SeriesFBApproximation.option2: + + # Note that P_A = A_1^T * A_0^(-1) and P_G = G_1^T * G_0^(-1), + # and K_A = A_0^(-1/2) * E_A and K_G = G_0^(-1/2) * E_G. + P_A, K_A, mu_A = self._input_factor.get_option2quants(self._damping_input) + P_G, K_G, mu_G = self._output_factor.get_option2quants( + self._damping_output) + + # Our approach differs superficially from the pseudo-code in the paper + # in order to reduce the total number of matrix-matrix multiplies. + # In particular, the first three computations in the pseudo code are + # Z = G0^(-1/2) * Z * A0^(-1/2) + # Z = Z - hPsi_G^T * Z * hPsi_A + # Z = E_G^T * Z * E_A + # Noting that hPsi = C0^(-1/2) * C1 * C0^(-1/2), so that + # C0^(-1/2) * hPsi = C0^(-1) * C1 * C0^(-1/2) = P^T * C0^(-1/2) + # the entire computation can be written as + # Z = E_G^T * (G0^(-1/2) * Z * A0^(-1/2) + # - hPsi_G^T * G0^(-1/2) * Z * A0^(-1/2) * hPsi_A) * E_A + # = E_G^T * (G0^(-1/2) * Z * A0^(-1/2) + # - G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2)) * E_A + # = E_G^T * G0^(-1/2) * Z * A0^(-1/2) * E_A + # - E_G^T* G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2) * E_A + # = K_G^T * Z * K_A - K_G^T * P_G * Z * P_A^T * K_A + # This final expression is computed by the following two lines: + # Z = Z - P_G * Z * P_A^T + Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A, transpose_b=True)) + # Z = K_G^T * Z * K_A + Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A), transpose_a=True) + + # Z = Z ./ (1*1^T - mu_G*mu_A^T) + # Be careful with the outer product. We don't want to accidentally + # make it an inner-product instead. + tmp = 1.0 - array_ops.reshape(mu_G, [int(mu_G.shape[0]), -1]) * mu_A + # Prevent some numerical issues by setting 0 eigs to 1.0 + tmp += 1.0 * array_ops.cast(math_ops.equal(tmp, 0.0), dtype=tmp.dtype) + Z /= tmp + + # We now perform the transpose/reverse version of the operations + # derived above, whose derivation from the original pseudo-code is + # analgous. + # Z = K_G * Z * K_A^T + Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A, transpose_b=True)) + + # Z = Z - P_G^T * Z * P_A + Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A), transpose_a=True) + + # Z = normalize (1/E[T]) * Z + # Note that this normalization is done because we compute the statistics + # by averaging, not summing, over time. (And the gradient is presumably + # summed over time, not averaged, and thus their scales are different.) + Z /= array_ops.cast(self._num_timesteps, Z.dtype) + + # Convert back to the "batch_dim==0" orientation. + Z = array_ops.transpose(Z) + + return utils.mat2d_to_layer_params(vector, Z) + + # pylint: enable=invalid-name + + def multiply(self, vector): + raise NotImplementedError + + def tensors_to_compute_grads(self): + return self._outputs + + def num_inputs(self): + return len(self._inputs) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index 6c1dd0ae40..ff8636785a 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -27,6 +27,8 @@ import six from tensorflow.contrib.kfac.python.ops import utils from tensorflow.python.framework import ops as tf_ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import special_math_ops @@ -101,7 +103,7 @@ def diagonal_covariance_initializer(shape, dtype, partition_info): # pylint: di return array_ops.ones(shape, dtype) -def _compute_cov(tensor, normalizer=None): +def _compute_cov(tensor, tensor_right=None, normalizer=None): """Compute the empirical second moment of the rows of a 2D Tensor. This function is meant to be applied to random matrices for which the true row @@ -109,6 +111,8 @@ def _compute_cov(tensor, normalizer=None): Args: tensor: A 2D Tensor. + tensor_right: An optional 2D Tensor. If provided, this function computes + the matrix product tensor^T * tensor_right instead of tensor^T * tensor. normalizer: optional scalar for the estimator (by default, the normalizer is the number of rows of tensor). @@ -117,9 +121,14 @@ def _compute_cov(tensor, normalizer=None): """ if normalizer is None: normalizer = array_ops.shape(tensor)[0] - cov = (math_ops.matmul(tensor, tensor, transpose_a=True) / math_ops.cast( - normalizer, tensor.dtype)) - return (cov + array_ops.transpose(cov)) / math_ops.cast(2, cov.dtype) + if tensor_right is None: + cov = ( + math_ops.matmul(tensor, tensor, transpose_a=True) / math_ops.cast( + normalizer, tensor.dtype)) + return (cov + array_ops.transpose(cov)) / math_ops.cast(2.0, cov.dtype) + else: + return (math_ops.matmul(tensor, tensor_right, transpose_a=True) / + math_ops.cast(normalizer, tensor.dtype)) def _append_homog(tensor): @@ -135,7 +144,7 @@ def _append_homog(tensor): rank = len(tensor.shape.as_list()) shape = array_ops.concat([array_ops.shape(tensor)[:-1], [1]], axis=0) ones = array_ops.ones(shape, dtype=tensor.dtype) - return array_ops.concat([tensor, ones], axis=rank-1) + return array_ops.concat([tensor, ones], axis=rank - 1) def scope_string_from_params(params): @@ -173,8 +182,8 @@ def scope_string_from_params(params): elif isinstance(param, (tf_ops.Tensor, variables.Variable)): name_parts.append(scope_string_from_name(param)) else: - raise ValueError( - "Encountered an unsupported param type {}".format(type(param))) + raise ValueError("Encountered an unsupported param type {}".format( + type(param))) return "_".join(name_parts) @@ -225,6 +234,10 @@ class FisherFactor(object): """ pass + @abc.abstractproperty + def _dtype(self): + pass + @property def _cov_initializer(self): return covariance_initializer @@ -236,7 +249,8 @@ class FisherFactor(object): "cov", initializer=self._cov_initializer, shape=self._cov_shape, - trainable=False) + trainable=False, + dtype=self._dtype) @abc.abstractmethod def _compute_new_cov(self, idx=0): @@ -273,6 +287,13 @@ class InverseProvidingFactor(FisherFactor): _cov_shape properties. """ + # TODO(b/69108481): This class (and its subclasses) should be refactored to + # serve the matrix quantities it computes as both (potentially stale) + # variables, updated by the inverse update ops, and fresh values stored in + # tensors that recomputed once every session.run() call. Currently matpower + # and damp_inverse have the former behavior, while eigendecomposition has + # the latter. + def __init__(self): self._inverses_by_damping = {} self._matpower_by_exp_and_damping = {} @@ -293,7 +314,8 @@ class InverseProvidingFactor(FisherFactor): "inv_damp{}".format(damping_string), initializer=inverse_initializer, shape=self._cov_shape, - trainable=False) + trainable=False, + dtype=self._dtype) self._inverses_by_damping[damping] = inv def register_matpower(self, exp, damping): @@ -311,7 +333,8 @@ class InverseProvidingFactor(FisherFactor): "matpower_exp{}_damp{}".format(exp_string, damping_string), initializer=inverse_initializer, shape=self._cov_shape, - trainable=False) + trainable=False, + dtype=self._dtype) self._matpower_by_exp_and_damping[(exp, damping)] = matpower def register_eigendecomp(self): @@ -325,8 +348,9 @@ class InverseProvidingFactor(FisherFactor): num_inverses = len(self._inverses_by_damping) matrix_power_registered = bool(self._matpower_by_exp_and_damping) - use_eig = (self._eigendecomp or matrix_power_registered or - num_inverses >= EIGENVALUE_DECOMPOSITION_THRESHOLD) + use_eig = ( + self._eigendecomp or matrix_power_registered or + num_inverses >= EIGENVALUE_DECOMPOSITION_THRESHOLD) if use_eig: self.register_eigendecomp() # ensures self._eigendecomp is set @@ -347,21 +371,30 @@ class InverseProvidingFactor(FisherFactor): for (exp, damping), matpower in self._matpower_by_exp_and_damping.items(): ops.append( matpower.assign( - math_ops.matmul(eigenvectors * (clipped_eigenvalues + damping)** - exp, array_ops.transpose(eigenvectors)))) + math_ops.matmul(eigenvectors * + (clipped_eigenvalues + damping)**exp, + array_ops.transpose(eigenvectors)))) + # These ops share computation and should be run on a single device. + ops = [control_flow_ops.group(ops)] else: for damping, inv in self._inverses_by_damping.items(): ops.append(inv.assign(utils.posdef_inv(self._cov, damping))) return ops - def get_inverse(self, damping): + def get_damped_inverse(self, damping): return self._inverses_by_damping[damping] def get_matpower(self, exp, damping): + # Note that this function returns a variable which gets updated by the + # inverse ops. It may be stale / inconsistent with the latest value of + # get_cov(). return self._matpower_by_exp_and_damping[(exp, damping)] def get_eigendecomp(self): + # Unlike get_inverse and get_matpower this doesn't retrieve a stored + # variable, but instead always computes a fresh version from the current + # value of get_cov(). return self._eigendecomp @@ -402,6 +435,10 @@ class FullFactor(InverseProvidingFactor): def _num_sources(self): return len(self._params_grads_flat) + @property + def _dtype(self): + return self._params_grads_flat[0].dtype + def _compute_new_cov(self, idx=0): # This will be a very basic rank 1 estimate with _maybe_colocate_with(self._params_grads_flat[idx], @@ -458,6 +495,10 @@ class NaiveDiagonalFactor(DiagonalFactor): def _num_sources(self): return len(self._params_grads) + @property + def _dtype(self): + return self._params_grads[0].dtype + def _compute_new_cov(self, idx=0): with _maybe_colocate_with(self._params_grads[idx], self._colocate_cov_ops_with_inputs): @@ -497,8 +538,8 @@ class FullyConnectedDiagonalFactor(DiagonalFactor): self._outputs_grads = outputs_grads self._colocate_cov_ops_with_inputs = colocate_cov_ops_with_inputs self._batch_size = array_ops.shape(inputs)[0] - self._orig_tensors_name = scope_string_from_params((inputs,) + - tuple(outputs_grads)) + self._orig_tensors_name = scope_string_from_params( + (inputs,) + tuple(outputs_grads)) # Note that we precompute the required operations on the inputs since the # inputs don't change with the 'idx' argument to _compute_new_cov. (Only @@ -522,6 +563,10 @@ class FullyConnectedDiagonalFactor(DiagonalFactor): def _num_sources(self): return len(self._outputs_grads) + @property + def _dtype(self): + return self._outputs_grads[0].dtype + def _compute_new_cov(self, idx=0): # The well-known special formula that uses the fact that the entry-wise # square of an outer product is the outer-product of the entry-wise squares. @@ -572,8 +617,8 @@ class ConvDiagonalFactor(DiagonalFactor): self._outputs_grads = outputs_grads self._colocate_cov_ops_with_inputs = colocate_cov_ops_with_inputs - self._orig_tensors_name = scope_string_from_name((inputs,) - + tuple(outputs_grads)) + self._orig_tensors_name = scope_string_from_name( + (inputs,) + tuple(outputs_grads)) # Note that we precompute the required operations on the inputs since the # inputs don't change with the 'idx' argument to _compute_new_cov. (Only @@ -604,13 +649,19 @@ class ConvDiagonalFactor(DiagonalFactor): @property def _cov_shape(self): filter_height, filter_width, in_channels, out_channels = self._filter_shape - return [filter_height * filter_width * in_channels + self._has_bias, - out_channels] + return [ + filter_height * filter_width * in_channels + self._has_bias, + out_channels + ] @property def _num_sources(self): return len(self._outputs_grads) + @property + def _dtype(self): + return self._outputs_grads[0].dtype + def _compute_new_cov(self, idx=0): with _maybe_colocate_with(self._outputs_grads[idx], self._colocate_cov_ops_with_inputs): @@ -644,8 +695,7 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): Args: tensors: List of Tensors of shape [batch_size, n]. Represents either a layer's inputs or its output's gradients. - has_bias: bool. If True, assume this factor is for the layer's inputs and - append '1' to each row. + has_bias: bool. If True, append '1' to each row. colocate_cov_ops_with_inputs: Whether to colocate cov_update ops with their inputs. """ @@ -670,6 +720,10 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): def _num_sources(self): return len(self._tensors) + @property + def _dtype(self): + return self._tensors[0].dtype + def _compute_new_cov(self, idx=0): with _maybe_colocate_with(self._tensors[idx], self._colocate_cov_ops_with_inputs): @@ -735,6 +789,10 @@ class ConvInputKroneckerFactor(InverseProvidingFactor): def _num_sources(self): return 1 + @property + def _dtype(self): + return self._inputs.dtype + def _compute_new_cov(self, idx=0): if idx != 0: raise ValueError("ConvInputKroneckerFactor only supports idx = 0") @@ -799,9 +857,288 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor): def _num_sources(self): return len(self._outputs_grads) + @property + def _dtype(self): + return self._outputs_grads[0].dtype + def _compute_new_cov(self, idx=0): with _maybe_colocate_with(self._outputs_grads[idx], self._colocate_cov_ops_with_inputs): reshaped_tensor = array_ops.reshape(self._outputs_grads[idx], [-1, self._out_channels]) return _compute_cov(reshaped_tensor) + + +class FullyConnectedMultiKF(InverseProvidingFactor): + """Kronecker factor for a fully connected recurrent layer.""" + + def __init__(self, + tensor_lists, + has_bias=False, + colocate_cov_ops_with_inputs=False): + """Constructs a new `FullyConnectedMultiKF`. + + Args: + tensor_lists: List of lists of Tensors of shape [batch_size, n]. Layer + inputs at each timestep. + has_bias: bool. If True, assume this factor is for the layer's inputs and + append '1' to each row. + colocate_cov_ops_with_inputs: Whether to colocate cov_update ops with + their inputs. + """ + + self._orig_tensors_name = scope_string_from_params(tensor_lists) + self._batch_size = array_ops.shape(tensor_lists[0][0])[0] + self._num_timesteps = len(tensor_lists[0]) + + tensors = tuple( + array_ops.concat(tensor_list, 0) for tensor_list in tensor_lists) + if has_bias: + tensors = tuple(_append_homog(tensor) for tensor in tensors) + self._tensors = tensors + + self._cov_dt1 = None + self._option1quants_by_damping = {} + self._option2quants_by_damping = {} + self._colocate_cov_ops_with_inputs = colocate_cov_ops_with_inputs + + super(FullyConnectedMultiKF, self).__init__() + + @property + def _var_scope(self): + return "ff_fc_multi/" + self._orig_tensors_name + + @property + def _num_sources(self): + return len(self._tensors) + + @property + def _dtype(self): + return self._tensors[0].dtype + + def make_covariance_update_op(self, ema_decay): + with _maybe_colocate_with(self._tensors, + self._colocate_cov_ops_with_inputs): + op = super(FullyConnectedMultiKF, + self).make_covariance_update_op(ema_decay) + + if self._cov_dt1 is not None: + new_cov_dt1 = math_ops.add_n( + tuple( + self._compute_new_cov_dt1(idx) + for idx in range(self._num_sources))) + op2 = moving_averages.assign_moving_average( + self._cov_dt1, new_cov_dt1, ema_decay, zero_debias=ZERO_DEBIAS) + + # TODO(b/69112164): + # It's important that _cov and _cov_dt1 remain consistent with each + # other while the inverse ops are happening. How can we ensure this? + # We will need to add explicit synchronization for this to + # work with asynchronous training. + op = control_flow_ops.group(op, op2) + + return op + + def _compute_new_cov(self, idx=0): + tensor = self._tensors[idx] + normalizer = self._num_timesteps * self._batch_size + return _compute_cov(tensor, normalizer=normalizer) + + def _compute_new_cov_dt1(self, idx=0): + tensor = self._tensors[idx] + normalizer = self._num_timesteps * self._batch_size + tensor_present = tensor[:-self._batch_size, :] + tensor_future = tensor[self._batch_size:, :] + return _compute_cov( + tensor_future, tensor_right=tensor_present, normalizer=normalizer) + + @property + def _cov_shape(self): + size = self._tensors[0].shape[1] + return [size, size] + + @property + def _vec_shape(self): + size = self._tensors[0].shape[1] + return [size] + + def get_option1quants(self, damping): + return self._option1quants_by_damping[damping] + + def get_option2quants(self, damping): + return self._option2quants_by_damping[damping] + + def get_cov_dt1(self): + assert self._cov_dt1 is not None + return self._cov_dt1 + + def register_cov_dt1(self): + """Create a variable representing temporal cross-covariance. + + This is technically the second moment, not covariance, since it's + not mean subtracted. + """ + if self._cov_dt1 is None: + with variable_scope.variable_scope(self._var_scope): + self._cov_dt1 = variable_scope.get_variable( + "cov_dt1", + initializer=self._cov_initializer, + shape=self._cov_shape, + trainable=False, + dtype=self._dtype) + + def register_option1quants(self, damping): + + self.register_eigendecomp() + self.register_cov_dt1() + + if damping not in self._option1quants_by_damping: + # It's questionable as to whether we should initialize with stuff like + # this at all. Ideally these values should never be used until they are + # updated at least once. + damping_string = scalar_or_tensor_to_string(damping) + with variable_scope.variable_scope(self._var_scope): + Lmat = variable_scope.get_variable( # pylint: disable=invalid-name + "Lmat_damp{}".format(damping_string), + initializer=inverse_initializer, + shape=self._cov_shape, + trainable=False, + dtype=self._dtype) + psi = variable_scope.get_variable( + "psi_damp{}".format(damping_string), + initializer=init_ops.ones_initializer, + shape=self._vec_shape, + trainable=False, + dtype=self._dtype) + + self._option1quants_by_damping[damping] = (Lmat, psi) + + def register_option2quants(self, damping): + + self.register_eigendecomp() + self.register_cov_dt1() + + if damping not in self._option2quants_by_damping: + # It's questionable as to whether we should initialize with stuff like + # this at all. Ideally these values should never be used until they are + # updated at least once. + damping_string = scalar_or_tensor_to_string(damping) + with variable_scope.variable_scope(self._var_scope): + Pmat = variable_scope.get_variable( # pylint: disable=invalid-name + "Lmat_damp{}".format(damping_string), + initializer=inverse_initializer, + shape=self._cov_shape, + trainable=False, + dtype=self._dtype) + Kmat = variable_scope.get_variable( # pylint: disable=invalid-name + "Kmat_damp{}".format(damping_string), + initializer=inverse_initializer, + shape=self._cov_shape, + trainable=False, + dtype=self._dtype) + mu = variable_scope.get_variable( + "mu_damp{}".format(damping_string), + initializer=init_ops.ones_initializer, + shape=self._vec_shape, + trainable=False, + dtype=self._dtype) + + self._option2quants_by_damping[damping] = (Pmat, Kmat, mu) + + def make_inverse_updates_ops(self): + """Create and return update ops corresponding to registered computations.""" + # TODO(b/69918258): Add correctness tests for this method. + # pylint: disable=invalid-name + + ops = super(FullyConnectedMultiKF, self).make_inverse_update_ops() + + if (len(self._option1quants_by_damping) + + len(self._option2quants_by_damping)): + + # Note that C0 and C1 are stand-ins for A0 and A1, or G0 and G1, from + # the pseudo-code in the original paper. Because the computations for + # the A and G case are essentially the same they can both be performed by + # the same class (this one). + + C1 = self.get_cov_dt1() + + # Get the eigendecomposition of C0 (= self.get_cov()) + eigen_e, eigen_V = self.get_eigendecomp() + + # TODO(b/69678661): Note, there is an implicit assumption here that C1 + # and C0 (as represented here by its eigen-decomp) are consistent. This + # could fail to be the case if self._cov and self._cov_dt1 are not updated + # consistently, or are somehow read between or during the cov updates. + # Can this possibly happen? Is there a way to prevent it? + + for damping, (Lmat_var, + psi_var) in self._option1quants_by_damping.items(): + + invsqrtC0 = math_ops.matmul( + eigen_V * (eigen_e + damping)**(-0.5), eigen_V, transpose_b=True) + + # Might need to enforce symmetry lost due to numerical issues. + invsqrtC0 = (invsqrtC0 + array_ops.transpose(invsqrtC0)) / 2.0 + + # The following line imposses the symmetry assumed by "Option 1" on C1. + # Stangely the code can work okay with this line commented out, + # depending on how psd_eig is defined. I'm not sure why. + C1 = (C1 + array_ops.transpose(C1)) / 2.0 + + # hPsi = C0^(-1/2) * C1 * C0^(-1/2) (hPsi means \hat{Psi}) + hPsi = math_ops.matmul(math_ops.matmul(invsqrtC0, C1), invsqrtC0) + + # Compute the decomposition U*diag(psi)*U^T = hPsi + psi, U = utils.psd_eig(hPsi) + + # L = C0^(-1/2) * U + Lmat = math_ops.matmul(invsqrtC0, U) + + ops.append(Lmat_var.assign(Lmat)) + ops.append(psi_var.assign(psi)) + + for damping, (Pmat_var, Kmat_var, + mu_var) in self._option2quants_by_damping.items(): + + # compute C0^(-1/2) + invsqrtC0 = math_ops.matmul( + eigen_V * (eigen_e + damping)**(-0.5), eigen_V, transpose_b=True) + + # Might need to enforce symmetry lost due to numerical issues. + invsqrtC0 = (invsqrtC0 + array_ops.transpose(invsqrtC0)) / 2.0 + + # Compute the product C0^(-1/2) * C1 + invsqrtC0C1 = math_ops.matmul(invsqrtC0, C1) + + # hPsi = C0^(-1/2) * C1 * C0^(-1/2) (hPsi means \hat{Psi}) + hPsi = math_ops.matmul(invsqrtC0C1, invsqrtC0) + + # Compute the decomposition E*diag(mu)*E^T = hPsi^T * hPsi + # Note that we using the notation mu instead of "m" for the eigenvalues. + # Instead of computing the product hPsi^T * hPsi and then doing an + # eigen-decomposition of this we just compute the SVD of hPsi and then + # square the singular values to get the eigenvalues. For a justification + # of this approach, see: + # https://en.wikipedia.org/wiki/Singular-value_decomposition#Relation_to_eigenvalue_decomposition + sqrtmu, _, E = linalg_ops.svd(hPsi) + mu = math_ops.square(sqrtmu) + + # Mathematically, the eigenvalues should not should not exceed 1.0, but + # due to numerical issues, or possible issues with inconsistent + # values of C1 and (the eigen-decomposition of) C0 they might. So + # we enforce this condition. + mu = math_ops.minimum(mu, 1.0) + + # P = (C0^(-1/2) * C1)^T * C0^(-1/2) = C_1^T * C_0^(-1) + Pmat = math_ops.matmul(invsqrtC0C1, invsqrtC0, transpose_a=True) + + # K = C_0^(-1/2) * E + Kmat = math_ops.matmul(invsqrtC0, E) + + ops.append(Pmat_var.assign(Pmat)) + ops.append(Kmat_var.assign(Kmat)) + ops.append(mu_var.assign(mu)) + + return [control_flow_ops.group(ops)] + + # pylint: enable=invalid-name diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py index d5461c9f2e..035f080fdb 100644 --- a/tensorflow/contrib/kfac/python/ops/utils.py +++ b/tensorflow/contrib/kfac/python/ops/utils.py @@ -30,6 +30,7 @@ from tensorflow.python.ops import random_ops # Method used for inverting matrices. POSDEF_INV_METHOD = "cholesky" +POSDEF_EIG_METHOD = "self_adjoint" def set_global_constants(posdef_inv_method=None): @@ -187,7 +188,7 @@ def posdef_inv(tensor, damping): """Computes the inverse of tensor + damping * identity.""" identity = linalg_ops.eye(tensor.shape.as_list()[0], dtype=tensor.dtype) damping = math_ops.cast(damping, dtype=tensor.dtype) - return posdef_inv_funcs[POSDEF_INV_METHOD](tensor, identity, damping) + return posdef_inv_functions[POSDEF_INV_METHOD](tensor, identity, damping) def posdef_inv_matrix_inverse(tensor, identity, damping): @@ -209,13 +210,39 @@ def posdef_inv_eig(tensor, identity, damping): eigenvectors / eigenvalues, eigenvectors, transpose_b=True) -posdef_inv_funcs = { +posdef_inv_functions = { "matrix_inverse": posdef_inv_matrix_inverse, "cholesky": posdef_inv_cholesky, "eig": posdef_inv_eig, } +def posdef_eig(mat): + """Computes the eigendecomposition of a positive semidefinite matrix.""" + return posdef_eig_functions[POSDEF_EIG_METHOD](mat) + + +def posdef_eig_svd(mat): + """Computes the singular values and left singular vectors of a matrix.""" + evals, evecs, _ = linalg_ops.svd(mat) + + return evals, evecs + + +def posdef_eig_self_adjoint(mat): + """Computes eigendecomposition using self_adjoint_eig.""" + evals, evecs = linalg_ops.self_adjoint_eig(mat) + evals = math_ops.abs(evals) # Should be equivalent to svd approach. + + return evals, evecs + + +posdef_eig_functions = { + "self_adjoint": posdef_eig_self_adjoint, + "svd": posdef_eig_svd, +} + + class SubGraph(object): """Defines a subgraph given by all the dependencies of a given set of outputs. """ -- GitLab From 6b6244c40197b34f49bb50aa52efb082380d4637 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 30 Nov 2017 23:58:26 -0800 Subject: [PATCH 0182/1924] Build demo app for SmartReply PiperOrigin-RevId: 177559103 --- tensorflow/contrib/lite/build_def.bzl | 5 +- .../contrib/lite/models/smartreply/BUILD | 85 ++++++++++++ .../demo/app/src/main/AndroidManifest.xml | 38 ++++++ .../models/smartreply/demo/app/src/main/BUILD | 65 +++++++++ .../smartreply/demo/app/src/main/assets/BUILD | 15 ++ .../app/src/main/assets/backoff_response.txt | 16 +++ .../android/smartreply/MainActivity.java | 99 ++++++++++++++ .../android/smartreply/SmartReply.java | 44 ++++++ .../android/smartreply/SmartReplyClient.java | 129 ++++++++++++++++++ .../app/src/main/res/layout/main_activity.xml | 44 ++++++ .../demo/app/src/main/smartreply_jni.cc | 129 ++++++++++++++++++ .../models/smartreply/ops/extract_feature.cc | 9 +- .../lite/models/smartreply/ops/normalize.cc | 7 +- .../lite/models/smartreply/predictor.cc | 21 +-- .../lite/models/smartreply/predictor.h | 12 +- .../lite/models/smartreply/predictor_test.cc | 9 +- tensorflow/contrib/lite/tools/BUILD | 1 + .../lite/tools/gen_op_registration_main.cc | 48 +++++-- .../contrib/lite/tools/mutable_op_resolver.h | 2 +- tensorflow/workspace.bzl | 18 ++- third_party/tflite_smartreply.BUILD | 13 ++ 21 files changed, 758 insertions(+), 51 deletions(-) create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/AndroidManifest.xml create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/BUILD create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/BUILD create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/backoff_response.txt create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/MainActivity.java create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReply.java create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/res/layout/main_activity.xml create mode 100644 tensorflow/contrib/lite/models/smartreply/demo/app/src/main/smartreply_jni.cc create mode 100644 third_party/tflite_smartreply.BUILD diff --git a/tensorflow/contrib/lite/build_def.bzl b/tensorflow/contrib/lite/build_def.bzl index e3c9cdd99b..5813b3de4d 100644 --- a/tensorflow/contrib/lite/build_def.bzl +++ b/tensorflow/contrib/lite/build_def.bzl @@ -223,11 +223,12 @@ def gen_selected_ops(name, model): """ out = name + "_registration.cc" tool = "//tensorflow/contrib/lite/tools:generate_op_registrations" + tflite_path = "//tensorflow/contrib/lite" native.genrule( name = name, srcs = [model], outs = [out], - cmd = ("$(location %s) --input_model=$(location %s) --output_registration=$(location %s)") - % (tool, model, out), + cmd = ("$(location %s) --input_model=$(location %s) --output_registration=$(location %s) --tflite_path=%s") + % (tool, model, out, tflite_path[2:]), tools = [tool], ) diff --git a/tensorflow/contrib/lite/models/smartreply/BUILD b/tensorflow/contrib/lite/models/smartreply/BUILD index fbdf19f205..733c3f4c7f 100644 --- a/tensorflow/contrib/lite/models/smartreply/BUILD +++ b/tensorflow/contrib/lite/models/smartreply/BUILD @@ -1,7 +1,92 @@ package(default_visibility = ["//visibility:public"]) +load("//tensorflow/contrib/lite:build_def.bzl", "tflite_copts", "gen_selected_ops") + licenses(["notice"]) # Apache 2.0 +gen_selected_ops( + name = "smartreply_ops", + model = "@tflite_smartreply//:smartreply.tflite", +) + +cc_library( + name = "custom_ops", + srcs = [ + "ops/extract_feature.cc", + "ops/normalize.cc", + "ops/predict.cc", + ":smartreply_ops", + ], + copts = tflite_copts(), + deps = [ + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite:string_util", + "//tensorflow/contrib/lite/kernels:builtin_ops", + "//tensorflow/contrib/lite/tools:mutable_op_resolver", + "@com_google_absl//absl/strings", + "@com_googlesource_code_re2//:re2", + "@farmhash_archive//:farmhash", + ], +) + +cc_library( + name = "predictor_lib", + srcs = ["predictor.cc"], + hdrs = ["predictor.h"], + copts = tflite_copts(), + deps = [ + ":custom_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite:string_util", + "//tensorflow/contrib/lite/kernels:builtin_ops", + "//tensorflow/contrib/lite/tools:mutable_op_resolver", + "@com_google_absl//absl/strings", + "@com_googlesource_code_re2//:re2", + ], +) + +cc_test( + name = "extract_feature_op_test", + size = "small", + srcs = ["ops/extract_feature_test.cc"], + deps = [ + ":custom_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite/kernels:builtin_ops", + "//tensorflow/contrib/lite/kernels:test_util", + "@com_google_googletest//:gtest", + "@farmhash_archive//:farmhash", + ], +) + +cc_test( + name = "normalize_op_test", + size = "small", + srcs = ["ops/normalize_test.cc"], + deps = [ + ":custom_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite:string_util", + "//tensorflow/contrib/lite/kernels:builtin_ops", + "//tensorflow/contrib/lite/kernels:test_util", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "predict_op_test", + size = "small", + srcs = ["ops/predict_test.cc"], + deps = [ + ":custom_ops", + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite:string_util", + "//tensorflow/contrib/lite/kernels:builtin_ops", + "//tensorflow/contrib/lite/kernels:test_util", + "@com_google_googletest//:gtest", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/AndroidManifest.xml b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..75ed9432c8 --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/AndroidManifest.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/BUILD b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/BUILD new file mode 100644 index 0000000000..f8767b443a --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/BUILD @@ -0,0 +1,65 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +load( + "//tensorflow/contrib/lite:build_def.bzl", + "tflite_copts", + "tflite_jni_binary", +) + +filegroup( + name = "assets", + srcs = [ + "@tflite_smartreply//:model_files", + ], +) + +android_binary( + name = "SmartReplyDemo", + srcs = glob(["java/**/*.java"]), + assets = [":assets"], + assets_dir = "", + custom_package = "com.example.android.smartreply", + manifest = "AndroidManifest.xml", + nocompress_extensions = [ + ".tflite", + ], + resource_files = glob(["res/**"]), + tags = ["manual"], + deps = [ + ":smartreply_runtime", + "@androidsdk//com.android.support:support-v13-25.2.0", + "@androidsdk//com.android.support:support-v4-25.2.0", + ], +) + +cc_library( + name = "smartreply_runtime", + srcs = ["libsmartreply_jni.so"], + visibility = ["//visibility:public"], +) + +tflite_jni_binary( + name = "libsmartreply_jni.so", + deps = [ + ":smartreply_jni_lib", + ], +) + +cc_library( + name = "smartreply_jni_lib", + srcs = [ + "smartreply_jni.cc", + ], + copts = tflite_copts(), + linkopts = [ + "-lm", + "-ldl", + ], + deps = [ + "//tensorflow/contrib/lite:framework", + "//tensorflow/contrib/lite/models/smartreply:predictor_lib", + ], + alwayslink = 1, +) diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/BUILD b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/BUILD new file mode 100644 index 0000000000..3c882ffc43 --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/BUILD @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(glob(["*"])) + +filegroup( + name = "assets_files", + srcs = glob( + ["**/*"], + exclude = [ + "BUILD", + ], + ), +) diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/backoff_response.txt b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/backoff_response.txt new file mode 100644 index 0000000000..a0a5b46b5f --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/assets/backoff_response.txt @@ -0,0 +1,16 @@ +Ok +Yes +No +👍 +☺ +😟 +❤️ +Lol +Thanks +Got it +Done +Nice +I don't know +What? +Why? +What's up? diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/MainActivity.java b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/MainActivity.java new file mode 100644 index 0000000000..02fec9ae5e --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/MainActivity.java @@ -0,0 +1,99 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package com.example.android.smartreply; + +import android.app.Activity; +import android.os.Bundle; +import android.os.Handler; +import android.util.Log; +import android.view.View; +import android.widget.Button; +import android.widget.EditText; +import android.widget.TextView; + +/** + * The main (and only) activity of this demo app. Displays a text box which updates as messages are + * received. + */ +public class MainActivity extends Activity { + private static final String TAG = "SmartReplyDemo"; + private SmartReplyClient client; + + private Button sendButton; + private TextView messageTextView; + private EditText messageInput; + + private Handler handler; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + Log.v(TAG, "onCreate"); + setContentView(R.layout.main_activity); + + client = new SmartReplyClient(getApplicationContext()); + handler = new Handler(); + + sendButton = (Button) findViewById(R.id.send_button); + sendButton.setOnClickListener( + (View v) -> { + send(messageInput.getText().toString()); + }); + + messageTextView = (TextView) findViewById(R.id.message_text); + messageInput = (EditText) findViewById(R.id.message_input); + } + + @Override + protected void onStart() { + super.onStart(); + Log.v(TAG, "onStart"); + handler.post( + () -> { + client.loadModel(); + }); + } + + @Override + protected void onStop() { + super.onStop(); + Log.v(TAG, "onStop"); + handler.post( + () -> { + client.unloadModel(); + }); + } + + private void send(final String message) { + handler.post( + () -> { + messageTextView.append("Input: " + message + "\n"); + + SmartReply[] ans = client.predict(new String[] {message}); + for (SmartReply reply : ans) { + appendMessage("Reply: " + reply.getText()); + } + appendMessage("------"); + }); + } + + private void appendMessage(final String message) { + handler.post( + () -> { + messageTextView.append(message + "\n"); + }); + } +} diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReply.java b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReply.java new file mode 100644 index 0000000000..3357fd17c1 --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReply.java @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package com.example.android.smartreply; + +import android.support.annotation.Keep; + +/** + * SmartReply contains predicted message, and confidence. + * + *

    NOTE: this class used by JNI, class name and constructor should not be obfuscated. + */ +@Keep +public class SmartReply { + + private final String text; + private final float score; + + @Keep + public SmartReply(String text, float score) { + this.text = text; + this.score = score; + } + + public String getText() { + return text; + } + + public float getScore() { + return score; + } +} diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java new file mode 100644 index 0000000000..d5b1ac0ffb --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/java/com/example/android/smartreply/SmartReplyClient.java @@ -0,0 +1,129 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package com.example.android.smartreply; + +import android.content.Context; +import android.content.res.AssetFileDescriptor; +import android.support.annotation.Keep; +import android.support.annotation.WorkerThread; +import android.util.Log; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +/** Interface to load TfLite model and provide predictions. */ +public class SmartReplyClient implements AutoCloseable { + private static final String TAG = "SmartReplyDemo"; + private static final String MODEL_PATH = "smartreply.tflite"; + private static final String BACKOFF_PATH = "backoff_response.txt"; + private static final String JNI_LIB = "smartreply_jni"; + + private final Context context; + private long storage; + private MappedByteBuffer model; + + private volatile boolean isLibraryLoaded; + + public SmartReplyClient(Context context) { + this.context = context; + } + + public boolean isLoaded() { + return storage != 0; + } + + @WorkerThread + public synchronized void loadModel() { + if (!isLibraryLoaded) { + System.loadLibrary(JNI_LIB); + isLibraryLoaded = true; + } + + try { + model = loadModelFile(); + String[] backoff = loadBackoffList(); + storage = loadJNI(model, backoff); + } catch (IOException e) { + Log.e(TAG, "Fail to load model", e); + return; + } + } + + @WorkerThread + public synchronized SmartReply[] predict(String[] input) { + if (storage != 0) { + return predictJNI(storage, input); + } else { + return new SmartReply[] {}; + } + } + + @WorkerThread + public synchronized void unloadModel() { + close(); + } + + @Override + public synchronized void close() { + if (storage != 0) { + unloadJNI(storage); + storage = 0; + } + } + + private MappedByteBuffer loadModelFile() throws IOException { + AssetFileDescriptor fileDescriptor = context.getAssets().openFd(MODEL_PATH); + FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor()); + try { + FileChannel fileChannel = inputStream.getChannel(); + long startOffset = fileDescriptor.getStartOffset(); + long declaredLength = fileDescriptor.getDeclaredLength(); + return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); + } finally { + inputStream.close(); + } + } + + private String[] loadBackoffList() throws IOException { + List labelList = new ArrayList(); + BufferedReader reader = + new BufferedReader(new InputStreamReader(context.getAssets().open(BACKOFF_PATH))); + String line; + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + labelList.add(line); + } + } + reader.close(); + String[] ans = new String[labelList.size()]; + labelList.toArray(ans); + return ans; + } + + @Keep + private native long loadJNI(MappedByteBuffer buffer, String[] backoff); + + @Keep + private native SmartReply[] predictJNI(long storage, String[] text); + + @Keep + private native void unloadJNI(long storage); +} diff --git a/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/res/layout/main_activity.xml b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/res/layout/main_activity.xml new file mode 100644 index 0000000000..23b4cadc00 --- /dev/null +++ b/tensorflow/contrib/lite/models/smartreply/demo/app/src/main/res/layout/main_activity.xml @@ -0,0 +1,44 @@ + + + + + + + + + + +