From c12bd64899b7e7d7a8c4f59e7c6fbbf507daa3ba Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 21 Oct 2013 01:10:35 -0700 Subject: [PATCH 001/295] Initial apitools commit. --- LICENSE.txt | 202 ++++++++++ README.md | 15 + apitools/__init__.py | 1 + apitools/base/__init__.py | 1 + apitools/base/py/__init__.py | 1 + apitools/base/py/app2.py | 337 +++++++++++++++++ apitools/base/py/base_api.py | 549 ++++++++++++++++++++++++++++ apitools/base/py/base_cli.py | 110 ++++++ apitools/base/py/credentials_lib.py | 208 +++++++++++ apitools/base/py/encoding.py | 173 +++++++++ apitools/base/py/exceptions.py | 78 ++++ apitools/base/py/transfer.py | 318 ++++++++++++++++ apitools/base/py/util.py | 51 +++ apitools/gen/__init__.py | 1 + apitools/gen/command_registry.py | 522 ++++++++++++++++++++++++++ apitools/gen/extended_descriptor.py | 434 ++++++++++++++++++++++ apitools/gen/gen_client.py | 210 +++++++++++ apitools/gen/gen_client_lib.py | 158 ++++++++ apitools/gen/message_registry.py | 340 +++++++++++++++++ apitools/gen/service_registry.py | 386 +++++++++++++++++++ apitools/gen/setup.py | 73 ++++ apitools/gen/util.py | 270 ++++++++++++++ setup.py | 71 ++++ 23 files changed, 4509 insertions(+) create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 apitools/__init__.py create mode 100644 apitools/base/__init__.py create mode 100644 apitools/base/py/__init__.py create mode 100644 apitools/base/py/app2.py create mode 100644 apitools/base/py/base_api.py create mode 100644 apitools/base/py/base_cli.py create mode 100644 apitools/base/py/credentials_lib.py create mode 100644 apitools/base/py/encoding.py create mode 100644 apitools/base/py/exceptions.py create mode 100644 apitools/base/py/transfer.py create mode 100644 apitools/base/py/util.py create mode 100644 apitools/gen/__init__.py create mode 100644 apitools/gen/command_registry.py create mode 100644 apitools/gen/extended_descriptor.py create mode 100644 apitools/gen/gen_client.py create mode 100644 apitools/gen/gen_client_lib.py create mode 100644 apitools/gen/message_registry.py create mode 100644 apitools/gen/service_registry.py create mode 100644 apitools/gen/setup.py create mode 100644 apitools/gen/util.py create mode 100644 setup.py diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..476ffd4 --- /dev/null +++ b/README.md @@ -0,0 +1,15 @@ +# apitools + +`apitools` is a collection of utilities to make it easier to build client-side +tools, especially those that talk to Google APIs. + +## Current status + +There are a few imminent large changes: + +* finish the protorpc -> proto2 transition +* switch from httplib2 to requests +* better retry support +* R client library generation +* optional support for `dict -> dict` as the signature on client methods, + doing the proto conversion (and validation!) under the hood. diff --git a/apitools/__init__.py b/apitools/__init__.py new file mode 100644 index 0000000..4265cc3 --- /dev/null +++ b/apitools/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/apitools/base/__init__.py b/apitools/base/__init__.py new file mode 100644 index 0000000..4265cc3 --- /dev/null +++ b/apitools/base/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py new file mode 100644 index 0000000..4265cc3 --- /dev/null +++ b/apitools/base/py/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py new file mode 100644 index 0000000..1afa7ae --- /dev/null +++ b/apitools/base/py/app2.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python +"""Appcommands-compatible command class with extra fixins.""" + +import cmd +import inspect +import pdb +import shlex +import sys +import traceback +import types + +from google.apputils import app +from google.apputils import appcommands +import gflags as flags + + +flags.DEFINE_boolean( + 'debug_mode', False, + 'Show tracebacks on Python exceptions.') +flags.DEFINE_boolean( + 'headless', False, + 'Assume no user is at the controlling console.') + + +FLAGS = flags.FLAGS + + +# TODO(craigcitro): This code uses more than the average amount of +# Python magic. Explain what the heck is going on throughout. +class NewCmd(appcommands.Cmd): + """Featureful extension of appcommands.Cmd.""" + + def __init__(self, name, flag_values): + super(NewCmd, self).__init__(name, flag_values) + run_with_args = getattr(self, 'RunWithArgs', None) + self._new_style = isinstance(run_with_args, types.MethodType) + if self._new_style: + func = run_with_args.im_func + + argspec = inspect.getargspec(func) + if argspec.args and argspec.args[0] == 'self': + argspec = argspec._replace(args=argspec.args[1:]) # pylint: disable=protected-access,g-line-too-long + self._argspec = argspec + # TODO(craigcitro): Do we really want to support all this + # nonsense? + self._star_args = self._argspec.varargs is not None + self._star_kwds = self._argspec.keywords is not None + self._max_args = len(self._argspec.args or ()) + self._min_args = self._max_args - len(self._argspec.defaults or ()) + if self._star_args: + self._max_args = sys.maxint + + self._debug_mode = FLAGS.debug_mode + self.surface_in_shell = True + self.__doc__ = self.RunWithArgs.__doc__ + + def __getattr__(self, name): + if name in self._command_flags: + return self._command_flags[name].value + return super(NewCmd, self).__getattribute__(name) + + def _GetFlag(self, flagname): + if flagname in self._command_flags: + return self._command_flags[flagname] + else: + return None + + def Run(self, argv): + """Run this command. + + If self is a new-style command, we set up arguments and call + self.RunWithArgs, gracefully handling exceptions. If not, we + simply call self.Run(argv). + + Args: + argv: List of arguments as strings. + + Returns: + 0 on success, nonzero on failure. + """ + if not self._new_style: + return super(NewCmd, self).Run(argv) + + # TODO(craigcitro): We need to save and restore flags each time so + # that we can per-command flags in the REPL. + args = argv[1:] + fail = None + if len(args) < self._min_args: + fail = 'Not enough positional args; found %d, expected at least %d' % ( + len(args), self._min_args) + if len(args) > self._max_args: + fail = 'Too many positional args; found %d, expected at most %d' % ( + len(args), self._max_args) + if fail: + print fail + if self.usage: + print 'Usage: %s' % (self.usage,) + return 1 + + if self._debug_mode: + return self.RunDebug(args, {}) + else: + return self.RunSafely(args, {}) + + def RunCmdLoop(self, argv): + """Hook for use in cmd.Cmd-based command shells.""" + try: + args = shlex.split(argv) + except ValueError as e: + raise SyntaxError(self.EncodeForPrinting(e)) + return self.Run([self._command_name] + args) + + @staticmethod + def EncodeForPrinting(s): + """Safely encode a string as the encoding for sys.stdout.""" + encoding = sys.stdout.encoding or 'ascii' + return unicode(s).encode(encoding, 'backslashreplace') + + def _FormatError(self, e): + """Hook for subclasses to modify how error messages are printed.""" + return str(e) + + def _HandleError(self, e): + message = self._FormatError(e) + print 'Exception raised in %s operation: %s' % (self._command_name, message) + return 1 + + def _IsDebuggableException(self, e): + """Hook for subclasses to skip debugging on certain exceptions.""" + return not isinstance(e, app.UsageError) + + def RunDebug(self, args, kwds): + """Run this command in debug mode.""" + try: + return_value = self.RunWithArgs(*args, **kwds) + except BaseException, e: + # Don't break into the debugger for expected exceptions. + if not self._IsDebuggableException(e): + return self._HandleError(e) + print + print '****************************************************' + print '** Unexpected Exception raised in execution! **' + if FLAGS.headless: + print '** --headless mode enabled, exiting. **' + print '** See STDERR for traceback. **' + else: + print '** --debug_mode enabled, starting pdb. **' + print '****************************************************' + print + traceback.print_exc() + print + if not FLAGS.headless: + pdb.post_mortem() + return 1 + return return_value + + def RunSafely(self, args, kwds): + """Run this command, turning exceptions into print statements.""" + try: + return_value = self.RunWithArgs(*args, **kwds) + except BaseException, e: + return self._HandleError(e) + return return_value + + +class CommandLoop(cmd.Cmd): + """Instance of cmd.Cmd built to work with NewCmd.""" + + class TerminateSignal(Exception): + """Exception type used for signaling loop completion.""" + + def __init__(self, commands, prompt): + cmd.Cmd.__init__(self) + self._commands = {'help': commands['help']} + self._special_command_names = ['help', 'repl', 'EOF'] + for name, command in commands.iteritems(): + if (name not in self._special_command_names and + isinstance(command, NewCmd) and + command.surface_in_shell): + self._commands[name] = command + setattr(self, 'do_%s' % (name,), command.RunCmdLoop) + self._default_prompt = prompt + self._set_prompt() + self._last_return_code = 0 + + @property + def last_return_code(self): + return self._last_return_code + + def _set_prompt(self): + self.prompt = self._default_prompt + + def do_EOF(self, *unused_args): + """Terminate the running command loop. + + This function raises an exception to avoid the need to do + potentially-error-prone string parsing inside onecmd. + + Args: + *unused_args: unused. + + Returns: + Never returns. + + Raises: + CommandLoop.TerminateSignal: always. + """ + raise CommandLoop.TerminateSignal() + + def postloop(self): + print 'Goodbye.' + + def completedefault(self, unused_text, line, unused_begidx, unused_endidx): + if not line: + return [] + else: + command_name = line.partition(' ')[0].lower() + usage = '' + if command_name in self._commands: + usage = self._commands[command_name].usage + if usage: + print + print usage + print '%s%s' % (self.prompt, line), + return [] + + def emptyline(self): + print 'Available commands:', + print ' '.join(list(self._commands)) + + def precmd(self, line): + """Preprocess the shell input.""" + if line == 'EOF': + return line + if line.startswith('exit') or line.startswith('quit'): + return 'EOF' + words = line.strip().split() + if len(words) == 1 and words[0] not in ['help', 'ls', 'version']: + return 'help %s' % (line.strip(),) + return line + + def onecmd(self, line): + """Process a single command. + + Runs a single command, and stores the return code in + self._last_return_code. Always returns False unless the command + was EOF. + + Args: + line: (str) Command line to process. + + Returns: + A bool signaling whether or not the command loop should terminate. + """ + try: + self._last_return_code = cmd.Cmd.onecmd(self, line) + except CommandLoop.TerminateSignal: + return True + except BaseException as e: + name = line.split(' ')[0] + print 'Error running %s:' % name + print e + self._last_return_code = 1 + return False + + def get_names(self): + names = dir(self) + commands = (name for name in self._commands + if name not in self._special_command_names) + names.extend('do_%s' % (name,) for name in commands) + names.remove('do_EOF') + return names + + def do_help(self, command_name): + """Print the help for command_name (if present) or general help.""" + + # TODO(craigcitro): Add command-specific flags. + def FormatOneCmd(name, command, command_names): + indent_size = appcommands.GetMaxCommandLength() + 3 + if len(command_names) > 1: + indent = ' ' * indent_size + command_help = flags.TextWrap( + command.CommandGetHelp('', cmd_names=command_names), + indent=indent, + firstline_indent='') + first_help_line, _, rest = command_help.partition('\n') + first_line = '%-*s%s' % (indent_size, name + ':', first_help_line) + return '\n'.join((first_line, rest)) + else: + default_indent = ' ' + return '\n' + flags.TextWrap( + command.CommandGetHelp('', cmd_names=command_names), + indent=default_indent, + firstline_indent=default_indent) + '\n' + + if not command_name: + print '\nHelp for commands:\n' + command_names = list(self._commands) + print '\n\n'.join( + FormatOneCmd(name, command, command_names) + for name, command in self._commands.iteritems() + if name not in self._special_command_names) + print + elif command_name in self._commands: + print FormatOneCmd(command_name, self._commands[command_name], + command_names=[command_name]) + return 0 + + def postcmd(self, stop, line): + return bool(stop) or line == 'EOF' +# pylint: enable=g-bad-name + + +class Repl(NewCmd): + """Start an interactive session.""" + PROMPT = '> ' + + def __init__(self, name, fv): + super(Repl, self).__init__(name, fv) + self.surface_in_shell = False + flags.DEFINE_string( + 'prompt', '', + 'Prompt to use for interactive shell.', + flag_values=fv) + + def RunWithArgs(self): + """Start an interactive session.""" + prompt = FLAGS.prompt or self.PROMPT + repl = CommandLoop(appcommands.GetCommandList(), prompt=prompt) + print 'Welcome! (Type help for more information.)' + while True: + try: + repl.cmdloop() + break + except KeyboardInterrupt: + print + return repl.last_return_code diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py new file mode 100644 index 0000000..0922696 --- /dev/null +++ b/apitools/base/py/base_api.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python +"""Base class for api services.""" + +import contextlib +import email.mime.multipart as mime_multipart +import email.mime.nonmultipart as mime_nonmultipart +import httplib +import logging +import types +import urllib +import urlparse + + +from apiclient import errors as apiclient_errors +from apiclient import http as apiclient_http +from apiclient import mimeparse +import apiclient.model +import httplib2 +from protorpc import message_types +from protorpc import messages + +import gflags as flags + +from apitools.base.py import credentials_lib +from apitools.base.py import encoding +from apitools.base.py import exceptions + +FLAGS = flags.FLAGS + +# TODO(craigcitro): Remove this once we quiet the spurious logging in +# oauth2client (or drop oauth2client). +logging.getLogger('oauth2client.util').setLevel(logging.ERROR) + + +class ApiUploadInfo(messages.Message): + """Media upload information for a method. + + Fields: + accept: (repeated) MIME Media Ranges for acceptable media uploads + to this method. + max_size: Maximum size of a media upload, such as "1MB" or "3TB". + resumable_path: Path to use for resumable uploads. + resumable_multipart: (boolean) Whether or not the resumable endpoint + supports multipart uploads. + simple_path: Path to use for simple uploads. + simple_multipart: (boolean) Whether or not the simple endpoint + supports multipart uploads. + """ + accept = messages.StringField(1, repeated=True) + max_size = messages.IntegerField(2) + resumable_path = messages.StringField(3) + resumable_multipart = messages.BooleanField(4) + simple_path = messages.StringField(5) + simple_multipart = messages.BooleanField(6) + + +class ApiMethodInfo(messages.Message): + """Configuration info for an API method. + + All fields are strings unless noted otherwise. + + Fields: + relative_path: Relative path for this method. + method_id: ID for this method. + http_method: HTTP verb to use for this method. + path_params: (repeated) path parameters for this method. + query_params: (repeated) query parameters for this method. + ordered_params: (repeated) ordered list of parameters for + this method. + description: description of this method. + request_type_name: name of the request type. + response_type_name: name of the response type. + request_field: if not null, the field to pass as the body + of this POST request. may also be the REQUEST_IS_BODY + value below to indicate the whole message is the body. + upload_config: (ApiUploadInfo) Information about the upload + configuration supported by this method. + supports_download: (boolean) If True, this method supports + downloading the request via the `alt=media` query + parameter. + """ + + relative_path = messages.StringField(1) + method_id = messages.StringField(2) + http_method = messages.StringField(3) + path_params = messages.StringField(4, repeated=True) + query_params = messages.StringField(5, repeated=True) + ordered_params = messages.StringField(6, repeated=True) + description = messages.StringField(7) + request_type_name = messages.StringField(8) + response_type_name = messages.StringField(9) + request_field = messages.StringField(10, default='') + upload_config = messages.MessageField(ApiUploadInfo, 11) + supports_download = messages.BooleanField(12, default=False) +REQUEST_IS_BODY = '' + + +def _LoadClass(name, messages_module): + if name.startswith('message_types.'): + _, _, classname = name.partition('.') + return getattr(message_types, classname) + elif '.' not in name: + return getattr(messages_module, name) + else: + raise exceptions.GeneratedClientError('Unknown class %s' % name) + + +def _RequireClassAttrs(obj, attrs): + for attr in attrs: + attr_name = attr.upper() + if not hasattr(obj, '%s' % attr_name) or not getattr(obj, attr_name): + msg = 'No %s specified for object of class %s.' % ( + attr_name, type(obj).__name__) + raise exceptions.GeneratedClientError(msg) + + +def _Typecheck(arg, arg_type, msg=None): + if not isinstance(arg, arg_type): + if msg is None: + if isinstance(arg_type, tuple): + msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) + else: + msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) + raise exceptions.TypecheckError(msg) + return arg + + +def NormalizeApiEndpoint(api_endpoint): + if not api_endpoint.endswith('/'): + api_endpoint += '/' + return api_endpoint + + +class BaseApiModel(apiclient.model.JsonModel): + """Base model for generated clients.""" + alt_param = None + + def __init__(self, request_type, response_type, log_request, log_response, + *args, **kwds): + self.__request_type = request_type + self.__response_type = response_type + self.__log_request = log_request + self.__log_response = log_response + # TODO(craigcitro): Remove this field when we switch to proto2. + self.include_fields = None + super(BaseApiModel, self).__init__(*args, **kwds) + + # TODO(craigcitro): Delete these methods once we don't have to + # support both variations of apiclient. + @staticmethod + def _GetDumpFlag(): + if hasattr(FLAGS, 'dump_request_response'): + return FLAGS.dump_request_response + else: + return apiclient.model.dump_request_response + + @staticmethod + def _SetDumpFlag(value): + if hasattr(FLAGS, 'dump_request_response'): + FLAGS.dump_request_response = value + else: + apiclient.model.dump_request_response = value + + def _log_request(self, *args, **kwds): + old_value = self._GetDumpFlag() + if self.__log_request: + self._SetDumpFlag(True) + super(BaseApiModel, self)._log_request(*args, **kwds) + self._SetDumpFlag(old_value) + + def _log_response(self, *args, **kwds): + old_value = self._GetDumpFlag() + if self.__log_response: + self._SetDumpFlag(True) + super(BaseApiModel, self)._log_response(*args, **kwds) + self._SetDumpFlag(old_value) + + def serialize(self, body_value): + """Serialize a message (which might involve ProtoRPC messages).""" + _Typecheck(body_value, self.__request_type) + return encoding.MessageToJson( + body_value, include_fields=self.include_fields) + + def deserialize(self, content): + """Deserialize a message (which might involve ProtoRPC messages).""" + try: + message = encoding.JsonToMessage(self.__response_type, content) + except (exceptions.InvalidDataFromServerError, + messages.ValidationError) as e: + raise exceptions.InvalidDataFromServerError( + 'Error decoding response "%s" as type %s: %s' % ( + content, self.__response_type, e)) + return message + + +class BaseMediaDownloadModel(BaseApiModel): + """Base class for requests that return media in the response.""" + alt_param = 'media' + + def deserialize(self, content): + return content + + @property + def no_content_response(self): + return '' + + +class BaseApiClient(object): + """Base class for client libraries.""" + MESSAGES_MODULE = None + + _API_KEY = '' + _CLIENT_ID = '' + _CLIENT_SECRET = '' + _PACKAGE = '' + _SCOPES = [] + _USER_AGENT = '' + + def __init__(self, url, credentials=None, get_credentials=True, http=None, + model=None, log_request=False, log_response=False, + default_global_params=None): + _RequireClassAttrs(self, ( + '_package', '_scopes', '_client_id', '_client_secret', + 'messages_module')) + if default_global_params is not None: + _Typecheck(default_global_params, self.params_type) + self.__default_global_params = default_global_params + self.log_request = log_request + self.log_response = log_response + self._base_model_class = model or BaseApiModel + self._url = url + self._credentials = credentials + if get_credentials and not credentials: + # TODO(craigcitro): It's a bit dangerous to pass this + # still-half-initialized self into this method, but we might need + # to set attributes on it associated with our credentials. + # Consider another way around this (maybe a callback?) and whether + # or not it's worth it. + self._credentials = credentials_lib.GetCredentials( + self._PACKAGE, self._SCOPES, self._CLIENT_ID, self._CLIENT_SECRET, + self._USER_AGENT, api_key=self._API_KEY, client=self) + self._http = http or httplib2.Http() + # Note that "no credentials" is totally possible. + if self._credentials is not None: + self._http = self._credentials.authorize(self._http) + # TODO(craigcitro): Remove this field when we switch to proto2. + self.__include_fields = None + + @property + def base_model_class(self): + return self._base_model_class + + @property + def http(self): + return self._http + + @property + def url(self): + return self._url + + @classmethod + def GetScopes(cls): + return cls._SCOPES + + @property + def params_type(self): + return _LoadClass('StandardQueryParameters', self.MESSAGES_MODULE) + + @property + def _default_global_params(self): + if self.__default_global_params is None: + self.__default_global_params = self.params_type() + return self.__default_global_params + + def AddGlobalParam(self, name, value): + params = self._default_global_params + setattr(params, name, value) + + @property + def global_params(self): + return encoding.CopyProtoMessage(self._default_global_params) + + def ConfigureModel(self, model): + model.include_fields = self.__include_fields + + @contextlib.contextmanager + def IncludeFields(self, include_fields): + self.__include_fields = include_fields + yield + self.__include_fields = None + + +class BaseApiService(object): + """Base class for generated API services.""" + + def __init__(self, client): + self.__client = client + + @property + def _client(self): + return self.__client + + def __CombineGlobalParams(self, global_params, default_params): + _Typecheck(global_params, (types.NoneType, self.__client.params_type)) + result = self.__client.params_type() + global_params = global_params or self.__client.params_type() + for field in result.all_fields(): + value = (global_params.get_assigned_value(field.name) or + default_params.get_assigned_value(field.name)) + if value not in (None, [], ()): + setattr(result, field.name, value) + return result + + def __ConstructQueryParams(self, query_params, request, global_params): + query_info = dict((field.name, getattr(global_params, field.name)) + for field in self.__client.params_type.all_fields()) + query_info.update( + (param, getattr(request, param, None)) for param in query_params) + query_info = dict((k, v) for k, v in query_info.iteritems() + if v is not None) + return query_info + + def __ConstructPathParams(self, method_config, request): + path = method_config.relative_path + path_params = {} + for param in method_config.path_params: + param_template = '{%s}' % param + if param_template not in path: + raise exceptions.InvalidUserInputError( + 'Missing path parameter %s' % param) + try: + # TODO(craigcitro): Do we want to support some sophisticated + # mapping here? + value = getattr(request, param) + except AttributeError: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + if value is None: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + try: + path = path.replace(param_template, + urllib.quote(value.encode('utf_8'), '')) + except TypeError as e: + raise exceptions.InvalidUserInputError( + 'Error setting required parameter %s to value %s: %s' % ( + param, value, e)) + path_params[param] = value + return path, path_params + + def __GetUploadStrategy(self, upload, upload_config): + # Choose a protocol: We generally prefer resumable, unless the + # server only supports simple, or if we have a small transfer. + strategy = 'resumable' + if upload_config.simple_path: + if not upload_config.resumable_path: + strategy = 'simple' + elif (upload.total_size is not None and + upload.total_size < 4 * 1 << 20 and + upload_config.simple_multipart): + strategy = 'simple' + return strategy + + def __GetUploadParams(self, upload, upload_config, body_value): + strategy = self.__GetUploadStrategy(upload, upload_config) + params = {} + if strategy == 'simple': + params['uploadType'] = 'multipart' if body_value else 'media' + else: + params['uploadType'] = 'resumable' + return params + + def __GetUploadPath(self, upload, upload_config): + # In theory, we should use the resumable path in the case that the + # strategy is 'resumable'. However, apiclient is designed around a + # flow that pushes the original body in the first request, and + # pumps the media bytes through on successive requests. + _ = self.__GetUploadStrategy(upload, upload_config) + return upload_config.simple_path + + def __SimpleMediaBody(self, upload, headers, body_value): + # Rewrite the body. (This section follows apiclient.discovery.) + upload.stream.seek(0) + if not body_value: + headers['content-type'] = upload.mime_type + body_value = upload.stream.read() + else: + # This is a multipart/related upload. + msg_root = mime_multipart.MIMEMultipart('related') + # msg_root should not write out it's own headers + setattr(msg_root, '_write_headers', lambda self: None) + + # attach the body as one part + msg = mime_nonmultipart.MIMENonMultipart( + *headers['content-type'].split('/')) + msg.set_payload(body_value) + msg_root.attach(msg) + + # attach the media as the second part + msg = mime_nonmultipart.MIMENonMultipart( + *upload.mime_type.split('/')) + msg['Content-Transfer-Encoding'] = 'binary' + msg.set_payload(upload.stream.read()) + msg_root.attach(msg) + + body_value = msg_root.as_string() + multipart_boundary = msg_root.get_boundary() + headers['content-type'] = ('multipart/related; ' + 'boundary=%r') % multipart_boundary + return headers, body_value + + def __CreateMediaUpload(self, upload, upload_config, headers, body_value): + # Validate total_size vs. max_size + if (upload.total_size and upload_config.max_size and + upload.total_size > upload_config.max_size): + raise exceptions.InvalidUserInputError( + 'Upload too big: %s larger than max size %s' % ( + upload.total_size, upload_config.max_size)) + # Validate mime type + if not mimeparse.best_match(upload_config.accept, upload.mime_type): + raise exceptions.InvalidUserInputError( + 'MIME type %s does not match any accepted MIME ranges %s' % ( + upload.mime_type, upload_config.accept)) + strategy = self.__GetUploadStrategy(upload, upload_config) + # Create a MediaIoBaseUpload + if strategy == 'simple': + media_upload = False + headers, body_value = self.__SimpleMediaBody(upload, headers, body_value) + elif strategy == 'resumable': + # Don't need to set a body value in this case, since the + # HttpRequest is in charge of uploading it. + media_upload = apiclient_http.MediaIoBaseUpload( + upload.stream, upload.mime_type, resumable=True) + return media_upload, headers, body_value + + def __IsRetryable(self, exc): + status = int(exc.resp.get('status')) + # 308 doesn't have a name in httplib. + retryable_status = status in ( + httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER, + httplib.TEMPORARY_REDIRECT, 308) + return retryable_status and 'location' in exc.resp + + def __ExecuteRequest(self, request, url): + try: + return request.execute() + except apiclient_errors.HttpError as e: + if self.__IsRetryable(e): + logging.info('Got redirect for %s', request.uri) + request.uri = e.resp['location'] + logging.info('Redirecting to %s', request.uri) + return self.__ExecuteRequest(request, request.uri) + e.content = e.content.decode('ascii', 'replace') + logging.error('Error making request to "%s": "%s", "%s"', + url, e, e.content) + raise exceptions.HttpError.FromApiclientError(e) + except httplib2.HttpLib2Error as e: + raise exceptions.CommunicationError( + 'Communication error making request to "%s": "%s"' % (url, e)) + + def _RunMethod(self, method_config, request, global_params=None, + upload=None, upload_config=None, download=None): + """Call this method with request.""" + global_params = self.__CombineGlobalParams( + global_params, self.__client.global_params) + request_type = _LoadClass( + method_config.request_type_name, self.__client.MESSAGES_MODULE) + response_type = _LoadClass( + method_config.response_type_name, self.__client.MESSAGES_MODULE) + _Typecheck(request, request_type) + if self.__client.log_request: + logging.info('Request of type %s: %s', + method_config.request_type_name, request) + body_type = None + if method_config.request_field == REQUEST_IS_BODY: + body_type = request_type + elif method_config.request_field: + body_field = request_type.field_by_name(method_config.request_field) + _Typecheck(body_field, messages.MessageField) + body_type = body_field.type + # TODO(craigcitro): Make the http and model objects configurable. + request_builder = apiclient_http.HttpRequest + model_class = self.__client.base_model_class + if download: + model_class = BaseMediaDownloadModel + api_model = model_class( + body_type, response_type, + self.__client.log_request, self.__client.log_response) + self.__client.ConfigureModel(api_model) + + body_value = None + if method_config.request_field == REQUEST_IS_BODY: + body_value = request + elif method_config.request_field: + body_value = getattr(request, method_config.request_field) + + query_params = self.__ConstructQueryParams( + method_config.query_params, request, global_params) + if upload: + query_params.update(self.__GetUploadParams( + upload, upload_config, body_value)) + method_config.relative_path = self.__GetUploadPath(upload, upload_config) + relative_path, path_params = self.__ConstructPathParams( + method_config, request) + + # Note that api_model.request side-effects the headers, so must + # be threaded through. + headers = {} + headers, path_params, query, body = api_model.request( + headers, path_params, query_params, body_value) + + resumable = False + if upload: + resumable, headers, body = self.__CreateMediaUpload( + upload, upload_config, headers, body) + + url = urlparse.urljoin(self.__client.url, ''.join((relative_path, query))) + if self.__client.log_request: + logging.info('%s %s', method_config.http_method, url) + request = request_builder( + self.__client.http, + api_model.response, + url, + method=method_config.http_method, + body=body, + headers=headers, + methodId=method_config.method_id, + resumable=resumable) + + # If we're downloading media, we want to just get the new URL and + # hand it back to the download object. + if download: + try: + request.http.request( + uri=str(request.uri), method='GET', headers=request.headers, + body='', redirections=0) + # TODO(craigcitro): Confirm that this is invalid. + raise exceptions.InvalidDataFromServerError( + 'No redirect received for media download') + except httplib2.RedirectLimit as e: + download.url = e.response['location'] + download.http = request.http + return + + response = self.__ExecuteRequest(request, url) + if self.__client.log_response: + logging.info('Response of type %s: %s', + method_config.response_type_name, response) + return response diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py new file mode 100644 index 0000000..17c8c97 --- /dev/null +++ b/apitools/base/py/base_cli.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +"""Base script for generated CLI.""" + + +import atexit +import code +import logging +import os +import readline +import rlcompleter +import sys + +from google.apputils import appcommands +import gflags as flags + +# TODO(craigcitro): We should move all the flags for the +# StandardQueryParameters into this file, so that they can be used +# elsewhere easily. + +# TODO(craigcitro): FlagValidators? +flags.DEFINE_boolean( + 'log_request', False, + 'Log requests.') +flags.DEFINE_boolean( + 'log_response', False, + 'Log responses.') +flags.DEFINE_boolean( + 'log_request_response', False, + 'Log requests and responses.') + +# NOTE: This is specified here so that it can be read by other files +# without depending on the flag to be registered. +TRACE_HELP = ( + 'A tracing token of the form "trace:" ' + 'to include in api requests.') +FLAGS = flags.FLAGS + + +def SetupLogger(): + if FLAGS.log_request or FLAGS.log_response or FLAGS.log_request_response: + logging.basicConfig() + logging.getLogger().setLevel(logging.INFO) + + +class _SmartCompleter(rlcompleter.Completer): + def _callable_postfix(self, val, word): + if ('(' in readline.get_line_buffer() or + not callable(val)): + return word + else: + return word + '(' + + def complete(self, text, state): + if not readline.get_line_buffer().strip(): + if not state: + return ' ' + else: + return None + return rlcompleter.Completer.complete(self, text, state) + + +class ConsoleWithReadline(code.InteractiveConsole): + """InteractiveConsole with readline, tab completion, and history.""" + + def __init__(self, env, filename='', histfile=None): + new_locals = dict(env) + new_locals.update({ + '_SmartCompleter': _SmartCompleter, + 'readline': readline, + 'rlcompleter': rlcompleter, + }) + code.InteractiveConsole.__init__(self, new_locals, filename) + readline.parse_and_bind('tab: complete') + readline.set_completer(_SmartCompleter(new_locals).complete) + if histfile is not None: + histfile = os.path.expanduser(histfile) + if os.path.exists(histfile): + readline.read_history_file(histfile) + atexit.register(lambda: readline.write_history_file(histfile)) + + +def run_main(): + """Function to be used as setuptools script entry point. + + Appcommands assumes that it always runs as __main__, but launching + via a setuptools-generated entry_point breaks this rule. We do some + trickery here to make sure that appcommands and flags find their + state where they expect to by faking ourselves as __main__. + """ + + # Put the flags for this module somewhere the flags module will look + # for them. + # pylint: disable=protected-access + new_name = flags._GetMainModule() + sys.modules[new_name] = sys.modules['__main__'] + for flag in FLAGS.FlagsByModuleDict().get(__name__, []): + FLAGS._RegisterFlagByModule(new_name, flag) + for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): + FLAGS._RegisterKeyFlagForModule(new_name, key_flag) + # pylint: enable=protected-access + + # Now set __main__ appropriately so that appcommands will be + # happy. + sys.modules['__main__'] = sys.modules[__name__] + appcommands.Run() + sys.modules['__main__'] = sys.modules.pop(new_name) + + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py new file mode 100644 index 0000000..12c0bf0 --- /dev/null +++ b/apitools/base/py/credentials_lib.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +"""Common credentials classes and constructors.""" + +import getpass +import httplib +import json +import os +import urllib2 + + +import httplib2 +import oauth2client +import oauth2client.client +import oauth2client.gce +import oauth2client.multistore_file +import oauth2client.tools +from protorpc import messages + +import gflags as flags + +from apitools.base.py import exceptions +from apitools.base.py import util + +FLAGS = flags.FLAGS + + +# TODO(craigcitro): Expose the extra args here somewhere higher up, +# possibly as flags in the generated CLI. +def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, + credentials_filename=None, service_account_info=None, + robot_email=None, api_key=None, client=None): + """Attempt to get credentials, using an oauth dance as the last resort.""" + scopes = util.NormalizeScopes(scopes) + # TODO(craigcitro): Error checking. + client_info = { + 'client_id': client_id, + 'client_secret': client_secret, + 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), + 'user_agent': user_agent or '%s-generated/0.1' % package_name, + } + if service_account_info is not None: + credentials = ServiceAccountCredentials(service_account_info, scopes) + if credentials is not None: + return credentials + credentials = GaeAssertionCredentials.Get(scopes) + if credentials is not None: + return credentials + credentials = GceAssertionCredentials.Get(scopes) + if credentials is not None: + return credentials + credentials_filename = credentials_filename or os.path.expanduser( + '~/.apitools.token') + credentials = CredentialsFromFile(credentials_filename, client_info) + if credentials is not None: + return credentials + raise exceptions.CredentialsError('Could not create valid credentials') + + +class ServiceAccountCredentialInfo(messages.Message): + """Information needed to create a signed service account credential.""" + service_account_name = messages.StringField(1) + private_key = messages.StringField(2) + + +def ServiceAccountInfoFromFile(service_account_name, key_filename): + with open(key_filename) as key_file: + return ServiceAccountCredentialInfo( + service_account_name=service_account_name, + private_key=key_file.read()) + + +def ServiceAccountCredentials(service_account_info, scopes): + scopes = util.NormalizeScopes(scopes) + return oauth2client.client.SignedJwtAssertionCredentials( + service_account_info.service_account_name, + service_account_info.private_key, + scopes) + + +# TODO(craigcitro): We override to add some utility code, and to +# update the old refresh implementation. Either push this code into +# oauth2client or drop oauth2client. +class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): + """Assertion credentials for GCE instances.""" + + def __init__(self, scopes, service_account_name='default', **kwds): + if not util.DetectGce(): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested outside a GCE instance') + self.__service_account_name = service_account_name + scope_ls = util.NormalizeScopes(scopes) + instance_scopes = self._GetInstanceScopes() + if scope_ls > instance_scopes: + raise exceptions.CredentialsError( + 'Instance did not have access to scopes %s' % ( + sorted(list(scope_ls - instance_scopes)),)) + super(GceAssertionCredentials, self).__init__(scopes, **kwds) + + @classmethod + def Get(cls, *args, **kwds): + try: + return cls(*args, **kwds) + except exceptions.Error: + return None + + def _GetInstanceScopes(self): + scopes_uri = ( + 'http://metadata.google.internal/computeMetadata/v1beta1/instance/' + 'service-accounts/%s/scopes') % self.__service_account_name + try: + response = urllib2.urlopen(scopes_uri) + except urllib2.URLError as e: + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + return util.NormalizeScopes(scope.strip() for scope in response.readlines()) + + def _refresh(self, do_request): + """Refresh self.access_token. + + Args: + do_request: A function matching httplib2.Http.request's signature. + """ + token_uri = ( + 'http://metadata.google.internal/computeMetadata/v1beta1/instance/' + 'service-accounts/%s/token') % self.__service_account_name + response, content = do_request(token_uri) + if response.status != httplib.OK: + raise exceptions.CredentialsError( + 'Error refreshing credentials: %s' % content) + try: + credential_info = json.loads(content) + except ValueError: + raise exceptions.CredentialsError( + 'Invalid credentials response: %s' % content) + self.access_token = credential_info['access_token'] + + +# TODO(craigcitro): Currently, we can't even *load* +# `oauth2client.appengine` without being on appengine, because of how +# it handles imports. Fix that by splitting that module into +# GAE-specific and GAE-independent bits, and guarding imports. +class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): + """Assertion credentials for Google App Engine apps.""" + + def __init__(self, scopes, **kwds): + if not util.DetectGae(): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested outside a GCE instance') + self._scopes = list(util.NormalizeScopes(scopes)) + super(GaeAssertionCredentials, self).__init__(None, **kwds) + + @classmethod + def Get(cls, *args, **kwds): + try: + return cls(*args, **kwds) + except exceptions.Error: + return None + + @classmethod + def from_json(cls, json_data): + data = json.loads(json_data) + return GaeAssertionCredentials(data['_scopes']) + + def _refresh(self, _): + """Refresh self.access_token. + + Args: + _: (ignored) A function matching httplib2.Http.request's signature. + """ + from google.appengine.api import app_identity + try: + token, _ = app_identity.get_access_token(self._scopes) + except app_identity.Error as e: + raise exceptions.CredentialsError(str(e)) + self.access_token = token + + +# TODO(craigcitro): Switch this from taking a path to taking a stream. +def CredentialsFromFile(path, client_info): + """Read credentials from a file.""" + credential_store = oauth2client.multistore_file.get_credential_storage( + path, + client_info['client_id'], + client_info['user_agent'], + client_info['scope']) + if hasattr(FLAGS, 'auth_local_webserver'): + FLAGS.auth_local_webserver = False + credentials = credential_store.get() + if credentials is None or credentials.invalid: + print 'Generating new OAuth credentials ...' + while True: + # If authorization fails, we want to retry, rather than let this + # cascade up and get caught elsewhere. If users want out of the + # retry loop, they can ^C. + try: + flow = oauth2client.client.OAuth2WebServerFlow(**client_info) + credentials = oauth2client.tools.run(flow, credential_store) + break + except (oauth2client.client.FlowExchangeError, SystemExit) as e: + # Here SystemExit is "no credential at all", and the + # FlowExchangeError is "invalid" -- usually because you reused + # a token. + print 'Invalid authorization: %s' % (e,) + except httplib2.HttpLib2Error as e: + print 'Communication error: %s' % (e,) + raise util.CredentialsError( + 'Communication error creating credentials: %s' % e) + return credentials diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py new file mode 100644 index 0000000..2b435c1 --- /dev/null +++ b/apitools/base/py/encoding.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +"""Common code for converting proto to other formats, such as JSON.""" + +import base64 +import json + + +from protorpc import messages +from protorpc import protojson + +from apitools.base.py import exceptions + +__all__ = [ + 'CopyProtoMessage', + 'JsonToMessage', + 'MessageToJson', + 'DictToMessage', + 'MessageToDict', + ] + + +# TODO(craigcitro): Delete this function with the switch to proto2. +def CopyProtoMessage(message): + codec = protojson.ProtoJson() + return codec.decode_message(type(message), codec.encode_message(message)) + + +# XXX json.dumps(body_value, cls=ApiJsonEncoder) +def MessageToJson(message, include_fields=None): + """Convert the given message to JSON.""" + result = _ProtoJsonApilib.Get().encode_message(message) + return _IncludeFields(result, message, include_fields) + + +def JsonToMessage(message_type, message): + """Convert the given JSON to a message of type message_type.""" + return _ProtoJsonApilib.Get().decode_message(message_type, message) + + +# TODO(craigcitro): Do this directly, instead of via JSON. +def DictToMessage(d, message_type): + """Convert the given dictionary to a message of type message_type.""" + return JsonToMessage(message_type, json.dumps(d)) + + +def MessageToDict(message): + """Convert the given message to a dictionary.""" + return json.loads(MessageToJson(message)) + + +def _IncludeFields(encoded_message, message, include_fields): + """Add the requested fields to the encoded message.""" + if include_fields is None: + return encoded_message + result = json.loads(encoded_message) + for field_name in include_fields: + try: + message.field_by_name(field_name) + except KeyError: + raise exceptions.InvalidDataError( + 'No field named %s in message of type %s' % ( + field_name, type(message))) + result[field_name] = None + return json.dumps(result) + + +class _ProtoJsonApilib(protojson.ProtoJson): + """JSON encoder used by apitools clients.""" + _INSTANCE = None + + @classmethod + def Get(cls): + if cls._INSTANCE is None: + cls._INSTANCE = cls() + return cls._INSTANCE + + def decode_message(self, message_type, encoded_message): # pylint: disable=invalid-name + result = super(_ProtoJsonApilib, self).decode_message( + message_type, encoded_message) + return _DecodeUnknownFields(result) + + def decode_field(self, field, value): + """Decode the given value as JSON.""" + if isinstance(field, messages.BytesField): + try: + return base64.urlsafe_b64decode(str(value)) + except TypeError: + pass + field_value = super(_ProtoJsonApilib, self).decode_field(field, value) + if isinstance(field, messages.MessageField): + field_value = _DecodeUnknownFields(field_value) + return field_value + + def encode_message(self, message): # pylint: disable=invalid-name + message = _EncodeUnknownFields(message) + return super(_ProtoJsonApilib, self).encode_message(message) + + def encode_field(self, field, value): + """Encode the given value as JSON.""" + if isinstance(field, messages.BytesField): + try: + if isinstance(field, messages.BytesField): + if field.repeated: + return [base64.urlsafe_b64encode(byte) for byte in value] + else: + return base64.urlsafe_b64encode(value) + except TypeError: + pass + if isinstance(field, messages.MessageField): + value = _EncodeUnknownFields(value) + return super(_ProtoJsonApilib, self).encode_field(field, value) + + +# TODO(craigcitro): Storing this in a global is a bad idea, for all +# the usual reasons. In particular, if we plan to make base_api a +# shared file, we need to fix this. +_UNRECOGNIZED_FIELD_MAPPINGS = {} + + +def _DecodeUnknownFields(message): + """Rewrite unknown fields in message into message.destination.""" + destination = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) + if destination is None: + return message + pair_field = message.field_by_name(destination) + if not isinstance(pair_field, messages.MessageField): + raise exceptions.InvalidDataFromServerError( + 'Unrecognized fields must be mapped to a compound ' + 'message type.') + pair_type = pair_field.message_type + # TODO(craigcitro): Add more error checking around the pair + # type being exactly what we suspect (field names, etc). + new_values = [] + for unknown_field in message.all_unrecognized_fields(): + # TODO(craigcitro): Consider validating the variant if + # the assignment below doesn't take care of it. It may + # also be necessary to check it in the case that the + # type has multiple encodings. + value, _ = message.get_unrecognized_field_info(unknown_field) + new_pair = pair_type(key=str(unknown_field), value=value) + new_values.append(new_pair) + setattr(message, destination, new_values) + # We could probably get away with not setting this, but + # why not clear it? + setattr(message, '_Message__unrecognized_fields', {}) + return message + + +def _EncodeUnknownFields(message): + """Remap unknown fields in message out of message.source.""" + source = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) + if source is None: + return message + result = CopyProtoMessage(message) + pairs_field = message.field_by_name(source) + if not isinstance(pairs_field, messages.MessageField): + raise exceptions.InvalidUserInputError( + 'Invalid pairs field %s' % pairs_field) + pairs_type = pairs_field.message_type + value_variant = pairs_type.field_by_name('value').variant + pairs = getattr(message, source) + for pair in pairs: + result.set_unrecognized_field(pair.key, pair.value, value_variant) + setattr(result, source, []) + return result + + +def MapUnrecognizedFields(field_name): + """Register field_name as a container for unrecognized fields in message.""" + def Register(cls): + _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name + return cls + return Register diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py new file mode 100644 index 0000000..28531b9 --- /dev/null +++ b/apitools/base/py/exceptions.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +"""Exceptions for generated client libraries.""" + +from apiclient import errors as apiclient_errors + + +class Error(Exception): + """Base class for all exceptions.""" + + +class TypecheckError(Error, TypeError): + """An object of an incorrect type is provided.""" + + +class NotFoundError(Error): + """A specified resource could not be found.""" + + +class UserError(Error): + """Base class for errors related to user input.""" + + +class InvalidDataError(Error): + """Base class for any invalid data error.""" + + +class CommunicationError(Error): + """Any communication error talking to an API server.""" + + +class HttpError(CommunicationError, apiclient_errors.HttpError): + """Error making a request, with a code.""" + + def __init__(self, *args, **kwds): + CommunicationError.__init__(self) # pylint: disable=non-parent-init-called + apiclient_errors.HttpError.__init__(self, *args, **kwds) + + @classmethod + def FromApiclientError(cls, e): + if not isinstance(e, apiclient_errors.HttpError): + raise TypecheckError('Invalid error type: %s', type(e).__name__) # pylint: disable=nonstandard-exception + return cls(e.resp, e.content, uri=e.uri) + + +class InvalidUserInputError(InvalidDataError): + """User-provided input is invalid.""" + + +class InvalidDataFromServerError(InvalidDataError, CommunicationError): + """Data received from the server is malformed.""" + + +class ConfigurationError(Error): + """Base class for configuration errors.""" + + +class GeneratedClientError(Error): + """The generated client configuration is invalid.""" + + +class ConfigurationValueError(UserError): + """Some part of the user-specified client configuration is invalid.""" + + +class ResourceUnavailableError(Error): + """User requested an unavailable resource.""" + + +class CredentialsError(Error): + """Errors related to invalid credentials.""" + + +class TransferError(CommunicationError): + """Errors related to transfers.""" + + +class TransferInvalidError(TransferError): + """The given transfer is invalid.""" diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py new file mode 100644 index 0000000..083473e --- /dev/null +++ b/apitools/base/py/transfer.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python +"""Upload and download support for apitools.""" + +import collections +import httplib +import io +import json +import mimetypes +import os +import threading + +from apitools.base.py import exceptions + +__all__ = [ + 'Download', + 'Upload', + ] + +# pylint: disable=slots-on-old-class + + +# Note: currently the order of fields here is important, since we want +# to be able to pass in the result from httplib2.request. +class _HttpResponse(collections.namedtuple( + '_HttpResponse', ['info', 'content'])): + __slots__ = () + + def __len__(self): + return int(self.info.get('content-length', len(self.content))) + + @property + def status_code(self): + return int(self.info['status']) + + +class _TransferSerializationData(collections.namedtuple( + '_TransferSerializationData', ['progress', 'total_size', 'url'])): + __slots__ = () + + def ToJson(self): + return json.dumps(self._asdict()) + + @classmethod + def FromJson(cls, json_data): + data = json.loads(json_data) + data_keys = set(('progress', 'url')) + if data_keys > set(cls._fields): + raise exceptions.InvalidDataError( + 'Invalid keys for Transfer: %s' % data_keys) + return cls._make(data[field] for field in cls._fields) + + +class _Transfer(object): + """Generic bits common to Uploads and Downloads.""" + + def __init__(self, stream, close_stream=False, chunksize=None): + self.__close_stream = close_stream + self.__http = None + self.__stream = stream + self.__total_size = None + self.__url = None + + self._progress = 0 + + self.chunksize = chunksize or 1048576L + + def __repr__(self): + return str(self) + + @property + def _type_name(self): + return type(self).__name__ + + @property + def url(self): + return self.__url + + @url.setter + def url(self, value): + if self.url is not None: + raise exceptions.ConfigurationValueError( + 'Cannot set download url on initialized %s', self._type_name) + self.__url = value + + @property + def http(self): + return self.__http + + @http.setter + def http(self, value): + if self.http is not None: + raise exceptions.ConfigurationValueError( + 'Cannot set http on initialized %s', self._type_name) + self.__http = value + + @property + def total_size(self): + return self.__total_size + + @total_size.setter + def total_size(self, value): + if self.total_size is not None: + raise exceptions.ConfigurationValueError( + 'Cannot set total_size on initialized %s', self._type_name) + self.__total_size = value + + @property + def initialized(self): + return self.url is not None and self.http is not None + + def EnsureInitialized(self): + if not self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot use uninitialized %s', self._type_name) + + @property + def close_stream(self): + return self.__close_stream + + @property + def stream(self): + return self.__stream + + @property + def serialization_data(self): + self.EnsureInitialized() + return { + 'progress': self._progress, + 'total_size': self.total_size, + 'url': self.url, + } + + def __del__(self): + if self.__close_stream: + self.__stream.close() + + +class Download(_Transfer): + """Data for a single download. + + Public attributes: + chunksize: default chunksize to use for transfers. + """ + + def __str__(self): + return 'Download for url %s' % self.url + + @classmethod + def FromFile(cls, filename, overwrite=False): + """Create a new download object from a filename.""" + path = os.path.expanduser(filename) + if os.path.exists(path) and not overwrite: + raise exceptions.InvalidUserInputError( + 'File %s exists and overwrite not specified' % path) + return cls(open(path, 'wb'), close_stream=True) + + @classmethod + def FromStream(cls, stream): + """Create a new Download object from a stream.""" + return cls(stream) + + @classmethod + def FromData(cls, stream, json_data, http=None): + """Create a new Download object from a stream and serialized data.""" + download = cls.FromStream(stream) + info = _TransferSerializationData.FromJson(json_data) + download._progress = info.progress # pylint: disable=protected-access + download.http = http + download.total_size = info.total_size + download.url = info.url + return download + + def __SetTotal(self, info): + if self.total_size is None and 'content-range' in info: + _, _, total = info['content-range'].rpartition('/') + if total != '*': + self.total_size = int(total) + + def __GetChunk(self, start, end=None, chunksize=None): + """Retrieve a chunk, and return the full response.""" + self.EnsureInitialized() + start = max(start, 0) + chunksize = chunksize or self.chunksize + if end and end < 0: + # Requesting range from end + start = '' + end = abs(end) + else: + max_end = start + chunksize - 1 + end = min(end or max_end, max_end) + if end < start: + raise exceptions.TransferInvalidError( + 'Range requested with end[%s] < start[%s]' % (end, start)) + headers = {'Range': 'bytes=%s-%d' % (start, end)} + # TODO(craigcitro): Add support for retries. + response = _HttpResponse(*self.http.request(self.url, headers=headers)) + self.__SetTotal(response.info) + if response.status_code not in (httplib.PARTIAL_CONTENT, + httplib.REQUESTED_RANGE_NOT_SATISFIABLE): + raise exceptions.TransferInvalidError(response.content) + if response.status_code == httplib.PARTIAL_CONTENT: + self.stream.write(response.content) + return response + + def GetRange(self, start, end, chunksize=None, exact_range=True): + """Retrieve a given byte range from this download.""" + progress = start + chunksize = chunksize or self.chunksize + while progress < end: + response = self.__GetChunk(progress, end=end) + if (response.status_code == httplib.REQUESTED_RANGE_NOT_SATISFIABLE and + exact_range): + raise exceptions.TransferInvalidError( + 'Could not fetch all requested bytes: ended at %d' % progress) + progress += len(response) + + def __ExecuteCallback(self, callback, response): + # TODO(craigcitro): Push these into a queue. + if callback is not None: + threading.Thread(target=callback, args=(response, self)).start() + + def StreamInChunks(self, callback=None, finish_callback=None, chunksize=None, + end=None): + """Stream the entire download.""" + + def ArgPrinter(response, unused_download): + print 'Received bytes %s' % response.info['content-range'] + + def CompletePrinter(*unused_args): + print 'Download complete' + + callback = callback or ArgPrinter + finish_callback = finish_callback or CompletePrinter + + self.EnsureInitialized() + while True: + response = self.__GetChunk(self._progress, chunksize=chunksize, end=end) + # TODO(craigcitro): Consider whether this update and writing + # the response to self.stream need to happen as a transaction. + self._progress += len(response) + if response.status_code == httplib.REQUESTED_RANGE_NOT_SATISFIABLE: + break + # Callback with the new chunk. + self.__ExecuteCallback(callback, response) + # Handle range requests + # TODO(craigcitro): Exert python mastery over the known universe by + # cleaning up this hackish implementation. + if end: + if end < 0: + if(self._progress) >= abs(end): + break + elif self._progress >= end: + break + self.__ExecuteCallback(finish_callback, response) + + +class Upload(_Transfer): + """Data for a single Upload. + + Fields: + stream: The stream to upload. + mime_type: MIME type of the upload. + mime_encoding: (optional) Encoding for the upload. Currently unused. + size_hint: (optional) Total upload size for the stream. + close_stream: (default: False) Whether or not we should close the + stream when finished with the upload. + """ + + def __init__(self, stream, mime_type, mime_encoding=None, size_hint=None, + close_stream=False, chunksize=None): + super(Upload, self).__init__(stream, close_stream=close_stream, + chunksize=chunksize) + self.__mime_type = mime_type + self.__mime_encoding = mime_encoding + + self.total_size = size_hint + + @property + def mime_type(self): + return self.__mime_type + + @property + def mime_encoding(self): + return self.__mime_encoding + + def __str__(self): + size = self.total_size or '' + return 'Upload of size %s and mime type %s' % (size, self.mime_type) + + @classmethod + def FromFile(cls, filename, mime_type=None, mime_encoding=None): + """Create a new Upload object from a filename.""" + path = os.path.expanduser(filename) + if not os.path.exists(path): + raise exceptions.NotFoundError('Could not find file %s' % path) + if not mime_type: + mime_type, mime_encoding = mimetypes.guess_type(path) + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'Could not guess mime type for %s' % path) + size = os.stat(path).st_size + return cls(open(path, 'rb'), mime_type, mime_encoding=mime_encoding, + size_hint=size, close_stream=True) + + @classmethod + def FromStream(cls, stream, mime_type, mime_encoding=None): + """Create a new Upload object from a seekable stream.""" + if isinstance(stream, io.IOBase) and not stream.seekable(): + raise exceptions.InvalidUserInputError('Stream not seekable') + # TODO(craigcitro): Consider checking the full interface we need + # for a stream here. + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'No mime_type specified for stream') + stream.seek(0, io.SEEK_END) + size = stream.tell() + return cls(stream, mime_type, mime_encoding=mime_encoding, + size_hint=size, close_stream=False) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py new file mode 100644 index 0000000..5589412 --- /dev/null +++ b/apitools/base/py/util.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""Assorted utilities shared between parts of apitools.""" + +import collections +import httplib +import os +import types +import urllib2 + +from apitools.base.py import exceptions + + +def DetectGae(): + """Determine whether or not we're running on GAE. + + This is based on: + https://developers.google.com/appengine/docs/python/#The_Environment + + Returns: + True iff we're running on GAE. + """ + server_software = os.environ.get('SERVER_SOFTWARE', '') + return (server_software.startswith('Development/') or + server_software.startswith('Google App Engine/')) + + +def DetectGce(): + """Determine whether or not we're running on GCE. + + This is based on: + https://developers.google.com/compute/docs/instances#dmi + + Returns: + True iff we're running on a GCE instance. + """ + try: + o = urllib2.urlopen('http://metadata.google.internal') + except urllib2.URLError: + return False + return o.getcode() == httplib.OK + + +def NormalizeScopes(scope_spec): + """Normalize scope_spec to a set of strings.""" + if isinstance(scope_spec, types.StringTypes): + return set(scope_spec.split(' ')) + elif isinstance(scope_spec, collections.Iterable): + return set(scope_spec) + raise exceptions.TypecheckError( + 'NormalizeScopes expected string or iterable, found %s' % ( + type(scope_spec),)) diff --git a/apitools/gen/__init__.py b/apitools/gen/__init__.py new file mode 100644 index 0000000..4265cc3 --- /dev/null +++ b/apitools/gen/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py new file mode 100644 index 0000000..2e888a7 --- /dev/null +++ b/apitools/gen/command_registry.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python +"""Command registry for apitools.""" + +import logging +import textwrap + +from protorpc import descriptor +from protorpc import messages + +from apitools.gen import extended_descriptor +from apitools.gen import util + + +_VARIANT_TO_FLAG_TYPE_MAP = { + messages.Variant.DOUBLE: 'float', + messages.Variant.FLOAT: 'float', + messages.Variant.INT64: 'string', + messages.Variant.UINT64: 'string', + messages.Variant.INT32: 'integer', + messages.Variant.BOOL: 'boolean', + messages.Variant.STRING: 'string', + messages.Variant.MESSAGE: 'string', + messages.Variant.BYTES: 'string', + messages.Variant.UINT32: 'integer', + messages.Variant.ENUM: 'enum', + messages.Variant.SINT32: 'integer', + messages.Variant.SINT64: 'integer', + } + + +class FlagInfo(messages.Message): + """Information about a flag and conversion to a message. + + Fields: + name: name of this flag. + type: type of the flag. + description: description of the flag. + default: default value for this flag. + enum_values: if this flag is an enum, the list of possible + values. + required: whether or not this flag is required. + fv: name of the flag_values object where this flag should + be registered. + conversion: template for type conversion. + special: (boolean, default: False) If True, this flag doesn't + correspond to an attribute on the request. + """ + name = messages.StringField(1) + type = messages.StringField(2) + description = messages.StringField(3) + default = messages.StringField(4) + enum_values = messages.StringField(5, repeated=True) + required = messages.BooleanField(6, default=False) + fv = messages.StringField(7) + conversion = messages.StringField(8) + special = messages.BooleanField(9, default=False) + + +class ArgInfo(messages.Message): + """Information about a single positional command argument. + + Fields: + name: argument name. + description: description of this argument. + conversion: template for type conversion. + """ + name = messages.StringField(1) + description = messages.StringField(2) + conversion = messages.StringField(3) + + +class CommandInfo(messages.Message): + """Information about a single command. + + Fields: + name: name of this command. + class_name: name of the app2.NewCmd class for this command. + description: description of this command. + flags: list of FlagInfo messages for the command-specific flags. + args: list of ArgInfo messages for the positional args. + request_type: name of the request type for this command. + client_method_path: path from the client object to the method + this command is wrapping. + """ + name = messages.StringField(1) + class_name = messages.StringField(2) + description = messages.StringField(3) + flags = messages.MessageField(FlagInfo, 4, repeated=True) + args = messages.MessageField(ArgInfo, 5, repeated=True) + request_type = messages.StringField(6) + client_method_path = messages.StringField(7) + has_upload = messages.BooleanField(8, default=False) + has_download = messages.BooleanField(9, default=False) + + +class CommandRegistry(object): + """Registry for CLI commands.""" + + def __init__(self, package, version, client_info, message_registry, + root_package, base_files_package, base_url, names): + self.__package = package + self.__version = version + self.__client_info = client_info + self.__names = names + self.__message_registry = message_registry + self.__root_package = root_package + self.__base_files_package = base_files_package + self.__base_url = base_url + self.__command_list = [] + self.__global_flags = [] + + def Validate(self): + self.__message_registry.Validate() + + def AddGlobalParameters(self, schema): + for field in schema.fields: + self.__global_flags.append(self.__FlagInfoFromField(field, schema)) + + def AddCommandForMethod(self, service_name, method_name, method_info, + request, _): + """Add the given method as a command.""" + command_name = self.__GetCommandName(method_info.method_id) + calling_path = '%s.%s' % (service_name, method_name) + request_type = self.__message_registry.LookupDescriptor(request) + description = method_info.description + if not description: + description = 'Call the %s method.' % method_info.method_id + field_map = dict((f.name, f) for f in request_type.fields) + args = [] + arg_names = [] + for field_name in method_info.ordered_params: + extended_field = field_map[field_name] + name = extended_field.name + args.append(ArgInfo( + name=name, + description=extended_field.description, + conversion=self.__GetConversion(extended_field, request_type), + )) + arg_names.append(name) + flags = [] + for extended_field in sorted(request_type.fields, key=lambda x: x.name): + field = extended_field.field_descriptor + if extended_field.name in arg_names: + continue + if self.__FieldIsRequired(field): + logging.warning( + 'Required field %s not in ordered_params for command %s', + extended_field.name, command_name) + flags.append(self.__FlagInfoFromField( + extended_field, request_type, fv='fv')) + if method_info.upload_config: + # TODO(craigcitro): Consider adding additional flags to allow + # determining the filename from the object metadata. + upload_flag_info = FlagInfo( + name='upload_filename', type='string', default='', + description='Filename to use for upload.', fv='fv', special=True) + flags.append(upload_flag_info) + mime_description = ( + 'MIME type to use for the upload. Only needed if ' + 'the extension on --upload_filename does not determine ' + 'the correct (or any) MIME type.') + mime_type_flag_info = FlagInfo( + name='upload_mime_type', type='string', default='', + description=mime_description, fv='fv', special=True) + flags.append(mime_type_flag_info) + if method_info.supports_download: + download_flag_info = FlagInfo( + name='download_filename', type='string', default='', + description='Filename to use for download.', fv='fv', special=True) + flags.append(download_flag_info) + overwrite_description = ( + 'If True, overwrite the existing file when downloading.') + overwrite_flag_info = FlagInfo( + name='overwrite', type='boolean', default='False', + description=overwrite_description, fv='fv', special=True) + flags.append(overwrite_flag_info) + command_info = CommandInfo( + name=command_name, + class_name=self.__names.ClassName(command_name), + description=description, + flags=flags, + args=args, + request_type=request_type.full_name, + client_method_path=calling_path, + has_upload=bool(method_info.upload_config), + has_download=bool(method_info.supports_download) + ) + self.__command_list.append(command_info) + + def __LookupMessage(self, message, field): + message_type = self.__message_registry.LookupDescriptor( + '%s.%s' % (message.name, field.type_name)) + if message_type is None: + message_type = self.__message_registry.LookupDescriptor(field.type_name) + return message_type + + def __GetCommandName(self, method_id): + command_name = method_id + prefix = '%s.' % self.__package + if command_name.startswith(prefix): + command_name = command_name[len(prefix):] + command_name = command_name.replace('.', '_') + return command_name + + def __GetConversion(self, extended_field, extended_message): + field = extended_field.field_descriptor + + type_name = '' + if field.variant in (messages.Variant.MESSAGE, messages.Variant.ENUM): + if field.type_name.startswith('protorpc.'): + type_name = field.type_name + else: + field_message = self.__LookupMessage(extended_message, field) + if field_message is None: + raise ValueError('Could not find type for field %s' % field.name) + type_name = 'messages.%s' % field_message.full_name + + template = '' + if field.variant in (messages.Variant.INT64, messages.Variant.UINT64): + template = 'int(%s)' + elif field.variant == messages.Variant.MESSAGE: + template = 'base_api.JsonToMessage(%s, %%s)' % type_name + elif field.variant == messages.Variant.ENUM: + template = '%s(%%s)' % type_name + + if self.__FieldIsRepeated(extended_field.field_descriptor): + if template: + template = '[%s for x in %%s]' % (template % 'x') + + return template + + def __FieldIsRequired(self, field): + return field.label == descriptor.FieldDescriptor.Label.REQUIRED + + def __FieldIsRepeated(self, field): + return field.label == descriptor.FieldDescriptor.Label.REPEATED + + def __FlagInfoFromField(self, extended_field, extended_message, fv=''): + field = extended_field.field_descriptor + flag_info = FlagInfo() + flag_info.name = str(field.name) + # TODO(craigcitro): We should key by variant. + flag_info.type = _VARIANT_TO_FLAG_TYPE_MAP[field.variant] + flag_info.description = extended_field.description + if field.default_value: + # TODO(craigcitro): Formatting? + flag_info.default = field.default_value + if flag_info.type == 'enum': + # TODO(craigcitro): Does protorpc do this for us? + enum_type = self.__LookupMessage(extended_message, field) + if enum_type is None: + raise ValueError('Cannot find enum type %s', field.type_name) + flag_info.enum_values = [x.name for x in enum_type.values] + # Note that this choice is completely arbitrary -- but we only + # push the value through if the user specifies it, so this + # doesn't hurt anything. + if flag_info.default is None: + flag_info.default = flag_info.enum_values[0] + if self.__FieldIsRequired(field): + flag_info.required = True + flag_info.fv = fv + flag_info.conversion = self.__GetConversion( + extended_field, extended_message) + return flag_info + + def __PrintGlobalFlags(self, printer): + for flag_info in self.__global_flags: + self.__PrintFlag(printer, flag_info) + + def __PrintGetGlobalParams(self, printer): + printer('def GetGlobalParamsFromFlags():') + with printer.Indent(): + printer('"""Return a StandardQueryParameters based on flags."""') + printer('result = messages.StandardQueryParameters()') + + for flag_info in self.__global_flags: + rhs = 'FLAGS.%s' % flag_info.name + if flag_info.conversion: + rhs = flag_info.conversion % rhs + printer('if FLAGS[%r].present:', flag_info.name) + with printer.Indent(): + printer('result.%s = %s', flag_info.name, rhs) + printer('return result') + printer() + printer() + + def __PrintGetClient(self, printer): + printer('def GetClientFromFlags():') + with printer.Indent(): + printer('"""Return a client object, configured from flags."""') + printer('log_request = FLAGS.log_request or FLAGS.log_request_response') + printer('log_response = FLAGS.log_response or FLAGS.log_request_response') + printer('api_endpoint = base_api.NormalizeApiEndpoint(' + 'FLAGS.api_endpoint)') + printer('try:') + with printer.Indent(): + printer('client = client_lib.%s(', self.__client_info.client_class_name) + with printer.Indent(indent=' '): + printer('api_endpoint, log_request=log_request,') + printer('log_response=log_response)') + printer('except exceptions.CredentialsError as e:') + with printer.Indent(): + printer("print 'Error creating credentials: %%s' %% e") + printer('sys.exit(1)') + printer('return client') + printer() + printer() + + def __PrintCommandDocstring(self, printer, command_info): + for line in textwrap.wrap('"""%s' % command_info.description, + printer.CalculateWidth()): + printer(line) + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.args, 'Args') + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.flags, 'Flags') + printer('"""') + + def __PrintFlag(self, printer, flag_info): + printer('flags.DEFINE_%s(', flag_info.type) + with printer.Indent(indent=' '): + printer('%r,', flag_info.name) + printer('%r,', flag_info.default) + if flag_info.type == 'enum': + printer('%r,', flag_info.enum_values) + + # TODO(craigcitro): Consider using 'drop_whitespace' elsewhere. + description_lines = textwrap.wrap( + flag_info.description, 75 - len(printer.indent), + drop_whitespace=False) + for line in description_lines[:-1]: + printer('%r', line) + printer('%r%s', description_lines[-1], ',' if flag_info.fv else ')') + if flag_info.fv: + printer('flag_values=%s)', flag_info.fv) + if flag_info.required: + printer('flags.MarkFlagAsRequired(%r)', flag_info.name) + + def __PrintPyShell(self, printer): + printer('class PyShell(appcommands.Cmd):') + with printer.Indent(): + printer('def Run(self, _):') + with printer.Indent(): + printer('"""Run an interactive python shell with the client."""') + printer('client = GetClientFromFlags()') + printer('params = GetGlobalParamsFromFlags()') + printer('for field in params.all_fields():') + with printer.Indent(): + printer('value = params.get_assigned_value(field.name)') + printer('if value != field.default:') + with printer.Indent(): + printer('client.AddGlobalParam(field.name, value)') + printer('banner = """') + printer(' == %s interactive console ==' % ( + self.__client_info.package)) + printer(' client: a %s client' % self.__client_info.package) + printer(' messages: the generated messages module') + printer('"""') + printer('local_vars = {') + with printer.Indent(indent=' '): + printer("'client': client,") + printer("'client_lib': client_lib,") + printer("'messages': messages,") + printer('}') + printer("if platform.system() == 'Linux':") + with printer.Indent(): + printer('console = base_cli.ConsoleWithReadline(') + with printer.Indent(indent=' '): + printer('local_vars, histfile=FLAGS.history_file)') + printer('else:') + with printer.Indent(): + printer('console = code.InteractiveConsole(local_vars)') + printer('try:') + with printer.Indent(): + printer('console.interact(banner)') + printer('except SystemExit as e:') + with printer.Indent(): + printer('return e.code') + printer() + printer() + + def WriteFile(self, out): + """Write a simple CLI (currently just a stub).""" + printer = util.SimplePrettyPrinter(out) + printer('"""CLI for %s, version %s."""', self.__package, self.__version) + # TODO(craigcitro): Add a build stamp, along with some other + # information. + printer() + printer('import code') + printer('import platform') + printer('import sys') + printer() + printer('import protorpc') + printer('from protorpc import message_types') + printer('from protorpc import messages') + printer() + printer( + 'from ' + 'google.apputils' + ' import appcommands') + printer( + 'import gflags as ' + 'flags') + printer() + printer('from %s import app2', self.__base_files_package) + printer('from %s import base_api', self.__base_files_package) + printer('from %s import base_cli', self.__base_files_package) + printer('from %s import exceptions', self.__base_files_package) + printer('from %s import transfer', self.__base_files_package) + printer('from %s import %s as client_lib', + self.__root_package, self.__client_info.client_rule_name) + printer('from %s import %s as messages', + self.__root_package, self.__client_info.messages_rule_name) + printer() + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'api_endpoint',") + printer('%r,', self.__base_url) + printer("'URL of the API endpoint to use.',") + printer("short_name='%s_url')", self.__package) + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'history_file',") + printer('%r,', '~/.%s.%s.history' % (self.__package, self.__version)) + printer("'File with interactive shell history.')") + printer() + self.__PrintGlobalFlags(printer) + printer('FLAGS = flags.FLAGS') + printer() + printer() + self.__PrintGetGlobalParams(printer) + self.__PrintGetClient(printer) + self.__PrintPyShell(printer) + self.__PrintCommands(printer) + printer('def main(_):') + with printer.Indent(): + printer("appcommands.AddCmd('pyshell', PyShell)") + for command_info in self.__command_list: + printer("appcommands.AddCmd('%s', %s)", + command_info.name, command_info.class_name) + printer() + printer('base_cli.SetupLogger()') + # TODO(craigcitro): Just call SetDefaultCommand as soon as + # another appcommands release happens and this exists + # externally. + printer("if hasattr(appcommands, 'SetDefaultCommand'):") + with printer.Indent(): + printer("appcommands.SetDefaultCommand('pyshell')") + printer() + printer() + printer('run_main = base_cli.run_main') + printer() + printer("if __name__ == '__main__':") + with printer.Indent(): + printer('appcommands.Run()') + + def __PrintCommands(self, printer): + """Print all commands in this registry using printer.""" + for command_info in self.__command_list: + arg_list = [arg_info.name for arg_info in command_info.args] + printer('class %s(app2.NewCmd):', command_info.class_name) + with printer.Indent(): + printer('"""Command wrapping %s."""', command_info.client_method_path) + printer() + printer('usage = """%s%s%s"""', + command_info.name, + ' ' if arg_list else '', + ' '.join('<%s>' % argname for argname in arg_list)) + printer() + printer('def __init__(self, name, fv):') + with printer.Indent(): + printer('super(%s, self).__init__(name, fv)', command_info.class_name) + for flag in command_info.flags: + self.__PrintFlag(printer, flag) + printer() + printer('def RunWithArgs(%s):', ', '.join(['self'] + arg_list)) + with printer.Indent(): + self.__PrintCommandDocstring(printer, command_info) + printer('client = GetClientFromFlags()') + printer('global_params = GetGlobalParamsFromFlags()') + printer('request = messages.%s(', command_info.request_type) + with printer.Indent(indent=' '): + for arg in command_info.args: + rhs = arg.name + if arg.conversion: + rhs = arg.conversion % arg.name + printer('%s=%s,', arg.name, rhs) + printer(')') + for flag_info in command_info.flags: + if flag_info.special: + continue + rhs = 'FLAGS.%s' % flag_info.name + if flag_info.conversion: + rhs = flag_info.conversion % rhs + printer('if FLAGS[%r].present:', flag_info.name) + with printer.Indent(): + printer('request.%s = %s', flag_info.name, rhs) + call_args = ['request', 'global_params=global_params'] + if command_info.has_upload: + call_args.append('upload=upload') + printer('upload = None') + printer('if FLAGS.upload_filename:') + with printer.Indent(): + printer('upload = transfer.Upload.FromFile(') + printer(' FLAGS.upload_filename, FLAGS.upload_mime_type)') + if command_info.has_download: + call_args.append('download=download') + printer('download = None') + printer('if FLAGS.download_filename:') + with printer.Indent(): + printer('download = transfer.Download.FromFile(' + 'FLAGS.download_filename, overwrite=FLAGS.overwrite)') + printer('result = client.%s(', command_info.client_method_path) + with printer.Indent(indent=' '): + printer('%s)', ', '.join(call_args)) + if command_info.has_download: + printer('if FLAGS.download_filename:') + with printer.Indent(): + printer('download.StreamInChunks()') + printer('print result') + printer() + printer() diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py new file mode 100644 index 0000000..f4e3ea4 --- /dev/null +++ b/apitools/gen/extended_descriptor.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python +"""Extended protorpc descriptors. + +This takes existing protorpc Descriptor classes and adds extra +properties not directly supported in proto itself, notably field and +message descriptions. We need this in order to generate protorpc +message files with comments. + +Note that for most of these classes, we can't simply wrap the existing +message, since we need to change the type of the subfields. We could +have a "plain" descriptor attached, but that seems like unnecessary +bookkeeping. Where possible, we purposely reuse existing tag numbers; +for new fields, we start numbering at 100. +""" +import abc +import operator +import textwrap + +from protorpc import descriptor +from protorpc import message_types +from protorpc import messages + +from apitools.gen import util + + +class ExtendedEnumValueDescriptor(messages.Message): + """Enum value descriptor with additional fields. + + Fields: + name: Name of enumeration value. + number: Number of enumeration value. + description: Description of this enum value. + """ + name = messages.StringField(1) + number = messages.IntegerField(2, variant=messages.Variant.INT32) + + description = messages.StringField(100) + + +class ExtendedEnumDescriptor(messages.Message): + """Enum class descriptor with additional fields. + + Fields: + name: Name of Enum without any qualification. + values: Values defined by Enum class. + description: Description of this enum class. + full_name: Fully qualified name of this enum class. + """ + name = messages.StringField(1) + values = messages.MessageField(ExtendedEnumValueDescriptor, 2, repeated=True) + + description = messages.StringField(100) + full_name = messages.StringField(101) + + +class ExtendedFieldDescriptor(messages.Message): + """Field descriptor with additional fields. + + Fields: + field_descriptor: The underlying field descriptor. + name: The name of this field. + description: Description of this field. + """ + field_descriptor = messages.MessageField(descriptor.FieldDescriptor, 100) + # We duplicate the names for easier bookkeeping. + name = messages.StringField(101) + description = messages.StringField(102) + + +class ExtendedMessageDescriptor(messages.Message): + """Message descriptor with additional fields. + + Fields: + name: Name of Message without any qualification. + fields: Fields defined for message. + message_types: Nested Message classes defined on message. + enum_types: Nested Enum classes defined on message. + description: Description of this message. + full_name: Full qualified name of this message. + decorators: Decorators to include in the definition when printing. + Printed in the given order from top to bottom (so the last entry + is the innermost decorator). + """ + name = messages.StringField(1) + fields = messages.MessageField(ExtendedFieldDescriptor, 2, repeated=True) + message_types = messages.MessageField( + 'extended_descriptor.ExtendedMessageDescriptor', 3, repeated=True) + enum_types = messages.MessageField(ExtendedEnumDescriptor, 4, repeated=True) + + description = messages.StringField(100) + full_name = messages.StringField(101) + decorators = messages.StringField(102, repeated=True) + + +class ExtendedFileDescriptor(messages.Message): + """File descriptor with additional fields. + + Fields: + package: Fully qualified name of package that definitions belong to. + message_types: Message definitions contained in file. + enum_types: Enum definitions contained in file. + description: Description of this file. + additional_imports: Extra imports used in this package. + """ + package = messages.StringField(2) + + message_types = messages.MessageField( + ExtendedMessageDescriptor, 4, repeated=True) + enum_types = messages.MessageField( + ExtendedEnumDescriptor, 5, repeated=True) + + description = messages.StringField(100) + additional_imports = messages.StringField(101, repeated=True) + + +def _WriteFile(file_descriptor, package, version, proto_printer): + """Write the given extended file descriptor to the printer.""" + proto_printer.PrintPreamble(package, version, file_descriptor) + _PrintEnums(proto_printer, file_descriptor.enum_types) + _PrintMessages(proto_printer, file_descriptor.message_types) + + +def WriteMessagesFile(file_descriptor, package, version, out): + """Write the given extended file descriptor to out as a message file.""" + _WriteFile(file_descriptor, package, version, + _Proto2Printer(util.SimplePrettyPrinter(out))) + + +def WritePythonFile(file_descriptor, package, version, out): + """Write the given extended file descriptor to out.""" + _WriteFile(file_descriptor, package, version, + _ProtoRpcPrinter(util.SimplePrettyPrinter(out))) + + +def PrintIndentedDescriptions(printer, ls, name, prefix=''): + if ls: + width = printer.CalculateWidth() - len(prefix) + printer(prefix) + printer('%s%s:', prefix, name) + for x in ls: + description = '%s: %s' % (x.name, x.description) + for line in textwrap.wrap(description, width, initial_indent=' ', + subsequent_indent=' '): + printer('%s%s', prefix, line) + + +def _EmptyMessage(message_type): + return not any((message_type.enum_types, + message_type.message_types, + message_type.fields)) + + +class ProtoPrinter(object): + """Interface for proto printers.""" + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def PrintPreamble(self, package, version, file_descriptor): + """Print the file docstring and import lines.""" + + @abc.abstractmethod + def PrintEnum(self, enum_type): + """Print the given enum declaration.""" + + @abc.abstractmethod + def PrintMessage(self, message_type): + """Print the given message declaration.""" + + +class _Proto2Printer(ProtoPrinter): + """Printer for proto2 definitions.""" + + def __init__(self, printer): + self.__printer = printer + + def __PrintEnumCommentLines(self, enum_type): + description = enum_type.description or '%s enum type.' % enum_type.name + for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + self.__printer('// %s', line) + PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values', + prefix='// ') + + def __PrintEnumValueCommentLines(self, enum_value): + if enum_value.description: + width = self.__printer.CalculateWidth() - 3 + for line in textwrap.wrap(enum_value.description, width): + self.__printer('// %s', line) + + def PrintEnum(self, enum_type): + self.__PrintEnumCommentLines(enum_type) + self.__printer('enum %s {', enum_type.name) + with self.__printer.Indent(): + enum_values = sorted(enum_type.values, key=operator.attrgetter('number')) + for enum_value in enum_values: + self.__printer() + self.__PrintEnumValueCommentLines(enum_value) + self.__printer('%s = %s;', enum_value.name, enum_value.number) + self.__printer('}') + self.__printer() + + def PrintPreamble(self, package, version, file_descriptor): + self.__printer('// Generated message classes for %s version %s.', + package, version) + description_lines = textwrap.wrap(file_descriptor.description, 75) + if description_lines: + self.__printer('//') + for line in description_lines: + self.__printer('// %s', line) + self.__printer() + self.__printer('syntax = "proto2";') + self.__printer('package %s;', file_descriptor.package) + + def __PrintMessageCommentLines(self, message_type): + """Print the description of this message.""" + description = message_type.description or '%s message type.' % ( + message_type.name) + width = self.__printer.CalculateWidth() - 3 + for line in textwrap.wrap(description, width): + self.__printer('// %s', line) + PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums', + prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.message_types, + 'Messages', prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields', + prefix='// ') + + def __PrintFieldDescription(self, description): + for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + self.__printer('// %s', line) + + def __PrintFields(self, fields): + for extended_field in fields: + field = extended_field.field_descriptor + field_type = messages.Field.lookup_field_type_by_variant(field.variant) + self.__printer() + self.__PrintFieldDescription(extended_field.description) + label = str(field.label).lower() + if field_type in (messages.EnumField, messages.MessageField): + proto_type = field.type_name + else: + proto_type = str(field.variant).lower() + default_statement = '' + if field.default_value: + if field_type in [messages.BytesField, messages.StringField]: + default_value = '"%s"' % field.default_value + elif field_type is messages.BooleanField: + default_value = str(field.default_value).lower() + else: + default_value = str(field.default_value) + + default_statement = ' [default = %s]' % default_value + self.__printer( + '%s %s %s = %d%s;', + label, proto_type, field.name, field.number, default_statement) + + def PrintMessage(self, message_type): + self.__printer() + self.__PrintMessageCommentLines(message_type) + if _EmptyMessage(message_type): + self.__printer('message %s {}', message_type.name) + return + self.__printer('message %s {', message_type.name) + with self.__printer.Indent(): + _PrintEnums(self, message_type.enum_types) + _PrintMessages(self, message_type.message_types) + self.__PrintFields(message_type.fields) + self.__printer('}') + + +class _ProtoRpcPrinter(ProtoPrinter): + """Printer for ProtoRPC definitions.""" + + def __init__(self, printer): + self.__printer = printer + + def __PrintClassSeparator(self): + self.__printer() + if not self.__printer.indent: + self.__printer() + + def __PrintEnumDocstringLines(self, enum_type): + description = enum_type.description or '%s enum type.' % enum_type.name + for line in textwrap.wrap('"""%s' % description, + self.__printer.CalculateWidth()): + self.__printer(line) + PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values') + self.__printer('"""') + + def PrintEnum(self, enum_type): + self.__printer('class %s(messages.Enum):', enum_type.name) + with self.__printer.Indent(): + self.__PrintEnumDocstringLines(enum_type) + enum_values = sorted(enum_type.values, key=operator.attrgetter('number')) + for enum_value in enum_values: + self.__printer('%s = %s', enum_value.name, enum_value.number) + if not enum_type.values: + self.__printer('pass') + self.__PrintClassSeparator() + + def __PrintAdditionalImports(self, imports): + """Print additional imports needed for protorpc.""" + google_imports = [x for x in imports if 'google' in x] + other_imports = [x for x in imports if 'google' not in x] + if other_imports: + for import_ in sorted(other_imports): + self.__printer(import_) + self.__printer() + # Note: If we ever were going to add imports from this package, we'd + # need to sort those out and put them at the end. + if google_imports: + for import_ in sorted(google_imports): + self.__printer(import_) + self.__printer() + + def PrintPreamble(self, package, version, file_descriptor): + self.__printer('"""Generated message classes for %s version %s.', + package, version) + self.__printer() + for line in textwrap.wrap(file_descriptor.description, 78): + self.__printer(line) + self.__printer('"""') + self.__printer() + self.__PrintAdditionalImports(file_descriptor.additional_imports) + self.__printer() + self.__printer("package = '%s'", file_descriptor.package) + self.__printer() + self.__printer() + + def __PrintMessageDocstringLines(self, message_type): + """Print the docstring for this message.""" + description = message_type.description or '%s message type.' % ( + message_type.name) + short_description = ( + _EmptyMessage(message_type) and + len(description) < (self.__printer.CalculateWidth() - 6)) + if short_description: + self.__printer('"""%s"""', description) + return + for line in textwrap.wrap('"""%s' % description, + self.__printer.CalculateWidth()): + self.__printer(line) + + PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums') + PrintIndentedDescriptions( + self.__printer, message_type.message_types, 'Messages') + PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields') + self.__printer('"""') + self.__printer() + + def PrintMessage(self, message_type): + for decorator in message_type.decorators: + self.__printer('@%s', decorator) + self.__printer('class %s(messages.Message):', message_type.name) + with self.__printer.Indent(): + self.__PrintMessageDocstringLines(message_type) + _PrintEnums(self, message_type.enum_types) + _PrintMessages(self, message_type.message_types) + _PrintFields(message_type.fields, self.__printer) + self.__PrintClassSeparator() + + +def _PrintEnums(proto_printer, enum_types): + """Print all enums to the given proto_printer.""" + enum_types = sorted(enum_types, key=operator.attrgetter('name')) + for enum_type in enum_types: + proto_printer.PrintEnum(enum_type) + + +def _PrintMessages(proto_printer, message_list): + message_list = sorted(message_list, key=operator.attrgetter('name')) + for message_type in message_list: + proto_printer.PrintMessage(message_type) + + +_MESSAGE_FIELD_MAP = { + message_types.DateTimeMessage.definition_name(): message_types.DateTimeField, + } + + +def _PrintFields(fields, printer): + for extended_field in fields: + field = extended_field.field_descriptor + printed_field_info = { + 'name': field.name, + 'module': 'messages', + 'type_name': '', + 'type_format': '', + 'number': field.number, + 'label_format': '', + 'variant_format': '', + 'default_format': '', + } + + message_field = _MESSAGE_FIELD_MAP.get(field.type_name) + if message_field: + printed_field_info['module'] = 'message_types' + field_type = message_field + else: + field_type = messages.Field.lookup_field_type_by_variant(field.variant) + + if field_type in (messages.EnumField, messages.MessageField): + printed_field_info['type_format'] = "'%s', " % field.type_name + + if field.label == descriptor.FieldDescriptor.Label.REQUIRED: + printed_field_info['label_format'] = ', required=True' + elif field.label == descriptor.FieldDescriptor.Label.REPEATED: + printed_field_info['label_format'] = ', repeated=True' + + if field_type.DEFAULT_VARIANT != field.variant: + printed_field_info['variant_format'] = ', variant=messages.Variant.%s' % ( + field.variant,) + + if field.default_value: + if field_type in [messages.BytesField, messages.StringField]: + default_value = repr(field.default_value) + elif field_type is messages.EnumField: + try: + default_value = str(int(field.default_value)) + except ValueError: + default_value = repr(field.default_value) + else: + default_value = field.default_value + + printed_field_info['default_format'] = ', default=%s' % (default_value,) + + printed_field_info['type_name'] = field_type.__name__ + args = ''.join('%%(%s)s' % field for field in ( + 'type_format', + 'number', + 'label_format', + 'variant_format', + 'default_format')) + format_str = '%%(name)s = %%(module)s.%%(type_name)s(%s)' % args + printer(format_str % printed_field_info) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py new file mode 100644 index 0000000..34287cd --- /dev/null +++ b/apitools/gen/gen_client.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +"""Command-line interface to gen_client.""" + +import contextlib +import json +import logging +import os +import pkgutil +import sys + +from google.apputils import appcommands +import gflags as flags + +from apitools.base.py import exceptions +from apitools.gen import gen_client_lib +from apitools.gen import util + +flags.DEFINE_string( + 'infile', '', + 'Filename for the discovery document. Mutually exclusive with ' + '--discovery_url.') +flags.DEFINE_string( + 'discovery_url', '', + 'URL of the discovery document to use. Mutually exclusive with --infile.') + +flags.DEFINE_string( + 'outdir', '', + 'Directory name for output files. (Defaults to the API name.)') +flags.DEFINE_boolean( + 'overwrite', False, + 'Only overwrite the output directory if this flag is specified.') +flags.DEFINE_string( + 'root_package_dir', '', + 'Ultimate destination for generated code (used for generating ' + 'correct import lines). Defaults to the value of FLAGS.outdir.' + ) + +flags.DEFINE_multistring( + 'strip_prefix', [], + 'Prefix to strip from type names in the discovery document. (May ' + 'be specified multiple times.)') +flags.DEFINE_string( + 'api_key', None, + 'API key to use for API access.') +flags.DEFINE_string( + 'client_id', None, + 'Client ID to use for the generated client.') +flags.DEFINE_string( + 'client_secret', None, + 'Client secret for the generated client.') +flags.DEFINE_multistring( + 'scope', [], + 'Scopes to request in the generated client. May be specified more than ' + 'once.') +flags.DEFINE_string( + 'user_agent', '', + 'User agent for the generated client. Defaults to -generated/0.1.') + +flags.DEFINE_boolean( + 'experimental_capitalize_enums', False, + 'Dangerous: attempt to rewrite enum values to be uppercase.') +flags.DEFINE_enum( + 'experimental_name_convention', util.Names.DEFAULT_NAME_CONVENTION, + util.Names.NAME_CONVENTIONS, + 'Dangerous: use a particular style for generated names.') +flags.DEFINE_boolean( + 'experimental_proto2_output', False, + 'Dangerous: also output a proto2 message file.') + +FLAGS = flags.FLAGS + +flags.MarkFlagAsRequired('client_id') +flags.MarkFlagAsRequired('client_secret') +flags.RegisterValidator( + 'infile', lambda i: not (i and FLAGS.discovery_url), + 'Cannot specify both --infile and --discovery_url') +flags.RegisterValidator( + 'discovery_url', lambda i: not (i and FLAGS.infile), + 'Cannot specify both --infile and --discovery_url') + + +def _CopyLocalFile(filename): + with contextlib.closing(open(filename, 'w')) as out: + src_data = pkgutil.get_data( + 'apitools.base.py', filename) + if src_data is None: + raise exceptions.GeneratedClientError('Could not find file %s' % filename) + out.write(src_data) + + +def _GetCodegenFromFlags(): + """Create a codegen object from flags.""" + if FLAGS.discovery_url: + try: + discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) + except exceptions.CommunicationError: + return None + else: + infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' + discovery_doc = json.load(open(infile)) + names = util.Names( + FLAGS.strip_prefix, + FLAGS.experimental_name_convention, + FLAGS.experimental_capitalize_enums) + client_info = util.ClientInfo.Create( + discovery_doc, FLAGS.scope, FLAGS.client_id, FLAGS.client_secret, + FLAGS.user_agent, names, FLAGS.api_key) + outdir = os.path.expanduser(FLAGS.outdir) or client_info.default_directory + if os.path.exists(outdir) and not FLAGS.overwrite: + raise exceptions.ConfigurationValueError( + 'Output directory exists, pass --overwrite to replace ' + 'the existing files.') + if not FLAGS.root_package_dir: + FLAGS.root_package_dir = outdir + FLAGS.root_package_dir = os.path.abspath(FLAGS.root_package_dir) + root_package = util.GetPackage(FLAGS.root_package_dir) + return gen_client_lib.DescriptorGenerator( + discovery_doc, client_info, names, root_package, outdir, + use_proto2=FLAGS.experimental_proto2_output) + + +# TODO(craigcitro): Delete this if we don't need this functionality. +def _WriteBaseFiles(codegen): + with util.Chdir(codegen.outdir): + _CopyLocalFile('app2.py') + _CopyLocalFile('base_api.py') + _CopyLocalFile('base_cli.py') + _CopyLocalFile('credentials_lib.py') + _CopyLocalFile('exceptions.py') + + +def _WriteProtoFiles(codegen): + with util.Chdir(codegen.outdir): + with open(codegen.client_info.messages_proto_file_name, 'w') as out: + codegen.WriteMessagesProtoFile(out) + with open(codegen.client_info.services_proto_file_name, 'w') as out: + codegen.WriteServicesProtoFile(out) + + +def _WriteGeneratedFiles(codegen): + if codegen.use_proto2: + _WriteProtoFiles(codegen) + with util.Chdir(codegen.outdir): + with open(codegen.client_info.messages_file_name, 'w') as out: + codegen.WriteMessagesFile(out) + with open(codegen.client_info.client_file_name, 'w') as out: + codegen.WriteClientLibrary(out) + with open(codegen.client_info.cli_file_name, 'w') as out: + codegen.WriteCli(out) + + +def _WriteInit(codegen): + with util.Chdir(codegen.outdir): + with open('__init__.py', 'w') as out: + codegen.WriteInit(out) + + +class GenerateClient(appcommands.Cmd): + """Driver for client code generation.""" + + def Run(self, _): + """Create a client library.""" + codegen = _GetCodegenFromFlags() + if codegen is None: + logging.error('Failed to create codegen, exiting.') + return 128 + _WriteGeneratedFiles(codegen) + _WriteInit(codegen) + + +class GenerateProto(appcommands.Cmd): + """Generate just the two proto files for a given API.""" + + def Run(self, _): + """Create proto definitions for an API.""" + codegen = _GetCodegenFromFlags() + _WriteProtoFiles(codegen) + + +# pylint: disable-msg=invalid-name + + +def run_main(): + """Function to be used as setuptools script entry point.""" + # Put the flags for this module somewhere the flags module will look + # for them. + + # pylint: disable-msg=protected-access + new_name = flags._GetMainModule() + sys.modules[new_name] = sys.modules['__main__'] + for flag in FLAGS.FlagsByModuleDict().get(__name__, []): + FLAGS._RegisterFlagByModule(new_name, flag) + for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): + FLAGS._RegisterKeyFlagForModule(new_name, key_flag) + # pylint: enable-msg=protected-access + + # Now set __main__ appropriately so that appcommands will be + # happy. + sys.modules['__main__'] = sys.modules[__name__] + appcommands.Run() + sys.modules['__main__'] = sys.modules.pop(new_name) + + +def main(_): + appcommands.AddCmd('client', GenerateClient) + appcommands.AddCmd('proto', GenerateProto) + + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py new file mode 100644 index 0000000..9e23485 --- /dev/null +++ b/apitools/gen/gen_client_lib.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +"""Simple tool for generating a client library. + +Relevant links: + https://developers.google.com/discovery/v1/reference/apis#resource +""" + +import json +import logging +import urlparse + +from apitools.base.py import base_cli +from apitools.gen import command_registry +from apitools.gen import message_registry +from apitools.gen import service_registry +from apitools.gen import util + + +def _StandardQueryParametersSchema(discovery_doc): + standard_query_schema = { + 'id': 'StandardQueryParameters', + 'type': 'object', + 'description': 'Query parameters accepted by all methods.', + 'properties': discovery_doc.get('parameters', {}), + } + # We add an entry for the trace, since Discovery doesn't. + standard_query_schema['properties']['trace'] = { + 'type': 'string', + 'description': base_cli.TRACE_HELP, + 'location': 'query', + } + return standard_query_schema + + +def _ComputePaths(package, version, discovery_doc): + full_path = urlparse.urljoin( + discovery_doc['rootUrl'], discovery_doc['servicePath']) + api_path_component = '/'.join((package, version, '')) + if api_path_component not in full_path: + logging.warning('Could not find path "%s" in API path "%s"', + api_path_component, full_path) + return full_path, '' + prefix, _, suffix = full_path.rpartition(api_path_component) + return prefix + api_path_component, suffix + + +class DescriptorGenerator(object): + """Code generator for a given discovery document.""" + + def __init__(self, discovery_doc, client_info, names, root_package, outdir, + use_proto2=False): + self.__discovery_doc = discovery_doc + self.__client_info = client_info + self.__outdir = outdir + self.__use_proto2 = use_proto2 + self.__description = self.__discovery_doc.get('description', '') + self.__package = self.__client_info.package + self.__version = self.__client_info.version + self.__root_package = root_package + # TODO(craigcitro): Centralize this information ... somewhere. + self.__base_files_package = 'apitools.base.py' + self.__base_files_target = ( + '//cloud/bigscience/apitools/base/py:apitools_base') + self.__names = names + self.__base_url, self.__base_path = _ComputePaths( + self.__package, self.__client_info.url_version, self.__discovery_doc) + + # Order is important here: we need the schemas before we can + # define the services. + self.__message_registry = message_registry.MessageRegistry( + self.__client_info, self.__names, self.__description, + self.__root_package, self.__base_files_package) + schemas = self.__discovery_doc.get('schemas', {}) + for schema_name, schema in schemas.iteritems(): + self.__message_registry.AddDescriptorFromSchema(schema_name, schema) + + # We need to add one more message type for the global parameters. + standard_query_schema = _StandardQueryParametersSchema( + self.__discovery_doc) + self.__message_registry.AddDescriptorFromSchema( + standard_query_schema['id'], standard_query_schema) + + self.__command_registry = command_registry.CommandRegistry( + self.__package, self.__version, self.__client_info, + self.__message_registry, self.__root_package, self.__base_files_package, + self.__base_url, self.__names) + self.__command_registry.AddGlobalParameters( + self.__message_registry.LookupDescriptorOrDie( + 'StandardQueryParameters')) + + self.__services_registry = service_registry.ServiceRegistry( + self.__client_info, + self.__message_registry, + self.__command_registry, + self.__base_url, + self.__base_path, + self.__names, + self.__root_package, + self.__base_files_package) + services = self.__discovery_doc.get('resources', {}) + for service_name, methods in sorted(services.iteritems()): + self.__services_registry.AddServiceFromResource(service_name, methods) + # We might also have top-level methods. + api_methods = self.__discovery_doc.get('methods', []) + if api_methods: + self.__services_registry.AddServiceFromResource( + 'api', {'methods': api_methods}) + self.__client_info = self.__client_info._replace(scopes=self.__services_registry.scopes) # pylint:disable=protected-access,g-line-too-long + + @property + def client_info(self): + return self.__client_info + + @property + def discovery_doc(self): + return self.__discovery_doc + + @property + def names(self): + return self.__names + + @property + def outdir(self): + return self.__outdir + + @property + def use_proto2(self): + return self.__use_proto2 + + def WriteInit(self, out): + """Write a simple __init__.py for the generated client.""" + printer = util.SimplePrettyPrinter(out) + printer('"""Common imports for generated %s client library."""', + self.__client_info.package) + printer() + printer('from %s import credentials_lib', self.__base_files_package) + printer('from %s.base_api import *', self.__base_files_package) + printer('from %s.exceptions import *', self.__base_files_package) + printer('from %s.transfer import *', self.__base_files_package) + printer('from %s.%s import *', + self.__root_package, self.__client_info.client_rule_name) + printer('from %s.%s import *', + self.__root_package, self.__client_info.messages_rule_name) + + def WriteMessagesFile(self, out): + self.__message_registry.WriteFile(out) + + def WriteMessagesProtoFile(self, out): + self.__message_registry.WriteProtoFile(out) + + def WriteServicesProtoFile(self, out): + self.__services_registry.WriteProtoFile(out) + + def WriteClientLibrary(self, out): + self.__services_registry.WriteFile(out) + + def WriteCli(self, out): + self.__command_registry.WriteFile(out) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py new file mode 100644 index 0000000..290e0de --- /dev/null +++ b/apitools/gen/message_registry.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python +"""Message registry for apitools.""" + +import collections +import contextlib +import json + +from protorpc import descriptor +from protorpc import messages + +from apitools.gen import extended_descriptor + +TypeInfo = collections.namedtuple('TypeInfo', ('type_name', 'variant')) + + +class MessageRegistry(object): + """Registry for message types. + + This closely mirrors a messages.FileDescriptor, but adds additional + attributes (such as message and field descriptions) and some extra + code for validation and cycle detection. + """ + + # Type information from these two maps comes from here: + # https://developers.google.com/discovery/v1/type-format + PRIMITIVE_TYPE_INFO_MAP = { + 'string': TypeInfo(type_name='string', + variant=messages.StringField.DEFAULT_VARIANT), + 'integer': TypeInfo(type_name='integer', + variant=messages.IntegerField.DEFAULT_VARIANT), + 'boolean': TypeInfo(type_name='boolean', + variant=messages.BooleanField.DEFAULT_VARIANT), + 'number': TypeInfo(type_name='number', + variant=messages.FloatField.DEFAULT_VARIANT), + } + + PRIMITIVE_FORMAT_MAP = { + 'int32': TypeInfo(type_name='integer', + variant=messages.Variant.INT32), + 'uint32': TypeInfo(type_name='integer', + variant=messages.Variant.UINT32), + 'int64': TypeInfo(type_name='string', + variant=messages.Variant.INT64), + 'uint64': TypeInfo(type_name='string', + variant=messages.Variant.UINT64), + 'double': TypeInfo(type_name='number', + variant=messages.Variant.DOUBLE), + 'float': TypeInfo(type_name='number', + variant=messages.Variant.FLOAT), + 'byte': TypeInfo(type_name='byte', + variant=messages.BytesField.DEFAULT_VARIANT), + 'date': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', + variant=messages.Variant.MESSAGE), + 'date-time': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', + variant=messages.Variant.MESSAGE), + } + + def __init__(self, client_info, names, description, + root_package_dir, base_files_package): + self.__names = names + self.__client_info = client_info + self.__package = client_info.package + self.__description = description + self.__root_package_dir = root_package_dir + self.__base_files_package = base_files_package + self.__file_descriptor = extended_descriptor.ExtendedFileDescriptor( + package=self.__package, description=self.__description) + # Add required imports + self.__file_descriptor.additional_imports = [ + 'from protorpc import messages', + ] + # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. + self.__message_registry = collections.OrderedDict() + # A set of types that we're currently adding (for cycle detection). + self.__nascent_types = set() + # A set of types for which we've seen a reference but no + # definition; if this set is nonempty, validation fails. + self.__unknown_types = set() + # Used for tracking paths during message creation + self.__current_path = [] + # Where to register created messages + self.__current_env = self.__file_descriptor + # TODO(craigcitro): Add a `Finalize` method. + + @property + def file_descriptor(self): + self.Validate() + return self.__file_descriptor + + def WriteProtoFile(self, out): + """Write the messages file to out as proto.""" + self.Validate() + extended_descriptor.WriteMessagesFile( + self.__file_descriptor, self.__package, self.__client_info.version, out) + + def WriteFile(self, out): + """Write the messages file to out.""" + self.Validate() + extended_descriptor.WritePythonFile( + self.__file_descriptor, self.__package, self.__client_info.version, out) + + def Validate(self): + mysteries = self.__nascent_types or self.__unknown_types + if mysteries: + raise ValueError('Malformed MessageRegistry: %s' % mysteries) + + def __ComputeFullName(self, name): + return '.'.join(map(unicode, self.__current_path[:] + [name])) + + def __AddImport(self, new_import): + if new_import not in self.__file_descriptor.additional_imports: + self.__file_descriptor.additional_imports.append(new_import) + + def __DeclareDescriptor(self, name): + self.__nascent_types.add(self.__ComputeFullName(name)) + + def __RegisterDescriptor(self, new_descriptor): + if not isinstance(new_descriptor, ( + extended_descriptor.ExtendedMessageDescriptor, + extended_descriptor.ExtendedEnumDescriptor)): + raise ValueError('Cannot add descriptor of type %s' % ( + type(new_descriptor),)) + full_name = self.__ComputeFullName(new_descriptor.name) + if full_name in self.__message_registry: + raise ValueError('Attempt to re-register descriptor %s' % full_name) + if full_name not in self.__nascent_types: + raise ValueError('Directly adding types is not supported') + new_descriptor.full_name = full_name + self.__message_registry[full_name] = new_descriptor + if isinstance(new_descriptor, + extended_descriptor.ExtendedMessageDescriptor): + self.__current_env.message_types.append(new_descriptor) + elif isinstance(new_descriptor, extended_descriptor.ExtendedEnumDescriptor): + self.__current_env.enum_types.append(new_descriptor) + self.__unknown_types.discard(full_name) + self.__nascent_types.remove(full_name) + + def LookupDescriptor(self, name): + return self.__GetDescriptorByName(name) + + def LookupDescriptorOrDie(self, name): + message_descriptor = self.LookupDescriptor(name) + if message_descriptor is None: + raise ValueError('No message descriptor named "%s"', name) + return message_descriptor + + def __GetDescriptor(self, name): + return self.__GetDescriptorByName(self.__ComputeFullName(name)) + + def __GetDescriptorByName(self, name): + if name in self.__message_registry: + return self.__message_registry[name] + if name in self.__nascent_types: + raise ValueError( + 'Cannot retrieve type currently being created: %s' % name) + return None + + @contextlib.contextmanager + def __DescriptorEnv(self, message_descriptor): + # TODO(craigcitro): Typecheck? + previous_env = self.__current_env + self.__current_path.append(message_descriptor.name) + self.__current_env = message_descriptor + yield + self.__current_path.pop() + self.__current_env = previous_env + + def AddEnumDescriptor(self, name, description, + enum_values, enum_descriptions): + """Add a new EnumDescriptor named name with the given enum values.""" + message = extended_descriptor.ExtendedEnumDescriptor() + message.name = self.__names.ClassName(name) + message.description = description + self.__DeclareDescriptor(message.name) + for index, (enum_name, enum_description) in enumerate( + zip(enum_values, enum_descriptions)): + enum_value = extended_descriptor.ExtendedEnumValueDescriptor() + enum_value.name = self.__names.NormalizeEnumName(enum_name) + enum_value.number = index + enum_value.description = enum_description or '' + message.values.append(enum_value) + self.__RegisterDescriptor(message) + + def AddDescriptorFromSchema(self, schema_name, schema): + """Add a new MessageDescriptor named schema_name based on schema.""" + # TODO(craigcitro): Is schema_name redundant? + if self.__GetDescriptor(schema_name): + return + if schema.get('type') not in ('object', 'any'): + raise ValueError( + 'Cannot create message descriptors for type %s', schema.get('type')) + message = extended_descriptor.ExtendedMessageDescriptor() + message.name = self.__names.ClassName(schema['id']) + message.description = schema.get( + 'description', 'A %s object.' % message.name) + self.__DeclareDescriptor(message.name) + with self.__DescriptorEnv(message): + properties = schema.get('properties', {}) + for index, (name, attrs) in enumerate(sorted(properties.iteritems())): + field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) + message.fields.append(field) + if 'additionalProperties' in schema: + additional_properties_info = schema['additionalProperties'] + entries_type_name = self.__AddAdditionalPropertyType( + message.name, additional_properties_info) + description = additional_properties_info.get('description') + if description is None: + description = 'Additional properties of type %s' % message.name + attrs = { + 'items': { + '$ref': entries_type_name, + }, + 'description': description, + 'type': 'array', + } + field_name = 'additionalProperties' + message.fields.append(self.__FieldDescriptorFromProperties( + field_name, len(properties) + 1, attrs)) + self.__AddImport( + 'from %s import encoding' % self.__base_files_package) + message.decorators.append( + 'encoding.MapUnrecognizedFields(%r)' % field_name) + self.__RegisterDescriptor(message) + + def __AddAdditionalPropertyType(self, name, property_schema): + new_type_name = 'AdditionalProperty' + property_schema = dict(property_schema) + # We drop the description here on purpose, so the resulting + # messages are less repetitive. + property_schema.pop('description', None) + description = 'An additional property for a %s object.' % name + schema = { + 'id': new_type_name, + 'type': 'object', + 'description': description, + 'properties': { + 'key': { + 'type': 'string', + 'description': 'Name of the additional property.', + }, + 'value': property_schema, + }, + } + self.AddDescriptorFromSchema(new_type_name, schema) + return new_type_name + + def __FieldDescriptorFromProperties(self, name, index, attrs): + field = descriptor.FieldDescriptor() + field.name = self.__names.CleanName(name) + field.number = index + field.label = self.__ComputeLabel(attrs) + new_type_name_hint = self.__names.ClassName( + '%sValue' % self.__names.ClassName(name)) + type_info = self.__GetTypeInfo(attrs, new_type_name_hint) + field.type_name = type_info.type_name + field.variant = type_info.variant + if 'default' in attrs: + # TODO(craigcitro): Correctly handle non-primitive default values. + default = attrs['default'] + if field.type_name != 'string' and field.variant != messages.Variant.ENUM: + default = str(json.loads(default)) + if field.variant == messages.Variant.ENUM: + default = self.__names.NormalizeEnumName(default) + field.default_value = default + extended_field = extended_descriptor.ExtendedFieldDescriptor() + extended_field.name = field.name + extended_field.description = attrs.get('description', 'A %s attribute.' % ( + field.type_name,)) + extended_field.field_descriptor = field + return extended_field + + @staticmethod + def __ComputeLabel(attrs): + if attrs.get('required', False): + return descriptor.FieldDescriptor.Label.REQUIRED + elif attrs.get('type') == 'array': + return descriptor.FieldDescriptor.Label.REPEATED + return descriptor.FieldDescriptor.Label.OPTIONAL + + def __GetTypeInfo(self, attrs, name_hint): + """Return a TypeInfo object for attrs, creating one if needed.""" + + def AddIfUnknown(type_name): + type_name = self.__names.ClassName(type_name) + full_type_name = self.__ComputeFullName(type_name) + if (full_type_name not in self.__message_registry.viewkeys() and + type_name not in self.__message_registry.viewkeys()): + self.__unknown_types.add(type_name) + + type_ref = self.__names.ClassName(attrs.get('$ref')) + type_name = attrs.get('type') + if not (type_ref or type_name): + raise ValueError('No type found for %s' % attrs) + + if type_ref: + AddIfUnknown(type_ref) + return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) + + if 'enum' in attrs: + enum_name = '%sValuesEnum' % name_hint + description = attrs.get('description', '') + self.AddEnumDescriptor(enum_name, description, + attrs['enum'], attrs['enumDescriptions']) + AddIfUnknown(enum_name) + return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) + + if 'format' in attrs: + type_info = self.PRIMITIVE_FORMAT_MAP.get(attrs['format']) + if (type_info.type_name.startswith('protorpc.message_types.') or + type_info.type_name.startswith('message_types.')): + self.__AddImport('from protorpc import message_types') + if type_info is None: + raise ValueError('Unknown format %s for type %s' % ( + attrs['format'], type_name)) + return type_info + + if type_name in self.PRIMITIVE_TYPE_INFO_MAP: + type_info = self.PRIMITIVE_TYPE_INFO_MAP[type_name] + return type_info + + if type_name == 'array': + items = attrs.get('items') + if not items: + raise ValueError('Array type with no item type: %s' % attrs) + item_name_hint = items.get('title') or '%sListEntry' % name_hint + item_name_hint = self.__names.ClassName(item_name_hint) + return self.__GetTypeInfo(items, item_name_hint) + elif type_name == 'any': + return self.PRIMITIVE_TYPE_INFO_MAP['string'] + elif type_name == 'object': + # TODO(craigcitro): Think of a better way to come up with names. + if not name_hint: + raise ValueError('Cannot create subtype without some name hint') + schema = dict(attrs) + schema['id'] = name_hint + self.AddDescriptorFromSchema(name_hint, schema) + AddIfUnknown(name_hint) + return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) + + raise ValueError('Unknown type: %s' % type_name) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py new file mode 100644 index 0000000..388c742 --- /dev/null +++ b/apitools/gen/service_registry.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python +"""Service registry for apitools.""" + +import collections +import logging +import re +import textwrap + + +from apitools.base.py import base_api +from apitools.gen import util + + +class ServiceRegistry(object): + """Registry for service types.""" + + def __init__(self, client_info, message_registry, command_registry, + base_url, base_path, names, + root_package_dir, base_files_package): + self.__client_info = client_info + self.__package = client_info.package + self.__names = names + self.__service_method_info_map = collections.OrderedDict() + self.__message_registry = message_registry + self.__command_registry = command_registry + self.__base_url = base_url + self.__base_path = base_path + self.__root_package_dir = root_package_dir + self.__base_files_package = base_files_package + self.__all_scopes = set(self.__client_info.scopes) + + def Validate(self): + self.__message_registry.Validate() + + @property + def scopes(self): + return sorted(list(self.__all_scopes)) + + def __GetServiceClassName(self, service_name): + return self.__names.ClassName( + '%sService' % self.__names.ClassName(service_name)) + + def __PrintDocstring(self, printer, method_info, method_name, name): + """Print a docstring for a service method.""" + if method_info.description: + description = method_info.description + first_line, newline, remaining = method_info.description.partition( + '\n') + if not first_line.endswith('.'): + first_line = '%s.' % first_line + description = '%s%s%s' % (first_line, newline, remaining) + else: + description = '%s method for the %s service.' % (method_name, name) + printer('"""%s', description) + printer() + printer('Args:') + printer(' request: (%s) input message', method_info.request_type_name) + printer(' global_params: (StandardQueryParameters, default: None) ' + 'global arguments') + if method_info.upload_config: + printer(' upload: (Upload, default: None) If present, upload') + printer(' this stream with the request.') + if method_info.supports_download: + printer(' download: (Download, default: None) If present, download') + printer(' data from the request via this stream.') + printer('Returns:') + printer(' (%s) The response message.', method_info.response_type_name) + printer('"""') + + def __WriteSingleService(self, printer, name, method_info_map): + printer() + class_name = self.__GetServiceClassName(name) + printer('class %s(base_api.BaseApiService):', class_name) + with printer.Indent(): + printer('"""Service class for the %s resource."""', name) + for method_name, method_info in method_info_map.iteritems(): + printer() + params = ['self', 'request', 'global_params=None'] + if method_info.upload_config: + params.append('upload=None') + if method_info.supports_download: + params.append('download=None') + printer('def %s(%s):', method_name, ', '.join(params)) + with printer.Indent(): + self.__PrintDocstring(printer, method_info, method_name, name) + printer('config = base_api.ApiMethodInfo(') + with printer.Indent(indent=' '): + attrs = sorted(x.name for x in method_info.all_fields()) + for attr in attrs: + if attr in ('upload_config', 'description'): + continue + printer('%s=%r,', attr, getattr(method_info, attr)) + printer(')') + + upload_config = method_info.upload_config + if upload_config is not None: + printer('upload_config = base_api.ApiUploadInfo(') + with printer.Indent(indent=' '): + attrs = sorted(x.name for x in upload_config.all_fields()) + for attr in attrs: + printer('%s=%r,', attr, getattr(upload_config, attr)) + printer(')') + + arg_lines = ['config, request, global_params=global_params'] + if method_info.upload_config: + arg_lines.append('upload=upload, upload_config=upload_config') + if method_info.supports_download: + arg_lines.append('download=download') + printer('return self._RunMethod(') + with printer.Indent(indent=' '): + for line in arg_lines[:-1]: + printer('%s,', line) + printer('%s)', arg_lines[-1]) + + def __WriteProtoServiceDeclaration(self, printer, name, method_info_map): + """Write a single service declaration to a proto file.""" + printer() + printer('service %s {', self.__GetServiceClassName(name)) + with printer.Indent(): + for method_name, method_info in method_info_map.iteritems(): + for line in textwrap.wrap(method_info.description, + printer.CalculateWidth() - 3): + printer('// %s', line) + printer('rpc %s (%s) returns (%s);', + method_name, + method_info.request_type_name, + method_info.response_type_name) + printer('}') + + def WriteProtoFile(self, out): + """Write the services in this registry to out as proto.""" + self.Validate() + client_info = self.__client_info + printer = util.SimplePrettyPrinter(out) + printer('// Generated services for %s version %s.', + client_info.package, client_info.version) + printer() + printer('syntax = "proto2";') + printer('package %s;', self.__package) + printer('import "%s";', client_info.messages_proto_file_name) + printer() + for name, method_info_map in self.__service_method_info_map.iteritems(): + self.__WriteProtoServiceDeclaration(printer, name, method_info_map) + + def WriteFile(self, out): + """Write the services in this registry to out.""" + self.Validate() + client_info = self.__client_info + printer = util.SimplePrettyPrinter(out) + printer('"""Generated client library for %s version %s."""', + client_info.package, client_info.version) + printer() + printer() + printer('class %s(base_api.BaseApiClient):', client_info.client_class_name) + with printer.Indent(): + printer('"""Generated client library for service %s version %s."""', + client_info.package, client_info.version) + printer() + printer('MESSAGES_MODULE = messages') + printer() + client_info_items = client_info._asdict().iteritems() # pylint:disable=protected-access + for attr, val in client_info_items: + if attr == 'api_key': + printer('# MOE:begin_strip') + printer('_%s = %r' % (attr.upper(), val)) + if attr == 'api_key': + printer('# MOE:end_strip') + printer() + printer("def __init__(self, url='', credentials=None,") + printer(' get_credentials=True, http=None, model=None,') + printer(' log_request=False, log_response=False,') + printer(' default_global_params=None):') + with printer.Indent(): + printer('"""Create a new %s handle."""', client_info.package) + printer('url = url or %r', self.__base_url) + printer('super(%s, self).__init__(', client_info.client_class_name) + printer(' url, credentials=credentials,') + printer(' get_credentials=get_credentials, http=http, model=model,') + printer(' log_request=log_request, log_response=log_response,') + printer(' default_global_params=default_global_params)') + for name in self.__service_method_info_map.iterkeys(): + printer('self.%s = self.%s(self)', + name, self.__GetServiceClassName(name)) + for name, method_info_map in self.__service_method_info_map.iteritems(): + self.__WriteSingleService(printer, name, method_info_map) + + def __RegisterService(self, service_name, method_info_map): + if service_name in self.__service_method_info_map: + raise ValueError('Attempt to re-register descriptor %s' % service_name) + self.__service_method_info_map[service_name] = method_info_map + + def __CreateRequestType(self, method_description, body_type=None): + """Create a request type for this method.""" + schema = {} + schema['id'] = self.__names.ClassName('%sRequest' % ( + self.__names.ClassName(method_description['id'], separator='.'),)) + schema['type'] = 'object' + schema['properties'] = collections.OrderedDict() + if 'parameterOrder' not in method_description: + ordered_parameters = list(method_description.get('parameters', [])) + else: + ordered_parameters = method_description['parameterOrder'][:] + for k in method_description['parameters']: + if k not in ordered_parameters: + ordered_parameters.append(k) + for parameter_name in ordered_parameters: + field_name = self.__names.CleanName(parameter_name) + field = dict(method_description['parameters'][parameter_name]) + if 'type' not in field: + raise ValueError('No type found in parameter %s' % field) + schema['properties'][field_name] = field + if body_type is not None: + body_field_name = self.__GetRequestField(method_description, body_type) + if body_field_name in schema['properties']: + raise ValueError('Failed to normalize request resource name') + if 'description' not in body_type: + body_type['description'] = ( + 'A %s resource to be passed as the request body.' % ( + self.__GetRequestType(body_type),)) + schema['properties'][body_field_name] = body_type + self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) + return schema['id'] + + def __CreateVoidResponseType(self, method_description): + """Create an empty response type.""" + schema = {} + method_name = self.__names.ClassName( + method_description['id'], separator='.') + schema['id'] = self.__names.ClassName('%sResponse' % method_name) + schema['type'] = 'object' + schema['description'] = 'An empty %s response.' % method_name + self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) + return schema['id'] + + def __NeedRequestType(self, method_description, request_type): + """Determine if this method needs a new request type created.""" + if not request_type: + return True + message = self.__message_registry.LookupDescriptorOrDie(request_type) + if message is None: + return True + field_names = [x.name for x in message.fields] + parameters = method_description.get('parameters', {}) + for param_name, param_info in parameters.iteritems(): + if (param_info.get('location') != 'path' or + self.__names.CleanName(param_name) not in field_names): + break + else: + return False + return True + + def __MaxSizeToInt(self, max_size): + """Convert max_size to an int.""" + size_groups = re.match(r'(?P\d+)(?P.B)?$', max_size) + if size_groups is None: + raise ValueError('Could not parse maxSize') + size, unit = size_groups.group('size', 'unit') + shift = 0 + if unit is not None: + unit_dict = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} + shift = unit_dict.get(unit.upper()) + if shift is None: + raise ValueError('Unknown unit %s' % unit) + return int(size) * (1 << shift) + + def __ComputeUploadConfig(self, media_upload_config, method_id): + """Fill out the upload config for this method.""" + config = base_api.ApiUploadInfo() + if 'maxSize' in media_upload_config: + config.max_size = self.__MaxSizeToInt( + media_upload_config['maxSize']) + if 'accept' not in media_upload_config: + logging.warn( + 'No accept types found for upload configuration in ' + 'method %s, using */*', method_id) + config.accept.extend([ + str(a) for a in media_upload_config.get('accept', '*/*')]) + protocols = media_upload_config.get('protocols', {}) + for protocol in ('simple', 'resumable'): + media = protocols.get(protocol, {}) + for attr in ('multipart', 'path'): + if attr in media: + setattr(config, '%s_%s' % (protocol, attr), media[attr]) + return config + + def __ComputeMethodInfo(self, method_description, request, response, + request_field): + """Compute the base_api.ApiMethodInfo for this method.""" + relative_path = self.__names.NormalizeRelativePath( + ''.join((self.__base_path, method_description['path']))) + method_id = method_description['id'] + method_info = base_api.ApiMethodInfo( + relative_path=relative_path, + method_id=method_id, + http_method=method_description['httpMethod'], + description=method_description.get('description', ''), + query_params=[], + path_params=[], + ordered_params=method_description.get('parameterOrder', []), + request_type_name=self.__names.ClassName(request), + response_type_name=self.__names.ClassName(response), + request_field=request_field, + ) + if method_description.get('supportsMediaUpload', False): + method_info.upload_config = self.__ComputeUploadConfig( + method_description.get('mediaUpload'), method_id) + method_info.supports_download = method_description.get( + 'supportsMediaDownload', False) + self.__all_scopes.update(method_description.get('scopes', ())) + for param, desc in method_description.get('parameters', {}).iteritems(): + param = self.__names.CleanName(param) + location = desc['location'] + if location == 'query': + method_info.query_params.append(param) + elif location == 'path': + method_info.path_params.append(param) + else: + raise ValueError('Unknown parameter location %s for parameter %s' % ( + location, param)) + method_info.path_params.sort() + method_info.query_params.sort() + return method_info + + def __BodyFieldName(self, body_type): + if body_type is None: + return '' + return self.__names.FieldName(body_type['$ref']) + + def __GetRequestType(self, body_type): + return self.__names.ClassName(body_type.get('$ref')) + + def __GetRequestField(self, method_description, body_type): + """Determine the request field for this method.""" + body_field_name = self.__BodyFieldName(body_type) + if body_field_name in method_description.get('parameters', {}): + body_field_name = self.__names.FieldName( + '%s_resource' % body_field_name) + # It's exceedingly unlikely that we'd get two name collisions, which + # means it's bound to happen at some point. + while body_field_name in method_description.get('parameters', {}): + body_field_name = self.__names.FieldName( + '%s_body' % body_field_name) + return body_field_name + + def AddServiceFromResource(self, service_name, methods): + """Add a new service named service_name with the given methods.""" + method_descriptions = methods.get('methods', {}) + method_info_map = collections.OrderedDict() + items = sorted(method_descriptions.iteritems()) + for method_name, method_description in items: + method_name = self.__names.MethodName(method_name) + + # NOTE: According to the discovery document, if the request or + # response is present, it will simply contain a `$ref`. + body_type = method_description.get('request') + if body_type is None: + request_type = None + else: + request_type = self.__GetRequestType(body_type) + if self.__NeedRequestType(method_description, request_type): + request = self.__CreateRequestType( + method_description, body_type=body_type) + request_field = self.__GetRequestField( + method_description, body_type) + else: + request = request_type + request_field = base_api.REQUEST_IS_BODY + + if 'response' in method_description: + response = method_description['response']['$ref'] + else: + response = self.__CreateVoidResponseType(method_description) + + method_info_map[method_name] = self.__ComputeMethodInfo( + method_description, request, response, request_field) + self.__command_registry.AddCommandForMethod( + service_name, method_name, method_info_map[method_name], + request, response) + + nested_services = methods.get('resources', {}) + services = sorted(nested_services.iteritems()) + for subservice_name, submethods in services: + new_service_name = '%s_%s' % (service_name, subservice_name) + self.AddServiceFromResource(new_service_name, submethods) + + self.__RegisterService(service_name, method_info_map) diff --git a/apitools/gen/setup.py b/apitools/gen/setup.py new file mode 100644 index 0000000..aad6bfc --- /dev/null +++ b/apitools/gen/setup.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup configuration.""" + +import platform + +from ez_setup import use_setuptools +# pylint:disable-msg=C6204 +use_setuptools() +import setuptools + +# Configure the required packages and scripts to install, depending on +# Python version and OS. +REQUIRED_PACKAGES = [ + 'apitools==0.1', + 'ez-setup==0.9', + 'google-api-python-client==1.2', + 'google-apputils==0.4.0', + 'protorpc==0.9.1', + 'python-dateutil==1.5', + 'python-gflags==2.0', + 'pytz==2013.7', + 'wsgiref==0.1.2', + ] + +CONSOLE_SCRIPTS = [ + 'gen_client = apitools.gen.gen_client:run_main', + ] + +py_version = platform.python_version() + +if py_version < '2.7': + REQUIRED_PACKAGES.append('argparse==1.2.1') + +_APITOOLS_GEN_VERSION = '0.1' + +setuptools.setup( + name='apitools_gen', + version=_APITOOLS_GEN_VERSION, + description='apitools client library generation tools', + url='http://github.com/craigcitro/apitools', + author='Craig Citro', + author_email='craigcitro@google.com', + # Contained modules and scripts. + packages=setuptools.find_packages(), + entry_points={ + 'console_scripts': CONSOLE_SCRIPTS, + }, + install_requires=REQUIRED_PACKAGES, + # PyPI package information. + classifiers=[ + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: Apache Software License', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache 2.0', + keywords='apitools', + ) diff --git a/apitools/gen/util.py b/apitools/gen/util.py new file mode 100644 index 0000000..2594936 --- /dev/null +++ b/apitools/gen/util.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python +"""Assorted utilities shared between parts of apitools.""" + +import collections +import contextlib +import json +import keyword +import logging +import os +import re +import urllib2 +import urlparse + + + +class Error(Exception): + """Base error for apitools generation.""" + + +class CommunicationError(Error): + """Error in network communication.""" + + +def _SortLengthFirst(a, b): + return -cmp(len(a), len(b)) or cmp(a, b) + + +class Names(object): + """Utility class for cleaning and normalizing names in a fixed style.""" + DEFAULT_NAME_CONVENTION = 'LOWER_CAMEL' + NAME_CONVENTIONS = ['LOWER_CAMEL', 'LOWER_WITH_UNDER', 'NONE'] + + def __init__(self, strip_prefixes, + name_convention=None, + capitalize_enums=False): + self.__strip_prefixes = sorted(strip_prefixes, cmp=_SortLengthFirst) + self.__name_convention = name_convention or self.DEFAULT_NAME_CONVENTION + self.__capitalize_enums = capitalize_enums + + @staticmethod + def __FromCamel(name, separator='_'): + name = re.sub(r'([a-z0-9])([A-Z])', r'\1%s\2' % separator, name) + return name.lower() + + @staticmethod + def __ToCamel(name, separator='_'): + return ''.join(s[0].upper() + s[1:] for s in name.split(separator)) + + @staticmethod + def __ToLowerCamel(name, separator='_'): + name = Names.__ToCamel(name, separator=separator) + return name[0].lower() + name[1:] + + def __StripName(self, name): + """Strip strip_prefix entries from name.""" + if not name: + return name + for prefix in self.__strip_prefixes: + if name.startswith(prefix): + return name[len(prefix):] + return name + + @staticmethod + def CleanName(name): + """Perform generic name cleaning.""" + name = re.sub('[^_A-Za-z0-9]', '_', name) + if name[0].isdigit(): + name = '_%s' % name + while name in keyword.kwlist: + name = '%s_' % name + return name + + @staticmethod + def NormalizeRelativePath(path): + """Normalize camelCase entries in path.""" + path_components = path.split('/') + normalized_components = [] + for component in path_components: + if re.match(r'{[A-Za-z0-9_]+}$', component): + normalized_components.append( + '{%s}' % Names.CleanName(component[1:-1])) + else: + normalized_components.append(component) + return '/'.join(normalized_components) + + def NormalizeEnumName(self, enum_name): + if self.__capitalize_enums: + return enum_name.upper() + return enum_name + + def ClassName(self, name, separator='_'): + """Generate a valid class name from name.""" + # TODO(craigcitro): Get rid of this case here and in MethodName. + if name is None: + return name + # TODO(craigcitro): This is a hack to handle the case of specific + # protorpc class names; clean this up. + if name.startswith('protorpc.') or name.startswith('message_types.'): + return name + name = self.__StripName(name) + name = self.__ToCamel(name, separator=separator) + return self.CleanName(name) + + def MethodName(self, name, separator='_'): + """Generate a valid method name from name.""" + if name is None: + return None + name = Names.__ToCamel(name, separator=separator) + return Names.CleanName(name) + + def FieldName(self, name): + """Generate a valid field name from name.""" + # TODO(craigcitro): We shouldn't need to strip this name, but some + # of the service names here are excessive. Fix the API and then + # remove this. + name = self.__StripName(name) + if self.__name_convention == 'LOWER_CAMEL': + name = Names.__ToLowerCamel(name) + elif self.__name_convention == 'LOWER_WITH_UNDER': + name = Names.__FromCamel(name) + return Names.CleanName(name) + + +@contextlib.contextmanager +def Chdir(dirname, create=True): + if not os.path.exists(dirname): + if not create: + raise OSError('Cannot find directory %s' % dirname) + else: + os.mkdir(dirname) + previous_directory = os.getcwd() + os.chdir(dirname) + yield + os.chdir(previous_directory) + + +def NormalizeVersion(version): + # Currently, '.' is the only character that might cause us trouble. + return version.replace('.', '_') + + +class ClientInfo(collections.namedtuple('ClientInfo', ( + 'package', 'scopes', 'version', 'client_id', 'client_secret', + 'user_agent', 'client_class_name', 'url_version', 'api_key'))): + """Container for client-related info and names.""" + + @classmethod + def Create(cls, discovery_doc, + scope_ls, client_id, client_secret, user_agent, names, api_key): + """Create a new ClientInfo object from a discovery document.""" + scopes = set( + discovery_doc.get('auth', {}).get('oauth2', {}).get('scopes', {})) + scopes.update(scope_ls) + client_info = { + 'package': discovery_doc['name'], + 'version': NormalizeVersion(discovery_doc['version']), + 'url_version': discovery_doc['version'], + 'scopes': sorted(list(scopes)), + 'client_id': client_id, + 'client_secret': client_secret, + 'user_agent': user_agent, + 'api_key': api_key, + } + client_class_name = ''.join( + map(names.ClassName, (client_info['package'], client_info['version']))) + client_info['client_class_name'] = client_class_name + return cls(**client_info) + + @property + def default_directory(self): + return self.package + + @property + def cli_rule_name(self): + return '%s_%s' % (self.package, self.version) + + @property + def cli_file_name(self): + return '%s.py' % self.cli_rule_name + + @property + def client_rule_name(self): + return '%s_%s_client' % (self.package, self.version) + + @property + def client_file_name(self): + return '%s.py' % self.client_rule_name + + @property + def messages_rule_name(self): + return '%s_%s_messages' % (self.package, self.version) + + @property + def services_rule_name(self): + return '%s_%s_services' % (self.package, self.version) + + @property + def messages_file_name(self): + return '%s.py' % self.messages_rule_name + + @property + def messages_proto_file_name(self): + return '%s.proto' % self.messages_rule_name + + @property + def services_proto_file_name(self): + return '%s.proto' % self.services_rule_name + + +def GetPackage(path): + path_components = path.split(os.path.sep) + return '.'.join(path_components) + + +class SimplePrettyPrinter(object): + """Simple pretty-printer that supports an indent contextmanager.""" + + def __init__(self, out): + self.__out = out + self.__indent = '' + + @property + def indent(self): + return self.__indent + + def CalculateWidth(self, max_width=78): + return max_width - len(self.indent) + + @contextlib.contextmanager + def Indent(self, indent=' '): + previous_indent = self.__indent + self.__indent = '%s%s' % (previous_indent, indent) + yield + self.__indent = previous_indent + + def __call__(self, *args): + if args and args[0]: + line = (args[0] % args[1:]).rstrip() + print >>self.__out, '%s%s' % (self.__indent, line) + else: + print >>self.__out, '' + + +def NormalizeDiscoveryUrl(discovery_url): + """Expands a few abbreviations into full discovery urls.""" + if discovery_url.startswith('http'): + return discovery_url + elif '.' not in discovery_url: + raise ValueError('Unrecognized value "%s" for discovery url') + api_name, _, api_version = discovery_url.partition('.') + return 'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % ( + api_name, api_version) + + +def FetchDiscoveryDoc(discovery_url, retries=5): + """Fetch the discovery document at the given url.""" + discovery_url = NormalizeDiscoveryUrl(discovery_url) + discovery_doc = None + last_exception = None + for _ in xrange(retries): + try: + discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) + break + except (urllib2.HTTPError, urllib2.URLError) as last_exception: + logging.warning('Attempting to fetch discovery doc again after "%s"', + last_exception) + if discovery_doc is None: + raise CommunicationError('Could not find discovery doc at url "%s": %s' % ( + discovery_url, last_exception)) + return discovery_doc diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..86f6469 --- /dev/null +++ b/setup.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup configuration.""" + +import platform + +from ez_setup import use_setuptools +use_setuptools() +# pylint:disable-msg=C6204 +import setuptools + +# Configure the required packages and scripts to install, depending on +# Python version and OS. +REQUIRED_PACKAGES = [ + 'ez-setup==0.9', + 'google-api-python-client==1.2', + 'google-apputils==0.4.0', + 'protorpc==0.9.1', + 'python-dateutil==1.5', + 'python-gflags==2.0', + 'pytz==2013.7', + 'wsgiref==0.1.2', + ] +CONSOLE_SCRIPTS = [] + +py_version = platform.python_version() + +if py_version < '2.7': + REQUIRED_PACKAGES.append('argparse==1.2.1') + +_APITOOLS_VERSION = '0.1' + +setuptools.setup( + name='apitools', + version=_APITOOLS_VERSION, + description='client libraries for humans', + url='http://github.com/craigcitro/apitools', + author='Craig Citro', + author_email='craigcitro@google.com', + # Contained modules and scripts. + packages=setuptools.find_packages(), + entry_points={ + 'console_scripts': CONSOLE_SCRIPTS, + }, + install_requires=REQUIRED_PACKAGES, + provides=[ + 'apitools (%s)' % (_APITOOLS_VERSION,), + ], + # PyPI package information. + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache 2.0', + keywords='apitools', + ) -- GitLab From 498872fb4877fb309e3ad2e753efe20ec296524c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 21 Oct 2013 01:12:20 -0700 Subject: [PATCH 002/295] Add a gitignore. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..02b2bb5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +apitools.egg-info/* +build/* +dist/* +distribute-* +*~ +*.pyc -- GitLab From 30de3e9caecac03cc2bbd48520e7697458d4b4f3 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 21 Oct 2013 10:48:26 -0700 Subject: [PATCH 003/295] Drop the extra setup.py file. --- apitools/gen/setup.py | 73 ------------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 apitools/gen/setup.py diff --git a/apitools/gen/setup.py b/apitools/gen/setup.py deleted file mode 100644 index aad6bfc..0000000 --- a/apitools/gen/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Setup configuration.""" - -import platform - -from ez_setup import use_setuptools -# pylint:disable-msg=C6204 -use_setuptools() -import setuptools - -# Configure the required packages and scripts to install, depending on -# Python version and OS. -REQUIRED_PACKAGES = [ - 'apitools==0.1', - 'ez-setup==0.9', - 'google-api-python-client==1.2', - 'google-apputils==0.4.0', - 'protorpc==0.9.1', - 'python-dateutil==1.5', - 'python-gflags==2.0', - 'pytz==2013.7', - 'wsgiref==0.1.2', - ] - -CONSOLE_SCRIPTS = [ - 'gen_client = apitools.gen.gen_client:run_main', - ] - -py_version = platform.python_version() - -if py_version < '2.7': - REQUIRED_PACKAGES.append('argparse==1.2.1') - -_APITOOLS_GEN_VERSION = '0.1' - -setuptools.setup( - name='apitools_gen', - version=_APITOOLS_GEN_VERSION, - description='apitools client library generation tools', - url='http://github.com/craigcitro/apitools', - author='Craig Citro', - author_email='craigcitro@google.com', - # Contained modules and scripts. - packages=setuptools.find_packages(), - entry_points={ - 'console_scripts': CONSOLE_SCRIPTS, - }, - install_requires=REQUIRED_PACKAGES, - # PyPI package information. - classifiers=[ - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: Apache Software License', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], - license='Apache 2.0', - keywords='apitools', - ) -- GitLab From 903c5d4d4293de1caece40ff8ab37ebdbbfb256e Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 23 Oct 2013 11:01:29 -0700 Subject: [PATCH 004/295] New apitools version from internal. * Fixed an install snafu with `ez_setup`. * Added a `MANIFEST.in`. * Working on better `Download` support. * Better errors for download failures. * Add a script at install. --- MANIFEST.in | 4 + apitools/base/py/base_api.py | 20 +-- apitools/base/py/transfer.py | 73 +++++++-- apitools/base/py/util.py | 9 ++ apitools/gen/command_registry.py | 4 - apitools/gen/service_registry.py | 4 - ez_setup.py | 251 +++++++++++++++++++++++++++++++ setup.py | 6 +- 8 files changed, 333 insertions(+), 38 deletions(-) create mode 100644 MANIFEST.in create mode 100755 ez_setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..fc80ed3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include *.py +include *.txt +include *.md +recursive-include apitools *.py diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 0922696..624e84e 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -4,7 +4,6 @@ import contextlib import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart -import httplib import logging import types import urllib @@ -24,6 +23,7 @@ import gflags as flags from apitools.base.py import credentials_lib from apitools.base.py import encoding from apitools.base.py import exceptions +from apitools.base.py import util FLAGS = flags.FLAGS @@ -436,9 +436,7 @@ class BaseApiService(object): def __IsRetryable(self, exc): status = int(exc.resp.get('status')) # 308 doesn't have a name in httplib. - retryable_status = status in ( - httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER, - httplib.TEMPORARY_REDIRECT, 308) + retryable_status = status in util.RETRYABLE_STATUS_CODES return retryable_status and 'location' in exc.resp def __ExecuteRequest(self, request, url): @@ -527,19 +525,9 @@ class BaseApiService(object): methodId=method_config.method_id, resumable=resumable) - # If we're downloading media, we want to just get the new URL and - # hand it back to the download object. if download: - try: - request.http.request( - uri=str(request.uri), method='GET', headers=request.headers, - body='', redirections=0) - # TODO(craigcitro): Confirm that this is invalid. - raise exceptions.InvalidDataFromServerError( - 'No redirect received for media download') - except httplib2.RedirectLimit as e: - download.url = e.response['location'] - download.http = request.http + download.InitializeDownload( + str(request.uri), request.http, headers=request.headers) return response = self.__ExecuteRequest(request, url) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 083473e..0deffd8 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -9,7 +9,10 @@ import mimetypes import os import threading +import httplib2 + from apitools.base.py import exceptions +from apitools.base.py import util __all__ = [ 'Download', @@ -34,7 +37,8 @@ class _HttpResponse(collections.namedtuple( class _TransferSerializationData(collections.namedtuple( - '_TransferSerializationData', ['progress', 'total_size', 'url'])): + '_TransferSerializationData', + ['auto_transfer', 'progress', 'total_size', 'url'])): __slots__ = () def ToJson(self): @@ -44,7 +48,7 @@ class _TransferSerializationData(collections.namedtuple( def FromJson(cls, json_data): data = json.loads(json_data) data_keys = set(('progress', 'url')) - if data_keys > set(cls._fields): + if not data_keys.issubset(set(cls._fields)): raise exceptions.InvalidDataError( 'Invalid keys for Transfer: %s' % data_keys) return cls._make(data[field] for field in cls._fields) @@ -53,7 +57,8 @@ class _TransferSerializationData(collections.namedtuple( class _Transfer(object): """Generic bits common to Uploads and Downloads.""" - def __init__(self, stream, close_stream=False, chunksize=None): + def __init__(self, stream, close_stream=False, chunksize=None, + auto_transfer=True): self.__close_stream = close_stream self.__http = None self.__stream = stream @@ -62,6 +67,7 @@ class _Transfer(object): self._progress = 0 + self.auto_transfer = auto_transfer self.chunksize = chunksize or 1048576L def __repr__(self): @@ -113,6 +119,11 @@ class _Transfer(object): raise exceptions.TransferInvalidError( 'Cannot use uninitialized %s', self._type_name) + def EnsureUninitialized(self): + if self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot re-initialize %s', self._type_name) + @property def close_stream(self): return self.__close_stream @@ -125,6 +136,7 @@ class _Transfer(object): def serialization_data(self): self.EnsureInitialized() return { + 'auto_transfer': self.auto_transfer, 'progress': self._progress, 'total_size': self.total_size, 'url': self.url, @@ -146,18 +158,18 @@ class Download(_Transfer): return 'Download for url %s' % self.url @classmethod - def FromFile(cls, filename, overwrite=False): + def FromFile(cls, filename, overwrite=False, auto_transfer=True): """Create a new download object from a filename.""" path = os.path.expanduser(filename) if os.path.exists(path) and not overwrite: raise exceptions.InvalidUserInputError( 'File %s exists and overwrite not specified' % path) - return cls(open(path, 'wb'), close_stream=True) + return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer) @classmethod - def FromStream(cls, stream): + def FromStream(cls, stream, auto_transfer=True): """Create a new Download object from a stream.""" - return cls(stream) + return cls(stream, auto_transfer=auto_transfer) @classmethod def FromData(cls, stream, json_data, http=None): @@ -170,13 +182,45 @@ class Download(_Transfer): download.url = info.url return download + def InitializeDownload(self, url, http, + method='GET', headers=None, body=''): + """Initialize this download by making a request. + + Args: + url: The URL to use for our initial request. + http: The httplib2.Http instance for this request. + method: The HTTP method used for the call. + headers: Additional headers to pass in the request. + body: The message body. + """ + self.EnsureUninitialized() + headers = headers or {} + try: + response, content = http.request( + uri=url, method=method, headers=headers, body=body, redirections=0) + if response['status'] not in util.RETRYABLE_STATUS_CODES: + raise exceptions.HttpError(response, content, uri=url) + else: + raise exceptions.InvalidDataFromServerError( + 'No redirect received for media download from url <%s>: %s' % ( + url, response)) + except httplib2.RedirectLimit as e: + self.url = e.response['location'] + self.http = http + + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + self.StreamInChunks() + def __SetTotal(self, info): if self.total_size is None and 'content-range' in info: _, _, total = info['content-range'].rpartition('/') if total != '*': self.total_size = int(total) - def __GetChunk(self, start, end=None, chunksize=None): + def __GetChunk(self, start, end=None, chunksize=None, + additional_headers=None): """Retrieve a chunk, and return the full response.""" self.EnsureInitialized() start = max(start, 0) @@ -192,6 +236,8 @@ class Download(_Transfer): raise exceptions.TransferInvalidError( 'Range requested with end[%s] < start[%s]' % (end, start)) headers = {'Range': 'bytes=%s-%d' % (start, end)} + if additional_headers is not None: + headers.update(additional_headers) # TODO(craigcitro): Add support for retries. response = _HttpResponse(*self.http.request(self.url, headers=headers)) self.__SetTotal(response.info) @@ -202,12 +248,14 @@ class Download(_Transfer): self.stream.write(response.content) return response - def GetRange(self, start, end, chunksize=None, exact_range=True): + def GetRange(self, start, end, chunksize=None, exact_range=True, + additional_headers=None): """Retrieve a given byte range from this download.""" progress = start chunksize = chunksize or self.chunksize while progress < end: - response = self.__GetChunk(progress, end=end) + response = self.__GetChunk(progress, end=end, + additional_headers=additional_headers) if (response.status_code == httplib.REQUESTED_RANGE_NOT_SATISFIABLE and exact_range): raise exceptions.TransferInvalidError( @@ -220,7 +268,7 @@ class Download(_Transfer): threading.Thread(target=callback, args=(response, self)).start() def StreamInChunks(self, callback=None, finish_callback=None, chunksize=None, - end=None): + end=None, additional_headers=None): """Stream the entire download.""" def ArgPrinter(response, unused_download): @@ -234,7 +282,8 @@ class Download(_Transfer): self.EnsureInitialized() while True: - response = self.__GetChunk(self._progress, chunksize=chunksize, end=end) + response = self.__GetChunk(self._progress, chunksize=chunksize, end=end, + additional_headers=additional_headers) # TODO(craigcitro): Consider whether this update and writing # the response to self.stream need to happen as a transaction. self._progress += len(response) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 5589412..5d8713d 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -9,6 +9,15 @@ import urllib2 from apitools.base.py import exceptions +RETRYABLE_STATUS_CODES = ( + httplib.MOVED_PERMANENTLY, + httplib.FOUND, + httplib.SEE_OTHER, + httplib.TEMPORARY_REDIRECT, + # 308 doesn't have a name in httplib. + 308, + ) + def DetectGae(): """Determine whether or not we're running on GAE. diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 2e888a7..e5355a8 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -513,10 +513,6 @@ class CommandRegistry(object): printer('result = client.%s(', command_info.client_method_path) with printer.Indent(indent=' '): printer('%s)', ', '.join(call_args)) - if command_info.has_download: - printer('if FLAGS.download_filename:') - with printer.Indent(): - printer('download.StreamInChunks()') printer('print result') printer() printer() diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 388c742..9e79a38 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -160,11 +160,7 @@ class ServiceRegistry(object): printer() client_info_items = client_info._asdict().iteritems() # pylint:disable=protected-access for attr, val in client_info_items: - if attr == 'api_key': - printer('# MOE:begin_strip') printer('_%s = %r' % (attr.upper(), val)) - if attr == 'api_key': - printer('# MOE:end_strip') printer() printer("def __init__(self, url='', credentials=None,") printer(' get_credentials=True, http=None, model=None,') diff --git a/ez_setup.py b/ez_setup.py new file mode 100755 index 0000000..3756829 --- /dev/null +++ b/ez_setup.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python +"""Bootstrap setuptools installation + +If you want to use setuptools in your package's setup.py, just include this +file in the same directory with it, and add this to the top of your setup.py:: + + from ez_setup import use_setuptools + use_setuptools() + +If you want to require a specific version of setuptools, set a download +mirror, or use an alternate download directory, you can do so by supplying +the appropriate options to ``use_setuptools()``. + +This file can also be run as a script to install or upgrade setuptools. +""" +import sys +DEFAULT_VERSION = "0.6c11" +DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] + +md5_data = { + 'setuptools-0.6c10-py2.3.egg': 'ce1e2ab5d3a0256456d9fc13800a7090', + 'setuptools-0.6c10-py2.4.egg': '57d6d9d6e9b80772c59a53a8433a5dd4', + 'setuptools-0.6c10-py2.5.egg': 'de46ac8b1c97c895572e5e8596aeb8c7', + 'setuptools-0.6c10-py2.6.egg': '58ea40aef06da02ce641495523a0b7f5', + 'setuptools-0.6c11-py2.3.egg': '2baeac6e13d414a9d28e7ba5b5a596de', + 'setuptools-0.6c11-py2.4.egg': 'bd639f9b0eac4c42497034dec2ec0c2b', + 'setuptools-0.6c11-py2.5.egg': '64c94f3bf7a72a13ec83e0b24f2749b2', + 'setuptools-0.6c11-py2.6.egg': 'bfa92100bd772d5a213eedd356d64086', + 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902', + 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de', + 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b', + 'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03', + 'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a', + 'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6', + 'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a', +} + +import sys, os +try: from hashlib import md5 +except ImportError: from md5 import md5 + +def _validate_md5(egg_name, data): + if egg_name in md5_data: + digest = md5(data).hexdigest() + if digest != md5_data[egg_name]: + print >>sys.stderr, ( + "md5 validation of %s failed! (Possible download problem?)" + % egg_name + ) + sys.exit(2) + return data + +def use_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, + download_delay=15 +): + """Automatically find/download setuptools and make it available on sys.path + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end with + a '/'). `to_dir` is the directory where setuptools will be downloaded, if + it is not already available. If `download_delay` is specified, it should + be the number of seconds that will be paused before initiating a download, + should one be required. If an older version of setuptools is installed, + this routine will print a message to ``sys.stderr`` and raise SystemExit in + an attempt to abort the calling script. + """ + was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules + def do_download(): + egg = download_setuptools(version, download_base, to_dir, download_delay) + sys.path.insert(0, egg) + import setuptools; setuptools.bootstrap_install_from = egg + try: + import pkg_resources + except ImportError: + return do_download() + try: + pkg_resources.require("setuptools>="+version); return + except pkg_resources.VersionConflict, e: + if was_imported: + print >>sys.stderr, ( + "The required version of setuptools (>=%s) is not available, and\n" + "can't be installed while this script is running. Please install\n" + " a more recent version first, using 'easy_install -U setuptools'." + "\n\n(Currently using %r)" + ) % (version, e.args[0]) + sys.exit(2) + except pkg_resources.DistributionNotFound: + pass + + del pkg_resources, sys.modules['pkg_resources'] # reload ok + return do_download() + +def download_setuptools( + version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, + delay = 15 +): + """Download setuptools from a specified location and return its filename + + `version` should be a valid setuptools version number that is available + as an egg for download under the `download_base` URL (which should end + with a '/'). `to_dir` is the directory where the egg will be downloaded. + `delay` is the number of seconds to pause before an actual download attempt. + """ + import urllib2, shutil + egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) + url = download_base + egg_name + saveto = os.path.join(to_dir, egg_name) + src = dst = None + if not os.path.exists(saveto): # Avoid repeated downloads + try: + from distutils import log + if delay: + log.warn(""" +--------------------------------------------------------------------------- +This script requires setuptools version %s to run (even to display +help). I will attempt to download it for you (from +%s), but +you may need to enable firewall access for this script first. +I will start the download in %d seconds. + +(Note: if this machine does not have network access, please obtain the file + + %s + +and place it in this directory before rerunning this script.) +---------------------------------------------------------------------------""", + version, download_base, delay, url + ); from time import sleep; sleep(delay) + log.warn("Downloading %s", url) + src = urllib2.urlopen(url) + # Read/write all in one block, so we don't create a corrupt file + # if the download is interrupted. + data = _validate_md5(egg_name, src.read()) + dst = open(saveto,"wb"); dst.write(data) + finally: + if src: src.close() + if dst: dst.close() + return os.path.realpath(saveto) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +def main(argv, version=DEFAULT_VERSION): + """Install or upgrade setuptools and EasyInstall""" + try: + import setuptools + except ImportError: + egg = None + try: + egg = download_setuptools(version, delay=0) + sys.path.insert(0,egg) + from setuptools.command.easy_install import main + return main(list(argv)+[egg]) # we're done here + finally: + if egg and os.path.exists(egg): + os.unlink(egg) + else: + if setuptools.__version__ == '0.0.1': + print >>sys.stderr, ( + "You have an obsolete version of setuptools installed. Please\n" + "remove it from your system entirely before rerunning this script." + ) + sys.exit(2) + + req = "setuptools>="+version + import pkg_resources + try: + pkg_resources.require(req) + except pkg_resources.VersionConflict: + try: + from setuptools.command.easy_install import main + except ImportError: + from easy_install import main + main(list(argv)+[download_setuptools(delay=0)]) + sys.exit(0) # try to force an exit + else: + if argv: + from setuptools.command.easy_install import main + main(argv) + else: + print "Setuptools version",version,"or greater has been installed." + print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' + +def update_md5(filenames): + """Update our built-in md5 registry""" + + import re + + for name in filenames: + base = os.path.basename(name) + f = open(name,'rb') + md5_data[base] = md5(f.read()).hexdigest() + f.close() + + data = [" %r: %r,\n" % it for it in md5_data.items()] + data.sort() + repl = "".join(data) + + import inspect + srcfile = inspect.getsourcefile(sys.modules[__name__]) + f = open(srcfile, 'rb'); src = f.read(); f.close() + + match = re.search("\nmd5_data = {\n([^}]+)}", src) + if not match: + print >>sys.stderr, "Internal error!" + sys.exit(2) + + src = src[:match.start(1)] + repl + src[match.end(1):] + f = open(srcfile,'w') + f.write(src) + f.close() + + +if __name__=='__main__': + if len(sys.argv)>2 and sys.argv[1]=='--md5update': + update_md5(sys.argv[2:]) + else: + main(sys.argv[1:]) diff --git a/setup.py b/setup.py index 86f6469..46977ff 100644 --- a/setup.py +++ b/setup.py @@ -35,14 +35,16 @@ REQUIRED_PACKAGES = [ 'pytz==2013.7', 'wsgiref==0.1.2', ] -CONSOLE_SCRIPTS = [] +CONSOLE_SCRIPTS = [ + 'gen_client = apitools.gen.gen_client:run_main', + ] py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse==1.2.1') -_APITOOLS_VERSION = '0.1' +_APITOOLS_VERSION = '0.1.1' setuptools.setup( name='apitools', -- GitLab From 01be33e3b46bbce3ed0b037861680ca71d1738d0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 01:38:09 -0700 Subject: [PATCH 005/295] Update to version 0.2. This is another large code drop from internal. --- apitools/base/py/__init__.py | 20 + apitools/base/py/app2.py | 17 +- apitools/base/py/base_api.py | 604 +++++++++--------- apitools/base/py/base_api_test.py | 46 ++ apitools/base/py/base_cli.py | 61 +- apitools/base/py/batch.py | 406 ++++++++++++ apitools/base/py/credentials_lib.py | 81 +-- apitools/base/py/credentials_lib_test.py | 54 ++ apitools/base/py/encoding.py | 224 +++++-- apitools/base/py/encoding_test.py | 147 +++++ apitools/base/py/exceptions.py | 40 +- apitools/base/py/extra_types.py | 232 +++++++ apitools/base/py/extra_types_test.py | 141 +++++ apitools/base/py/http_wrapper.py | 164 +++++ apitools/base/py/transfer.py | 775 ++++++++++++++++------- apitools/base/py/util.py | 23 +- apitools/gen/command_registry.py | 96 +-- apitools/gen/extended_descriptor.py | 16 +- apitools/gen/gen_client.py | 20 +- apitools/gen/gen_client_lib.py | 35 +- apitools/gen/message_registry.py | 108 +++- apitools/gen/service_registry.py | 89 ++- apitools/gen/util.py | 5 +- setup.py | 11 +- 24 files changed, 2669 insertions(+), 746 deletions(-) create mode 100644 apitools/base/py/base_api_test.py create mode 100644 apitools/base/py/batch.py create mode 100644 apitools/base/py/credentials_lib_test.py create mode 100644 apitools/base/py/encoding_test.py create mode 100644 apitools/base/py/extra_types.py create mode 100644 apitools/base/py/extra_types_test.py create mode 100644 apitools/base/py/http_wrapper.py diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py index 4265cc3..a0f920e 100644 --- a/apitools/base/py/__init__.py +++ b/apitools/base/py/__init__.py @@ -1 +1,21 @@ #!/usr/bin/env python +"""Top-level imports for apitools base files.""" + +from apitools.base.py.base_api import * +from apitools.base.py.batch import * +from apitools.base.py.credentials_lib import * +from apitools.base.py.encoding import * +from apitools.base.py.exceptions import * +from apitools.base.py.extra_types import * +from apitools.base.py.http_wrapper import * +from apitools.base.py.transfer import * +from apitools.base.py.util import * + +try: + from apitools.base.py.app2 import * + from apitools.base.py.base_cli import * + # pylint: enable=g-import-not-at-top +except ImportError: + # We want to allow this to fail in some cases, such as importing on + # GAE. + pass diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 1afa7ae..80b36ed 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -13,6 +13,10 @@ from google.apputils import app from google.apputils import appcommands import gflags as flags +__all__ = [ + 'NewCmd', + 'Repl', + ] flags.DEFINE_boolean( 'debug_mode', False, @@ -20,13 +24,18 @@ flags.DEFINE_boolean( flags.DEFINE_boolean( 'headless', False, 'Assume no user is at the controlling console.') +FLAGS = flags.FLAGS -FLAGS = flags.FLAGS +def _SafeMakeAscii(s): + if isinstance(s, unicode): + return s.encode('ascii') + elif isinstance(s, str): + return s.decode('ascii') + else: + return unicode(s).encode('ascii', 'backslashreplace') -# TODO(craigcitro): This code uses more than the average amount of -# Python magic. Explain what the heck is going on throughout. class NewCmd(appcommands.Cmd): """Featureful extension of appcommands.Cmd.""" @@ -118,7 +127,7 @@ class NewCmd(appcommands.Cmd): def _FormatError(self, e): """Hook for subclasses to modify how error messages are printed.""" - return str(e) + return _SafeMakeAscii(e) def _HandleError(self, e): message = self._FormatError(e) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 624e84e..ad1eafd 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -2,35 +2,37 @@ """Base class for api services.""" import contextlib -import email.mime.multipart as mime_multipart -import email.mime.nonmultipart as mime_nonmultipart +import httplib import logging +import pprint import types import urllib import urlparse -from apiclient import errors as apiclient_errors -from apiclient import http as apiclient_http -from apiclient import mimeparse -import apiclient.model -import httplib2 from protorpc import message_types from protorpc import messages -import gflags as flags - from apitools.base.py import credentials_lib from apitools.base.py import encoding from apitools.base.py import exceptions +from apitools.base.py import http_wrapper from apitools.base.py import util -FLAGS = flags.FLAGS +__all__ = [ + 'ApiMethodInfo', + 'ApiUploadInfo', + 'BaseApiClient', + 'BaseApiService', + 'NormalizeApiEndpoint', + ] # TODO(craigcitro): Remove this once we quiet the spurious logging in # oauth2client (or drop oauth2client). logging.getLogger('oauth2client.util').setLevel(logging.ERROR) +_MAX_URL_LENGTH = 2048 + class ApiUploadInfo(messages.Message): """Media upload information for a method. @@ -38,7 +40,8 @@ class ApiUploadInfo(messages.Message): Fields: accept: (repeated) MIME Media Ranges for acceptable media uploads to this method. - max_size: Maximum size of a media upload, such as "1MB" or "3TB". + max_size: (integer) Maximum size of a media upload, such as 3MB + or 1TB (converted to an integer). resumable_path: Path to use for resumable uploads. resumable_multipart: (boolean) Whether or not the resumable endpoint supports multipart uploads. @@ -114,95 +117,64 @@ def _RequireClassAttrs(obj, attrs): raise exceptions.GeneratedClientError(msg) -def _Typecheck(arg, arg_type, msg=None): - if not isinstance(arg, arg_type): - if msg is None: - if isinstance(arg_type, tuple): - msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) - else: - msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) - raise exceptions.TypecheckError(msg) - return arg - - def NormalizeApiEndpoint(api_endpoint): if not api_endpoint.endswith('/'): api_endpoint += '/' return api_endpoint -class BaseApiModel(apiclient.model.JsonModel): - """Base model for generated clients.""" - alt_param = None +class _UrlBuilder(object): + """Convenient container for url data.""" - def __init__(self, request_type, response_type, log_request, log_response, - *args, **kwds): - self.__request_type = request_type - self.__response_type = response_type - self.__log_request = log_request - self.__log_response = log_response - # TODO(craigcitro): Remove this field when we switch to proto2. - self.include_fields = None - super(BaseApiModel, self).__init__(*args, **kwds) - - # TODO(craigcitro): Delete these methods once we don't have to - # support both variations of apiclient. - @staticmethod - def _GetDumpFlag(): - if hasattr(FLAGS, 'dump_request_response'): - return FLAGS.dump_request_response - else: - return apiclient.model.dump_request_response + def __init__(self, base_url, relative_path=None, query_params=None): + components = urlparse.urlsplit(urlparse.urljoin( + base_url, relative_path or '')) + if components.fragment: + raise exceptions.ConfigurationValueError( + 'Unexpected url fragment: %s' % components.fragment) + self.query_params = urlparse.parse_qs(components.query or '') + if query_params is not None: + self.query_params.update(query_params) + self.__scheme = components.scheme + self.__netloc = components.netloc + self.relative_path = components.path - @staticmethod - def _SetDumpFlag(value): - if hasattr(FLAGS, 'dump_request_response'): - FLAGS.dump_request_response = value - else: - apiclient.model.dump_request_response = value - - def _log_request(self, *args, **kwds): - old_value = self._GetDumpFlag() - if self.__log_request: - self._SetDumpFlag(True) - super(BaseApiModel, self)._log_request(*args, **kwds) - self._SetDumpFlag(old_value) - - def _log_response(self, *args, **kwds): - old_value = self._GetDumpFlag() - if self.__log_response: - self._SetDumpFlag(True) - super(BaseApiModel, self)._log_response(*args, **kwds) - self._SetDumpFlag(old_value) - - def serialize(self, body_value): - """Serialize a message (which might involve ProtoRPC messages).""" - _Typecheck(body_value, self.__request_type) - return encoding.MessageToJson( - body_value, include_fields=self.include_fields) - - def deserialize(self, content): - """Deserialize a message (which might involve ProtoRPC messages).""" - try: - message = encoding.JsonToMessage(self.__response_type, content) - except (exceptions.InvalidDataFromServerError, - messages.ValidationError) as e: - raise exceptions.InvalidDataFromServerError( - 'Error decoding response "%s" as type %s: %s' % ( - content, self.__response_type, e)) - return message + @classmethod + def FromUrl(cls, url): + urlparts = urlparse.urlsplit(url) + query_params = urlparse.parse_qs(urlparts.query) + base_url = urlparse.urlunsplit(( + urlparts.scheme, urlparts.netloc, '', None, None)) + relative_path = urlparts.path + return cls(base_url, relative_path=relative_path, query_params=query_params) + @property + def base_url(self): + return urlparse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) -class BaseMediaDownloadModel(BaseApiModel): - """Base class for requests that return media in the response.""" - alt_param = 'media' + @base_url.setter + def base_url(self, value): + components = urlparse.urlsplit(value) + if components.path or components.query or components.fragment: + raise exceptions.ConfigurationValueError('Invalid base url: %s' % value) + self.__scheme = components.scheme + self.__netloc = components.netloc - def deserialize(self, content): - return content + @property + def query(self): + # TODO(craigcitro): In the case that some of the query params are + # non-ASCII, we may silently fail to encode correctly. We should + # figure out who is responsible for owning the object -> str + # conversion. + return urllib.urlencode(self.query_params, doseq=True) @property - def no_content_response(self): - return '' + def url(self): + if '{' in self.relative_path or '}' in self.relative_path: + raise exceptions.ConfigurationValueError( + 'Cannot create url with relative path %s' % self.relative_path) + return urlparse.urlunsplit(( + self.__scheme, self.__netloc, self.relative_path, self.query, '')) class BaseApiClient(object): @@ -217,38 +189,75 @@ class BaseApiClient(object): _USER_AGENT = '' def __init__(self, url, credentials=None, get_credentials=True, http=None, - model=None, log_request=False, log_response=False, - default_global_params=None): + model=None, log_request=False, log_response=False, num_retries=5, + credentials_args=None, default_global_params=None): _RequireClassAttrs(self, ( '_package', '_scopes', '_client_id', '_client_secret', 'messages_module')) if default_global_params is not None: - _Typecheck(default_global_params, self.params_type) + util.Typecheck(default_global_params, self.params_type) self.__default_global_params = default_global_params self.log_request = log_request self.log_response = log_response - self._base_model_class = model or BaseApiModel + self.__num_retries = 5 + # We let the @property machinery below do our validation. + self.num_retries = num_retries self._url = url self._credentials = credentials if get_credentials and not credentials: - # TODO(craigcitro): It's a bit dangerous to pass this - # still-half-initialized self into this method, but we might need - # to set attributes on it associated with our credentials. - # Consider another way around this (maybe a callback?) and whether - # or not it's worth it. - self._credentials = credentials_lib.GetCredentials( - self._PACKAGE, self._SCOPES, self._CLIENT_ID, self._CLIENT_SECRET, - self._USER_AGENT, api_key=self._API_KEY, client=self) - self._http = http or httplib2.Http() + credentials_args = credentials_args or {} + self._SetCredentials(**credentials_args) + self._http = http or http_wrapper.GetHttp() # Note that "no credentials" is totally possible. if self._credentials is not None: self._http = self._credentials.authorize(self._http) # TODO(craigcitro): Remove this field when we switch to proto2. self.__include_fields = None + # TODO(craigcitro): Finish deprecating these fields. + _ = model + + def _SetCredentials(self, **kwds): + """Fetch credentials, and set them for this client. + + Note that we can't simply return credentials, since creating them + may involve side-effecting self. + + Args: + **kwds: Additional keyword arguments are passed on to GetCredentials. + + Returns: + None. Sets self._credentials. + """ + args = { + 'api_key': self._API_KEY, + 'client': self, + 'client_id': self._CLIENT_ID, + 'client_secret': self._CLIENT_SECRET, + 'package_name': self._PACKAGE, + 'scopes': self._SCOPES, + 'user_agent': self._USER_AGENT, + } + args.update(kwds) + # TODO(craigcitro): It's a bit dangerous to pass this + # still-half-initialized self into this method, but we might need + # to set attributes on it associated with our credentials. + # Consider another way around this (maybe a callback?) and whether + # or not it's worth it. + self._credentials = credentials_lib.GetCredentials(**args) + + @classmethod + def ClientInfo(cls): + return { + 'client_id': cls._CLIENT_ID, + 'client_secret': cls._CLIENT_SECRET, + 'scope': ' '.join(sorted(util.NormalizeScopes(cls._SCOPES))), + 'user_agent': cls._USER_AGENT, + } + @property def base_model_class(self): - return self._base_model_class + return None @property def http(self): @@ -266,6 +275,10 @@ class BaseApiClient(object): def params_type(self): return _LoadClass('StandardQueryParameters', self.MESSAGES_MODULE) + @property + def user_agent(self): + return self._USER_AGENT + @property def _default_global_params(self): if self.__default_global_params is None: @@ -280,15 +293,81 @@ class BaseApiClient(object): def global_params(self): return encoding.CopyProtoMessage(self._default_global_params) - def ConfigureModel(self, model): - model.include_fields = self.__include_fields - @contextlib.contextmanager def IncludeFields(self, include_fields): self.__include_fields = include_fields yield self.__include_fields = None + @property + def num_retries(self): + return self.__num_retries + + @num_retries.setter + def num_retries(self, value): + util.Typecheck(value, (int, long)) + if value < 0: + raise exceptions.InvalidDataError( + 'Cannot have negative value for num_retries') + self.__num_retries = value + + @contextlib.contextmanager + def WithRetries(self, num_retries): + old_num_retries = self.num_retries + self.num_retries = num_retries + yield + self.num_retries = old_num_retries + + def ProcessRequest(self, method_config, request): + """Hook for pre-processing of requests.""" + if self.log_request: + logging.info( + 'Calling method %s with %s: %s', method_config.method_id, + method_config.request_type_name, request) + return request + + def ProcessHttpRequest(self, http_request): + """Hook for pre-processing of http requests.""" + if self.log_request: + logging.info('Making http %s to %s', + http_request.http_method, http_request.url) + logging.info('Headers: %s', pprint.pformat(http_request.headers)) + if http_request.body: + # TODO(craigcitro): Make this safe to print in the case of + # non-printable body characters. + logging.info('Body:\n%s', http_request.body) + else: + logging.info('Body: (none)') + return http_request + + def ProcessResponse(self, method_config, response): + if self.log_response: + logging.info('Response of type %s: %s', + method_config.response_type_name, response) + return response + + # TODO(craigcitro): Decide where these two functions should live. + def SerializeMessage(self, message): + return encoding.MessageToJson(message, include_fields=self.__include_fields) + + def DeserializeMessage(self, response_type, data): + """Deserialize the given data as method_config.response_type.""" + try: + message = encoding.JsonToMessage(response_type, data) + except (exceptions.InvalidDataFromServerError, + messages.ValidationError) as e: + raise exceptions.InvalidDataFromServerError( + 'Error decoding response "%s" as type %s: %s' % ( + data, response_type.__name__, e)) + return message + + def FinalizeTransferUrl(self, url): + """Modify the url for a given transfer, based on auth and version.""" + url_builder = _UrlBuilder.FromUrl(url) + if self.global_params.key: + url_builder.query_params['key'] = self.global_params.key + return url_builder.url + class BaseApiService(object): """Base class for generated API services.""" @@ -301,7 +380,7 @@ class BaseApiService(object): return self.__client def __CombineGlobalParams(self, global_params, default_params): - _Typecheck(global_params, (types.NoneType, self.__client.params_type)) + util.Typecheck(global_params, (types.NoneType, self.__client.params_type)) result = self.__client.params_type() global_params = global_params or self.__client.params_type() for field in result.all_fields(): @@ -312,17 +391,29 @@ class BaseApiService(object): return result def __ConstructQueryParams(self, query_params, request, global_params): + """Construct a dictionary of query parameters for this request.""" + global_params = self.__CombineGlobalParams( + global_params, self.__client.global_params) query_info = dict((field.name, getattr(global_params, field.name)) for field in self.__client.params_type.all_fields()) query_info.update( (param, getattr(request, param, None)) for param in query_params) query_info = dict((k, v) for k, v in query_info.iteritems() if v is not None) + for k, v in query_info.iteritems(): + if isinstance(v, unicode): + query_info[k] = v.encode('utf8') + elif isinstance(v, str): + query_info[k] = v.decode('utf8') return query_info - def __ConstructPathParams(self, method_config, request): - path = method_config.relative_path - path_params = {} + def __ConstructRelativePath(self, method_config, request, relative_path=None): + """Determine the relative path for request.""" + path = relative_path or method_config.relative_path + # TODO(user): Why does the discovery document have pluses? + # Figure this out. + path = path.replace('+', '') + for param in method_config.path_params: param_template = '{%s}' % param if param_template not in path: @@ -339,199 +430,138 @@ class BaseApiService(object): raise exceptions.InvalidUserInputError( 'Request missing required parameter %s' % param) try: + # TODO(user): this isn't likely to break anything, but if you notice + # that it does please contact me and I'll get a concrete fix in ASAP + if not isinstance(value, basestring): + value = str(value) path = path.replace(param_template, urllib.quote(value.encode('utf_8'), '')) except TypeError as e: raise exceptions.InvalidUserInputError( 'Error setting required parameter %s to value %s: %s' % ( param, value, e)) - path_params[param] = value - return path, path_params - - def __GetUploadStrategy(self, upload, upload_config): - # Choose a protocol: We generally prefer resumable, unless the - # server only supports simple, or if we have a small transfer. - strategy = 'resumable' - if upload_config.simple_path: - if not upload_config.resumable_path: - strategy = 'simple' - elif (upload.total_size is not None and - upload.total_size < 4 * 1 << 20 and - upload_config.simple_multipart): - strategy = 'simple' - return strategy - - def __GetUploadParams(self, upload, upload_config, body_value): - strategy = self.__GetUploadStrategy(upload, upload_config) - params = {} - if strategy == 'simple': - params['uploadType'] = 'multipart' if body_value else 'media' - else: - params['uploadType'] = 'resumable' - return params - - def __GetUploadPath(self, upload, upload_config): - # In theory, we should use the resumable path in the case that the - # strategy is 'resumable'. However, apiclient is designed around a - # flow that pushes the original body in the first request, and - # pumps the media bytes through on successive requests. - _ = self.__GetUploadStrategy(upload, upload_config) - return upload_config.simple_path - - def __SimpleMediaBody(self, upload, headers, body_value): - # Rewrite the body. (This section follows apiclient.discovery.) - upload.stream.seek(0) - if not body_value: - headers['content-type'] = upload.mime_type - body_value = upload.stream.read() - else: - # This is a multipart/related upload. - msg_root = mime_multipart.MIMEMultipart('related') - # msg_root should not write out it's own headers - setattr(msg_root, '_write_headers', lambda self: None) - - # attach the body as one part - msg = mime_nonmultipart.MIMENonMultipart( - *headers['content-type'].split('/')) - msg.set_payload(body_value) - msg_root.attach(msg) - - # attach the media as the second part - msg = mime_nonmultipart.MIMENonMultipart( - *upload.mime_type.split('/')) - msg['Content-Transfer-Encoding'] = 'binary' - msg.set_payload(upload.stream.read()) - msg_root.attach(msg) - - body_value = msg_root.as_string() - multipart_boundary = msg_root.get_boundary() - headers['content-type'] = ('multipart/related; ' - 'boundary=%r') % multipart_boundary - return headers, body_value - - def __CreateMediaUpload(self, upload, upload_config, headers, body_value): - # Validate total_size vs. max_size - if (upload.total_size and upload_config.max_size and - upload.total_size > upload_config.max_size): - raise exceptions.InvalidUserInputError( - 'Upload too big: %s larger than max size %s' % ( - upload.total_size, upload_config.max_size)) - # Validate mime type - if not mimeparse.best_match(upload_config.accept, upload.mime_type): - raise exceptions.InvalidUserInputError( - 'MIME type %s does not match any accepted MIME ranges %s' % ( - upload.mime_type, upload_config.accept)) - strategy = self.__GetUploadStrategy(upload, upload_config) - # Create a MediaIoBaseUpload - if strategy == 'simple': - media_upload = False - headers, body_value = self.__SimpleMediaBody(upload, headers, body_value) - elif strategy == 'resumable': - # Don't need to set a body value in this case, since the - # HttpRequest is in charge of uploading it. - media_upload = apiclient_http.MediaIoBaseUpload( - upload.stream, upload.mime_type, resumable=True) - return media_upload, headers, body_value - - def __IsRetryable(self, exc): - status = int(exc.resp.get('status')) - # 308 doesn't have a name in httplib. - retryable_status = status in util.RETRYABLE_STATUS_CODES - return retryable_status and 'location' in exc.resp - - def __ExecuteRequest(self, request, url): - try: - return request.execute() - except apiclient_errors.HttpError as e: - if self.__IsRetryable(e): - logging.info('Got redirect for %s', request.uri) - request.uri = e.resp['location'] - logging.info('Redirecting to %s', request.uri) - return self.__ExecuteRequest(request, request.uri) - e.content = e.content.decode('ascii', 'replace') - logging.error('Error making request to "%s": "%s", "%s"', - url, e, e.content) - raise exceptions.HttpError.FromApiclientError(e) - except httplib2.HttpLib2Error as e: - raise exceptions.CommunicationError( - 'Communication error making request to "%s": "%s"' % (url, e)) + return path + + def __FinalizeRequest(self, http_request, url_builder): + """Make any final general adjustments to the request.""" + if (http_request.http_method == 'GET' and + len(http_request.url) > _MAX_URL_LENGTH): + http_request.http_method = 'POST' + http_request.headers['x-http-method-override'] = 'GET' + http_request.headers['content-type'] = 'application/x-www-form-urlencoded' + http_request.body = url_builder.query + url_builder.query_params = {} + http_request.url = url_builder.url + + def __ProcessHttpResponse(self, method_config, http_response): + """Process the given http response.""" + if http_response.status_code not in (httplib.OK, httplib.NO_CONTENT): + raise exceptions.HttpError.FromResponse(http_response) + if http_response.status_code == httplib.NO_CONTENT: + # TODO(craigcitro): Find out why _replace doesn't seem to work here. + http_response = http_wrapper.Response( + info=http_response.info, content='{}', + request_url=http_response.request_url) + response_type = _LoadClass( + method_config.response_type_name, self.__client.MESSAGES_MODULE) + return self.__client.DeserializeMessage( + response_type, http_response.content) + + def __SetBaseHeaders(self, http_request, client): + """Fill in the basic headers on http_request.""" + # TODO(craigcitro): Make the default a little better here, and + # include the apitools version. + user_agent = client.user_agent or 'apitools-client/1.0' + http_request.headers['user-agent'] = user_agent + http_request.headers['accept'] = 'application/json' + http_request.headers['accept-encoding'] = 'gzip, deflate' + + def __SetBody(self, http_request, method_config, request, upload): + """Fill in the body on http_request.""" + if not method_config.request_field: + return - def _RunMethod(self, method_config, request, global_params=None, - upload=None, upload_config=None, download=None): - """Call this method with request.""" - global_params = self.__CombineGlobalParams( - global_params, self.__client.global_params) request_type = _LoadClass( method_config.request_type_name, self.__client.MESSAGES_MODULE) - response_type = _LoadClass( - method_config.response_type_name, self.__client.MESSAGES_MODULE) - _Typecheck(request, request_type) - if self.__client.log_request: - logging.info('Request of type %s: %s', - method_config.request_type_name, request) - body_type = None if method_config.request_field == REQUEST_IS_BODY: + body_value = request body_type = request_type - elif method_config.request_field: + else: + body_value = getattr(request, method_config.request_field) body_field = request_type.field_by_name(method_config.request_field) - _Typecheck(body_field, messages.MessageField) + util.Typecheck(body_field, messages.MessageField) body_type = body_field.type - # TODO(craigcitro): Make the http and model objects configurable. - request_builder = apiclient_http.HttpRequest - model_class = self.__client.base_model_class - if download: - model_class = BaseMediaDownloadModel - api_model = model_class( - body_type, response_type, - self.__client.log_request, self.__client.log_response) - self.__client.ConfigureModel(api_model) - - body_value = None - if method_config.request_field == REQUEST_IS_BODY: - body_value = request - elif method_config.request_field: - body_value = getattr(request, method_config.request_field) - query_params = self.__ConstructQueryParams( + if upload and not body_value: + # We're going to fill in the body later. + return + util.Typecheck(body_value, body_type) + http_request.headers['content-type'] = 'application/json' + http_request.body = self.__client.SerializeMessage(body_value) + + def PrepareHttpRequest(self, method_config, request, global_params=None, + upload=None, upload_config=None, download=None): + """Prepares an HTTP request to be sent.""" + request_type = _LoadClass( + method_config.request_type_name, self.__client.MESSAGES_MODULE) + util.Typecheck(request, request_type) + request = self.__client.ProcessRequest(method_config, request) + + http_request = http_wrapper.Request(http_method=method_config.http_method) + self.__SetBaseHeaders(http_request, self.__client) + self.__SetBody(http_request, method_config, request, upload) + + url_builder = _UrlBuilder( + self.__client.url, relative_path=method_config.relative_path) + url_builder.query_params = self.__ConstructQueryParams( method_config.query_params, request, global_params) - if upload: - query_params.update(self.__GetUploadParams( - upload, upload_config, body_value)) - method_config.relative_path = self.__GetUploadPath(upload, upload_config) - relative_path, path_params = self.__ConstructPathParams( - method_config, request) - - # Note that api_model.request side-effects the headers, so must - # be threaded through. - headers = {} - headers, path_params, query, body = api_model.request( - headers, path_params, query_params, body_value) - - resumable = False - if upload: - resumable, headers, body = self.__CreateMediaUpload( - upload, upload_config, headers, body) - - url = urlparse.urljoin(self.__client.url, ''.join((relative_path, query))) - if self.__client.log_request: - logging.info('%s %s', method_config.http_method, url) - request = request_builder( - self.__client.http, - api_model.response, - url, - method=method_config.http_method, - body=body, - headers=headers, - methodId=method_config.method_id, - resumable=resumable) - - if download: - download.InitializeDownload( - str(request.uri), request.http, headers=request.headers) + + # It's important that upload and download go before we fill in the + # relative path, so that they can replace it. + if upload is not None: + upload.ConfigureRequest(upload_config, http_request, url_builder) + if download is not None: + download.ConfigureRequest(http_request, url_builder) + + url_builder.relative_path = self.__ConstructRelativePath( + method_config, request, relative_path=url_builder.relative_path) + self.__FinalizeRequest(http_request, url_builder) + + return self.__client.ProcessHttpRequest(http_request) + + def _RunMethod(self, method_config, request, global_params=None, + upload=None, upload_config=None, download=None): + """Call this method with request.""" + if upload is not None and download is not None: + # TODO(craigcitro): This just involves refactoring the logic + # below into callbacks that we can pass around; in particular, + # the order should be that the upload gets the initial request, + # and then passes its reply to a download if one exists, and + # then that goes to ProcessResponse and is returned. + raise exceptions.NotYetImplementedError( + 'Cannot yet use both upload and download at once') + + http_request = self.PrepareHttpRequest( + method_config, request, global_params, upload, upload_config, download) + + # TODO(craigcitro): Make num_retries customizable on Transfer + # objects, and pass in self.__client.num_retries when initializing + # an upload or download. + if download is not None: + download.InitializeDownload(http_request, client=self._client) return - response = self.__ExecuteRequest(request, url) - if self.__client.log_response: - logging.info('Response of type %s: %s', - method_config.response_type_name, response) - return response + http_response = None + if upload is not None: + http_response = upload.InitializeUpload(http_request, client=self._client) + if http_response is None: + http_response = http_wrapper.MakeRequest( + self.__client.http, http_request, retries=self.__client.num_retries) + + return self.ProcessHttpResponse(method_config, http_response) + + def ProcessHttpResponse(self, method_config, http_response): + """Convert an HTTP response to the expected message type.""" + return self.__client.ProcessResponse( + method_config, + self.__ProcessHttpResponse(method_config, http_response)) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py new file mode 100644 index 0000000..9f971c9 --- /dev/null +++ b/apitools/base/py/base_api_test.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + + +from protorpc import messages + +from google.apputils import basetest as googletest +from apitools.base.py import base_api + + +class SimpleMessage(messages.Message): + field = messages.StringField(1) + + +class FakeCredentials(object): + def authorize(self, _): # pylint: disable=invalid-name + return None + + +class FakeClient(base_api.BaseApiClient): + MESSAGES_MODULE = 'message_module' + _PACKAGE = 'package' + _SCOPES = ['scope1'] + _CLIENT_ID = 'client_id' + _CLIENT_SECRET = 'client_secret' + + +class BaseApiTest(googletest.TestCase): + + def __GetFakeClient(self): + return FakeClient('', credentials=FakeCredentials()) + + def testNoCredentials(self): + client = FakeClient('', get_credentials=False) + self.assertIsNotNone(client) + self.assertIsNone(client._credentials) + + def testIncludeEmptyFieldsClient(self): + msg = SimpleMessage() + client = self.__GetFakeClient() + self.assertEqual('{}', client.SerializeMessage(msg)) + with client.IncludeFields(('field',)): + self.assertEqual('{"field": null}', client.SerializeMessage(msg)) + + +if __name__ == '__main__': + googletest.main() diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index 17c8c97..ee5192b 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -13,25 +13,55 @@ import sys from google.apputils import appcommands import gflags as flags +from apitools.base.py import encoding +from apitools.base.py import exceptions + +__all__ = [ + 'ConsoleWithReadline', + 'DeclareBaseFlags', + 'FormatOutput', + 'SetupLogger', + 'run_main', + ] + + # TODO(craigcitro): We should move all the flags for the # StandardQueryParameters into this file, so that they can be used # elsewhere easily. -# TODO(craigcitro): FlagValidators? -flags.DEFINE_boolean( - 'log_request', False, - 'Log requests.') -flags.DEFINE_boolean( - 'log_response', False, - 'Log responses.') -flags.DEFINE_boolean( - 'log_request_response', False, - 'Log requests and responses.') +_BASE_FLAGS_DECLARED = False +_OUTPUT_FORMATTER_MAP = { + 'protorpc': lambda x: x, + 'json': encoding.MessageToJson, + } + + +def DeclareBaseFlags(): + """Declare base flags for all CLIs.""" + # TODO(craigcitro): FlagValidators? + global _BASE_FLAGS_DECLARED + if _BASE_FLAGS_DECLARED: + return + flags.DEFINE_boolean( + 'log_request', False, + 'Log requests.') + flags.DEFINE_boolean( + 'log_response', False, + 'Log responses.') + flags.DEFINE_boolean( + 'log_request_response', False, + 'Log requests and responses.') + flags.DEFINE_enum( + 'output_format', + 'protorpc', + _OUTPUT_FORMATTER_MAP.viewkeys(), + 'Display format for results.') + _BASE_FLAGS_DECLARED = True # NOTE: This is specified here so that it can be read by other files # without depending on the flag to be registered. TRACE_HELP = ( - 'A tracing token of the form "trace:" ' + 'A tracing token of the form "token:" ' 'to include in api requests.') FLAGS = flags.FLAGS @@ -42,6 +72,15 @@ def SetupLogger(): logging.getLogger().setLevel(logging.INFO) +def FormatOutput(message, output_format=None): + """Convert the output to the user-specified format.""" + output_format = output_format or FLAGS.output_format + formatter = _OUTPUT_FORMATTER_MAP.get(FLAGS.output_format) + if formatter is None: + raise exceptions.UserError('Unknown output format: %s' % output_format) + return formatter(message) + + class _SmartCompleter(rlcompleter.Completer): def _callable_postfix(self, val, word): if ('(' in readline.get_line_buffer() or diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py new file mode 100644 index 0000000..c5f8567 --- /dev/null +++ b/apitools/base/py/batch.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python +"""HTTP batch requests for apitools. + +This library is copied heavily from apiclient's BatchHttpRequest. +Some unneeded parts are removed, and apitools' BatchHttpRequest is +modified to work with the classes in http_wrapper instead of httplib2. +""" + +import collections +import email.generator as generator +import email.mime.multipart as mime_multipart +import email.mime.nonmultipart as mime_nonmultipart +import email.parser as email_parser +import itertools +import StringIO +import time +import urllib +import urlparse +import uuid + +from apitools.base.py import exceptions +from apitools.base.py import http_wrapper + +__all__ = [ + 'BatchApiRequest', + ] + + +RequestResponseHandler = collections.namedtuple( + 'RequestResponseHandler', ['request', 'response', 'handler']) + + +class BatchApiRequest(object): + """Friendly interface for batching API requests.""" + + class ApiRequestResponse(object): + """Each individual request is stored in its own object.""" + + def __init__(self, request, retryable_codes, service, method_config): + """Initialize an individual API request. + + Args: + request: An http_wrapper.Request object. + retryable_codes: A list of HTTP codes that can be retried. + service: A service inheriting from base_api.BaseApiService. + method_config: Method config for the desired API request. + """ + self.__retryable_codes = retryable_codes + self.__http_response = None + self.__service = service + self.__method_config = method_config + + self.http_request = request + # TODO(user): Add some validation to these fields. + self.__response = None + self.__exception = None + + @property + def is_error(self): + return self.exception is not None + + @property + def response(self): + return self.__response + + @property + def exception(self): + return self.__exception + + @property + def terminal_state(self): + return (self.__http_response and ( + self.__http_response.status_code not in self.__retryable_codes)) + + def HandleResponse(self, http_response, exception): + """Callback used with BatchApiRequest. + + Args: + http_response: Deserialized http_wrapper.Response object. + exception: apiclient.errors.HttpError object if an error occurred. + """ + self.__http_response = http_response + self.__exception = exception + if self.terminal_state: + self.__response = self.__service.ProcessHttpResponse( + self.__method_config, self.__http_response) + + def __init__(self, batch_url=None, retryable_codes=None): + """Initialize a batch API request object. + + Args: + batch_url: Base URL for batch API calls. + retryable_codes: A list of HTTP codes that can be retried. + """ + self.api_requests = [] + self.retryable_codes = retryable_codes or [] + self.batch_url = batch_url or 'https://www.googleapis.com/batch' + + def Add(self, service, method, request, global_params=None): + """Add a request to the batch. + + Args: + service: A class inheriting base_api.BaseApiService. + method: The desired method from the service. + request: An input message appropriate for the specified service.method. + global_params: Optional additional parameters to pass into + method.PrepareHttpRequest. + + Returns: + None + """ + # Retrieve the configs for the desired method and service. + method_config = service.GetMethodConfig(method) + upload_config = service.GetMethodUploadConfig(method) + + # Prepare the HTTP Request. + http_request = service.PrepareHttpRequest( + method_config, request, global_params=global_params, + upload_config=upload_config) + + # Create the request and add it to our master list. + api_request = self.ApiRequestResponse( + http_request, self.retryable_codes, service, method_config) + self.api_requests.append(api_request) + + def Execute(self, http, sleep_between_polls=5, max_retries=5): + """Execute all of the requests in the batch. + + Args: + http: httplib2.Http object for use in the request. + sleep_between_polls: How long to sleep between polls, in seconds. + max_retries: Max retries. Any requests that have not succeeded by + this number of retries simply report the last response or + exception, whatever it happened to be. + + Returns: + List of ApiRequestResponses. + """ + requests = [request for request in self.api_requests if not + request.terminal_state] + + for attempt in xrange(max_retries): + if attempt: + time.sleep(sleep_between_polls) + + # Create a batch_http_request object and populate it with incomplete + # requests. + batch_http_request = BatchHttpRequest(batch_url=self.batch_url) + for request in requests: + batch_http_request.Add(request.http_request, request.HandleResponse) + batch_http_request.Execute(http) + + # Collect retryable requests. + requests = [request for request in self.api_requests if not + request.terminal_state] + + if not requests: + break + + return self.api_requests + + +class BatchHttpRequest(object): + """Batches multiple http_wrapper.Request objects into a single request.""" + + def __init__(self, batch_url, callback=None): + """Constructor for a BatchHttpRequest. + + Args: + batch_url: string, URL to send batch requests to. + callback: callable, A callback to be called for each response, of the + form callback(response, exception). The first parameter is + the deserialized Response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no error occurred. + """ + # Endpoint to which these requests are sent. + self.__batch_url = batch_url + + # Global callback to be called for each individual response in the batch. + self.__callback = callback + + # List of requests, responses and handlers. + self.__request_response_handlers = {} + + # The last auto generated id. + self.__last_auto_id = itertools.count() + + # Unique ID on which to base the Content-ID headers. + self.__base_id = uuid.uuid4() + + def _IdToHeader(self, request_id): + """Convert an id to a Content-ID header value. + + Args: + request_id: string, identifier of individual request. + + Returns: + A Content-ID header with the id_ encoded into it. A UUID is prepended to + the value because Content-ID headers are supposed to be universally + unique. + """ + return '<%s+%s>' % (self.__base_id, urllib.quote(request_id)) + + def _HeaderToId(self, header): + """Convert a Content-ID header value to an id. + + Presumes the Content-ID header conforms to the format that _IdToHeader() + returns. + + Args: + header: string, Content-ID header value. + + Returns: + The extracted id value. + + Raises: + BatchError if the header is not in the expected format. + """ + if not (header.startswith('<') or header.endswith('>')): + raise exceptions.BatchError('Invalid value for Content-ID: %s' % header) + if '+' not in header: + raise exceptions.BatchError('Invalid value for Content-ID: %s' % header) + _, request_id = header[1:-1].rsplit('+', 1) + + return urllib.unquote(request_id) + + def _SerializeRequest(self, request): + """Convert a http_wrapper.Request object into a string. + + Args: + request: http_wrapper.Request, the request to serialize. + + Returns: + The request as a string in application/http format. + """ + # Construct status line + parsed = urlparse.urlsplit(request.url) + request_line = urlparse.urlunsplit( + (None, None, parsed.path, parsed.query, None)) + status_line = request.http_method + ' ' + request_line + ' HTTP/1.1\n' + major, minor = request.headers.get( + 'content-type', 'application/json').split('/') + msg = mime_nonmultipart.MIMENonMultipart(major, minor) + headers = request.headers.copy() + + # MIMENonMultipart adds its own Content-Type header. + # Keep all of the other headers in headers. + for key, value in headers.iteritems(): + if key == 'content-type': + continue + msg[key] = value + + msg['Host'] = parsed.netloc + msg.set_unixfrom(None) + + if request.body is not None: + msg.set_payload(request.body) + + # Serialize the mime message. + fp = StringIO.StringIO() + # maxheaderlen=0 means don't line wrap headers. + g = generator.Generator(fp, maxheaderlen=0) + g.flatten(msg, unixfrom=False) + body = fp.getvalue() + + # Strip off the \n\n that the MIME lib tacks onto the end of the payload. + if request.body is None: + body = body[:-2] + + return status_line.encode('utf-8') + body + + def _DeserializeResponse(self, payload): + """Convert string into Response and content. + + Args: + payload: string, headers and body as a string. + + Returns: + A Response object + """ + # Strip off the status line. + status_line, payload = payload.split('\n', 1) + _, status, _ = status_line.split(' ', 2) + + # Parse the rest of the response. + parser = email_parser.Parser() + msg = parser.parsestr(payload) + + # Get the headers. + info = dict(msg) + info['status'] = status + + # Create Response from the parsed headers. + content = msg.get_payload() + + return http_wrapper.Response(info, content, self.__batch_url) + + def _NewId(self): + """Create a new id. + + Auto incrementing number that avoids conflicts with ids already used. + + Returns: + string, a new unique id. + """ + return str(self.__last_auto_id.next()) + + def Add(self, request, callback=None): + """Add a new request. + + Args: + request: http_wrapper.Request, http_wrapper.Request to add to the batch. + callback: callable, A callback to be called for this response, of the + form callback(response, exception). The first parameter is the + deserialized response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no errors occurred. + + Returns: + None + """ + self.__request_response_handlers[self._NewId()] = RequestResponseHandler( + request, None, callback) + + def _Execute(self, http): + """Serialize batch request, send to server, process response. + + Args: + http: httplib2.Http, an http object to be used to make the request with. + + Raises: + httplib2.HttpLib2Error if a transport error has occured. + apiclient.errors.BatchError if the response is the wrong format. + """ + message = mime_multipart.MIMEMultipart('mixed') + # Message should not write out its own headers. + setattr(message, '_write_headers', lambda self: None) + + # Add all the individual requests. + for key in self.__request_response_handlers: + msg = mime_nonmultipart.MIMENonMultipart('application', 'http') + msg['Content-Transfer-Encoding'] = 'binary' + msg['Content-ID'] = self._IdToHeader(key) + + body = self._SerializeRequest( + self.__request_response_handlers[key].request) + msg.set_payload(body) + message.attach(msg) + + request = http_wrapper.Request(self.__batch_url, 'POST') + request.body = message.as_string() + request.headers['content-type'] = ( + 'multipart/mixed; boundary="%s"') % message.get_boundary() + + response = http_wrapper.MakeRequest(http, request) + + if response.status_code >= 300: + raise exceptions.HttpError.FromResponse(response) + + # Prepend with a content-type header so Parser can handle it. + header = 'content-type: %s\r\n\r\n' % response.info['content-type'] + + parser = email_parser.Parser() + mime_response = parser.parsestr(header + response.content) + + if not mime_response.is_multipart(): + raise exceptions.BatchError('Response not in multipart/mixed format.') + + for part in mime_response.get_payload(): + request_id = self._HeaderToId(part['Content-ID']) + response = self._DeserializeResponse(part.get_payload()) + + self.__request_response_handlers[request_id] = ( + self.__request_response_handlers[request_id]._replace( # pylint: disable=protected-access + response=response)) + + def Execute(self, http): + """Execute all the requests as a single batched HTTP request. + + Args: + http: httplib2.Http object to be used with the request. + + Returns: + None + + Raises: + BatchError if the response is the wrong format. + """ + + self._Execute(http) + + for key in self.__request_response_handlers: + response = self.__request_response_handlers[key].response + callback = self.__request_response_handlers[key].handler + + exception = None + + if response.status_code >= 300: + exception = exceptions.HttpError.FromResponse(response) + response = None + + if callback is not None: + callback(response, exception) + if self.__callback is not None: + self.__callback(response, exception) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 12c0bf0..6c673b0 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """Common credentials classes and constructors.""" -import getpass import httplib import json import os @@ -14,21 +13,30 @@ import oauth2client.client import oauth2client.gce import oauth2client.multistore_file import oauth2client.tools -from protorpc import messages import gflags as flags +import logging from apitools.base.py import exceptions from apitools.base.py import util -FLAGS = flags.FLAGS +__all__ = [ + 'CredentialsFromFile', + 'GaeAssertionCredentials', + 'GceAssertionCredentials', + 'GetCredentials', + 'ServiceAccountCredentials', + 'ServiceAccountCredentialsFromFile', + ] + # TODO(craigcitro): Expose the extra args here somewhere higher up, # possibly as flags in the generated CLI. def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, - credentials_filename=None, service_account_info=None, - robot_email=None, api_key=None, client=None): + credentials_filename=None, + service_account_name=None, service_account_keyfile=None, + api_key=None, client=None): """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) # TODO(craigcitro): Error checking. @@ -38,8 +46,9 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), 'user_agent': user_agent or '%s-generated/0.1' % package_name, } - if service_account_info is not None: - credentials = ServiceAccountCredentials(service_account_info, scopes) + if service_account_name is not None: + credentials = ServiceAccountCredentialsFromFile( + service_account_name, service_account_keyfile, scopes) if credentials is not None: return credentials credentials = GaeAssertionCredentials.Get(scopes) @@ -56,25 +65,17 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, raise exceptions.CredentialsError('Could not create valid credentials') -class ServiceAccountCredentialInfo(messages.Message): - """Information needed to create a signed service account credential.""" - service_account_name = messages.StringField(1) - private_key = messages.StringField(2) - - -def ServiceAccountInfoFromFile(service_account_name, key_filename): - with open(key_filename) as key_file: - return ServiceAccountCredentialInfo( - service_account_name=service_account_name, - private_key=key_file.read()) +def ServiceAccountCredentialsFromFile( + service_account_name, private_key_filename, scopes): + with open(private_key_filename) as key_file: + return ServiceAccountCredentials( + service_account_name, key_file.read(), scopes) -def ServiceAccountCredentials(service_account_info, scopes): +def ServiceAccountCredentials(service_account_name, private_key, scopes): scopes = util.NormalizeScopes(scopes) return oauth2client.client.SignedJwtAssertionCredentials( - service_account_info.service_account_name, - service_account_info.private_key, - scopes) + service_account_name, private_key, scopes) # TODO(craigcitro): We override to add some utility code, and to @@ -83,17 +84,20 @@ def ServiceAccountCredentials(service_account_info, scopes): class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """Assertion credentials for GCE instances.""" - def __init__(self, scopes, service_account_name='default', **kwds): + def __init__(self, scopes=None, service_account_name='default', **kwds): if not util.DetectGce(): raise exceptions.ResourceUnavailableError( 'GCE credentials requested outside a GCE instance') self.__service_account_name = service_account_name - scope_ls = util.NormalizeScopes(scopes) - instance_scopes = self._GetInstanceScopes() - if scope_ls > instance_scopes: - raise exceptions.CredentialsError( - 'Instance did not have access to scopes %s' % ( - sorted(list(scope_ls - instance_scopes)),)) + if scopes: + scope_ls = util.NormalizeScopes(scopes) + instance_scopes = self.GetInstanceScopes() + if scope_ls > instance_scopes: + raise exceptions.CredentialsError( + 'Instance did not have access to scopes %s' % ( + sorted(list(scope_ls - instance_scopes)),)) + else: + scopes = self.GetInstanceScopes() super(GceAssertionCredentials, self).__init__(scopes, **kwds) @classmethod @@ -103,12 +107,16 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): except exceptions.Error: return None - def _GetInstanceScopes(self): + def GetInstanceScopes(self): + # Extra header requirement can be found here: + # https://developers.google.com/compute/docs/metadata scopes_uri = ( - 'http://metadata.google.internal/computeMetadata/v1beta1/instance/' + 'http://metadata.google.internal/computeMetadata/v1/instance/' 'service-accounts/%s/scopes') % self.__service_account_name + additional_headers = {'X-Google-Metadata-Request': 'True'} + request = urllib2.Request(scopes_uri, headers=additional_headers) try: - response = urllib2.urlopen(scopes_uri) + response = urllib2.urlopen(request) except urllib2.URLError as e: raise exceptions.CommunicationError( 'Could not reach metadata service: %s' % e.reason) @@ -123,7 +131,8 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): token_uri = ( 'http://metadata.google.internal/computeMetadata/v1beta1/instance/' 'service-accounts/%s/token') % self.__service_account_name - response, content = do_request(token_uri) + extra_headers = {'X-Google-Metadata-Request': 'True'} + response, content = do_request(token_uri, headers=extra_headers) if response.status != httplib.OK: raise exceptions.CredentialsError( 'Error refreshing credentials: %s' % content) @@ -183,8 +192,8 @@ def CredentialsFromFile(path, client_info): client_info['client_id'], client_info['user_agent'], client_info['scope']) - if hasattr(FLAGS, 'auth_local_webserver'): - FLAGS.auth_local_webserver = False + if hasattr(flags.FLAGS, 'auth_local_webserver'): + flags.FLAGS.auth_local_webserver = False credentials = credential_store.get() if credentials is None or credentials.invalid: print 'Generating new OAuth credentials ...' @@ -203,6 +212,6 @@ def CredentialsFromFile(path, client_info): print 'Invalid authorization: %s' % (e,) except httplib2.HttpLib2Error as e: print 'Communication error: %s' % (e,) - raise util.CredentialsError( + raise exceptions.CredentialsError( 'Communication error creating credentials: %s' % e) return credentials diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py new file mode 100644 index 0000000..ecebb25 --- /dev/null +++ b/apitools/base/py/credentials_lib_test.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python + + +import httplib +import re +import StringIO +import urllib2 + +import mock + +from google.apputils import basetest as googletest +from apitools.base.py import credentials_lib +from apitools.base.py import util + + +def CreateUriValidator(uri_regexp, content=''): + def CheckUri(uri, headers=None): + if 'X-Google-Metadata-Request' not in headers: + raise ValueError('Missing required header') + if uri_regexp.match(uri): + message = content + status = httplib.OK + else: + message = 'Expected uri matching pattern %s' % uri_regexp.pattern + status = httplib.BAD_REQUEST + return type('HttpResponse', (object,), {'status': status})(), message + return CheckUri + + +class CredentialsLibTest(googletest.TestCase): + + def _GetServiceCreds(self, service_account_name=None, scopes=None): + scopes = scopes or ['scope1'] + kwargs = {} + if service_account_name is not None: + kwargs['service_account_name'] = service_account_name + service_account_name = service_account_name or 'default' + with mock.patch.object(urllib2, 'urlopen', autospec=True) as urllib_mock: + urllib_mock.return_value = StringIO.StringIO(''.join(scopes)) + with mock.patch.object(util, 'DetectGce', autospec=True) as mock_util: + mock_util.return_value = True + validator = CreateUriValidator( + re.compile(r'.*/%s/.*' % service_account_name), + content='{"access_token": "token"}') + credentials = credentials_lib.GceAssertionCredentials(scopes, **kwargs) + self.assertIsNone(credentials._refresh(validator)) + + def testGceServiceAccounts(self): + self._GetServiceCreds() + self._GetServiceCreds(service_account_name='my_service_account') + + +if __name__ == '__main__': + googletest.main() diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 2b435c1..6853cc8 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -2,6 +2,7 @@ """Common code for converting proto to other formats, such as JSON.""" import base64 +import collections import json @@ -16,25 +17,69 @@ __all__ = [ 'MessageToJson', 'DictToMessage', 'MessageToDict', + 'PyValueToMessage', + 'MessageToPyValue', ] +_Codec = collections.namedtuple('_Codec', ['encoder', 'decoder']) +CodecResult = collections.namedtuple('CodecResult', ['value', 'complete']) + + +# TODO(craigcitro): Make these non-global. +_UNRECOGNIZED_FIELD_MAPPINGS = {} +_CUSTOM_MESSAGE_CODECS = {} +_CUSTOM_FIELD_CODECS = {} +_FIELD_TYPE_CODECS = {} + + +def MapUnrecognizedFields(field_name): + """Register field_name as a container for unrecognized fields in message.""" + def Register(cls): + _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name + return cls + return Register + + +def RegisterCustomMessageCodec(encoder, decoder): + """Register a custom encoder/decoder for this message class.""" + def Register(cls): + _CUSTOM_MESSAGE_CODECS[cls] = _Codec(encoder=encoder, decoder=decoder) + return cls + return Register + + +def RegisterCustomFieldCodec(encoder, decoder): + """Register a custom encoder/decoder for this field.""" + def Register(field): + _CUSTOM_FIELD_CODECS[field] = _Codec(encoder=encoder, decoder=decoder) + return field + return Register + + +def RegisterFieldTypeCodec(encoder, decoder): + """Register a custom encoder/decoder for all fields of this type.""" + def Register(field_type): + _FIELD_TYPE_CODECS[field_type] = _Codec(encoder=encoder, decoder=decoder) + return field_type + return Register + + # TODO(craigcitro): Delete this function with the switch to proto2. def CopyProtoMessage(message): codec = protojson.ProtoJson() return codec.decode_message(type(message), codec.encode_message(message)) -# XXX json.dumps(body_value, cls=ApiJsonEncoder) def MessageToJson(message, include_fields=None): """Convert the given message to JSON.""" - result = _ProtoJsonApilib.Get().encode_message(message) + result = _ProtoJsonApiTools.Get().encode_message(message) return _IncludeFields(result, message, include_fields) def JsonToMessage(message_type, message): """Convert the given JSON to a message of type message_type.""" - return _ProtoJsonApilib.Get().decode_message(message_type, message) + return _ProtoJsonApiTools.Get().decode_message(message_type, message) # TODO(craigcitro): Do this directly, instead of via JSON. @@ -48,6 +93,16 @@ def MessageToDict(message): return json.loads(MessageToJson(message)) +def PyValueToMessage(message_type, value): + """Convert the given python value to a message of type message_type.""" + return JsonToMessage(message_type, json.dumps(value)) + + +def MessageToPyValue(message): + """Convert the given message to a python value.""" + return json.loads(MessageToJson(message)) + + def _IncludeFields(encoded_message, message, include_fields): """Add the requested fields to the encoded message.""" if include_fields is None: @@ -64,7 +119,15 @@ def _IncludeFields(encoded_message, message, include_fields): return json.dumps(result) -class _ProtoJsonApilib(protojson.ProtoJson): +def _GetFieldCodecs(field, attr): + result = [ + getattr(_CUSTOM_FIELD_CODECS.get(field), attr, None), + getattr(_FIELD_TYPE_CODECS.get(type(field)), attr, None), + ] + return [x for x in result if x is not None] + + +class _ProtoJsonApiTools(protojson.ProtoJson): """JSON encoder used by apitools clients.""" _INSTANCE = None @@ -75,49 +138,63 @@ class _ProtoJsonApilib(protojson.ProtoJson): return cls._INSTANCE def decode_message(self, message_type, encoded_message): # pylint: disable=invalid-name - result = super(_ProtoJsonApilib, self).decode_message( + if message_type in _CUSTOM_MESSAGE_CODECS: + return _CUSTOM_MESSAGE_CODECS[message_type].decoder(encoded_message) + result = super(_ProtoJsonApiTools, self).decode_message( message_type, encoded_message) - return _DecodeUnknownFields(result) + return _DecodeUnknownFields(result, encoded_message) def decode_field(self, field, value): - """Decode the given value as JSON.""" - if isinstance(field, messages.BytesField): - try: - return base64.urlsafe_b64decode(str(value)) - except TypeError: - pass - field_value = super(_ProtoJsonApilib, self).decode_field(field, value) + """Decode the given JSON value. + + Args: + field: a messages.Field for the field we're decoding. + value: a python value we'd like to decode. + + Returns: + A value suitable for assignment to field. + """ + for decoder in _GetFieldCodecs(field, 'decoder'): + result = decoder(field, value) + value = result.value + if result.complete: + return value if isinstance(field, messages.MessageField): - field_value = _DecodeUnknownFields(field_value) + field_value = self.decode_message(field.message_type, json.dumps(value)) + else: + field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) return field_value def encode_message(self, message): # pylint: disable=invalid-name + if isinstance(message, messages.FieldList): + return '[%s]' % (', '.join(self.encode_message(x) for x in message)) + if type(message) in _CUSTOM_MESSAGE_CODECS: + return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) message = _EncodeUnknownFields(message) - return super(_ProtoJsonApilib, self).encode_message(message) + return super(_ProtoJsonApiTools, self).encode_message(message) def encode_field(self, field, value): - """Encode the given value as JSON.""" - if isinstance(field, messages.BytesField): - try: - if isinstance(field, messages.BytesField): - if field.repeated: - return [base64.urlsafe_b64encode(byte) for byte in value] - else: - return base64.urlsafe_b64encode(value) - except TypeError: - pass + """Encode the given value as JSON. + + Args: + field: a messages.Field for the field we're encoding. + value: a value for field. + + Returns: + A python value suitable for json.dumps. + """ + for encoder in _GetFieldCodecs(field, 'encoder'): + result = encoder(field, value) + value = result.value + if result.complete: + return value if isinstance(field, messages.MessageField): - value = _EncodeUnknownFields(value) - return super(_ProtoJsonApilib, self).encode_field(field, value) + value = json.loads(self.encode_message(value)) + return super(_ProtoJsonApiTools, self).encode_field(field, value) -# TODO(craigcitro): Storing this in a global is a bad idea, for all -# the usual reasons. In particular, if we plan to make base_api a -# shared file, we need to fix this. -_UNRECOGNIZED_FIELD_MAPPINGS = {} - - -def _DecodeUnknownFields(message): +# TODO(craigcitro): Fold this and _IncludeFields in as codecs. +def _DecodeUnknownFields(message, encoded_message): """Rewrite unknown fields in message into message.destination.""" destination = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) if destination is None: @@ -130,6 +207,34 @@ def _DecodeUnknownFields(message): pair_type = pair_field.message_type # TODO(craigcitro): Add more error checking around the pair # type being exactly what we suspect (field names, etc). + if isinstance(pair_type.value, messages.MessageField): + new_values = _DecodeUnknownMessages( + message, json.loads(encoded_message), pair_type) + else: + new_values = _DecodeUnrecognizedFields(message, pair_type) + setattr(message, destination, new_values) + # We could probably get away with not setting this, but + # why not clear it? + setattr(message, '_Message__unrecognized_fields', {}) + return message + + +def _DecodeUnknownMessages(message, encoded_message, pair_type): + """Process unknown fields in encoded_message of a message type.""" + field_type = pair_type.value.type + new_values = [] + all_field_names = [x.name for x in message.all_fields()] + for name, value_dict in encoded_message.iteritems(): + if name in all_field_names: + continue + value = PyValueToMessage(field_type, value_dict) + new_pair = pair_type(key=name, value=value) + new_values.append(new_pair) + return new_values + + +def _DecodeUnrecognizedFields(message, pair_type): + """Process unrecognized fields in message.""" new_values = [] for unknown_field in message.all_unrecognized_fields(): # TODO(craigcitro): Consider validating the variant if @@ -137,13 +242,14 @@ def _DecodeUnknownFields(message): # also be necessary to check it in the case that the # type has multiple encodings. value, _ = message.get_unrecognized_field_info(unknown_field) - new_pair = pair_type(key=str(unknown_field), value=value) + value_type = pair_type.field_by_name('value') + if isinstance(value_type, messages.MessageField): + decoded_value = DictToMessage(value, pair_type.value.message_type) + else: + decoded_value = value + new_pair = pair_type(key=str(unknown_field), value=decoded_value) new_values.append(new_pair) - setattr(message, destination, new_values) - # We could probably get away with not setting this, but - # why not clear it? - setattr(message, '_Message__unrecognized_fields', {}) - return message + return new_values def _EncodeUnknownFields(message): @@ -160,14 +266,38 @@ def _EncodeUnknownFields(message): value_variant = pairs_type.field_by_name('value').variant pairs = getattr(message, source) for pair in pairs: - result.set_unrecognized_field(pair.key, pair.value, value_variant) + if value_variant == messages.Variant.MESSAGE: + encoded_value = MessageToDict(pair.value) + else: + encoded_value = pair.value + result.set_unrecognized_field(pair.key, encoded_value, value_variant) setattr(result, source, []) return result -def MapUnrecognizedFields(field_name): - """Register field_name as a container for unrecognized fields in message.""" - def Register(cls): - _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name - return cls - return Register +def _SafeEncodeBytes(field, value): + """Encode the bytes in value as urlsafe base64.""" + try: + if field.repeated: + result = [base64.urlsafe_b64encode(byte) for byte in value] + else: + result = base64.urlsafe_b64encode(value) + complete = True + except TypeError: + result = value + complete = False + return CodecResult(value=result, complete=complete) + + +def _SafeDecodeBytes(unused_field, value): + """Decode the urlsafe base64 value into bytes.""" + try: + result = base64.urlsafe_b64decode(str(value)) + complete = True + except TypeError: + result = value + complete = False + return CodecResult(value=result, complete=complete) + + +RegisterFieldTypeCodec(_SafeEncodeBytes, _SafeDecodeBytes)(messages.BytesField) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py new file mode 100644 index 0000000..de00c8e --- /dev/null +++ b/apitools/base/py/encoding_test.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + + +import base64 +import json + +from protorpc import messages + +from google.apputils import basetest as googletest +from apitools.base.py import encoding + + +class SimpleMessage(messages.Message): + field = messages.StringField(1) + repfield = messages.StringField(2, repeated=True) + + +class BytesMessage(messages.Message): + field = messages.BytesField(1) + repfield = messages.BytesField(2, repeated=True) + + +@encoding.MapUnrecognizedFields('additional_properties') +class AdditionalPropertiesMessage(messages.Message): + + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.StringField(2) + + additional_properties = messages.MessageField( + AdditionalProperty, 1, repeated=True) + + +class CompoundPropertyType(messages.Message): + index = messages.IntegerField(1) + name = messages.StringField(2) + + +@encoding.MapUnrecognizedFields('additional_properties') +class AdditionalMessagePropertiesMessage(messages.Message): + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.MessageField(CompoundPropertyType, 2) + + additional_properties = messages.MessageField( + 'AdditionalProperty', 1, repeated=True) + + +class HasNestedMessage(messages.Message): + nested = messages.MessageField(AdditionalPropertiesMessage, 1) + + +class EncodingTest(googletest.TestCase): + + def testCopyProtoMessage(self): + msg = SimpleMessage(field='abc') + new_msg = encoding.CopyProtoMessage(msg) + self.assertEqual(msg.field, new_msg.field) + msg.field = 'def' + self.assertNotEqual(msg.field, new_msg.field) + + def testBytesEncoding(self): + b64_str = 'AAc+' + b64_msg = '{"field": "%s"}' % b64_str + urlsafe_b64_str = 'AAc-' + urlsafe_b64_msg = '{"field": "%s"}' % urlsafe_b64_str + data = base64.b64decode(b64_str) + msg = BytesMessage(field=data) + self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, urlsafe_b64_msg)) + self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, b64_msg)) + self.assertEqual(urlsafe_b64_msg, encoding.MessageToJson(msg)) + + enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % { + 'b': urlsafe_b64_str, + } + rep_msg = BytesMessage(repfield=[data, data]) + self.assertEqual(rep_msg, encoding.JsonToMessage(BytesMessage, enc_rep_msg)) + self.assertEqual(enc_rep_msg, encoding.MessageToJson(rep_msg)) + + def testIncludeFields(self): + msg = SimpleMessage() + self.assertEqual('{}', encoding.MessageToJson(msg)) + self.assertEqual( + '{"field": null}', + encoding.MessageToJson(msg, include_fields=['field'])) + self.assertEqual( + '{"repfield": null}', + encoding.MessageToJson(msg, include_fields=['repfield'])) + + def testAdditionalPropertyMapping(self): + msg = AdditionalPropertiesMessage() + msg.additional_properties = [ + AdditionalPropertiesMessage.AdditionalProperty( + key='key_one', value='value_one'), + AdditionalPropertiesMessage.AdditionalProperty( + key='key_two', value='value_two'), + ] + + encoded_msg = encoding.MessageToJson(msg) + self.assertEqual( + {'key_one': 'value_one', 'key_two': 'value_two'}, + json.loads(encoded_msg)) + + new_msg = encoding.JsonToMessage(type(msg), encoded_msg) + self.assertEqual( + set(('key_one', 'key_two')), + {x.key for x in new_msg.additional_properties}) + self.assertIsNot(msg, new_msg) + + new_msg.additional_properties.pop() + self.assertEqual(1, len(new_msg.additional_properties)) + self.assertEqual(2, len(msg.additional_properties)) + + def testAdditionalMessageProperties(self): + json_msg = '{"input": {"index": 0, "name": "output"}}' + result = encoding.JsonToMessage( + AdditionalMessagePropertiesMessage, json_msg) + self.assertEqual(1, len(result.additional_properties)) + self.assertEqual(0, result.additional_properties[0].value.index) + + def testNestedFieldMapping(self): + nested_msg = AdditionalPropertiesMessage() + nested_msg.additional_properties = [ + AdditionalPropertiesMessage.AdditionalProperty( + key='key_one', value='value_one'), + AdditionalPropertiesMessage.AdditionalProperty( + key='key_two', value='value_two'), + ] + msg = HasNestedMessage(nested=nested_msg) + + encoded_msg = encoding.MessageToJson(msg) + self.assertEqual( + {'nested': {'key_one': 'value_one', 'key_two': 'value_two'}}, + json.loads(encoded_msg)) + + new_msg = encoding.JsonToMessage(type(msg), encoded_msg) + self.assertEqual( + set(('key_one', 'key_two')), + {x.key for x in new_msg.nested.additional_properties}) + + new_msg.nested.additional_properties.pop() + self.assertEqual(1, len(new_msg.nested.additional_properties)) + self.assertEqual(2, len(msg.nested.additional_properties)) + + +if __name__ == '__main__': + googletest.main() diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index 28531b9..04a00a4 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -1,8 +1,6 @@ #!/usr/bin/env python """Exceptions for generated client libraries.""" -from apiclient import errors as apiclient_errors - class Error(Exception): """Base class for all exceptions.""" @@ -28,18 +26,30 @@ class CommunicationError(Error): """Any communication error talking to an API server.""" -class HttpError(CommunicationError, apiclient_errors.HttpError): - """Error making a request, with a code.""" +class HttpError(CommunicationError): + """Error making a request. Soon to be HttpError.""" + + def __init__(self, response, content, url): + super(HttpError, self).__init__() + self.response = response + self.content = content + self.url = url - def __init__(self, *args, **kwds): - CommunicationError.__init__(self) # pylint: disable=non-parent-init-called - apiclient_errors.HttpError.__init__(self, *args, **kwds) + def __str__(self): + content = self.content.decode('ascii', 'replace') + return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( + self.url, self.response, content) + + @property + def status_code(self): + # TODO(craigcitro): Turn this into something better than a + # KeyError if there is no status. + return int(self.response['status']) @classmethod - def FromApiclientError(cls, e): - if not isinstance(e, apiclient_errors.HttpError): - raise TypecheckError('Invalid error type: %s', type(e).__name__) # pylint: disable=nonstandard-exception - return cls(e.resp, e.content, uri=e.uri) + def FromResponse(cls, http_response): + return cls(http_response.info, http_response.content, + http_response.request_url) class InvalidUserInputError(InvalidDataError): @@ -50,6 +60,10 @@ class InvalidDataFromServerError(InvalidDataError, CommunicationError): """Data received from the server is malformed.""" +class BatchError(Error): + """Error generated while constructing a batch request.""" + + class ConfigurationError(Error): """Base class for configuration errors.""" @@ -76,3 +90,7 @@ class TransferError(CommunicationError): class TransferInvalidError(TransferError): """The given transfer is invalid.""" + + +class NotYetImplementedError(GeneratedClientError): + """This functionality is not yet implemented.""" diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py new file mode 100644 index 0000000..15783ac --- /dev/null +++ b/apitools/base/py/extra_types.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +"""Extra types understood by apitools. + +This file will be replaced by a .proto file when we switch to proto2 +from protorpc. +""" + +import collections +import json +import numbers + +from protorpc import message_types +from protorpc import messages +from protorpc import protojson + +from apitools.base.py import encoding +from apitools.base.py import exceptions +from apitools.base.py import util + +__all__ = [ + 'DateTimeMessage', + 'JsonArray', + 'JsonObject', + 'JsonValue', + 'JsonProtoEncoder', + 'JsonProtoDecoder', + ] + +# We import from protorpc. +# pylint:disable=invalid-name +DateTimeMessage = message_types.DateTimeMessage +# pylint:enable=invalid-name + + +def _ValidateJsonValue(json_value): + entries = [(f, json_value.get_assigned_value(f.name)) + for f in json_value.all_fields()] + assigned_entries = [(f, value) for f, value in entries if value is not None] + if len(assigned_entries) != 1: + raise exceptions.InvalidDataError('Malformed JsonValue: %s' % json_value) + + +def _JsonValueToPythonValue(json_value): + """Convert the given JsonValue to a json string.""" + util.Typecheck(json_value, JsonValue) + _ValidateJsonValue(json_value) + if json_value.is_null: + return None + entries = [(f, json_value.get_assigned_value(f.name)) + for f in json_value.all_fields()] + assigned_entries = [(f, value) for f, value in entries if value is not None] + field, value = assigned_entries[0] + if not isinstance(field, messages.MessageField): + return value + elif field.message_type is JsonObject: + return _JsonObjectToPythonValue(value) + elif field.message_type is JsonArray: + return _JsonArrayToPythonValue(value) + + +def _JsonObjectToPythonValue(json_value): + util.Typecheck(json_value, JsonObject) + return dict([(prop.key, _JsonValueToPythonValue(prop.value)) for prop + in json_value.properties]) + + +def _JsonArrayToPythonValue(json_value): + util.Typecheck(json_value, JsonArray) + return [_JsonValueToPythonValue(e) for e in json_value.entries] + + +_MAXINT64 = 2 << 63 - 1 +_MININT64 = -(2 << 63) + + +def _PythonValueToJsonValue(py_value): + """Convert the given python value to a JsonValue.""" + if py_value is None: + return JsonValue(is_null=True) + if isinstance(py_value, bool): + return JsonValue(boolean_value=py_value) + if isinstance(py_value, basestring): + return JsonValue(string_value=py_value) + if isinstance(py_value, numbers.Number): + if isinstance(py_value, (int, long)): + if _MININT64 < py_value < _MAXINT64: + return JsonValue(integer_value=py_value) + return JsonValue(double_value=float(py_value)) + if isinstance(py_value, dict): + return JsonValue(object_value=_PythonValueToJsonObject(py_value)) + if isinstance(py_value, collections.Iterable): + return JsonValue(array_value=_PythonValueToJsonArray(py_value)) + raise exceptions.InvalidDataError( + 'Cannot convert "%s" to JsonValue' % py_value) + + +def _PythonValueToJsonObject(py_value): + util.Typecheck(py_value, dict) + return JsonObject( + properties=[ + JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) + for key, value in py_value.iteritems()]) + + +def _PythonValueToJsonArray(py_value): + return JsonArray(entries=map(_PythonValueToJsonValue, py_value)) + + +class JsonValue(messages.Message): + """Any valid JSON value.""" + # Is this JSON object `null`? + is_null = messages.BooleanField(1, default=False) + + # Exactly one of the following is provided if is_null is False; none + # should be provided if is_null is True. + boolean_value = messages.BooleanField(2) + string_value = messages.StringField(3) + # We keep two numeric fields to keep int64 round-trips exact. + double_value = messages.FloatField(4, variant=messages.Variant.DOUBLE) + integer_value = messages.IntegerField(5, variant=messages.Variant.INT64) + # Compound types + object_value = messages.MessageField('JsonObject', 6) + array_value = messages.MessageField('JsonArray', 7) + + +class JsonObject(messages.Message): + """A JSON object value. + + Messages: + Property: A property of a JsonObject. + + Fields: + properties: A list of properties of a JsonObject. + """ + + class Property(messages.Message): + """A property of a JSON object. + + Fields: + key: Name of the property. + value: A JsonValue attribute. + """ + key = messages.StringField(1) + value = messages.MessageField(JsonValue, 2) + + properties = messages.MessageField(Property, 1, repeated=True) + + +class JsonArray(messages.Message): + """A JSON array value.""" + entries = messages.MessageField(JsonValue, 1, repeated=True) + + +_JSON_PROTO_TO_PYTHON_MAP = { + JsonArray: _JsonArrayToPythonValue, + JsonObject: _JsonObjectToPythonValue, + JsonValue: _JsonValueToPythonValue, + } +_JSON_PROTO_TYPES = tuple(_JSON_PROTO_TO_PYTHON_MAP.keys()) + + +def _JsonProtoToPythonValue(json_proto): + util.Typecheck(json_proto, _JSON_PROTO_TYPES) + return _JSON_PROTO_TO_PYTHON_MAP[type(json_proto)](json_proto) + + +def _PythonValueToJsonProto(py_value): + if isinstance(py_value, dict): + return _PythonValueToJsonObject(py_value) + if (isinstance(py_value, collections.Iterable) and + not isinstance(py_value, basestring)): + return _PythonValueToJsonArray(py_value) + return _PythonValueToJsonValue(py_value) + + +def _JsonProtoToJson(json_proto, unused_encoder=None): + return json.dumps(_JsonProtoToPythonValue(json_proto)) + + +def _JsonToJsonProto(json_data, unused_decoder=None): + return _PythonValueToJsonProto(json.loads(json_data)) + + +# pylint:disable=invalid-name +JsonProtoEncoder = _JsonProtoToJson +JsonProtoDecoder = _JsonToJsonProto +# pylint:enable=invalid-name +encoding.RegisterCustomMessageCodec( + encoder=JsonProtoEncoder, decoder=JsonProtoDecoder)(JsonValue) +encoding.RegisterCustomMessageCodec( + encoder=JsonProtoEncoder, decoder=JsonProtoDecoder)(JsonObject) +encoding.RegisterCustomMessageCodec( + encoder=JsonProtoEncoder, decoder=JsonProtoDecoder)(JsonArray) + + +def _EncodeDateTimeField(field, value): + result = protojson.ProtoJson().encode_field(field, value) + return encoding.CodecResult(value=result, complete=True) + + +def _DecodeDateTimeField(unused_field, value): + result = protojson.ProtoJson().decode_field( + message_types.DateTimeField(1), value) + return encoding.CodecResult(value=result, complete=True) + + +encoding.RegisterFieldTypeCodec(_EncodeDateTimeField, _DecodeDateTimeField)( + message_types.DateTimeField) + + +def _EncodeInt64Field(field, value): + """Handle the special case of int64 as a string.""" + capabilities = [ + messages.Variant.INT64, + messages.Variant.UINT64, + ] + if field.variant not in capabilities: + return encoding.CodecResult(value=value, complete=False) + + if field.repeated: + result = [str(x) for x in value] + else: + result = str(value) + return encoding.CodecResult(value=result, complete=True) + + +def _DecodeInt64Field(unused_field, value): + # Don't need to do anything special, they're decoded just fine + return encoding.CodecResult(value=value, complete=False) + +encoding.RegisterFieldTypeCodec(_EncodeInt64Field, _DecodeInt64Field)( + messages.IntegerField) diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py new file mode 100644 index 0000000..7275671 --- /dev/null +++ b/apitools/base/py/extra_types_test.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + + +import json +import math + +from protorpc import messages + +from google.apputils import basetest as googletest +from apitools.base.py import encoding +from apitools.base.py import exceptions +from apitools.base.py import extra_types + + +class ExtraTypesTest(googletest.TestCase): + + def assertRoundTrip(self, value): + if isinstance(value, extra_types._JSON_PROTO_TYPES): + self.assertEqual( + value, + extra_types._PythonValueToJsonProto( + extra_types._JsonProtoToPythonValue(value))) + else: + self.assertEqual( + value, + extra_types._JsonProtoToPythonValue( + extra_types._PythonValueToJsonProto(value))) + + def assertTranslations(self, py_value, json_proto): + self.assertEqual(py_value, extra_types._JsonProtoToPythonValue(json_proto)) + self.assertEqual(json_proto, extra_types._PythonValueToJsonProto(py_value)) + + def testInvalidProtos(self): + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue(extra_types.JsonValue()) + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue( + extra_types.JsonValue(is_null=True, string_value='a')) + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue( + extra_types.JsonValue(integer_value=3, string_value='a')) + + def testNullEncoding(self): + self.assertTranslations(None, extra_types.JsonValue(is_null=True)) + + def testJsonNumberEncoding(self): + seventeen = extra_types.JsonValue(integer_value=17) + self.assertRoundTrip(17) + self.assertRoundTrip(seventeen) + self.assertTranslations(17, seventeen) + + json_pi = extra_types.JsonValue(double_value=math.pi) + self.assertRoundTrip(math.pi) + self.assertRoundTrip(json_pi) + self.assertTranslations(math.pi, json_pi) + + def testArrayEncoding(self): + array = [3, 'four', False] + json_array = extra_types.JsonArray(entries=[ + extra_types.JsonValue(integer_value=3), + extra_types.JsonValue(string_value='four'), + extra_types.JsonValue(boolean_value=False), + ]) + self.assertRoundTrip(array) + self.assertRoundTrip(json_array) + self.assertTranslations(array, json_array) + + def testDictEncoding(self): + d = {'a': 6, 'b': 'eleventeen'} + json_d = extra_types.JsonObject(properties=[ + extra_types.JsonObject.Property( + key='a', value=extra_types.JsonValue(integer_value=6)), + extra_types.JsonObject.Property( + key='b', value=extra_types.JsonValue(string_value='eleventeen')), + ]) + self.assertRoundTrip(d) + # We don't know json_d will round-trip, because of randomness in + # python dictionary iteration ordering. We also need to force + # comparison as lists, since hashing protos isn't helpful. + translated_properties = extra_types._PythonValueToJsonProto(d).properties + for p in json_d.properties: + self.assertIn(p, translated_properties) + for p in translated_properties: + self.assertIn(p, json_d.properties) + + def testJsonObjectPropertyTranslation(self): + value = extra_types.JsonValue(string_value='abc') + obj = extra_types.JsonObject(properties=[ + extra_types.JsonObject.Property(key='attr_name', value=value)]) + json_value = '"abc"' + json_obj = '{"attr_name": "abc"}' + + self.assertRoundTrip(value) + self.assertRoundTrip(obj) + self.assertRoundTrip(json_value) + self.assertRoundTrip(json_obj) + + self.assertEqual(json_value, encoding.MessageToJson(value)) + self.assertEqual(json_obj, encoding.MessageToJson(obj)) + + def testInt64(self): + # Testing roundtrip of type 'long' + + class DogeMsg(messages.Message): + such_string = messages.StringField(1) + wow = messages.IntegerField(2, variant=messages.Variant.INT64) + very_unsigned = messages.IntegerField(3, variant=messages.Variant.UINT64) + much_repeated = messages.IntegerField( + 4, variant=messages.Variant.INT64, repeated=True) + + def MtoJ(msg): + return encoding.MessageToJson(msg) + + def JtoM(class_type, json_str): + return encoding.JsonToMessage(class_type, json_str) + + def DoRoundtrip(class_type, json_msg=None, message=None, times=4): + if json_msg: + json_msg = MtoJ(JtoM(class_type, json_msg)) + if message: + message = JtoM(class_type, MtoJ(message)) + if times == 0: + result = json_msg if json_msg else message + return result + return DoRoundtrip(class_type=class_type, json_msg=json_msg, + message=message, times=times-1) + + # Single + json_msg = ('{"such_string": "poot", "wow": "-1234",' + ' "very_unsigned": "999", "much_repeated": ["123", "456"]}') + out_json = MtoJ(JtoM(DogeMsg, json_msg)) + self.assertEqual(json.loads(out_json)['wow'], '-1234') + + # Repeated test case + msg = DogeMsg(such_string='wow', wow=-1234, + very_unsigned=800, much_repeated=[123, 456]) + self.assertEqual(msg, DoRoundtrip(DogeMsg, message=msg)) + + +if __name__ == '__main__': + googletest.main() diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py new file mode 100644 index 0000000..5b911c7 --- /dev/null +++ b/apitools/base/py/http_wrapper.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +"""HTTP wrapper for apitools. + +This library wraps the underlying http library we use, which is +currently httplib2. +""" + +import collections +import httplib +import logging +import socket +import time +import urlparse + +import httplib2 + +from apitools.base.py import exceptions + +__all__ = [ + 'GetHttp', + 'MakeRequest', + ] + + +# 308 and 429 don't have names in httplib. +RESUME_INCOMPLETE = 308 +TOO_MANY_REQUESTS = 429 +_REDIRECT_STATUS_CODES = ( + httplib.MOVED_PERMANENTLY, + httplib.FOUND, + httplib.SEE_OTHER, + httplib.TEMPORARY_REDIRECT, + RESUME_INCOMPLETE, + ) + + +class Request(object): + """Class encapsulating the data for an HTTP request.""" + + def __init__(self, url='', http_method='GET', headers=None, body=''): + self.url = url + self.http_method = http_method + self.headers = headers or {} + self.__body = None + self.body = body + + @property + def body(self): + return self.__body + + @body.setter + def body(self, value): + self.__body = value + if value is not None: + self.headers['content-length'] = str(len(self.__body)) + else: + self.headers.pop('content-length', None) + + +# Note: currently the order of fields here is important, since we want +# to be able to pass in the result from httplib2.request. +class Response(collections.namedtuple( + 'HttpResponse', ['info', 'content', 'request_url'])): + """Class encapsulating data for an HTTP response.""" + __slots__ = () + + def __len__(self): + if '-content-encoding' in self.info and 'content-range' in self.info: + # httplib2 rewrites content-length in the case of a compressed + # transfer; we can't trust the content-length header in that + # case, but we *can* trust content-range, if it's present. + _, _, range_spec = self.info['content-range'].partition(' ') + byte_range, _, _ = range_spec.partition('/') + start, _, end = byte_range.partition('-') + return int(end) - int(start) + 1 + return int(self.info.get('content-length', len(self.content))) + + @property + def status_code(self): + return int(self.info['status']) + + @property + def retry_after(self): + if 'retry-after' in self.info: + return int(self.info['retry-after']) + + @property + def is_redirect(self): + return (self.status_code in _REDIRECT_STATUS_CODES and + 'location' in self.info) + + +def MakeRequest(http, http_request, retries=5, redirections=5): + """Send http_request via the given http. + + This wrapper exists to handle translation between the plain httplib2 + request/response types and the Request and Response types above. + This will also be the hook for error/retry handling. + + Args: + http: An httplib2.Http instance, or a http multiplexer that delegates to + an underlying http, for example, HTTPMultiplexer. + http_request: A Request to send. + retries: (int, default 5) Number of retries to attempt on 5XX replies. + redirections: (int, default 5) Number of redirects to follow. + + Returns: + A Response object. + + Raises: + InvalidDataFromServerError: if there is no response after retries. + """ + response = None + exc = None + connection_type = None + # Handle overrides for connection types. This is used if the caller + # wants control over the underlying connection for managing callbacks + # or hash digestion. + if getattr(http, 'connections', None): + url_scheme = urlparse.urlsplit(http_request.url).scheme + if url_scheme and url_scheme in http.connections: + connection_type = http.connections[url_scheme] + for retry in xrange(retries + 1): + # Note that the str() calls here are important for working around + # some funny business with message construction and unicode in + # httplib itself. See, eg, + # http://bugs.python.org/issue11898 + info = None + try: + info, content = http.request( + str(http_request.url), method=str(http_request.http_method), + body=http_request.body, headers=http_request.headers, + redirections=redirections, connection_type=connection_type) + except httplib.BadStatusLine as e: + logging.error('Caught BadStatusLine from httplib, retrying: %s', e) + exc = e + except socket.error as e: + if http_request.http_method != 'GET': + raise + logging.error('Caught socket error, retrying: %s', e) + exc = e + if info is not None: + response = Response(info, content, http_request.url) + if (response.status_code < 500 and response.status_code != 429 and + not response.retry_after): + break + logging.info('Retrying request to url <%s> after status code %s', + response.request_url, response.status_code) + else: + logging.info('Retrying request to url <%s> after connection break', + str(http_request.url)) + # TODO(craigcitro): Make this timeout configurable. + if response: + time.sleep(response.retry_after or 2 ** retry) + else: + time.sleep(2 ** retry) + if response is None: + raise exceptions.InvalidDataFromServerError( + 'HTTP error on final retry: %s' % exc) + return response + + +def GetHttp(): + return httplib2.Http() diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 0deffd8..0ccd8e6 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -1,7 +1,8 @@ #!/usr/bin/env python """Upload and download support for apitools.""" -import collections +import email.mime.multipart as mime_multipart +import email.mime.nonmultipart as mime_nonmultipart import httplib import io import json @@ -9,64 +10,32 @@ import mimetypes import os import threading -import httplib2 +import mimeparse from apitools.base.py import exceptions -from apitools.base.py import util +from apitools.base.py import http_wrapper __all__ = [ 'Download', 'Upload', ] -# pylint: disable=slots-on-old-class - - -# Note: currently the order of fields here is important, since we want -# to be able to pass in the result from httplib2.request. -class _HttpResponse(collections.namedtuple( - '_HttpResponse', ['info', 'content'])): - __slots__ = () - - def __len__(self): - return int(self.info.get('content-length', len(self.content))) - - @property - def status_code(self): - return int(self.info['status']) - - -class _TransferSerializationData(collections.namedtuple( - '_TransferSerializationData', - ['auto_transfer', 'progress', 'total_size', 'url'])): - __slots__ = () - - def ToJson(self): - return json.dumps(self._asdict()) - - @classmethod - def FromJson(cls, json_data): - data = json.loads(json_data) - data_keys = set(('progress', 'url')) - if not data_keys.issubset(set(cls._fields)): - raise exceptions.InvalidDataError( - 'Invalid keys for Transfer: %s' % data_keys) - return cls._make(data[field] for field in cls._fields) +_RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 +_SIMPLE_UPLOAD = 'simple' +_RESUMABLE_UPLOAD = 'resumable' class _Transfer(object): """Generic bits common to Uploads and Downloads.""" def __init__(self, stream, close_stream=False, chunksize=None, - auto_transfer=True): + auto_transfer=True, http=None): + self.__bytes_http = None self.__close_stream = close_stream - self.__http = None + self.__http = http self.__stream = stream - self.__total_size = None self.__url = None - self._progress = 0 - self.auto_transfer = auto_transfer self.chunksize = chunksize or 1048576L @@ -74,46 +43,56 @@ class _Transfer(object): return str(self) @property - def _type_name(self): - return type(self).__name__ - - @property - def url(self): - return self.__url - - @url.setter - def url(self, value): - if self.url is not None: - raise exceptions.ConfigurationValueError( - 'Cannot set download url on initialized %s', self._type_name) - self.__url = value + def close_stream(self): + return self.__close_stream @property def http(self): return self.__http - @http.setter - def http(self, value): - if self.http is not None: - raise exceptions.ConfigurationValueError( - 'Cannot set http on initialized %s', self._type_name) - self.__http = value + @property + def bytes_http(self): + return self.__bytes_http or self.http + + @bytes_http.setter + def bytes_http(self, value): + self.__bytes_http = value @property - def total_size(self): - return self.__total_size + def stream(self): + return self.__stream - @total_size.setter - def total_size(self, value): - if self.total_size is not None: - raise exceptions.ConfigurationValueError( - 'Cannot set total_size on initialized %s', self._type_name) - self.__total_size = value + @property + def url(self): + return self.__url + + def _Initialize(self, http, url): + """Initialize this download by setting self.http and self.url. + + We want the user to be able to override self.http by having set + the value in the constructor; in that case, we ignore the provided + http. + + Args: + http: An httplib2.Http instance or None. + url: The url for this transfer. + + Returns: + None. Initializes self. + """ + self.EnsureUninitialized() + if self.http is None: + self.__http = http or http_wrapper.GetHttp() + self.__url = url @property def initialized(self): return self.url is not None and self.http is not None + @property + def _type_name(self): + return type(self).__name__ + def EnsureInitialized(self): if not self.initialized: raise exceptions.TransferInvalidError( @@ -124,28 +103,15 @@ class _Transfer(object): raise exceptions.TransferInvalidError( 'Cannot re-initialize %s', self._type_name) - @property - def close_stream(self): - return self.__close_stream - - @property - def stream(self): - return self.__stream - - @property - def serialization_data(self): - self.EnsureInitialized() - return { - 'auto_transfer': self.auto_transfer, - 'progress': self._progress, - 'total_size': self.total_size, - 'url': self.url, - } - def __del__(self): if self.__close_stream: self.__stream.close() + def _ExecuteCallback(self, callback, response): + # TODO(craigcitro): Push these into a queue. + if callback is not None: + threading.Thread(target=callback, args=(response, self)).start() + class Download(_Transfer): """Data for a single download. @@ -153,9 +119,24 @@ class Download(_Transfer): Public attributes: chunksize: default chunksize to use for transfers. """ + _ACCEPTABLE_STATUSES = set(( + httplib.OK, + httplib.NO_CONTENT, + httplib.PARTIAL_CONTENT, + httplib.REQUESTED_RANGE_NOT_SATISFIABLE, + )) + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'progress', 'total_size', 'url')) + + def __init__(self, *args, **kwds): + super(Download, self).__init__(*args, **kwds) + self.__initial_response = None + self.__progress = 0 + self.__total_size = None - def __str__(self): - return 'Download for url %s' % self.url + @property + def progress(self): + return self.__progress @classmethod def FromFile(cls, filename, overwrite=False, auto_transfer=True): @@ -172,135 +153,204 @@ class Download(_Transfer): return cls(stream, auto_transfer=auto_transfer) @classmethod - def FromData(cls, stream, json_data, http=None): + def FromData(cls, stream, json_data, http=None, auto_transfer=None): """Create a new Download object from a stream and serialized data.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) download = cls.FromStream(stream) - info = _TransferSerializationData.FromJson(json_data) - download._progress = info.progress # pylint: disable=protected-access - download.http = http - download.total_size = info.total_size - download.url = info.url + if auto_transfer is not None: + download.auto_transfer = auto_transfer + else: + download.auto_transfer = info['auto_transfer'] + setattr(download, '_Download__progress', info['progress']) + setattr(download, '_Download__total_size', info['total_size']) + download._Initialize(http, info['url']) # pylint: disable=protected-access return download - def InitializeDownload(self, url, http, - method='GET', headers=None, body=''): + @property + def serialization_data(self): + self.EnsureInitialized() + return { + 'auto_transfer': self.auto_transfer, + 'progress': self.progress, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def total_size(self): + return self.__total_size + + def __str__(self): + if not self.initialized: + return 'Download (uninitialized)' + else: + return 'Download with %d/%s bytes transferred from url %s' % ( + self.progress, self.total_size, self.url) + + def ConfigureRequest(self, http_request, url_builder): + url_builder.query_params['alt'] = 'media' + http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) + + def __SetTotal(self, info): + if 'content-range' in info: + _, _, total = info['content-range'].rpartition('/') + if total != '*': + self.__total_size = int(total) + # Note "total_size is None" means we don't know it; if no size + # info was returned on our initial range request, that means we + # have a 0-byte file. (That last statement has been verified + # empirically, but is not clearly documented anywhere.) + if self.total_size is None: + self.__total_size = 0 + + def InitializeDownload(self, http_request, http=None, client=None): """Initialize this download by making a request. Args: - url: The URL to use for our initial request. + http_request: The HttpRequest to use to initialize this download. http: The httplib2.Http instance for this request. - method: The HTTP method used for the call. - headers: Additional headers to pass in the request. - body: The message body. + client: If provided, let this client process the final URL before + sending any additional requests. If client is provided and + http is not, client.http will be used instead. """ self.EnsureUninitialized() - headers = headers or {} - try: - response, content = http.request( - uri=url, method=method, headers=headers, body=body, redirections=0) - if response['status'] not in util.RETRYABLE_STATUS_CODES: - raise exceptions.HttpError(response, content, uri=url) - else: - raise exceptions.InvalidDataFromServerError( - 'No redirect received for media download from url <%s>: %s' % ( - url, response)) - except httplib2.RedirectLimit as e: - self.url = e.response['location'] - self.http = http - + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) + if response.status_code not in self._ACCEPTABLE_STATUSES: + raise exceptions.HttpError.FromResponse(response) + self.__initial_response = response + self.__SetTotal(response.info) + url = response.info.get('content-location', response.request_url) + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) # Unless the user has requested otherwise, we want to just # go ahead and pump the bytes now. if self.auto_transfer: self.StreamInChunks() - def __SetTotal(self, info): - if self.total_size is None and 'content-range' in info: - _, _, total = info['content-range'].rpartition('/') - if total != '*': - self.total_size = int(total) - - def __GetChunk(self, start, end=None, chunksize=None, - additional_headers=None): - """Retrieve a chunk, and return the full response.""" - self.EnsureInitialized() - start = max(start, 0) - chunksize = chunksize or self.chunksize - if end and end < 0: - # Requesting range from end - start = '' - end = abs(end) + @staticmethod + def _ArgPrinter(response, unused_download): + if 'content-range' in response.info: + print 'Received %s' % response.info['content-range'] else: - max_end = start + chunksize - 1 - end = min(end or max_end, max_end) + print 'Received %d bytes' % len(response) + + @staticmethod + def _CompletePrinter(*unused_args): + print 'Download complete' + + def __NormalizeStartEnd(self, start, end=None): + if end is not None: + if start < 0: + raise exceptions.TransferInvalidError( + 'Cannot have end index with negative start index') + elif start >= self.total_size: + raise exceptions.TransferInvalidError( + 'Cannot have start index greater than total size') + end = min(end, self.total_size - 1) if end < start: raise exceptions.TransferInvalidError( 'Range requested with end[%s] < start[%s]' % (end, start)) - headers = {'Range': 'bytes=%s-%d' % (start, end)} + return start, end + else: + if start < 0: + start = max(0, start + self.total_size) + return start, self.total_size + + def __SetRangeHeader(self, request, start, end=None): + if start < 0: + request.headers['range'] = 'bytes=%d' % start + elif end is None: + request.headers['range'] = 'bytes=%d-' % start + else: + request.headers['range'] = 'bytes=%d-%d' % (start, end) + + def __GetChunk(self, start, end=None, additional_headers=None): + """Retrieve a chunk, and return the full response.""" + self.EnsureInitialized() + end_byte = min(end or start + self.chunksize, self.total_size) + request = http_wrapper.Request(url=self.url) + self.__SetRangeHeader(request, start, end=end_byte) if additional_headers is not None: - headers.update(additional_headers) - # TODO(craigcitro): Add support for retries. - response = _HttpResponse(*self.http.request(self.url, headers=headers)) - self.__SetTotal(response.info) - if response.status_code not in (httplib.PARTIAL_CONTENT, - httplib.REQUESTED_RANGE_NOT_SATISFIABLE): + request.headers.update(additional_headers) + return http_wrapper.MakeRequest(self.bytes_http, request) + + def __ProcessResponse(self, response): + """Process this response (by updating self and writing to self.stream).""" + if response.status_code not in self._ACCEPTABLE_STATUSES: raise exceptions.TransferInvalidError(response.content) - if response.status_code == httplib.PARTIAL_CONTENT: + if response.status_code in (httplib.OK, httplib.PARTIAL_CONTENT): self.stream.write(response.content) + self.__progress += len(response) + elif response.status_code == httplib.NO_CONTENT: + # It's important to write something to the stream for the case + # of a 0-byte download to a file, as otherwise python won't + # create the file. + self.stream.write('') return response - def GetRange(self, start, end, chunksize=None, exact_range=True, - additional_headers=None): - """Retrieve a given byte range from this download.""" - progress = start - chunksize = chunksize or self.chunksize - while progress < end: - response = self.__GetChunk(progress, end=end, - additional_headers=additional_headers) - if (response.status_code == httplib.REQUESTED_RANGE_NOT_SATISFIABLE and - exact_range): - raise exceptions.TransferInvalidError( - 'Could not fetch all requested bytes: ended at %d' % progress) - progress += len(response) + def GetRange(self, start, end=None, additional_headers=None): + """Retrieve a given byte range from this download, inclusive. - def __ExecuteCallback(self, callback, response): - # TODO(craigcitro): Push these into a queue. - if callback is not None: - threading.Thread(target=callback, args=(response, self)).start() + Range must be of one of these three forms: + * 0 <= start, end = None: Fetch from start to the end of the file. + * 0 <= start <= end: Fetch the bytes from start to end. + * start < 0, end = None: Fetch the last -start bytes of the file. - def StreamInChunks(self, callback=None, finish_callback=None, chunksize=None, - end=None, additional_headers=None): - """Stream the entire download.""" + (These variations correspond to those described in the HTTP 1.1 + protocol for range headers in RFC 2616, sec. 14.35.1.) - def ArgPrinter(response, unused_download): - print 'Received bytes %s' % response.info['content-range'] + Args: + start: (int) Where to start fetching bytes. (See above.) + end: (int, optional) Where to stop fetching bytes. (See above.) + additional_headers: (bool, optional) Any additional headers to + pass with the request. - def CompletePrinter(*unused_args): - print 'Download complete' + Returns: + None. Streams bytes into self.stream. + """ + self.EnsureInitialized() + progress, end = self.__NormalizeStartEnd(start, end) + while progress < end: + chunk_end = min(progress + self.chunksize, end) + response = self.__GetChunk(progress, end=chunk_end, + additional_headers=additional_headers) + response = self.__ProcessResponse(response) + progress += len(response) + if not response: + raise exceptions.TransferInvalidError( + 'Zero bytes unexpectedly returned in download response') - callback = callback or ArgPrinter - finish_callback = finish_callback or CompletePrinter + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Stream the entire download.""" + callback = callback or self._ArgPrinter + finish_callback = finish_callback or self._CompletePrinter self.EnsureInitialized() while True: - response = self.__GetChunk(self._progress, chunksize=chunksize, end=end, - additional_headers=additional_headers) - # TODO(craigcitro): Consider whether this update and writing - # the response to self.stream need to happen as a transaction. - self._progress += len(response) - if response.status_code == httplib.REQUESTED_RANGE_NOT_SATISFIABLE: + if self.__initial_response is not None: + response = self.__initial_response + self.__initial_response = None + else: + response = self.__GetChunk(self.progress, + additional_headers=additional_headers) + response = self.__ProcessResponse(response) + self._ExecuteCallback(callback, response) + if (response.status_code == httplib.OK or + self.progress >= self.total_size): break - # Callback with the new chunk. - self.__ExecuteCallback(callback, response) - # Handle range requests - # TODO(craigcitro): Exert python mastery over the known universe by - # cleaning up this hackish implementation. - if end: - if end < 0: - if(self._progress) >= abs(end): - break - elif self._progress >= end: - break - self.__ExecuteCallback(finish_callback, response) + self._ExecuteCallback(finish_callback, response) class Upload(_Transfer): @@ -309,59 +359,350 @@ class Upload(_Transfer): Fields: stream: The stream to upload. mime_type: MIME type of the upload. - mime_encoding: (optional) Encoding for the upload. Currently unused. - size_hint: (optional) Total upload size for the stream. + total_size: (optional) Total upload size for the stream. close_stream: (default: False) Whether or not we should close the stream when finished with the upload. + auto_transfer: (default: True) If True, stream all bytes as soon as + the upload is created. """ - - def __init__(self, stream, mime_type, mime_encoding=None, size_hint=None, - close_stream=False, chunksize=None): - super(Upload, self).__init__(stream, close_stream=close_stream, - chunksize=chunksize) + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'mime_type', 'total_size', 'url')) + + def __init__(self, stream, mime_type, total_size=None, http=None, + close_stream=False, chunksize=None, auto_transfer=True): + super(Upload, self).__init__( + stream, close_stream=close_stream, chunksize=chunksize, + auto_transfer=auto_transfer, http=http) + self.__complete = False self.__mime_type = mime_type - self.__mime_encoding = mime_encoding + self.__progress = 0 + self.__server_chunk_granularity = None + self.__strategy = None - self.total_size = size_hint + self.total_size = total_size @property - def mime_type(self): - return self.__mime_type - - @property - def mime_encoding(self): - return self.__mime_encoding - - def __str__(self): - size = self.total_size or '' - return 'Upload of size %s and mime type %s' % (size, self.mime_type) + def progress(self): + return self.__progress @classmethod - def FromFile(cls, filename, mime_type=None, mime_encoding=None): + def FromFile(cls, filename, mime_type=None, auto_transfer=True): """Create a new Upload object from a filename.""" path = os.path.expanduser(filename) if not os.path.exists(path): raise exceptions.NotFoundError('Could not find file %s' % path) if not mime_type: - mime_type, mime_encoding = mimetypes.guess_type(path) + mime_type, _ = mimetypes.guess_type(path) if mime_type is None: raise exceptions.InvalidUserInputError( 'Could not guess mime type for %s' % path) size = os.stat(path).st_size - return cls(open(path, 'rb'), mime_type, mime_encoding=mime_encoding, - size_hint=size, close_stream=True) + return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, + auto_transfer=auto_transfer) @classmethod - def FromStream(cls, stream, mime_type, mime_encoding=None): - """Create a new Upload object from a seekable stream.""" - if isinstance(stream, io.IOBase) and not stream.seekable(): - raise exceptions.InvalidUserInputError('Stream not seekable') - # TODO(craigcitro): Consider checking the full interface we need - # for a stream here. + def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True): + """Create a new Upload object from a stream.""" if mime_type is None: raise exceptions.InvalidUserInputError( 'No mime_type specified for stream') - stream.seek(0, io.SEEK_END) - size = stream.tell() - return cls(stream, mime_type, mime_encoding=mime_encoding, - size_hint=size, close_stream=False) + return cls(stream, mime_type, total_size=total_size, close_stream=False, + auto_transfer=auto_transfer) + + @classmethod + def FromData(cls, stream, json_data, http, auto_transfer=None): + """Create a new Upload of stream from serialized json_data using http.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) + upload = cls.FromStream(stream, info['mime_type'], + total_size=info.get('total_size')) + if isinstance(stream, io.IOBase) and not stream.seekable(): + raise exceptions.InvalidUserInputError( + 'Cannot restart resumable upload on non-seekable stream') + if auto_transfer is not None: + upload.auto_transfer = auto_transfer + else: + upload.auto_transfer = info['auto_transfer'] + upload.strategy = _RESUMABLE_UPLOAD + upload._Initialize(http, info['url']) # pylint: disable=protected-access + upload._RefreshResumableUploadState() # pylint: disable=protected-access + upload.EnsureInitialized() + if upload.auto_transfer: + upload.StreamInChunks() + return upload + + @property + def serialization_data(self): + self.EnsureInitialized() + if self.strategy != _RESUMABLE_UPLOAD: + raise exceptions.InvalidDataError( + 'Serialization only supported for resumable uploads') + return { + 'auto_transfer': self.auto_transfer, + 'mime_type': self.mime_type, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def complete(self): + return self.__complete + + @property + def mime_type(self): + return self.__mime_type + + def __str__(self): + if not self.initialized: + return 'Upload (uninitialized)' + else: + return 'Upload with %d/%s bytes transferred for url %s' % ( + self.progress, self.total_size or '???', self.url) + + @property + def strategy(self): + return self.__strategy + + @strategy.setter + def strategy(self, value): + if value not in (_SIMPLE_UPLOAD, _RESUMABLE_UPLOAD): + raise exceptions.UserError(( + 'Invalid value "%s" for upload strategy, must be one of ' + '"simple" or "resumable".') % value) + self.__strategy = value + + @property + def total_size(self): + return self.__total_size + + @total_size.setter + def total_size(self, value): + self.EnsureUninitialized() + self.__total_size = value + + def __SetDefaultUploadStrategy(self, upload_config, http_request): + """Determine and set the default upload strategy for this upload. + + We generally prefer simple or multipart, unless we're forced to + use resumable. This happens when any of (1) the upload is too + large, (2) the simple endpoint doesn't support multipart requests + and we have metadata, or (3) there is no simple upload endpoint. + + Args: + upload_config: Configuration for the upload endpoint. + http_request: The associated http request. + + Returns: + None. + """ + if self.strategy is not None: + return + strategy = _SIMPLE_UPLOAD + if (self.total_size is not None and + self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): + strategy = _RESUMABLE_UPLOAD + if http_request.body and not upload_config.simple_multipart: + strategy = _RESUMABLE_UPLOAD + if not upload_config.simple_path: + strategy = _RESUMABLE_UPLOAD + self.strategy = strategy + + def ConfigureRequest(self, upload_config, http_request, url_builder): + """Configure the request and url for this upload.""" + # Validate total_size vs. max_size + if (self.total_size and upload_config.max_size and + self.total_size > upload_config.max_size): + raise exceptions.InvalidUserInputError( + 'Upload too big: %s larger than max size %s' % ( + self.total_size, upload_config.max_size)) + # Validate mime type + if not mimeparse.best_match(upload_config.accept, self.mime_type): + raise exceptions.InvalidUserInputError( + 'MIME type %s does not match any accepted MIME ranges %s' % ( + self.mime_type, upload_config.accept)) + + self.__SetDefaultUploadStrategy(upload_config, http_request) + if self.strategy == _SIMPLE_UPLOAD: + url_builder.relative_path = upload_config.simple_path + if http_request.body: + url_builder.query_params['uploadType'] = 'multipart' + self.__ConfigureMultipartRequest(http_request) + else: + url_builder.query_params['uploadType'] = 'media' + self.__ConfigureMediaRequest(http_request) + else: + url_builder.relative_path = upload_config.resumable_path + url_builder.query_params['uploadType'] = 'resumable' + self.__ConfigureResumableRequest(http_request) + + def __ConfigureMediaRequest(self, http_request): + """Configure http_request as a simple request for this upload.""" + http_request.headers['content-type'] = self.mime_type + http_request.body = self.stream.read() + + def __ConfigureMultipartRequest(self, http_request): + """Configure http_request as a multipart request for this upload.""" + # This is a multipart/related upload. + msg_root = mime_multipart.MIMEMultipart('related') + # msg_root should not write out its own headers + setattr(msg_root, '_write_headers', lambda self: None) + + # attach the body as one part + msg = mime_nonmultipart.MIMENonMultipart( + *http_request.headers['content-type'].split('/')) + msg.set_payload(http_request.body) + msg_root.attach(msg) + + # attach the media as the second part + msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) + msg['Content-Transfer-Encoding'] = 'binary' + msg.set_payload(self.stream.read()) + msg_root.attach(msg) + + http_request.body = msg_root.as_string() + multipart_boundary = msg_root.get_boundary() + http_request.headers['content-type'] = ( + 'multipart/related; boundary=%r' % multipart_boundary) + + def __ConfigureResumableRequest(self, http_request): + http_request.headers['X-Upload-Content-Type'] = self.mime_type + if self.total_size is not None: + http_request.headers['X-Upload-Content-Length'] = self.total_size + + def _RefreshResumableUploadState(self): + """Talk to the server and refresh the state of this resumable upload.""" + if self.strategy != _RESUMABLE_UPLOAD: + return + self.EnsureInitialized() + refresh_request = http_wrapper.Request( + url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) + refresh_response = http_wrapper.MakeRequest( + self.http, refresh_request, redirections=0) + range_header = refresh_response.info.get( + 'Range', refresh_response.info.get('range')) + if refresh_response.status_code in (httplib.OK, httplib.CREATED): + self.__complete = True + elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: + if range_header is None: + self.__progress = 0 + else: + self.__progress = self.__GetLastByte(range_header) + 1 + self.stream.seek(self.progress) + else: + raise exceptions.HttpError.FromResponse(refresh_response) + + def InitializeUpload(self, http_request, http=None, client=None): + """Initialize this upload from the given http_request.""" + if self.strategy is None: + raise exceptions.UserError( + 'No upload strategy set; did you call ConfigureRequest?') + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + if self.strategy != _RESUMABLE_UPLOAD: + return + if self.total_size is None: + raise exceptions.InvalidUserInputError( + 'Cannot stream upload without total size') + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + self.EnsureUninitialized() + http_response = http_wrapper.MakeRequest(http, http_request) + if http_response.status_code != httplib.OK: + raise exceptions.HttpError.FromResponse(http_response) + + self.__server_chunk_granularity = http_response.info.get( + 'X-Goog-Upload-Chunk-Granularity') + self.__ValidateChunksize() + url = http_response.info['location'] + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) + + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + return self.StreamInChunks() + + def __GetLastByte(self, range_header): + _, _, end = range_header.partition('-') + # TODO(craigcitro): Validate start == 0? + return int(end) + + def __ValidateChunksize(self, chunksize=None): + if self.__server_chunk_granularity is None: + return + chunksize = chunksize or self.chunksize + if chunksize % self.__server_chunk_granularity: + raise exceptions.ConfigurationValueError( + 'Server requires chunksize to be a multiple of %d', + self.__server_chunk_granularity) + + @staticmethod + def _ArgPrinter(response, unused_upload): + print 'Sent %s' % response.info['range'] + + @staticmethod + def _CompletePrinter(*unused_args): + print 'Upload complete' + + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this (resumable) upload in chunks.""" + if self.strategy != _RESUMABLE_UPLOAD: + raise exceptions.InvalidUserInputError( + 'Cannot stream non-resumable upload') + if self.total_size is None: + raise exceptions.InvalidUserInputError( + 'Cannot stream upload without total size') + callback = callback or self._ArgPrinter + finish_callback = finish_callback or self._CompletePrinter + response = None + self.__ValidateChunksize(self.chunksize) + self.EnsureInitialized() + while not self.complete: + response = self.__SendChunk(self.stream.tell(), + additional_headers=additional_headers) + if response.status_code in (httplib.OK, httplib.CREATED): + self.__complete = True + break + self.__progress = self.__GetLastByte(response.info['range']) + if self.progress + 1 != self.stream.tell(): + # TODO(craigcitro): Add a better way to recover here. + raise exceptions.CommunicationError( + 'Failed to transfer all bytes in chunk, upload paused at byte ' + '%d' % self.progress) + self._ExecuteCallback(callback, response) + self._ExecuteCallback(finish_callback, response) + return response + + def __SendChunk(self, start, additional_headers=None, data=None): + """Send the specified chunk.""" + self.EnsureInitialized() + if data is None: + data = self.stream.read(self.chunksize) + end = start + len(data) + + request = http_wrapper.Request(url=self.url, http_method='PUT', body=data) + request.headers['Content-Type'] = self.mime_type + if data: + request.headers['Content-Range'] = 'bytes %s-%s/%s' % ( + start, end - 1, self.total_size) + if additional_headers: + request.headers.update(additional_headers) + + response = http_wrapper.MakeRequest(self.bytes_http, request) + if response.status_code not in (httplib.OK, httplib.CREATED, + http_wrapper.RESUME_INCOMPLETE): + raise exceptions.HttpError.FromResponse(response) + if response.status_code in (httplib.OK, httplib.CREATED): + return response + # TODO(craigcitro): Add retries on no progress? + last_byte = self.__GetLastByte(response.info['range']) + if last_byte + 1 != end: + response = self.__SendChunk(last_byte + 1, data[last_byte + 1 - start:]) + return response diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 5d8713d..8246231 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -9,14 +9,10 @@ import urllib2 from apitools.base.py import exceptions -RETRYABLE_STATUS_CODES = ( - httplib.MOVED_PERMANENTLY, - httplib.FOUND, - httplib.SEE_OTHER, - httplib.TEMPORARY_REDIRECT, - # 308 doesn't have a name in httplib. - 308, - ) +__all__ = [ + 'DetectGae', + 'DetectGce', + ] def DetectGae(): @@ -58,3 +54,14 @@ def NormalizeScopes(scope_spec): raise exceptions.TypecheckError( 'NormalizeScopes expected string or iterable, found %s' % ( type(scope_spec),)) + + +def Typecheck(arg, arg_type, msg=None): + if not isinstance(arg, arg_type): + if msg is None: + if isinstance(arg_type, tuple): + msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) + else: + msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) + raise exceptions.TypecheckError(msg) + return arg diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index e5355a8..ad3f898 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -8,7 +8,6 @@ from protorpc import descriptor from protorpc import messages from apitools.gen import extended_descriptor -from apitools.gen import util _VARIANT_TO_FLAG_TYPE_MAP = { @@ -74,7 +73,7 @@ class CommandInfo(messages.Message): Fields: name: name of this command. - class_name: name of the app2.NewCmd class for this command. + class_name: name of the apitools_base.NewCmd class for this command. description: description of this command. flags: list of FlagInfo messages for the command-specific flags. args: list of ArgInfo messages for the positional args. @@ -219,9 +218,11 @@ class CommandRegistry(object): if field.variant in (messages.Variant.INT64, messages.Variant.UINT64): template = 'int(%s)' elif field.variant == messages.Variant.MESSAGE: - template = 'base_api.JsonToMessage(%s, %%s)' % type_name + template = 'apitools_base.JsonToMessage(%s, %%s)' % type_name elif field.variant == messages.Variant.ENUM: template = '%s(%%s)' % type_name + elif field.variant == messages.Variant.STRING: + template = "%s.decode('utf8')" if self.__FieldIsRepeated(extended_field.field_descriptor): if template: @@ -263,9 +264,35 @@ class CommandRegistry(object): extended_field, extended_message) return flag_info - def __PrintGlobalFlags(self, printer): - for flag_info in self.__global_flags: - self.__PrintFlag(printer, flag_info) + def __PrintFlagDeclarations(self, printer): + package = self.__client_info.package + function_name = '_Declare%sFlags' % (package[0].upper() + package[1:]) + printer() + printer() + printer('def %s():', function_name) + with printer.Indent(): + printer('"""Declare global flags in an idempotent way."""') + printer("if 'api_endpoint' in flags.FLAGS:") + with printer.Indent(): + printer('return') + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'api_endpoint',") + printer('%r,', self.__base_url) + printer("'URL of the API endpoint to use.',") + printer("short_name='%s_url')", self.__package) + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'history_file',") + printer('%r,', '~/.%s.%s.history' % (self.__package, self.__version)) + printer("'File with interactive shell history.')") + for flag_info in self.__global_flags: + self.__PrintFlag(printer, flag_info) + printer() + printer() + printer('FLAGS = flags.FLAGS') + printer('apitools_base.DeclareBaseFlags()') + printer('%s()', function_name) def __PrintGetGlobalParams(self, printer): printer('def GetGlobalParamsFromFlags():') @@ -290,7 +317,7 @@ class CommandRegistry(object): printer('"""Return a client object, configured from flags."""') printer('log_request = FLAGS.log_request or FLAGS.log_request_response') printer('log_response = FLAGS.log_response or FLAGS.log_request_response') - printer('api_endpoint = base_api.NormalizeApiEndpoint(' + printer('api_endpoint = apitools_base.NormalizeApiEndpoint(' 'FLAGS.api_endpoint)') printer('try:') with printer.Indent(): @@ -298,7 +325,7 @@ class CommandRegistry(object): with printer.Indent(indent=' '): printer('api_endpoint, log_request=log_request,') printer('log_response=log_response)') - printer('except exceptions.CredentialsError as e:') + printer('except apitools_base.CredentialsError as e:') with printer.Indent(): printer("print 'Error creating credentials: %%s' %% e") printer('sys.exit(1)') @@ -354,17 +381,19 @@ class CommandRegistry(object): printer(' == %s interactive console ==' % ( self.__client_info.package)) printer(' client: a %s client' % self.__client_info.package) + printer(' apitools_base: base apitools module') printer(' messages: the generated messages module') printer('"""') printer('local_vars = {') with printer.Indent(indent=' '): + printer("'apitools_base': apitools_base,") printer("'client': client,") printer("'client_lib': client_lib,") printer("'messages': messages,") printer('}') printer("if platform.system() == 'Linux':") with printer.Indent(): - printer('console = base_cli.ConsoleWithReadline(') + printer('console = apitools_base.ConsoleWithReadline(') with printer.Indent(indent=' '): printer('local_vars, histfile=FLAGS.history_file)') printer('else:') @@ -379,9 +408,8 @@ class CommandRegistry(object): printer() printer() - def WriteFile(self, out): + def WriteFile(self, printer): """Write a simple CLI (currently just a stub).""" - printer = util.SimplePrettyPrinter(out) printer('"""CLI for %s, version %s."""', self.__package, self.__version) # TODO(craigcitro): Add a build stamp, along with some other # information. @@ -394,38 +422,18 @@ class CommandRegistry(object): printer('from protorpc import message_types') printer('from protorpc import messages') printer() - printer( - 'from ' - 'google.apputils' - ' import appcommands') - printer( - 'import gflags as ' - 'flags') + appcommands_import = 'from google.apputils import appcommands' + printer(appcommands_import) + + flags_import = 'import gflags as flags' + printer(flags_import) printer() - printer('from %s import app2', self.__base_files_package) - printer('from %s import base_api', self.__base_files_package) - printer('from %s import base_cli', self.__base_files_package) - printer('from %s import exceptions', self.__base_files_package) - printer('from %s import transfer', self.__base_files_package) + printer('import %s as apitools_base', self.__base_files_package) printer('from %s import %s as client_lib', self.__root_package, self.__client_info.client_rule_name) printer('from %s import %s as messages', self.__root_package, self.__client_info.messages_rule_name) - printer() - printer('flags.DEFINE_string(') - with printer.Indent(' '): - printer("'api_endpoint',") - printer('%r,', self.__base_url) - printer("'URL of the API endpoint to use.',") - printer("short_name='%s_url')", self.__package) - printer('flags.DEFINE_string(') - with printer.Indent(' '): - printer("'history_file',") - printer('%r,', '~/.%s.%s.history' % (self.__package, self.__version)) - printer("'File with interactive shell history.')") - printer() - self.__PrintGlobalFlags(printer) - printer('FLAGS = flags.FLAGS') + self.__PrintFlagDeclarations(printer) printer() printer() self.__PrintGetGlobalParams(printer) @@ -439,7 +447,7 @@ class CommandRegistry(object): printer("appcommands.AddCmd('%s', %s)", command_info.name, command_info.class_name) printer() - printer('base_cli.SetupLogger()') + printer('apitools_base.SetupLogger()') # TODO(craigcitro): Just call SetDefaultCommand as soon as # another appcommands release happens and this exists # externally. @@ -448,7 +456,7 @@ class CommandRegistry(object): printer("appcommands.SetDefaultCommand('pyshell')") printer() printer() - printer('run_main = base_cli.run_main') + printer('run_main = apitools_base.run_main') printer() printer("if __name__ == '__main__':") with printer.Indent(): @@ -458,7 +466,7 @@ class CommandRegistry(object): """Print all commands in this registry using printer.""" for command_info in self.__command_list: arg_list = [arg_info.name for arg_info in command_info.args] - printer('class %s(app2.NewCmd):', command_info.class_name) + printer('class %s(apitools_base.NewCmd):', command_info.class_name) with printer.Indent(): printer('"""Command wrapping %s."""', command_info.client_method_path) printer() @@ -501,18 +509,18 @@ class CommandRegistry(object): printer('upload = None') printer('if FLAGS.upload_filename:') with printer.Indent(): - printer('upload = transfer.Upload.FromFile(') + printer('upload = apitools_base.Upload.FromFile(') printer(' FLAGS.upload_filename, FLAGS.upload_mime_type)') if command_info.has_download: call_args.append('download=download') printer('download = None') printer('if FLAGS.download_filename:') with printer.Indent(): - printer('download = transfer.Download.FromFile(' + printer('download = apitools_base.Download.FromFile(' 'FLAGS.download_filename, overwrite=FLAGS.overwrite)') printer('result = client.%s(', command_info.client_method_path) with printer.Indent(indent=' '): printer('%s)', ', '.join(call_args)) - printer('print result') + printer('print apitools_base.FormatOutput(result)') printer() printer() diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index f4e3ea4..69e3787 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -20,8 +20,6 @@ from protorpc import descriptor from protorpc import message_types from protorpc import messages -from apitools.gen import util - class ExtendedEnumValueDescriptor(messages.Message): """Enum value descriptor with additional fields. @@ -80,6 +78,7 @@ class ExtendedMessageDescriptor(messages.Message): decorators: Decorators to include in the definition when printing. Printed in the given order from top to bottom (so the last entry is the innermost decorator). + alias_for: This type is just an alias for the named type. """ name = messages.StringField(1) fields = messages.MessageField(ExtendedFieldDescriptor, 2, repeated=True) @@ -90,6 +89,7 @@ class ExtendedMessageDescriptor(messages.Message): description = messages.StringField(100) full_name = messages.StringField(101) decorators = messages.StringField(102, repeated=True) + alias_for = messages.StringField(103) class ExtendedFileDescriptor(messages.Message): @@ -120,16 +120,16 @@ def _WriteFile(file_descriptor, package, version, proto_printer): _PrintMessages(proto_printer, file_descriptor.message_types) -def WriteMessagesFile(file_descriptor, package, version, out): +def WriteMessagesFile(file_descriptor, package, version, printer): """Write the given extended file descriptor to out as a message file.""" _WriteFile(file_descriptor, package, version, - _Proto2Printer(util.SimplePrettyPrinter(out))) + _Proto2Printer(printer)) -def WritePythonFile(file_descriptor, package, version, out): +def WritePythonFile(file_descriptor, package, version, printer): """Write the given extended file descriptor to out.""" _WriteFile(file_descriptor, package, version, - _ProtoRpcPrinter(util.SimplePrettyPrinter(out))) + _ProtoRpcPrinter(printer)) def PrintIndentedDescriptions(printer, ls, name, prefix=''): @@ -348,6 +348,10 @@ class _ProtoRpcPrinter(ProtoPrinter): self.__printer() def PrintMessage(self, message_type): + if message_type.alias_for: + self.__printer('%s = %s', message_type.name, message_type.alias_for) + self.__PrintClassSeparator() + return for decorator in message_type.decorators: self.__printer('@%s', decorator) self.__printer('class %s(messages.Message):', message_type.name) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 34287cd..28664c8 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -23,6 +23,12 @@ flags.DEFINE_string( 'discovery_url', '', 'URL of the discovery document to use. Mutually exclusive with --infile.') +flags.DEFINE_string( + 'base_package', + 'apitools.base.py', + 'Base package path of apitools (defaults to ' + 'apitools.base.py)' + ) flags.DEFINE_string( 'outdir', '', 'Directory name for output files. (Defaults to the API name.)') @@ -35,6 +41,7 @@ flags.DEFINE_string( 'correct import lines). Defaults to the value of FLAGS.outdir.' ) + flags.DEFINE_multistring( 'strip_prefix', [], 'Prefix to strip from type names in the discovery document. (May ' @@ -55,6 +62,8 @@ flags.DEFINE_multistring( flags.DEFINE_string( 'user_agent', '', 'User agent for the generated client. Defaults to -generated/0.1.') +flags.DEFINE_boolean( + 'generate_cli', True, 'If True, a CLI is also generated.') flags.DEFINE_boolean( 'experimental_capitalize_enums', False, @@ -113,9 +122,13 @@ def _GetCodegenFromFlags(): if not FLAGS.root_package_dir: FLAGS.root_package_dir = outdir FLAGS.root_package_dir = os.path.abspath(FLAGS.root_package_dir) - root_package = util.GetPackage(FLAGS.root_package_dir) + root_package = ( + util.GetPackage(FLAGS.root_package_dir)) + base_package = FLAGS.base_package return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, + base_package=base_package, + generate_cli=FLAGS.generate_cli, use_proto2=FLAGS.experimental_proto2_output) @@ -145,8 +158,9 @@ def _WriteGeneratedFiles(codegen): codegen.WriteMessagesFile(out) with open(codegen.client_info.client_file_name, 'w') as out: codegen.WriteClientLibrary(out) - with open(codegen.client_info.cli_file_name, 'w') as out: - codegen.WriteCli(out) + if FLAGS.generate_cli: + with open(codegen.client_info.cli_file_name, 'w') as out: + codegen.WriteCli(out) def _WriteInit(codegen): diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 9e23485..58e8fc3 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -17,6 +17,7 @@ from apitools.gen import util def _StandardQueryParametersSchema(discovery_doc): + """Sets up dict of standard query parameters.""" standard_query_schema = { 'id': 'StandardQueryParameters', 'type': 'object', @@ -48,7 +49,7 @@ class DescriptorGenerator(object): """Code generator for a given discovery document.""" def __init__(self, discovery_doc, client_info, names, root_package, outdir, - use_proto2=False): + base_package, generate_cli=False, use_proto2=False): self.__discovery_doc = discovery_doc self.__client_info = client_info self.__outdir = outdir @@ -56,9 +57,9 @@ class DescriptorGenerator(object): self.__description = self.__discovery_doc.get('description', '') self.__package = self.__client_info.package self.__version = self.__client_info.version + self.__generate_cli = generate_cli self.__root_package = root_package - # TODO(craigcitro): Centralize this information ... somewhere. - self.__base_files_package = 'apitools.base.py' + self.__base_files_package = base_package self.__base_files_target = ( '//cloud/bigscience/apitools/base/py:apitools_base') self.__names = names @@ -127,32 +128,40 @@ class DescriptorGenerator(object): def use_proto2(self): return self.__use_proto2 + def _GetPrinter(self, out): + printer = util.SimplePrettyPrinter(out) + return printer + def WriteInit(self, out): """Write a simple __init__.py for the generated client.""" - printer = util.SimplePrettyPrinter(out) + printer = self._GetPrinter(out) printer('"""Common imports for generated %s client library."""', self.__client_info.package) printer() - printer('from %s import credentials_lib', self.__base_files_package) - printer('from %s.base_api import *', self.__base_files_package) - printer('from %s.exceptions import *', self.__base_files_package) - printer('from %s.transfer import *', self.__base_files_package) + printer('import pkgutil') + printer() + printer('from %s import *', self.__base_files_package) + if self.__generate_cli: + printer('from %s.%s import *', + self.__root_package, self.__client_info.cli_rule_name) printer('from %s.%s import *', self.__root_package, self.__client_info.client_rule_name) printer('from %s.%s import *', self.__root_package, self.__client_info.messages_rule_name) + printer() + printer('__path__ = pkgutil.extend_path(__path__, __name__)') def WriteMessagesFile(self, out): - self.__message_registry.WriteFile(out) + self.__message_registry.WriteFile(self._GetPrinter(out)) def WriteMessagesProtoFile(self, out): - self.__message_registry.WriteProtoFile(out) + self.__message_registry.WriteProtoFile(self._GetPrinter(out)) def WriteServicesProtoFile(self, out): - self.__services_registry.WriteProtoFile(out) + self.__services_registry.WriteProtoFile(self._GetPrinter(out)) def WriteClientLibrary(self, out): - self.__services_registry.WriteFile(out) + self.__services_registry.WriteFile(self._GetPrinter(out)) def WriteCli(self, out): - self.__command_registry.WriteFile(out) + self.__command_registry.WriteFile(self._GetPrinter(out)) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 290e0de..757adb4 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -32,6 +32,8 @@ class MessageRegistry(object): variant=messages.BooleanField.DEFAULT_VARIANT), 'number': TypeInfo(type_name='number', variant=messages.FloatField.DEFAULT_VARIANT), + 'any': TypeInfo(type_name='extra_types.JsonValue', + variant=messages.Variant.MESSAGE), } PRIMITIVE_FORMAT_MAP = { @@ -87,17 +89,19 @@ class MessageRegistry(object): self.Validate() return self.__file_descriptor - def WriteProtoFile(self, out): + def WriteProtoFile(self, printer): """Write the messages file to out as proto.""" self.Validate() extended_descriptor.WriteMessagesFile( - self.__file_descriptor, self.__package, self.__client_info.version, out) + self.__file_descriptor, self.__package, self.__client_info.version, + printer) - def WriteFile(self, out): + def WriteFile(self, printer): """Write the messages file to out.""" self.Validate() extended_descriptor.WritePythonFile( - self.__file_descriptor, self.__package, self.__client_info.version, out) + self.__file_descriptor, self.__package, self.__client_info.version, + printer) def Validate(self): mysteries = self.__nascent_types or self.__unknown_types @@ -115,6 +119,7 @@ class MessageRegistry(object): self.__nascent_types.add(self.__ComputeFullName(name)) def __RegisterDescriptor(self, new_descriptor): + """Register the given descriptor in this registry.""" if not isinstance(new_descriptor, ( extended_descriptor.ExtendedMessageDescriptor, extended_descriptor.ExtendedEnumDescriptor)): @@ -181,12 +186,47 @@ class MessageRegistry(object): message.values.append(enum_value) self.__RegisterDescriptor(message) + def __DeclareMessageAlias(self, schema, alias_for): + """Declare schema as an alias for alias_for.""" + # TODO(craigcitro): This is a hack. Remove it. + message = extended_descriptor.ExtendedMessageDescriptor() + message.name = self.__names.ClassName(schema['id']) + message.alias_for = alias_for + self.__DeclareDescriptor(message.name) + self.__AddImport('from %s import extra_types' % self.__base_files_package) + self.__RegisterDescriptor(message) + + def __AddAdditionalProperties(self, message, schema, properties): + """Add an additionalProperties field to message.""" + additional_properties_info = schema['additionalProperties'] + entries_type_name = self.__AddAdditionalPropertyType( + message.name, additional_properties_info) + description = additional_properties_info.get('description') + if description is None: + description = 'Additional properties of type %s' % message.name + attrs = { + 'items': { + '$ref': entries_type_name, + }, + 'description': description, + 'type': 'array', + } + field_name = 'additionalProperties' + message.fields.append(self.__FieldDescriptorFromProperties( + field_name, len(properties) + 1, attrs)) + self.__AddImport('from %s import encoding' % self.__base_files_package) + message.decorators.append( + 'encoding.MapUnrecognizedFields(%r)' % field_name) + def AddDescriptorFromSchema(self, schema_name, schema): """Add a new MessageDescriptor named schema_name based on schema.""" # TODO(craigcitro): Is schema_name redundant? if self.__GetDescriptor(schema_name): return - if schema.get('type') not in ('object', 'any'): + if schema.get('type') == 'any': + self.__DeclareMessageAlias(schema, 'extra_types.JsonValue') + return + if schema.get('type') != 'object': raise ValueError( 'Cannot create message descriptors for type %s', schema.get('type')) message = extended_descriptor.ExtendedMessageDescriptor() @@ -200,29 +240,11 @@ class MessageRegistry(object): field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) message.fields.append(field) if 'additionalProperties' in schema: - additional_properties_info = schema['additionalProperties'] - entries_type_name = self.__AddAdditionalPropertyType( - message.name, additional_properties_info) - description = additional_properties_info.get('description') - if description is None: - description = 'Additional properties of type %s' % message.name - attrs = { - 'items': { - '$ref': entries_type_name, - }, - 'description': description, - 'type': 'array', - } - field_name = 'additionalProperties' - message.fields.append(self.__FieldDescriptorFromProperties( - field_name, len(properties) + 1, attrs)) - self.__AddImport( - 'from %s import encoding' % self.__base_files_package) - message.decorators.append( - 'encoding.MapUnrecognizedFields(%r)' % field_name) + self.__AddAdditionalProperties(message, schema, properties) self.__RegisterDescriptor(message) def __AddAdditionalPropertyType(self, name, property_schema): + """Add a new nested AdditionalProperty message.""" new_type_name = 'AdditionalProperty' property_schema = dict(property_schema) # We drop the description here on purpose, so the resulting @@ -244,7 +266,26 @@ class MessageRegistry(object): self.AddDescriptorFromSchema(new_type_name, schema) return new_type_name + def __AddEntryType(self, entry_type_name, entry_schema, parent_name): + """Add a type for a list entry.""" + entry_schema.pop('description', None) + description = 'Single entry in a %s.' % parent_name + schema = { + 'id': entry_type_name, + 'type': 'object', + 'description': description, + 'properties': { + 'entry': { + 'type': 'array', + 'items': entry_schema, + }, + }, + } + self.AddDescriptorFromSchema(entry_type_name, schema) + return entry_type_name + def __FieldDescriptorFromProperties(self, name, index, attrs): + """Create a field descriptor for these attrs.""" field = descriptor.FieldDescriptor() field.name = self.__names.CleanName(name) field.number = index @@ -322,11 +363,20 @@ class MessageRegistry(object): items = attrs.get('items') if not items: raise ValueError('Array type with no item type: %s' % attrs) - item_name_hint = items.get('title') or '%sListEntry' % name_hint - item_name_hint = self.__names.ClassName(item_name_hint) - return self.__GetTypeInfo(items, item_name_hint) + entry_name_hint = self.__names.ClassName( + items.get('title') or '%sListEntry' % name_hint) + entry_label = self.__ComputeLabel(items) + if entry_label == descriptor.FieldDescriptor.Label.REPEATED: + parent_name = self.__names.ClassName(items.get('title') or name_hint) + entry_type_name = self.__AddEntryType( + entry_name_hint, items.get('items'), parent_name) + return TypeInfo( + type_name=entry_type_name, variant=messages.Variant.MESSAGE) + else: + return self.__GetTypeInfo(items, entry_name_hint) elif type_name == 'any': - return self.PRIMITIVE_TYPE_INFO_MAP['string'] + self.__AddImport('from %s import extra_types' % self.__base_files_package) + return self.PRIMITIVE_TYPE_INFO_MAP['any'] elif type_name == 'object': # TODO(craigcitro): Think of a better way to come up with names. if not name_hint: diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 9e79a38..8123388 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -8,7 +8,6 @@ import textwrap from apitools.base.py import base_api -from apitools.gen import util class ServiceRegistry(object): @@ -67,12 +66,57 @@ class ServiceRegistry(object): printer(' (%s) The response message.', method_info.response_type_name) printer('"""') - def __WriteSingleService(self, printer, name, method_info_map): + def __WriteSingleService( + self, printer, name, method_info_map, client_class_name): printer() class_name = self.__GetServiceClassName(name) printer('class %s(base_api.BaseApiService):', class_name) with printer.Indent(): printer('"""Service class for the %s resource."""', name) + + # Print the configs for the methods first. + printer() + printer('def __init__(self, client):') + with printer.Indent(): + printer('super(%s.%s, self).__init__(client)', + client_class_name, class_name) + printer('self.__configs = {') + with printer.Indent(indent=' '): + for method_name, method_info in method_info_map.iteritems(): + printer("'%s': base_api.ApiMethodInfo(", method_name) + with printer.Indent(indent=' '): + attrs = sorted(x.name for x in method_info.all_fields()) + for attr in attrs: + if attr in ('upload_config', 'description'): + continue + printer('%s=%r,', attr, getattr(method_info, attr)) + printer('),') + printer('}') + printer() + printer('self.__upload_configs = {') + with printer.Indent(indent=' '): + for method_name, method_info in method_info_map.iteritems(): + upload_config = method_info.upload_config + if upload_config is not None: + printer("'%s': base_api.ApiUploadInfo(", method_name) + with printer.Indent(indent=' '): + attrs = sorted(x.name for x in upload_config.all_fields()) + for attr in attrs: + printer('%s=%r,', attr, getattr(upload_config, attr)) + printer('),') + printer('}') + printer() + + printer('def GetMethodConfig(self, method):') + with printer.Indent(): + printer('return self.__configs.get(method)') + printer() + + printer('def GetMethodUploadConfig(self, method):') + with printer.Indent(): + printer('return self.__upload_configs.get(method)') + + # Now write each method in turn. for method_name, method_info in method_info_map.iteritems(): printer() params = ['self', 'request', 'global_params=None'] @@ -83,24 +127,11 @@ class ServiceRegistry(object): printer('def %s(%s):', method_name, ', '.join(params)) with printer.Indent(): self.__PrintDocstring(printer, method_info, method_name, name) - printer('config = base_api.ApiMethodInfo(') - with printer.Indent(indent=' '): - attrs = sorted(x.name for x in method_info.all_fields()) - for attr in attrs: - if attr in ('upload_config', 'description'): - continue - printer('%s=%r,', attr, getattr(method_info, attr)) - printer(')') - + printer("config = self.GetMethodConfig('%s')", method_name) upload_config = method_info.upload_config if upload_config is not None: - printer('upload_config = base_api.ApiUploadInfo(') - with printer.Indent(indent=' '): - attrs = sorted(x.name for x in upload_config.all_fields()) - for attr in attrs: - printer('%s=%r,', attr, getattr(upload_config, attr)) - printer(')') - + printer("upload_config = self.GetMethodUploadConfig('%s')", + method_name) arg_lines = ['config, request, global_params=global_params'] if method_info.upload_config: arg_lines.append('upload=upload, upload_config=upload_config') @@ -127,11 +158,10 @@ class ServiceRegistry(object): method_info.response_type_name) printer('}') - def WriteProtoFile(self, out): + def WriteProtoFile(self, printer): """Write the services in this registry to out as proto.""" self.Validate() client_info = self.__client_info - printer = util.SimplePrettyPrinter(out) printer('// Generated services for %s version %s.', client_info.package, client_info.version) printer() @@ -142,11 +172,10 @@ class ServiceRegistry(object): for name, method_info_map in self.__service_method_info_map.iteritems(): self.__WriteProtoServiceDeclaration(printer, name, method_info_map) - def WriteFile(self, out): + def WriteFile(self, printer): """Write the services in this registry to out.""" self.Validate() client_info = self.__client_info - printer = util.SimplePrettyPrinter(out) printer('"""Generated client library for %s version %s."""', client_info.package, client_info.version) printer() @@ -160,12 +189,18 @@ class ServiceRegistry(object): printer() client_info_items = client_info._asdict().iteritems() # pylint:disable=protected-access for attr, val in client_info_items: + if attr == 'scopes': + # We want to drop one scope as a special case. + extra_scope = 'https://www.googleapis.com/auth/cloud-platform' + if extra_scope in val: + val.remove(extra_scope) printer('_%s = %r' % (attr.upper(), val)) printer() printer("def __init__(self, url='', credentials=None,") - printer(' get_credentials=True, http=None, model=None,') - printer(' log_request=False, log_response=False,') - printer(' default_global_params=None):') + with printer.Indent(indent=' '): + printer('get_credentials=True, http=None, model=None,') + printer('log_request=False, log_response=False,') + printer('credentials_args=None, default_global_params=None):') with printer.Indent(): printer('"""Create a new %s handle."""', client_info.package) printer('url = url or %r', self.__base_url) @@ -173,12 +208,14 @@ class ServiceRegistry(object): printer(' url, credentials=credentials,') printer(' get_credentials=get_credentials, http=http, model=model,') printer(' log_request=log_request, log_response=log_response,') + printer(' credentials_args=credentials_args,') printer(' default_global_params=default_global_params)') for name in self.__service_method_info_map.iterkeys(): printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) for name, method_info_map in self.__service_method_info_map.iteritems(): - self.__WriteSingleService(printer, name, method_info_map) + self.__WriteSingleService( + printer, name, method_info_map, client_info.client_class_name) def __RegisterService(self, service_name, method_info_map): if service_name in self.__service_method_info_map: diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 2594936..32839de 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -218,6 +218,7 @@ class SimplePrettyPrinter(object): def __init__(self, out): self.__out = out self.__indent = '' + self.__skip = False @property def indent(self): @@ -233,12 +234,14 @@ class SimplePrettyPrinter(object): yield self.__indent = previous_indent + def __call__(self, *args): if args and args[0]: line = (args[0] % args[1:]).rstrip() + line = line.encode('ascii', 'backslashreplace') print >>self.__out, '%s%s' % (self.__indent, line) else: - print >>self.__out, '' + print >>self.__out, line def NormalizeDiscoveryUrl(discovery_url): diff --git a/setup.py b/setup.py index 46977ff..2e44583 100644 --- a/setup.py +++ b/setup.py @@ -20,15 +20,18 @@ import platform from ez_setup import use_setuptools use_setuptools() -# pylint:disable-msg=C6204 +# pylint:disable=C6204 import setuptools # Configure the required packages and scripts to install, depending on # Python version and OS. REQUIRED_PACKAGES = [ 'ez-setup==0.9', - 'google-api-python-client==1.2', 'google-apputils==0.4.0', + 'httplib2==0.8', + 'mimeparse==0.1.3', + 'mock==1.0.1', + 'oauth2client==1.2', 'protorpc==0.9.1', 'python-dateutil==1.5', 'python-gflags==2.0', @@ -44,7 +47,8 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse==1.2.1') -_APITOOLS_VERSION = '0.1.1' +_NAMESPACE = 'apitools' +_APITOOLS_VERSION = '0.2' setuptools.setup( name='apitools', @@ -59,6 +63,7 @@ setuptools.setup( 'console_scripts': CONSOLE_SCRIPTS, }, install_requires=REQUIRED_PACKAGES, + namespace_packages=[_NAMESPACE], provides=[ 'apitools (%s)' % (_APITOOLS_VERSION,), ], -- GitLab From 7a9c968692c8cc621d4bff1dea918fb92fe9e683 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 01:40:43 -0700 Subject: [PATCH 006/295] Add .travis.yml. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d8d5ea8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,2 @@ +language: python +script: nosetests -- GitLab From 7ebe97a937a11157030c92edffbe713acd65247a Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 01:43:24 -0700 Subject: [PATCH 007/295] fix travis config --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d8d5ea8..c632e53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,3 @@ language: python +install: pip install . script: nosetests -- GitLab From 35203f850aba669204ae498908ea72f306dedbd8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 01:46:27 -0700 Subject: [PATCH 008/295] fix travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c632e53..0bd50ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: python -install: pip install . +install: pip install --allow-external . script: nosetests -- GitLab From c051e8e0c44643367d84e13f2b0f3e79907934d1 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 10:17:07 -0700 Subject: [PATCH 009/295] Fix "insecure" dependency. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0bd50ff..579bba7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ language: python -install: pip install --allow-external . +install: + - pip install google-apputils --allow-insecure google-apputils + - pip install . script: nosetests -- GitLab From 4de1b5bee064bbdde1c1326cdded4f8e8098eafb Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 10:29:11 -0700 Subject: [PATCH 010/295] final cleanup --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 579bba7..7891a99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ language: python -install: - - pip install google-apputils --allow-insecure google-apputils - - pip install . +install: pip install . --allow-insecure google-apputils script: nosetests -- GitLab From c56f678083680e2dd98ea5942907d12af7ef4d3f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 7 Apr 2014 17:01:02 -0700 Subject: [PATCH 011/295] Remove now-unnecessary insecure flag. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7891a99..c632e53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: python -install: pip install . --allow-insecure google-apputils +install: pip install . script: nosetests -- GitLab From e83be4359a7d4504c04b1d61e5ef81db47c1a9dd Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 8 Apr 2014 00:38:11 -0700 Subject: [PATCH 012/295] try another python version --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c632e53..5e8db86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ language: python +python: + - "2.6" + - "2.7" install: pip install . script: nosetests -- GitLab From 687e0f110b7d5331456b97f668771dd7b0073686 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 8 Apr 2014 01:07:19 -0700 Subject: [PATCH 013/295] Drop silly argparse requirement. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2e44583..8568a15 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ CONSOLE_SCRIPTS = [ py_version = platform.python_version() -if py_version < '2.7': - REQUIRED_PACKAGES.append('argparse==1.2.1') +# if py_version < '2.7': +# REQUIRED_PACKAGES.append('argparse==1.2.1') _NAMESPACE = 'apitools' _APITOOLS_VERSION = '0.2' -- GitLab From 0cdd765ff93a4111e347cab3a030be6903fba72a Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 8 Apr 2014 10:01:45 -0700 Subject: [PATCH 014/295] tweak setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8568a15..f2c68f7 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,8 @@ CONSOLE_SCRIPTS = [ py_version = platform.python_version() -# if py_version < '2.7': +if py_version < '2.7': + REQUIRED_PACKAGES.append('unittest2==0.5.1') # REQUIRED_PACKAGES.append('argparse==1.2.1') _NAMESPACE = 'apitools' -- GitLab From 8528c243980fb1162f7bd4bcda9891311ac71d02 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 8 Apr 2014 10:14:19 -0700 Subject: [PATCH 015/295] 2.6-safe --- apitools/base/py/encoding_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index de00c8e..5c28d60 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -104,7 +104,7 @@ class EncodingTest(googletest.TestCase): new_msg = encoding.JsonToMessage(type(msg), encoded_msg) self.assertEqual( set(('key_one', 'key_two')), - {x.key for x in new_msg.additional_properties}) + set([x.key for x in new_msg.additional_properties])) self.assertIsNot(msg, new_msg) new_msg.additional_properties.pop() @@ -136,7 +136,7 @@ class EncodingTest(googletest.TestCase): new_msg = encoding.JsonToMessage(type(msg), encoded_msg) self.assertEqual( set(('key_one', 'key_two')), - {x.key for x in new_msg.nested.additional_properties}) + set([x.key for x in new_msg.nested.additional_properties])) new_msg.nested.additional_properties.pop() self.assertEqual(1, len(new_msg.nested.additional_properties)) -- GitLab From 3aa1bd91346a13d6030bddac8c1a63ebf76373cf Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 9 Apr 2014 00:22:25 -0700 Subject: [PATCH 016/295] try pypy --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5e8db86..cd251ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,5 +2,6 @@ language: python python: - "2.6" - "2.7" + - "pypy" install: pip install . script: nosetests -- GitLab From 93816681ed7ecf92ee49e12971a74a9d1be6b5e2 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 9 Apr 2014 10:10:30 -0700 Subject: [PATCH 017/295] 2.6 options --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd251ea..1c88779 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,5 @@ python: - "2.6" - "2.7" - "pypy" -install: pip install . +install: pip install . --allow-external argparse script: nosetests diff --git a/setup.py b/setup.py index f2c68f7..1793b77 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('unittest2==0.5.1') -# REQUIRED_PACKAGES.append('argparse==1.2.1') + REQUIRED_PACKAGES.append('argparse==1.2.1') _NAMESPACE = 'apitools' _APITOOLS_VERSION = '0.2' -- GitLab From a77212d29ccaf60cf16f573f98b9413431999e0e Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 9 Apr 2014 13:13:02 -0700 Subject: [PATCH 018/295] Add .travis.yml. --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..84dd8e9 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.6" + - "2.7" +# command to install dependencies +install: "pip install ." +# command to run tests +script: nosetests -- GitLab From 2ce2196691619a6e1a5bbd14809e5c2323111d71 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 9 Apr 2014 17:00:19 -0700 Subject: [PATCH 019/295] Update .travis.yml. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 84dd8e9..1c88779 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.6" - "2.7" -# command to install dependencies -install: "pip install ." -# command to run tests + - "pypy" +install: pip install . --allow-external argparse script: nosetests -- GitLab From 9ada0fcfd1519dbec2067b329846fc0f19ed2da1 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 10 Apr 2014 22:33:34 -0700 Subject: [PATCH 020/295] Add travis build shield. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 476ffd4..b1e2b48 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # apitools +[![Build Status](https://travis-ci.org/craigcitro/apitools.svg?branch=master)](https://travis-ci.org/craigcitro/apitools) + `apitools` is a collection of utilities to make it easier to build client-side tools, especially those that talk to Google APIs. -- GitLab From 6b6ca68384d453b22a42b719effb197b8b8d8046 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sat, 31 May 2014 00:58:06 -0700 Subject: [PATCH 021/295] Fix two codegen issues. This fixes a missing shebang in generated CLIs and missing imports in the generated client file. --- apitools/gen/command_registry.py | 1 + apitools/gen/gen_client.py | 6 +++--- apitools/gen/service_registry.py | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index ad3f898..c4dc685 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -410,6 +410,7 @@ class CommandRegistry(object): def WriteFile(self, printer): """Write a simple CLI (currently just a stub).""" + printer('#!/usr/bin/env python') printer('"""CLI for %s, version %s."""', self.__package, self.__version) # TODO(craigcitro): Add a build stamp, along with some other # information. diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 28664c8..b3b4b29 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -191,7 +191,7 @@ class GenerateProto(appcommands.Cmd): _WriteProtoFiles(codegen) -# pylint: disable-msg=invalid-name +# pylint:disable=invalid-name def run_main(): @@ -199,14 +199,14 @@ def run_main(): # Put the flags for this module somewhere the flags module will look # for them. - # pylint: disable-msg=protected-access + # pylint:disable=protected-access new_name = flags._GetMainModule() sys.modules[new_name] = sys.modules['__main__'] for flag in FLAGS.FlagsByModuleDict().get(__name__, []): FLAGS._RegisterFlagByModule(new_name, flag) for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): FLAGS._RegisterKeyFlagForModule(new_name, key_flag) - # pylint: enable-msg=protected-access + # pylint:enable=protected-access # Now set __main__ appropriately so that appcommands will be # happy. diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 8123388..ffcde17 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -178,6 +178,9 @@ class ServiceRegistry(object): client_info = self.__client_info printer('"""Generated client library for %s version %s."""', client_info.package, client_info.version) + printer('from %s import base_api', self.__base_files_package) + printer('from %s import %s as messages', self.__root_package_dir, + client_info.messages_rule_name) printer() printer() printer('class %s(base_api.BaseApiClient):', client_info.client_class_name) -- GitLab From 872abff94a1405b10a0dff807405bc8364940cf0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sat, 31 May 2014 01:02:59 -0700 Subject: [PATCH 022/295] Fix an unbound local error. (I swear I fixed this before.) --- apitools/gen/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 32839de..a517176 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -241,7 +241,7 @@ class SimplePrettyPrinter(object): line = line.encode('ascii', 'backslashreplace') print >>self.__out, '%s%s' % (self.__indent, line) else: - print >>self.__out, line + print >>self.__out, '' def NormalizeDiscoveryUrl(discovery_url): -- GitLab From 7e6dd7c36466a6d3270f0fe61704c5c450f1d9a9 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sat, 31 May 2014 14:53:43 -0700 Subject: [PATCH 023/295] Last cleanup for working package generation. This makes two changes: * Switch to a --root_package flag that we use for imports, which we elide if it's empty, and * chmod the generated CLI. Generated clients *should* work out of the box now. --- apitools/gen/command_registry.py | 11 +++++++---- apitools/gen/gen_client.py | 10 +++++----- apitools/gen/service_registry.py | 5 ++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index c4dc685..37c6183 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -430,10 +430,13 @@ class CommandRegistry(object): printer(flags_import) printer() printer('import %s as apitools_base', self.__base_files_package) - printer('from %s import %s as client_lib', - self.__root_package, self.__client_info.client_rule_name) - printer('from %s import %s as messages', - self.__root_package, self.__client_info.messages_rule_name) + import_prefix = '' + if self.__root_package: + import_prefix = 'from %s ' % self.__root_package + printer('%simport %s as client_lib', + import_prefix, self.__client_info.client_rule_name) + printer('%simport %s as messages', + import_prefix, self.__client_info.messages_rule_name) self.__PrintFlagDeclarations(printer) printer() printer() diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index b3b4b29..2ef4d93 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -40,6 +40,9 @@ flags.DEFINE_string( 'Ultimate destination for generated code (used for generating ' 'correct import lines). Defaults to the value of FLAGS.outdir.' ) +flags.DEFINE_string( + 'root_package', '', + 'Python import path for where these modules should be imported from.') flags.DEFINE_multistring( @@ -119,11 +122,7 @@ def _GetCodegenFromFlags(): raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') - if not FLAGS.root_package_dir: - FLAGS.root_package_dir = outdir - FLAGS.root_package_dir = os.path.abspath(FLAGS.root_package_dir) - root_package = ( - util.GetPackage(FLAGS.root_package_dir)) + root_package = FLAGS.root_package base_package = FLAGS.base_package return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, @@ -161,6 +160,7 @@ def _WriteGeneratedFiles(codegen): if FLAGS.generate_cli: with open(codegen.client_info.cli_file_name, 'w') as out: codegen.WriteCli(out) + os.chmod(codegen.client_info.cli_file_name, 0755) def _WriteInit(codegen): diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index ffcde17..de6a652 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -179,7 +179,10 @@ class ServiceRegistry(object): printer('"""Generated client library for %s version %s."""', client_info.package, client_info.version) printer('from %s import base_api', self.__base_files_package) - printer('from %s import %s as messages', self.__root_package_dir, + import_prefix = '' + if self.__root_package_dir: + import_prefix = 'from %s ' % self.__root_package_dir + printer('%simport %s as messages', import_prefix, client_info.messages_rule_name) printer() printer() -- GitLab From ebc95451cf364958d0e6db831a04982017e8522f Mon Sep 17 00:00:00 2001 From: Mike Schwartz Date: Tue, 10 Jun 2014 08:34:29 -0700 Subject: [PATCH 024/295] Fixed int/str data type bug in __ConfigureResumableRequest --- apitools/base/py/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 0ccd8e6..37f6526 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -571,7 +571,7 @@ class Upload(_Transfer): def __ConfigureResumableRequest(self, http_request): http_request.headers['X-Upload-Content-Type'] = self.mime_type if self.total_size is not None: - http_request.headers['X-Upload-Content-Length'] = self.total_size + http_request.headers['X-Upload-Content-Length'] = str(self.total_size) def _RefreshResumableUploadState(self): """Talk to the server and refresh the state of this resumable upload.""" -- GitLab From f24f570896c41bcfa8c9139a7985011bb0f53e08 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 14 Jul 2014 22:31:23 -0700 Subject: [PATCH 025/295] Drop the package namespace. This switches from using setuptools-style namespace packages to pkgutil-style ones. --- apitools/__init__.py | 3 +++ setup.py | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apitools/__init__.py b/apitools/__init__.py index 4265cc3..1847842 100644 --- a/apitools/__init__.py +++ b/apitools/__init__.py @@ -1 +1,4 @@ #!/usr/bin/env python + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/setup.py b/setup.py index 1793b77..2ec1eb3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,6 @@ import platform from ez_setup import use_setuptools use_setuptools() -# pylint:disable=C6204 import setuptools # Configure the required packages and scripts to install, depending on @@ -48,7 +47,6 @@ if py_version < '2.7': REQUIRED_PACKAGES.append('unittest2==0.5.1') REQUIRED_PACKAGES.append('argparse==1.2.1') -_NAMESPACE = 'apitools' _APITOOLS_VERSION = '0.2' setuptools.setup( @@ -64,7 +62,6 @@ setuptools.setup( 'console_scripts': CONSOLE_SCRIPTS, }, install_requires=REQUIRED_PACKAGES, - namespace_packages=[_NAMESPACE], provides=[ 'apitools (%s)' % (_APITOOLS_VERSION,), ], -- GitLab From 1a523321ad9d175140e2f3a9d85ddede10632555 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 17:12:44 -0700 Subject: [PATCH 026/295] Add a simple test for generated clients. --- apitools/gen/client_generation_test.py | 69 ++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 apitools/gen/client_generation_test.py diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py new file mode 100644 index 0000000..767c188 --- /dev/null +++ b/apitools/gen/client_generation_test.py @@ -0,0 +1,69 @@ +"""Test gen_client against all the APIs we use regularly.""" + +import contextlib +import logging +import os +import shutil +import subprocess +import tempfile + +from google.apputils import basetest as googletest + +from apitools.gen import util + +_API_LIST = [ + 'drive.v2', + 'bigquery.v2', + 'compute.v1', + 'storage.v1', +] + +@contextlib.contextmanager +def TempDir(): + original_dir = os.getcwd() + path = tempfile.mkdtemp() + try: + os.chdir(path) + yield path + finally: + os.chdir(original_dir) + shutil.rmtree(path) + + +class ClientGenerationTest(googletest.TestCase): + + def setUp(self): + super(ClientGenerationTest, self).setUp() + self.gen_client_binary = 'gen_client' + + def testGeneration(self): + for api in _API_LIST: + with TempDir() as path: + args = [ + self.gen_client_binary, + '--client_id=12345', + '--client_secret=67890', + '--discovery_url=%s' % api, + '--outdir=generated', + '--overwrite', + 'client', + ] + logging.info('Testing API %s with command line: %s', api, ' '.join(args)) + retcode = subprocess.call(args) + if retcode == 128: + logging.error('Failed to fetch discovery doc, continuing.') + continue + self.assertEqual(0, retcode) + + with tempfile.NamedTemporaryFile() as out: + cmdline_args = [ + os.path.join('generated', api.replace('.', '_') + '.py'), + 'help', + ] + retcode = subprocess.call(cmdline_args, stdout=out) + # appcommands returns 1 on help + self.assertEqual(1, retcode) + + +if __name__ == '__main__': + googletest.main() -- GitLab From f1c8ef024e6d4a15dffa6cde571424a3f08eb1d0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 17:27:02 -0700 Subject: [PATCH 027/295] Delete new test. --- apitools/gen/client_generation_test.py | 69 -------------------------- 1 file changed, 69 deletions(-) delete mode 100644 apitools/gen/client_generation_test.py diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py deleted file mode 100644 index 767c188..0000000 --- a/apitools/gen/client_generation_test.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Test gen_client against all the APIs we use regularly.""" - -import contextlib -import logging -import os -import shutil -import subprocess -import tempfile - -from google.apputils import basetest as googletest - -from apitools.gen import util - -_API_LIST = [ - 'drive.v2', - 'bigquery.v2', - 'compute.v1', - 'storage.v1', -] - -@contextlib.contextmanager -def TempDir(): - original_dir = os.getcwd() - path = tempfile.mkdtemp() - try: - os.chdir(path) - yield path - finally: - os.chdir(original_dir) - shutil.rmtree(path) - - -class ClientGenerationTest(googletest.TestCase): - - def setUp(self): - super(ClientGenerationTest, self).setUp() - self.gen_client_binary = 'gen_client' - - def testGeneration(self): - for api in _API_LIST: - with TempDir() as path: - args = [ - self.gen_client_binary, - '--client_id=12345', - '--client_secret=67890', - '--discovery_url=%s' % api, - '--outdir=generated', - '--overwrite', - 'client', - ] - logging.info('Testing API %s with command line: %s', api, ' '.join(args)) - retcode = subprocess.call(args) - if retcode == 128: - logging.error('Failed to fetch discovery doc, continuing.') - continue - self.assertEqual(0, retcode) - - with tempfile.NamedTemporaryFile() as out: - cmdline_args = [ - os.path.join('generated', api.replace('.', '_') + '.py'), - 'help', - ] - retcode = subprocess.call(cmdline_args, stdout=out) - # appcommands returns 1 on help - self.assertEqual(1, retcode) - - -if __name__ == '__main__': - googletest.main() -- GitLab From 40fe6a0e2a230175bba34019c976c05fe63cb4d6 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 19:10:45 -0700 Subject: [PATCH 028/295] Add a tox.ini. --- .gitignore | 10 ++++++---- tox.ini | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 02b2bb5..536e655 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +*~ +*.py[cod] apitools.egg-info/* -build/* -dist/* +build/ +dist/ distribute-* -*~ -*.pyc +# Test files +.tox/ diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5704043 --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py26, py27 + +[testenv] +deps = nose +commands = nosetests -- GitLab From 9f2043f35f34b7e8ddab1d03e569db966059e73c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 19:11:44 -0700 Subject: [PATCH 029/295] Update .travis.yml. --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c88779..216a8ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python -python: - - "2.6" - - "2.7" - - "pypy" +env: + - TOX_ENV=py26 + - TOX_ENV=py27 + - TOX_ENV=pypy install: pip install . --allow-external argparse -script: nosetests +script: tox -e $TOX_ENV -- GitLab From d3e9c963e7c30736e87db6d185e6c4cac3fb85d6 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 19:12:08 -0700 Subject: [PATCH 030/295] Skip an newer test for older pythons. --- apitools/gen/client_generation_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 767c188..141eb4f 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -5,6 +5,7 @@ import logging import os import shutil import subprocess +import sys import tempfile from google.apputils import basetest as googletest @@ -37,6 +38,12 @@ class ClientGenerationTest(googletest.TestCase): self.gen_client_binary = 'gen_client' def testGeneration(self): + if sys.version < (2, 7): + # TODO(craigcitro): Make apitools codegen support python 2.6. + # Maybe. + # + # unittest in 2.6 doesn't have skipIf. + return for api in _API_LIST: with TempDir() as path: args = [ -- GitLab From 01e9a4c2c8a3ae08f239b532a8edc3ad132bc613 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 21 Aug 2014 19:19:02 -0700 Subject: [PATCH 031/295] Add tox to .travis.yml. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 216a8ec..a593c03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,7 @@ env: - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=pypy -install: pip install . --allow-external argparse +install: + - pip install tox + - pip install . --allow-external argparse script: tox -e $TOX_ENV -- GitLab From b07a5a1c6cdd35e96bf7816f4f74c03feb19b6d7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 6 Oct 2014 23:10:05 -0400 Subject: [PATCH 032/295] Don't pin specific version dependencies by default. Also, avoid depending on ez_setup: use it at setup time only if setuptools is not already importable. --- setup.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 2ec1eb3..235ad74 100644 --- a/setup.py +++ b/setup.py @@ -18,14 +18,28 @@ import platform -from ez_setup import use_setuptools -use_setuptools() -import setuptools +try: + import setuptools +except ImportError: + from ez_setup import use_setuptools + use_setuptools() + import setuptools # Configure the required packages and scripts to install, depending on # Python version and OS. REQUIRED_PACKAGES = [ - 'ez-setup==0.9', + 'google-apputils', + 'httplib2', + 'mimeparse', + 'mock', + 'oauth2client', + 'protorpc', + 'python-dateutil', + 'python-gflags', + 'pytz', + ] + +PINNED_PACKAGES = [ 'google-apputils==0.4.0', 'httplib2==0.8', 'mimeparse==0.1.3', @@ -35,8 +49,8 @@ REQUIRED_PACKAGES = [ 'python-dateutil==1.5', 'python-gflags==2.0', 'pytz==2013.7', - 'wsgiref==0.1.2', ] + CONSOLE_SCRIPTS = [ 'gen_client = apitools.gen.gen_client:run_main', ] @@ -62,6 +76,9 @@ setuptools.setup( 'console_scripts': CONSOLE_SCRIPTS, }, install_requires=REQUIRED_PACKAGES, + extras_require = { + 'pinned': PINNED_PACKAGES, + }, provides=[ 'apitools (%s)' % (_APITOOLS_VERSION,), ], -- GitLab From d15f4c73fa684343100eec3a0897d313eba2dd06 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 8 Oct 2014 12:28:58 -0400 Subject: [PATCH 033/295] Break out non-core dependencies. 'cli' extra maps dependencies needed by the CLI. 'testing' extra maps dependencies needed by tests. --- README.md | 12 ++++++++++++ setup.py | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b1e2b48..606a3a5 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,18 @@ `apitools` is a collection of utilities to make it easier to build client-side tools, especially those that talk to Google APIs. +## Installing as a library + +* `pip install apitools` + +## Installing the command-line tools + +* `pip install apitools[cli]` + +## Installing the testing dependencies + +* `pip install apitools[testing]` + ## Current status There are a few imminent large changes: diff --git a/setup.py b/setup.py index 235ad74..9aa8619 100644 --- a/setup.py +++ b/setup.py @@ -28,17 +28,24 @@ except ImportError: # Configure the required packages and scripts to install, depending on # Python version and OS. REQUIRED_PACKAGES = [ - 'google-apputils', 'httplib2', 'mimeparse', - 'mock', 'oauth2client', 'protorpc', 'python-dateutil', - 'python-gflags', 'pytz', ] +CLI_PACKAGES = [ + 'google-apputils', + 'python-gflags', +] + +TESTING_PACKAGES = [ + 'google-apputils', + 'mock', +] + PINNED_PACKAGES = [ 'google-apputils==0.4.0', 'httplib2==0.8', @@ -76,8 +83,11 @@ setuptools.setup( 'console_scripts': CONSOLE_SCRIPTS, }, install_requires=REQUIRED_PACKAGES, + tests_require=REQUIRED_PACKAGES + TESTING_PACKAGES, extras_require = { 'pinned': PINNED_PACKAGES, + 'cli': CLI_PACKAGES, + 'testing': TESTING_PACKAGES, }, provides=[ 'apitools (%s)' % (_APITOOLS_VERSION,), -- GitLab From 3f015e94efdc5344c035470b029482272dc7f595 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 8 Oct 2014 14:49:06 -0400 Subject: [PATCH 034/295] Ensure tox passes. --- setup.py | 2 +- tox.ini | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9aa8619..bc0f16a 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ setuptools.setup( 'console_scripts': CONSOLE_SCRIPTS, }, install_requires=REQUIRED_PACKAGES, - tests_require=REQUIRED_PACKAGES + TESTING_PACKAGES, + tests_require=REQUIRED_PACKAGES + CLI_PACKAGES + TESTING_PACKAGES, extras_require = { 'pinned': PINNED_PACKAGES, 'cli': CLI_PACKAGES, diff --git a/tox.ini b/tox.ini index 5704043..a4e70e7 100644 --- a/tox.ini +++ b/tox.ini @@ -3,4 +3,6 @@ envlist = py26, py27 [testenv] deps = nose -commands = nosetests +commands = + pip install apitools[testing] + nosetests -- GitLab From f94f5f010565e341aca8e823752ed5345f9cadd4 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 17 Oct 2014 14:48:39 -0700 Subject: [PATCH 035/295] Add a sample of using uploads and downloads. This pulls two integration tests into the repo as a sample, and (unfortunately) just adds a copy of the generated storage client. --- samples/storage_sample/downloads_test.py | 175 + samples/storage_sample/storage/__init__.py | 10 + samples/storage_sample/storage/storage_v1.py | 2867 +++++++++++++++++ .../storage/storage_v1_client.py | 1017 ++++++ .../storage/storage_v1_messages.py | 1632 ++++++++++ .../storage_sample/testdata/fifteen_byte_file | 4 + .../testdata/filename_with_spaces | 4 + samples/storage_sample/uploads_test.py | 141 + 8 files changed, 5850 insertions(+) create mode 100644 samples/storage_sample/downloads_test.py create mode 100644 samples/storage_sample/storage/__init__.py create mode 100755 samples/storage_sample/storage/storage_v1.py create mode 100644 samples/storage_sample/storage/storage_v1_client.py create mode 100644 samples/storage_sample/storage/storage_v1_messages.py create mode 100644 samples/storage_sample/testdata/fifteen_byte_file create mode 100644 samples/storage_sample/testdata/filename_with_spaces create mode 100644 samples/storage_sample/uploads_test.py diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py new file mode 100644 index 0000000..e8e52c3 --- /dev/null +++ b/samples/storage_sample/downloads_test.py @@ -0,0 +1,175 @@ +"""Integration tests for uploading and downloading to GCS. + +These tests exercise most of the corner cases for upload/download of +files in apitools, via GCS. There are no performance tests here yet. +""" + +import json +import os +import pkgutil +import StringIO +import unittest + +import apitools.base.py as apitools_base +import storage + +_CLIENT = None +def _GetClient(): + global _CLIENT + if _CLIENT is None: + _CLIENT = storage.StorageV1() + return _CLIENT + +class DownloadsTest(unittest.TestCase): + _DEFAULT_BUCKET = 'apitools' + _TESTDATA_PREFIX = 'testdata' + + def setUp(self): + self.__client = _GetClient() + self.__ResetDownload() + + def __ResetDownload(self, auto_transfer=False): + self.__buffer = StringIO.StringIO() + self.__download = storage.Download.FromStream( + self.__buffer, auto_transfer=auto_transfer) + + def __GetTestdataFileContents(self, filename): + file_contents = open('testdata/%s' % filename).read() + self.assertIsNotNone( + file_contents, msg=('Could not read file %s' % filename)) + return file_contents + + @classmethod + def __GetRequest(cls, filename): + object_name = os.path.join(cls._TESTDATA_PREFIX, filename) + return storage.StorageObjectsGetRequest( + bucket=cls._DEFAULT_BUCKET, object=object_name) + + def __GetFile(self, request): + response = self.__client.objects.Get(request, download=self.__download) + self.assertIsNone(response, msg=( + 'Unexpected nonempty response for file download: %s' % response)) + + def __GetAndStream(self, request): + self.__GetFile(request) + self.__download.StreamInChunks() + + def testZeroBytes(self): + request = self.__GetRequest('zero_byte_file') + self.__GetAndStream(request) + self.assertEqual(0, self.__buffer.tell()) + + def testObjectDoesNotExist(self): + with self.assertRaises(apitools_base.HttpError): + self.__GetFile(self.__GetRequest('nonexistent_file')) + + def testAutoTransfer(self): + self.__ResetDownload(auto_transfer=True) + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read()) + + def testFilenameWithSpaces(self): + self.__ResetDownload(auto_transfer=True) + self.__GetFile(self.__GetRequest('filename with spaces')) + # NOTE(craigcitro): We add _ here to make this play nice with blaze. + file_contents = self.__GetTestdataFileContents('filename_with_spaces') + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read()) + + def testGetRange(self): + # TODO(craigcitro): Test about a thousand more corner cases. + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(5, 10) + self.assertEqual(6, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[5:11], self.__buffer.read()) + + def testGetRangeWithNegativeStart(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(-3) + self.assertEqual(3, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[-3:], self.__buffer.read()) + + def testGetRangeWithPositiveStart(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(2) + self.assertEqual(13, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[2:15], self.__buffer.read()) + + def testSmallChunksizes(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + request = self.__GetRequest('fifteen_byte_file') + for chunksize in (2, 3, 15, 100): + self.__ResetDownload() + self.__download.chunksize = chunksize + self.__GetAndStream(request) + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read(15)) + + def testLargeFileChunksizes(self): + request = self.__GetRequest('thirty_meg_file') + for chunksize in (1048576, 40 * 1048576): + self.__ResetDownload() + self.__download.chunksize = chunksize + self.__GetAndStream(request) + self.__buffer.seek(0) + + def testAutoGzipObject(self): + # TODO(craigcitro): Move this to a new object once we have a more + # permanent one, see: http://b/12250275 + request = storage.StorageObjectsGetRequest( + bucket='ottenl-gzip', object='50K.txt') + # First, try without auto-transfer. + self.__GetFile(request) + self.assertEqual(0, self.__buffer.tell()) + self.__download.StreamInChunks() + self.assertEqual(50000, self.__buffer.tell()) + # Next, try with auto-transfer. + self.__ResetDownload(auto_transfer=True) + self.__GetFile(request) + self.assertEqual(50000, self.__buffer.tell()) + + def testSmallGzipObject(self): + request = self.__GetRequest('zero-gzipd.html') + self.__GetFile(request) + self.assertEqual(0, self.__buffer.tell()) + additional_headers = {'accept-encoding': 'gzip, deflate'} + self.__download.StreamInChunks(additional_headers=additional_headers) + self.assertEqual(0, self.__buffer.tell()) + + def testSerializedDownload(self): + + def _ProgressCallback(unused_response, download_object): + print 'Progress %s' % download_object.progress + + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + object_name = os.path.join(self._TESTDATA_PREFIX, 'fifteen_byte_file') + request = storage.StorageObjectsGetRequest( + bucket=self._DEFAULT_BUCKET, object=object_name) + response = self.__client.objects.Get(request) + self.__buffer = StringIO.StringIO() + download_data = json.dumps({ + 'auto_transfer': False, + 'progress': 0, + 'total_size': response.size, + 'url': response.mediaLink, + }) + self.__download = storage.Download.FromData(self.__buffer, download_data, + http=self.__client.http) + self.__download.StreamInChunks(callback=_ProgressCallback) + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read(15)) + +if __name__ == '__main__': + unittest.main() diff --git a/samples/storage_sample/storage/__init__.py b/samples/storage_sample/storage/__init__.py new file mode 100644 index 0000000..5ff5bb5 --- /dev/null +++ b/samples/storage_sample/storage/__init__.py @@ -0,0 +1,10 @@ +"""Common imports for generated storage client library.""" + +import pkgutil + +from apitools.base.py import * +from storage_v1 import * +from storage_v1_client import * +from storage_v1_messages import * + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage/storage_v1.py new file mode 100755 index 0000000..a882e17 --- /dev/null +++ b/samples/storage_sample/storage/storage_v1.py @@ -0,0 +1,2867 @@ +#!/usr/bin/env python +"""CLI for storage, version v1.""" + +import code +import platform +import sys + +import protorpc +from protorpc import message_types +from protorpc import messages + +from google.apputils import appcommands +import gflags as flags + +import apitools.base.py as apitools_base +import storage_v1_client as client_lib +import storage_v1_messages as messages + + +def _DeclareStorageFlags(): + """Declare global flags in an idempotent way.""" + if 'api_endpoint' in flags.FLAGS: + return + flags.DEFINE_string( + 'api_endpoint', + u'https://www.googleapis.com/storage/v1/', + 'URL of the API endpoint to use.', + short_name='storage_url') + flags.DEFINE_string( + 'history_file', + u'~/.storage.v1.history', + 'File with interactive shell history.') + flags.DEFINE_enum( + 'alt', + u'json', + [u'json'], + u'Data format for the response.') + flags.DEFINE_string( + 'fields', + None, + u'Selector specifying which fields to include in a partial response.') + flags.DEFINE_string( + 'key', + None, + u'API key. Your API key identifies your project and provides you with ' + u'API access, quota, and reports. Required unless you provide an OAuth ' + u'2.0 token.') + flags.DEFINE_string( + 'oauth_token', + None, + u'OAuth 2.0 token for the current user.') + flags.DEFINE_boolean( + 'prettyPrint', + 'True', + u'Returns response with indentations and line breaks.') + flags.DEFINE_string( + 'quotaUser', + None, + u'Available to use for quota purposes for server-side applications. Can' + u' be any arbitrary string assigned to a user, but should not exceed 40' + u' characters. Overrides userIp if both are provided.') + flags.DEFINE_string( + 'trace', + None, + 'A tracing token of the form "token:" to include in api ' + 'requests.') + flags.DEFINE_string( + 'userIp', + None, + u'IP address of the site where the request originates. Use this if you ' + u'want to enforce per-user limits.') + + +FLAGS = flags.FLAGS +apitools_base.DeclareBaseFlags() +_DeclareStorageFlags() + + +def GetGlobalParamsFromFlags(): + """Return a StandardQueryParameters based on flags.""" + result = messages.StandardQueryParameters() + if FLAGS['alt'].present: + result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt) + if FLAGS['fields'].present: + result.fields = FLAGS.fields.decode('utf8') + if FLAGS['key'].present: + result.key = FLAGS.key.decode('utf8') + if FLAGS['oauth_token'].present: + result.oauth_token = FLAGS.oauth_token.decode('utf8') + if FLAGS['prettyPrint'].present: + result.prettyPrint = FLAGS.prettyPrint + if FLAGS['quotaUser'].present: + result.quotaUser = FLAGS.quotaUser.decode('utf8') + if FLAGS['trace'].present: + result.trace = FLAGS.trace.decode('utf8') + if FLAGS['userIp'].present: + result.userIp = FLAGS.userIp.decode('utf8') + return result + + +def GetClientFromFlags(): + """Return a client object, configured from flags.""" + log_request = FLAGS.log_request or FLAGS.log_request_response + log_response = FLAGS.log_response or FLAGS.log_request_response + api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) + try: + client = client_lib.StorageV1( + api_endpoint, log_request=log_request, + log_response=log_response) + except apitools_base.CredentialsError as e: + print 'Error creating credentials: %s' % e + sys.exit(1) + return client + + +class PyShell(appcommands.Cmd): + def Run(self, _): + """Run an interactive python shell with the client.""" + client = GetClientFromFlags() + params = GetGlobalParamsFromFlags() + for field in params.all_fields(): + value = params.get_assigned_value(field.name) + if value != field.default: + client.AddGlobalParam(field.name, value) + banner = """ + == storage interactive console == + client: a storage client + apitools_base: base apitools module + messages: the generated messages module + """ + local_vars = { + 'apitools_base': apitools_base, + 'client': client, + 'client_lib': client_lib, + 'messages': messages, + } + if platform.system() == 'Linux': + console = apitools_base.ConsoleWithReadline( + local_vars, histfile=FLAGS.history_file) + else: + console = code.InteractiveConsole(local_vars) + try: + console.interact(banner) + except SystemExit as e: + return e.code + + +class BucketAccessControlsDelete(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.Delete.""" + + usage = """bucketAccessControls_delete """ + + def __init__(self, name, fv): + super(BucketAccessControlsDelete, self).__init__(name, fv) + + def RunWithArgs(self, bucket, entity): + """Permanently deletes the ACL entry for the specified entity on the + specified bucket. + + Args: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketAccessControlsDeleteRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + result = client.bucketAccessControls.Delete( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketAccessControlsGet(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.Get.""" + + usage = """bucketAccessControls_get """ + + def __init__(self, name, fv): + super(BucketAccessControlsGet, self).__init__(name, fv) + + def RunWithArgs(self, bucket, entity): + """Returns the ACL entry for the specified entity on the specified bucket. + + Args: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketAccessControlsGetRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + result = client.bucketAccessControls.Get( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketAccessControlsInsert(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.Insert.""" + + usage = """bucketAccessControls_insert """ + + def __init__(self, name, fv): + super(BucketAccessControlsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entity', + None, + u'The entity holding the permission, in one of the following forms: ' + u'- user-userId - user-email - group-groupId - group-email - ' + u'domain-domain - project-team-projectId - allUsers - ' + u'allAuthenticatedUsers Examples: - The user liz@example.com would ' + u'be user-liz@example.com. - The group example@googlegroups.com ' + u'would be group-example@googlegroups.com. - To refer to all members' + u' of the Google Apps for Business domain example.com, the entity ' + u'would be domain-example.com.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Creates a new ACL entry on the specified bucket. + + Args: + bucket: The name of the bucket. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.BucketAccessControl( + bucket=bucket.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entity'].present: + request.entity = FLAGS.entity.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Insert( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketAccessControlsList(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.List.""" + + usage = """bucketAccessControls_list """ + + def __init__(self, name, fv): + super(BucketAccessControlsList, self).__init__(name, fv) + + def RunWithArgs(self, bucket): + """Retrieves ACL entries on the specified bucket. + + Args: + bucket: Name of a bucket. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketAccessControlsListRequest( + bucket=bucket.decode('utf8'), + ) + result = client.bucketAccessControls.List( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketAccessControlsPatch(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.Patch.""" + + usage = """bucketAccessControls_patch """ + + def __init__(self, name, fv): + super(BucketAccessControlsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket, entity): + """Updates an ACL entry on the specified bucket. This method supports + patch semantics. + + Args: + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.BucketAccessControl( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Patch( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketAccessControlsUpdate(apitools_base.NewCmd): + """Command wrapping bucketAccessControls.Update.""" + + usage = """bucketAccessControls_update """ + + def __init__(self, name, fv): + super(BucketAccessControlsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket, entity): + """Updates an ACL entry on the specified bucket. + + Args: + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.BucketAccessControl( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Update( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsDelete(apitools_base.NewCmd): + """Command wrapping buckets.Delete.""" + + usage = """buckets_delete """ + + def __init__(self, name, fv): + super(BucketsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u'If set, only deletes the bucket if its metageneration matches this ' + u'value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u'If set, only deletes the bucket if its metageneration does not ' + u'match this value.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Permanently deletes an empty bucket. + + Args: + bucket: Name of a bucket. + + Flags: + ifMetagenerationMatch: If set, only deletes the bucket if its + metageneration matches this value. + ifMetagenerationNotMatch: If set, only deletes the bucket if its + metageneration does not match this value. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsDeleteRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.buckets.Delete( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsGet(apitools_base.NewCmd): + """Command wrapping buckets.Get.""" + + usage = """buckets_get """ + + def __init__(self, name, fv): + super(BucketsGet, self).__init__(name, fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Returns metadata for the specified bucket. + + Args: + bucket: Name of a bucket. + + Flags: + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + projection: Set of properties to return. Defaults to noAcl. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsGetRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Get( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsInsert(apitools_base.NewCmd): + """Command wrapping buckets.Insert.""" + + usage = """buckets_insert """ + + def __init__(self, name, fv): + super(BucketsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'bucket', + None, + u'A Bucket resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the bucket ' + u'resource specifies acl or defaultObjectAcl properties, when it ' + u'defaults to full.', + flag_values=fv) + + def RunWithArgs(self, project): + """Creates a new bucket. + + Args: + project: A valid API project identifier. + + Flags: + bucket: A Bucket resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this bucket. + projection: Set of properties to return. Defaults to noAcl, unless the + bucket resource specifies acl or defaultObjectAcl properties, when it + defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsInsertRequest( + project=project.decode('utf8'), + ) + if FLAGS['bucket'].present: + request.bucket = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucket) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Insert( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsList(apitools_base.NewCmd): + """Command wrapping buckets.List.""" + + usage = """buckets_list """ + + def __init__(self, name, fv): + super(BucketsList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of buckets to return.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + + def RunWithArgs(self, project): + """Retrieves a list of buckets for a given project. + + Args: + project: A valid API project identifier. + + Flags: + maxResults: Maximum number of buckets to return. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + projection: Set of properties to return. Defaults to noAcl. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsListRequest( + project=project.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.List( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsPatch(apitools_base.NewCmd): + """Command wrapping buckets.Patch.""" + + usage = """buckets_patch """ + + def __init__(self, name, fv): + super(BucketsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'bucketResource', + None, + u'A Bucket resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Updates a bucket. This method supports patch semantics. + + Args: + bucket: Name of a bucket. + + Flags: + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + projection: Set of properties to return. Defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsPatchRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Patch( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class BucketsUpdate(apitools_base.NewCmd): + """Command wrapping buckets.Update.""" + + usage = """buckets_update """ + + def __init__(self, name, fv): + super(BucketsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'bucketResource', + None, + u'A Bucket resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Updates a bucket. + + Args: + bucket: Name of a bucket. + + Flags: + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + projection: Set of properties to return. Defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsUpdateRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Update( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ChannelsStop(apitools_base.NewCmd): + """Command wrapping channels.Stop.""" + + usage = """channels_stop""" + + def __init__(self, name, fv): + super(ChannelsStop, self).__init__(name, fv) + flags.DEFINE_string( + 'address', + None, + u'The address where notifications are delivered for this channel.', + flag_values=fv) + flags.DEFINE_string( + 'expiration', + None, + u'Date and time of notification channel expiration, expressed as a ' + u'Unix timestamp, in milliseconds. Optional.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'A UUID or similar unique string that identifies this channel.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'api#channel', + u'Identifies this as a notification channel used to watch for changes' + u' to a resource. Value: the fixed string "api#channel".', + flag_values=fv) + flags.DEFINE_string( + 'params', + None, + u'Additional parameters controlling delivery channel behavior. ' + u'Optional.', + flag_values=fv) + flags.DEFINE_boolean( + 'payload', + None, + u'A Boolean value to indicate whether payload is wanted. Optional.', + flag_values=fv) + flags.DEFINE_string( + 'resourceId', + None, + u'An opaque ID that identifies the resource being watched on this ' + u'channel. Stable across different API versions.', + flag_values=fv) + flags.DEFINE_string( + 'resourceUri', + None, + u'A version-specific identifier for the watched resource.', + flag_values=fv) + flags.DEFINE_string( + 'token', + None, + u'An arbitrary string delivered to the target address with each ' + u'notification delivered over this channel. Optional.', + flag_values=fv) + flags.DEFINE_string( + 'type', + None, + u'The type of delivery mechanism used for this channel.', + flag_values=fv) + + def RunWithArgs(self): + """Stop watching resources through this channel + + Flags: + address: The address where notifications are delivered for this channel. + expiration: Date and time of notification channel expiration, expressed + as a Unix timestamp, in milliseconds. Optional. + id: A UUID or similar unique string that identifies this channel. + kind: Identifies this as a notification channel used to watch for + changes to a resource. Value: the fixed string "api#channel". + params: Additional parameters controlling delivery channel behavior. + Optional. + payload: A Boolean value to indicate whether payload is wanted. + Optional. + resourceId: An opaque ID that identifies the resource being watched on + this channel. Stable across different API versions. + resourceUri: A version-specific identifier for the watched resource. + token: An arbitrary string delivered to the target address with each + notification delivered over this channel. Optional. + type: The type of delivery mechanism used for this channel. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Channel( + ) + if FLAGS['address'].present: + request.address = FLAGS.address.decode('utf8') + if FLAGS['expiration'].present: + request.expiration = int(FLAGS.expiration) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['params'].present: + request.params = apitools_base.JsonToMessage(messages.Channel.ParamsValue, FLAGS.params) + if FLAGS['payload'].present: + request.payload = FLAGS.payload + if FLAGS['resourceId'].present: + request.resourceId = FLAGS.resourceId.decode('utf8') + if FLAGS['resourceUri'].present: + request.resourceUri = FLAGS.resourceUri.decode('utf8') + if FLAGS['token'].present: + request.token = FLAGS.token.decode('utf8') + if FLAGS['type'].present: + request.type = FLAGS.type.decode('utf8') + result = client.channels.Stop( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsDelete(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.Delete.""" + + usage = """defaultObjectAccessControls_delete """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsDelete, self).__init__(name, fv) + + def RunWithArgs(self, bucket, entity): + """Permanently deletes the default object ACL entry for the specified + entity on the specified bucket. + + Args: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageDefaultObjectAccessControlsDeleteRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + result = client.defaultObjectAccessControls.Delete( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsGet(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.Get.""" + + usage = """defaultObjectAccessControls_get """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsGet, self).__init__(name, fv) + + def RunWithArgs(self, bucket, entity): + """Returns the default object ACL entry for the specified entity on the + specified bucket. + + Args: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageDefaultObjectAccessControlsGetRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + result = client.defaultObjectAccessControls.Get( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsInsert(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.Insert.""" + + usage = """defaultObjectAccessControls_insert """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entity', + None, + u'The entity holding the permission, in one of the following forms: ' + u'- user-userId - user-email - group-groupId - group-email - ' + u'domain-domain - project-team-projectId - allUsers - ' + u'allAuthenticatedUsers Examples: - The user liz@example.com would ' + u'be user-liz@example.com. - The group example@googlegroups.com ' + u'would be group-example@googlegroups.com. - To refer to all members' + u' of the Google Apps for Business domain example.com, the entity ' + u'would be domain-example.com.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Creates a new default object ACL entry on the specified bucket. + + Args: + bucket: The name of the bucket. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.ObjectAccessControl( + bucket=bucket.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entity'].present: + request.entity = FLAGS.entity.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Insert( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsList(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.List.""" + + usage = """defaultObjectAccessControls_list """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsList, self).__init__(name, fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"If present, only return default ACL listing if the bucket's current" + u' metageneration matches this value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"If present, only return default ACL listing if the bucket's current" + u' metageneration does not match the given value.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Retrieves default object ACL entries on the specified bucket. + + Args: + bucket: Name of a bucket. + + Flags: + ifMetagenerationMatch: If present, only return default ACL listing if + the bucket's current metageneration matches this value. + ifMetagenerationNotMatch: If present, only return default ACL listing if + the bucket's current metageneration does not match the given value. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageDefaultObjectAccessControlsListRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.defaultObjectAccessControls.List( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsPatch(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.Patch.""" + + usage = """defaultObjectAccessControls_patch """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket, entity): + """Updates a default object ACL entry on the specified bucket. This method + supports patch semantics. + + Args: + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.ObjectAccessControl( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Patch( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class DefaultObjectAccessControlsUpdate(apitools_base.NewCmd): + """Command wrapping defaultObjectAccessControls.Update.""" + + usage = """defaultObjectAccessControls_update """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'domain', + None, + u'The domain associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', + flag_values=fv) + + def RunWithArgs(self, bucket, entity): + """Updates a default object ACL entry on the specified bucket. + + Args: + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + + Flags: + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.ObjectAccessControl( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Update( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsDelete(apitools_base.NewCmd): + """Command wrapping objectAccessControls.Delete.""" + + usage = """objectAccessControls_delete """ + + def __init__(self, name, fv): + super(ObjectAccessControlsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + + def RunWithArgs(self, bucket, object, entity): + """Permanently deletes the ACL entry for the specified entity on the + specified object. + + Args: + bucket: Name of a bucket. + object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsDeleteRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.Delete( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsGet(apitools_base.NewCmd): + """Command wrapping objectAccessControls.Get.""" + + usage = """objectAccessControls_get """ + + def __init__(self, name, fv): + super(ObjectAccessControlsGet, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + + def RunWithArgs(self, bucket, object, entity): + """Returns the ACL entry for the specified entity on the specified object. + + Args: + bucket: Name of a bucket. + object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsGetRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.Get( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsInsert(apitools_base.NewCmd): + """Command wrapping objectAccessControls.Insert.""" + + usage = """objectAccessControls_insert """ + + def __init__(self, name, fv): + super(ObjectAccessControlsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'objectAccessControl', + None, + u'A ObjectAccessControl resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Creates a new ACL entry on the specified object. + + Args: + bucket: Name of a bucket. + object: Name of the object. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsInsertRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Insert( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsList(apitools_base.NewCmd): + """Command wrapping objectAccessControls.List.""" + + usage = """objectAccessControls_list """ + + def __init__(self, name, fv): + super(ObjectAccessControlsList, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Retrieves ACL entries on the specified object. + + Args: + bucket: Name of a bucket. + object: Name of the object. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsListRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.List( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsPatch(apitools_base.NewCmd): + """Command wrapping objectAccessControls.Patch.""" + + usage = """objectAccessControls_patch """ + + def __init__(self, name, fv): + super(ObjectAccessControlsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'objectAccessControl', + None, + u'A ObjectAccessControl resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, bucket, object, entity): + """Updates an ACL entry on the specified object. This method supports + patch semantics. + + Args: + bucket: Name of a bucket. + object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsPatchRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Patch( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectAccessControlsUpdate(apitools_base.NewCmd): + """Command wrapping objectAccessControls.Update.""" + + usage = """objectAccessControls_update """ + + def __init__(self, name, fv): + super(ObjectAccessControlsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'objectAccessControl', + None, + u'A ObjectAccessControl resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, bucket, object, entity): + """Updates an ACL entry on the specified object. + + Args: + bucket: Name of a bucket. + object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsUpdateRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Update( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectsCompose(apitools_base.NewCmd): + """Command wrapping objects.Compose.""" + + usage = """objects_compose """ + + def __init__(self, name, fv): + super(ObjectsCompose, self).__init__(name, fv) + flags.DEFINE_string( + 'composeRequest', + None, + u'A ComposeRequest resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, destinationBucket, destinationObject): + """Concatenates a list of existing objects into a new object in the same + bucket. + + Args: + destinationBucket: Name of the bucket in which to store the new object. + destinationObject: Name of the new object. + + Flags: + composeRequest: A ComposeRequest resource to be passed as the request + body. + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsComposeRequest( + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), + ) + if FLAGS['composeRequest'].present: + request.composeRequest = apitools_base.JsonToMessage(messages.ComposeRequest, FLAGS.composeRequest) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Compose( + request, global_params=global_params, download=download) + print apitools_base.FormatOutput(result) + + +class ObjectsCopy(apitools_base.NewCmd): + """Command wrapping objects.Copy.""" + + usage = """objects_copy """ + + def __init__(self, name, fv): + super(ObjectsCopy, self).__init__(name, fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'sourceGeneration', + None, + u'If present, selects a specific revision of the source object (as ' + u'opposed to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): + """Copies an object to a specified location. Optionally overrides + metadata. + + Args: + sourceBucket: Name of the bucket in which to find the source object. + sourceObject: Name of the source object. + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's + name value, if any. + + Flags: + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given + value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether + the source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on + whether the source object's current metageneration does not match the + given value. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsCopyRequest( + sourceBucket=sourceBucket.decode('utf8'), + sourceObject=sourceObject.decode('utf8'), + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), + ) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['ifSourceGenerationMatch'].present: + request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) + if FLAGS['ifSourceGenerationNotMatch'].present: + request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) + if FLAGS['ifSourceMetagenerationMatch'].present: + request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) + if FLAGS['ifSourceMetagenerationNotMatch'].present: + request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsCopyRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['sourceGeneration'].present: + request.sourceGeneration = int(FLAGS.sourceGeneration) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Copy( + request, global_params=global_params, download=download) + print apitools_base.FormatOutput(result) + + +class ObjectsDelete(apitools_base.NewCmd): + """Command wrapping objects.Delete.""" + + usage = """objects_delete """ + + def __init__(self, name, fv): + super(ObjectsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, permanently deletes a specific revision of this object ' + u'(as opposed to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Deletes an object and its metadata. Deletions are permanent if + versioning is not enabled for the bucket, or if the generation parameter + is used. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. + + Flags: + generation: If present, permanently deletes a specific revision of this + object (as opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsDeleteRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.objects.Delete( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectsGet(apitools_base.NewCmd): + """Command wrapping objects.Get.""" + + usage = """objects_get """ + + def __init__(self, name, fv): + super(ObjectsGet, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's generation " + u'matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's generation " + u'does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Retrieves an object or its metadata. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + projection: Set of properties to return. Defaults to noAcl. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsGetRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Get( + request, global_params=global_params, download=download) + print apitools_base.FormatOutput(result) + + +class ObjectsInsert(apitools_base.NewCmd): + """Command wrapping objects.Insert.""" + + usage = """objects_insert """ + + def __init__(self, name, fv): + super(ObjectsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'contentEncoding', + None, + u'If set, sets the contentEncoding property of the final object to ' + u'this value. Setting this parameter is equivalent to setting the ' + u'contentEncoding metadata property. This can be useful when ' + u'uploading an object with uploadType=media to indicate the encoding ' + u'of the content being uploaded.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Name of the object. Required when the object metadata is not ' + u"otherwise provided. Overrides the object metadata's name value, if " + u'any.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'upload_filename', + '', + 'Filename to use for upload.', + flag_values=fv) + flags.DEFINE_string( + 'upload_mime_type', + '', + 'MIME type to use for the upload. Only needed if the extension on ' + '--upload_filename does not determine the correct (or any) MIME ' + 'type.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Stores a new object and metadata. + + Args: + bucket: Name of the bucket in which to store the new object. Overrides + the provided object metadata's bucket value, if any. + + Flags: + contentEncoding: If set, sets the contentEncoding property of the final + object to this value. Setting this parameter is equivalent to setting + the contentEncoding metadata property. This can be useful when + uploading an object with uploadType=media to indicate the encoding of + the content being uploaded. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + name: Name of the object. Required when the object metadata is not + otherwise provided. Overrides the object metadata's name value, if + any. + object: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + upload_filename: Filename to use for upload. + upload_mime_type: MIME type to use for the upload. Only needed if the + extension on --upload_filename does not determine the correct (or any) + MIME type. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsInsertRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['contentEncoding'].present: + request.contentEncoding = FLAGS.contentEncoding.decode('utf8') + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + upload = None + if FLAGS.upload_filename: + upload = apitools_base.Upload.FromFile( + FLAGS.upload_filename, FLAGS.upload_mime_type) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Insert( + request, global_params=global_params, upload=upload, download=download) + print apitools_base.FormatOutput(result) + + +class ObjectsList(apitools_base.NewCmd): + """Command wrapping objects.List.""" + + usage = """objects_list """ + + def __init__(self, name, fv): + super(ObjectsList, self).__init__(name, fv) + flags.DEFINE_string( + 'delimiter', + None, + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of a file as distinct results.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Retrieves a list of objects matching the criteria. + + Args: + bucket: Name of the bucket in which to look for objects. + + Flags: + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsListRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.List( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectsPatch(apitools_base.NewCmd): + """Command wrapping objects.Patch.""" + + usage = """objects_patch """ + + def __init__(self, name, fv): + super(ObjectsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Updates an object's metadata. This method supports patch semantics. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsPatchRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.objects.Patch( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +class ObjectsUpdate(apitools_base.NewCmd): + """Command wrapping objects.Update.""" + + usage = """objects_update """ + + def __init__(self, name, fv): + super(ObjectsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Updates an object's metadata. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsUpdateRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Update( + request, global_params=global_params, download=download) + print apitools_base.FormatOutput(result) + + +class ObjectsWatchAll(apitools_base.NewCmd): + """Command wrapping objects.WatchAll.""" + + usage = """objects_watchAll """ + + def __init__(self, name, fv): + super(ObjectsWatchAll, self).__init__(name, fv) + flags.DEFINE_string( + 'channel', + None, + u'A Channel resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_string( + 'delimiter', + None, + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of a file as distinct results.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Watch for changes on all objects in a bucket. + + Args: + bucket: Name of the bucket in which to look for objects. + + Flags: + channel: A Channel resource to be passed as the request body. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsWatchAllRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['channel'].present: + request.channel = apitools_base.JsonToMessage(messages.Channel, FLAGS.channel) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsWatchAllRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.WatchAll( + request, global_params=global_params) + print apitools_base.FormatOutput(result) + + +def main(_): + appcommands.AddCmd('pyshell', PyShell) + appcommands.AddCmd('bucketAccessControls_delete', BucketAccessControlsDelete) + appcommands.AddCmd('bucketAccessControls_get', BucketAccessControlsGet) + appcommands.AddCmd('bucketAccessControls_insert', BucketAccessControlsInsert) + appcommands.AddCmd('bucketAccessControls_list', BucketAccessControlsList) + appcommands.AddCmd('bucketAccessControls_patch', BucketAccessControlsPatch) + appcommands.AddCmd('bucketAccessControls_update', BucketAccessControlsUpdate) + appcommands.AddCmd('buckets_delete', BucketsDelete) + appcommands.AddCmd('buckets_get', BucketsGet) + appcommands.AddCmd('buckets_insert', BucketsInsert) + appcommands.AddCmd('buckets_list', BucketsList) + appcommands.AddCmd('buckets_patch', BucketsPatch) + appcommands.AddCmd('buckets_update', BucketsUpdate) + appcommands.AddCmd('channels_stop', ChannelsStop) + appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) + appcommands.AddCmd('defaultObjectAccessControls_get', DefaultObjectAccessControlsGet) + appcommands.AddCmd('defaultObjectAccessControls_insert', DefaultObjectAccessControlsInsert) + appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) + appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) + appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) + appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) + appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) + appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) + appcommands.AddCmd('objectAccessControls_list', ObjectAccessControlsList) + appcommands.AddCmd('objectAccessControls_patch', ObjectAccessControlsPatch) + appcommands.AddCmd('objectAccessControls_update', ObjectAccessControlsUpdate) + appcommands.AddCmd('objects_compose', ObjectsCompose) + appcommands.AddCmd('objects_copy', ObjectsCopy) + appcommands.AddCmd('objects_delete', ObjectsDelete) + appcommands.AddCmd('objects_get', ObjectsGet) + appcommands.AddCmd('objects_insert', ObjectsInsert) + appcommands.AddCmd('objects_list', ObjectsList) + appcommands.AddCmd('objects_patch', ObjectsPatch) + appcommands.AddCmd('objects_update', ObjectsUpdate) + appcommands.AddCmd('objects_watchAll', ObjectsWatchAll) + + apitools_base.SetupLogger() + if hasattr(appcommands, 'SetDefaultCommand'): + appcommands.SetDefaultCommand('pyshell') + + +run_main = apitools_base.run_main + +if __name__ == '__main__': + appcommands.Run() diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage/storage_v1_client.py new file mode 100644 index 0000000..630fa96 --- /dev/null +++ b/samples/storage_sample/storage/storage_v1_client.py @@ -0,0 +1,1017 @@ +"""Generated client library for storage version v1.""" +from apitools.base.py import base_api +import storage_v1_messages as messages + + +class StorageV1(base_api.BaseApiClient): + """Generated client library for service storage version v1.""" + + MESSAGES_MODULE = messages + + _PACKAGE = u'storage' + _SCOPES = [u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] + _VERSION = u'v1' + _CLIENT_ID = '1042881264118.apps.googleusercontent.com' + _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _USER_AGENT = '' + _CLIENT_CLASS_NAME = u'StorageV1' + _URL_VERSION = u'v1' + _API_KEY = 'AIzaSyCBivqENU2RLBuz8XWfAjQiT5SAFCHHg6U' + + def __init__(self, url='', credentials=None, + get_credentials=True, http=None, model=None, + log_request=False, log_response=False, + credentials_args=None, default_global_params=None): + """Create a new storage handle.""" + url = url or u'https://www.googleapis.com/storage/v1/' + super(StorageV1, self).__init__( + url, credentials=credentials, + get_credentials=get_credentials, http=http, model=model, + log_request=log_request, log_response=log_response, + credentials_args=credentials_args, + default_global_params=default_global_params) + self.bucketAccessControls = self.BucketAccessControlsService(self) + self.buckets = self.BucketsService(self) + self.channels = self.ChannelsService(self) + self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) + self.objectAccessControls = self.ObjectAccessControlsService(self) + self.objects = self.ObjectsService(self) + + class BucketAccessControlsService(base_api.BaseApiService): + """Service class for the bucketAccessControls resource.""" + + def __init__(self, client): + super(StorageV1.BucketAccessControlsService, self).__init__(client) + self.__configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.bucketAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'StorageBucketAccessControlsDeleteRequest', + response_type_name=u'StorageBucketAccessControlsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.bucketAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'StorageBucketAccessControlsGetRequest', + response_type_name=u'BucketAccessControl', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.bucketAccessControls.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.bucketAccessControls.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', + request_field='', + request_type_name=u'StorageBucketAccessControlsListRequest', + response_type_name=u'BucketAccessControls', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.bucketAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.bucketAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ), + } + + self.__upload_configs = { + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Delete(self, request, global_params=None): + """Permanently deletes the ACL entry for the specified entity on the specified bucket. + + Args: + request: (StorageBucketAccessControlsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageBucketAccessControlsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Returns the ACL entry for the specified entity on the specified bucket. + + Args: + request: (StorageBucketAccessControlsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (BucketAccessControl) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a new ACL entry on the specified bucket. + + Args: + request: (BucketAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (BucketAccessControl) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves ACL entries on the specified bucket. + + Args: + request: (StorageBucketAccessControlsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (BucketAccessControls) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an ACL entry on the specified bucket. This method supports patch semantics. + + Args: + request: (BucketAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (BucketAccessControl) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates an ACL entry on the specified bucket. + + Args: + request: (BucketAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (BucketAccessControl) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class BucketsService(base_api.BaseApiService): + """Service class for the buckets resource.""" + + def __init__(self, client): + super(StorageV1.BucketsService, self).__init__(client) + self.__configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.buckets.delete', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}', + request_field='', + request_type_name=u'StorageBucketsDeleteRequest', + response_type_name=u'StorageBucketsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.get', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}', + request_field='', + request_type_name=u'StorageBucketsGetRequest', + response_type_name=u'Bucket', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.buckets.insert', + ordered_params=[u'project'], + path_params=[], + query_params=[u'predefinedAcl', u'project', u'projection'], + relative_path=u'b', + request_field=u'bucket', + request_type_name=u'StorageBucketsInsertRequest', + response_type_name=u'Bucket', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.list', + ordered_params=[u'project'], + path_params=[], + query_params=[u'maxResults', u'pageToken', u'project', u'projection'], + relative_path=u'b', + request_field='', + request_type_name=u'StorageBucketsListRequest', + response_type_name=u'Buckets', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.buckets.patch', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsPatchRequest', + response_type_name=u'Bucket', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.buckets.update', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsUpdateRequest', + response_type_name=u'Bucket', + supports_download=False, + ), + } + + self.__upload_configs = { + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Delete(self, request, global_params=None): + """Permanently deletes an empty bucket. + + Args: + request: (StorageBucketsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageBucketsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Returns metadata for the specified bucket. + + Args: + request: (StorageBucketsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Bucket) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a new bucket. + + Args: + request: (StorageBucketsInsertRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Bucket) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of buckets for a given project. + + Args: + request: (StorageBucketsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Buckets) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates a bucket. This method supports patch semantics. + + Args: + request: (StorageBucketsPatchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Bucket) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates a bucket. + + Args: + request: (StorageBucketsUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Bucket) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class ChannelsService(base_api.BaseApiService): + """Service class for the channels resource.""" + + def __init__(self, client): + super(StorageV1.ChannelsService, self).__init__(client) + self.__configs = { + 'Stop': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.channels.stop', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'channels/stop', + request_field='', + request_type_name=u'Channel', + response_type_name=u'StorageChannelsStopResponse', + supports_download=False, + ), + } + + self.__upload_configs = { + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Stop(self, request, global_params=None): + """Stop watching resources through this channel. + + Args: + request: (Channel) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageChannelsStopResponse) The response message. + """ + config = self.GetMethodConfig('Stop') + return self._RunMethod( + config, request, global_params=global_params) + + class DefaultObjectAccessControlsService(base_api.BaseApiService): + """Service class for the defaultObjectAccessControls resource.""" + + def __init__(self, client): + super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) + self.__configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.defaultObjectAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', + response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.defaultObjectAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.defaultObjectAccessControls.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.defaultObjectAccessControls.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/defaultObjectAcl', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.defaultObjectAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.defaultObjectAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + } + + self.__upload_configs = { + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Delete(self, request, global_params=None): + """Permanently deletes the default object ACL entry for the specified entity on the specified bucket. + + Args: + request: (StorageDefaultObjectAccessControlsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageDefaultObjectAccessControlsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Returns the default object ACL entry for the specified entity on the specified bucket. + + Args: + request: (StorageDefaultObjectAccessControlsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a new default object ACL entry on the specified bucket. + + Args: + request: (ObjectAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves default object ACL entries on the specified bucket. + + Args: + request: (StorageDefaultObjectAccessControlsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControls) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. + + Args: + request: (ObjectAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates a default object ACL entry on the specified bucket. + + Args: + request: (ObjectAccessControl) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class ObjectAccessControlsService(base_api.BaseApiService): + """Service class for the objectAccessControls resource.""" + + def __init__(self, client): + super(StorageV1.ObjectAccessControlsService, self).__init__(client) + self.__configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.objectAccessControls.delete', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field='', + request_type_name=u'StorageObjectAccessControlsDeleteRequest', + response_type_name=u'StorageObjectAccessControlsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objectAccessControls.get', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field='', + request_type_name=u'StorageObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objectAccessControls.insert', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsInsertRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objectAccessControls.list', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl', + request_field='', + request_type_name=u'StorageObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.objectAccessControls.patch', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsPatchRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objectAccessControls.update', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsUpdateRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ), + } + + self.__upload_configs = { + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Delete(self, request, global_params=None): + """Permanently deletes the ACL entry for the specified entity on the specified object. + + Args: + request: (StorageObjectAccessControlsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageObjectAccessControlsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Returns the ACL entry for the specified entity on the specified object. + + Args: + request: (StorageObjectAccessControlsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a new ACL entry on the specified object. + + Args: + request: (StorageObjectAccessControlsInsertRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves ACL entries on the specified object. + + Args: + request: (StorageObjectAccessControlsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControls) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an ACL entry on the specified object. This method supports patch semantics. + + Args: + request: (StorageObjectAccessControlsPatchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates an ACL entry on the specified object. + + Args: + request: (StorageObjectAccessControlsUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ObjectAccessControl) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class ObjectsService(base_api.BaseApiService): + """Service class for the objects resource.""" + + def __init__(self, client): + super(StorageV1.ObjectsService, self).__init__(client) + self.__configs = { + 'Compose': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.compose', + ordered_params=[u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], + relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', + request_field=u'composeRequest', + request_type_name=u'StorageObjectsComposeRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'Copy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.copy', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsCopyRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.objects.delete', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/o/{object}', + request_field='', + request_type_name=u'StorageObjectsDeleteRequest', + response_type_name=u'StorageObjectsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.get', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field='', + request_type_name=u'StorageObjectsGetRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o', + request_field=u'object', + request_type_name=u'StorageObjectsInsertRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o', + request_field='', + request_type_name=u'StorageObjectsListRequest', + response_type_name=u'Objects', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.objects.patch', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsPatchRequest', + response_type_name=u'Object', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objects.update', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsUpdateRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'WatchAll': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.watchAll', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o/watch', + request_field=u'channel', + request_type_name=u'StorageObjectsWatchAllRequest', + response_type_name=u'Channel', + supports_download=False, + ), + } + + self.__upload_configs = { + 'Insert': base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload/storage/v1/b/{bucket}/o', + simple_multipart=True, + simple_path=u'/upload/storage/v1/b/{bucket}/o', + ), + } + + def GetMethodConfig(self, method): + return self.__configs.get(method) + + def GetMethodUploadConfig(self, method): + return self.__upload_configs.get(method) + + def Compose(self, request, global_params=None, download=None): + """Concatenates a list of existing objects into a new object in the same bucket. + + Args: + request: (StorageObjectsComposeRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Compose') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def Copy(self, request, global_params=None, download=None): + """Copies an object to a specified location. Optionally overrides metadata. + + Args: + request: (StorageObjectsCopyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Copy') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def Delete(self, request, global_params=None): + """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. + + Args: + request: (StorageObjectsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageObjectsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None, download=None): + """Retrieves an object or its metadata. + + Args: + request: (StorageObjectsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def Insert(self, request, global_params=None, upload=None, download=None): + """Stores a new object and metadata. + + Args: + request: (StorageObjectsInsertRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + upload: (Upload, default: None) If present, upload + this stream with the request. + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Insert') + upload_config = self.GetMethodUploadConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params, + upload=upload, upload_config=upload_config, + download=download) + + def List(self, request, global_params=None): + """Retrieves a list of objects matching the criteria. + + Args: + request: (StorageObjectsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Objects) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an object's metadata. This method supports patch semantics. + + Args: + request: (StorageObjectsPatchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None, download=None): + """Updates an object's metadata. + + Args: + request: (StorageObjectsUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def WatchAll(self, request, global_params=None): + """Watch for changes on all objects in a bucket. + + Args: + request: (StorageObjectsWatchAllRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Channel) The response message. + """ + config = self.GetMethodConfig('WatchAll') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py new file mode 100644 index 0000000..764fcd8 --- /dev/null +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -0,0 +1,1632 @@ +"""Generated message classes for storage version v1. + +Lets you store and retrieve potentially-large, immutable data objects. +""" + +from apitools.base.py import encoding +from protorpc import message_types +from protorpc import messages + + +package = 'storage' + + +class Bucket(messages.Message): + """A bucket. + + Messages: + CorsValueListEntry: A CorsValueListEntry object. + LifecycleValue: The bucket's lifecycle configuration. See lifecycle + management for more information. + LoggingValue: The bucket's logging configuration, which defines the + destination bucket and optional name prefix for the current bucket's + logs. + OwnerValue: The owner of the bucket. This is always the project team's + owner group. + VersioningValue: The bucket's versioning configuration. + WebsiteValue: The bucket's website configuration. + + Fields: + acl: Access controls on the bucket. + cors: The bucket's Cross-Origin Resource Sharing (CORS) configuration. + defaultObjectAcl: Default access controls to apply to new objects when no + ACL is provided. + etag: HTTP 1.1 Entity tag for the bucket. + id: The ID of the bucket. + kind: The kind of item this is. For buckets, this is always + storage#bucket. + lifecycle: The bucket's lifecycle configuration. See lifecycle management + for more information. + location: The location of the bucket. Object data for objects in the + bucket resides in physical storage within this region. Defaults to US. + See the developer's guide for the authoritative list. + logging: The bucket's logging configuration, which defines the destination + bucket and optional name prefix for the current bucket's logs. + metageneration: The metadata generation of this bucket. + name: The name of the bucket. + owner: The owner of the bucket. This is always the project team's owner + group. + projectNumber: The project number of the project the bucket belongs to. + selfLink: The URI of this bucket. + storageClass: The bucket's storage class. This defines how objects in the + bucket are stored and determines the SLA and the cost of storage. + Typical values are STANDARD and DURABLE_REDUCED_AVAILABILITY. Defaults + to STANDARD. See the developer's guide for the authoritative list. + timeCreated: Creation time of the bucket in RFC 3339 format. + versioning: The bucket's versioning configuration. + website: The bucket's website configuration. + """ + + class CorsValueListEntry(messages.Message): + """A CorsValueListEntry object. + + Fields: + maxAgeSeconds: The value, in seconds, to return in the Access-Control- + Max-Age header used in preflight responses. + method: The list of HTTP methods on which to include CORS response + headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list + of methods, and means "any method". + origin: The list of Origins eligible to receive CORS response headers. + Note: "*" is permitted in the list of origins, and means "any Origin". + responseHeader: The list of HTTP headers other than the simple response + headers to give permission for the user-agent to share across domains. + """ + + maxAgeSeconds = messages.IntegerField(1, variant=messages.Variant.INT32) + method = messages.StringField(2, repeated=True) + origin = messages.StringField(3, repeated=True) + responseHeader = messages.StringField(4, repeated=True) + + class LifecycleValue(messages.Message): + """The bucket's lifecycle configuration. See lifecycle management for more + information. + + Messages: + RuleValueListEntry: A RuleValueListEntry object. + + Fields: + rule: A lifecycle management rule, which is made of an action to take + and the condition(s) under which the action will be taken. + """ + + class RuleValueListEntry(messages.Message): + """A RuleValueListEntry object. + + Messages: + ActionValue: The action to take. + ConditionValue: The condition(s) under which the action will be taken. + + Fields: + action: The action to take. + condition: The condition(s) under which the action will be taken. + """ + + class ActionValue(messages.Message): + """The action to take. + + Fields: + type: Type of the action. Currently, only Delete is supported. + """ + + type = messages.StringField(1) + + class ConditionValue(messages.Message): + """The condition(s) under which the action will be taken. + + Fields: + age: Age of an object (in days). This condition is satisfied when an + object reaches the specified age. + createdBefore: A date in RFC 3339 format with only the date part + (for instance, "2013-01-15"). This condition is satisfied when an + object is created before midnight of the specified date in UTC. + isLive: Relevant only for versioned objects. If the value is true, + this condition matches live objects; if the value is false, it + matches archived objects. + numNewerVersions: Relevant only for versioned objects. If the value + is N, this condition is satisfied when there are at least N + versions (including the live version) newer than this version of + the object. + """ + + age = messages.IntegerField(1, variant=messages.Variant.INT32) + createdBefore = message_types.DateTimeField(2) + isLive = messages.BooleanField(3) + numNewerVersions = messages.IntegerField(4, variant=messages.Variant.INT32) + + action = messages.MessageField('ActionValue', 1) + condition = messages.MessageField('ConditionValue', 2) + + rule = messages.MessageField('RuleValueListEntry', 1, repeated=True) + + class LoggingValue(messages.Message): + """The bucket's logging configuration, which defines the destination + bucket and optional name prefix for the current bucket's logs. + + Fields: + logBucket: The destination bucket where the current bucket's logs should + be placed. + logObjectPrefix: A prefix for log object names. + """ + + logBucket = messages.StringField(1) + logObjectPrefix = messages.StringField(2) + + class OwnerValue(messages.Message): + """The owner of the bucket. This is always the project team's owner group. + + Fields: + entity: The entity, in the form project-owner-projectId. + entityId: The ID for the entity. + """ + + entity = messages.StringField(1) + entityId = messages.StringField(2) + + class VersioningValue(messages.Message): + """The bucket's versioning configuration. + + Fields: + enabled: While set to true, versioning is fully enabled for this bucket. + """ + + enabled = messages.BooleanField(1) + + class WebsiteValue(messages.Message): + """The bucket's website configuration. + + Fields: + mainPageSuffix: Behaves as the bucket's directory index where missing + objects are treated as potential directories. + notFoundPage: The custom object to return when a requested resource is + not found. + """ + + mainPageSuffix = messages.StringField(1) + notFoundPage = messages.StringField(2) + + acl = messages.MessageField('BucketAccessControl', 1, repeated=True) + cors = messages.MessageField('CorsValueListEntry', 2, repeated=True) + defaultObjectAcl = messages.MessageField('ObjectAccessControl', 3, repeated=True) + etag = messages.StringField(4) + id = messages.StringField(5) + kind = messages.StringField(6, default=u'storage#bucket') + lifecycle = messages.MessageField('LifecycleValue', 7) + location = messages.StringField(8) + logging = messages.MessageField('LoggingValue', 9) + metageneration = messages.IntegerField(10) + name = messages.StringField(11) + owner = messages.MessageField('OwnerValue', 12) + projectNumber = messages.IntegerField(13, variant=messages.Variant.UINT64) + selfLink = messages.StringField(14) + storageClass = messages.StringField(15) + timeCreated = message_types.DateTimeField(16) + versioning = messages.MessageField('VersioningValue', 17) + website = messages.MessageField('WebsiteValue', 18) + + +class BucketAccessControl(messages.Message): + """An access-control entry. + + Messages: + ProjectTeamValue: The project team associated with the entity, if any. + + Fields: + bucket: The name of the bucket. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following forms: + - user-userId - user-email - group-groupId - group-email - domain- + domain - project-team-projectId - allUsers - allAuthenticatedUsers + Examples: - The user liz@example.com would be user-liz@example.com. - + The group example@googlegroups.com would be group- + example@googlegroups.com. - To refer to all members of the Google Apps + for Business domain example.com, the entity would be domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this is + always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. + """ + + class ProjectTeamValue(messages.Message): + """The project team associated with the entity, if any. + + Fields: + projectNumber: The project number. + team: The team. Can be owners, editors, or viewers. + """ + + projectNumber = messages.StringField(1) + team = messages.StringField(2) + + bucket = messages.StringField(1) + domain = messages.StringField(2) + email = messages.StringField(3) + entity = messages.StringField(4) + entityId = messages.StringField(5) + etag = messages.StringField(6) + id = messages.StringField(7) + kind = messages.StringField(8, default=u'storage#bucketAccessControl') + projectTeam = messages.MessageField('ProjectTeamValue', 9) + role = messages.StringField(10) + selfLink = messages.StringField(11) + + +class BucketAccessControls(messages.Message): + """An access-control list. + + Fields: + items: The list of items. + kind: The kind of item this is. For lists of bucket access control + entries, this is always storage#bucketAccessControls. + """ + + items = messages.MessageField('BucketAccessControl', 1, repeated=True) + kind = messages.StringField(2, default=u'storage#bucketAccessControls') + + +class Buckets(messages.Message): + """A list of buckets. + + Fields: + items: The list of items. + kind: The kind of item this is. For lists of buckets, this is always + storage#buckets. + nextPageToken: The continuation token, used to page through large result + sets. Provide this value in a subsequent request to return the next page + of results. + """ + + items = messages.MessageField('Bucket', 1, repeated=True) + kind = messages.StringField(2, default=u'storage#buckets') + nextPageToken = messages.StringField(3) + + +class Channel(messages.Message): + """An notification channel used to watch for resource changes. + + Messages: + ParamsValue: Additional parameters controlling delivery channel behavior. + Optional. + + Fields: + address: The address where notifications are delivered for this channel. + expiration: Date and time of notification channel expiration, expressed as + a Unix timestamp, in milliseconds. Optional. + id: A UUID or similar unique string that identifies this channel. + kind: Identifies this as a notification channel used to watch for changes + to a resource. Value: the fixed string "api#channel". + params: Additional parameters controlling delivery channel behavior. + Optional. + payload: A Boolean value to indicate whether payload is wanted. Optional. + resourceId: An opaque ID that identifies the resource being watched on + this channel. Stable across different API versions. + resourceUri: A version-specific identifier for the watched resource. + token: An arbitrary string delivered to the target address with each + notification delivered over this channel. Optional. + type: The type of delivery mechanism used for this channel. + """ + + @encoding.MapUnrecognizedFields('additionalProperties') + class ParamsValue(messages.Message): + """Additional parameters controlling delivery channel behavior. Optional. + + Messages: + AdditionalProperty: An additional property for a ParamsValue object. + + Fields: + additionalProperties: Declares a new parameter by name. + """ + + class AdditionalProperty(messages.Message): + """An additional property for a ParamsValue object. + + Fields: + key: Name of the additional property. + value: A string attribute. + """ + + key = messages.StringField(1) + value = messages.StringField(2) + + additionalProperties = messages.MessageField('AdditionalProperty', 1, repeated=True) + + address = messages.StringField(1) + expiration = messages.IntegerField(2) + id = messages.StringField(3) + kind = messages.StringField(4, default=u'api#channel') + params = messages.MessageField('ParamsValue', 5) + payload = messages.BooleanField(6) + resourceId = messages.StringField(7) + resourceUri = messages.StringField(8) + token = messages.StringField(9) + type = messages.StringField(10) + + +class ComposeRequest(messages.Message): + """A Compose request. + + Messages: + SourceObjectsValueListEntry: A SourceObjectsValueListEntry object. + + Fields: + destination: Properties of the resulting object. + kind: The kind of item this is. + sourceObjects: The list of source objects that will be concatenated into a + single object. + """ + + class SourceObjectsValueListEntry(messages.Message): + """A SourceObjectsValueListEntry object. + + Messages: + ObjectPreconditionsValue: Conditions that must be met for this operation + to execute. + + Fields: + generation: The generation of this object to use as the source. + name: The source object's name. The source object's bucket is implicitly + the destination bucket. + objectPreconditions: Conditions that must be met for this operation to + execute. + """ + + class ObjectPreconditionsValue(messages.Message): + """Conditions that must be met for this operation to execute. + + Fields: + ifGenerationMatch: Only perform the composition if the generation of + the source object that would be used matches this value. If this + value and a generation are both specified, they must be the same + value or the call will fail. + """ + + ifGenerationMatch = messages.IntegerField(1) + + generation = messages.IntegerField(1) + name = messages.StringField(2) + objectPreconditions = messages.MessageField('ObjectPreconditionsValue', 3) + + destination = messages.MessageField('Object', 1) + kind = messages.StringField(2, default=u'storage#composeRequest') + sourceObjects = messages.MessageField('SourceObjectsValueListEntry', 3, repeated=True) + + +class Object(messages.Message): + """An object. + + Messages: + MetadataValue: User-provided metadata, in key/value pairs. + OwnerValue: The owner of the object. This will always be the uploader of + the object. + + Fields: + acl: Access controls on the object. + bucket: The name of the bucket containing this object. + cacheControl: Cache-Control directive for the object data. + componentCount: Number of underlying components that make up this object. + Components are accumulated by compose operations. + contentDisposition: Content-Disposition of the object data. + contentEncoding: Content-Encoding of the object data. + contentLanguage: Content-Language of the object data. + contentType: Content-Type of the object data. + crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded + using base64. + etag: HTTP 1.1 Entity tag for the object. + generation: The content generation of this object. Used for object + versioning. + id: The ID of the object. + kind: The kind of item this is. For objects, this is always + storage#object. + md5Hash: MD5 hash of the data; encoded using base64. + mediaLink: Media download link. + metadata: User-provided metadata, in key/value pairs. + metageneration: The version of the metadata for this object at this + generation. Used for preconditions and for detecting changes in + metadata. A metageneration number is only meaningful in the context of a + particular generation of a particular object. + name: The name of this object. Required if not specified by URL parameter. + owner: The owner of the object. This will always be the uploader of the + object. + selfLink: The link to this object. + size: Content-Length of the data in bytes. + storageClass: Storage class of the object. + timeDeleted: The deletion time of the object in RFC 3339 format. Will be + returned if and only if this version of the object has been deleted. + updated: The creation or modification time of the object in RFC 3339 + format. For buckets with versioning enabled, changing an object's + metadata does not change this property. + """ + + @encoding.MapUnrecognizedFields('additionalProperties') + class MetadataValue(messages.Message): + """User-provided metadata, in key/value pairs. + + Messages: + AdditionalProperty: An additional property for a MetadataValue object. + + Fields: + additionalProperties: An individual metadata entry. + """ + + class AdditionalProperty(messages.Message): + """An additional property for a MetadataValue object. + + Fields: + key: Name of the additional property. + value: A string attribute. + """ + + key = messages.StringField(1) + value = messages.StringField(2) + + additionalProperties = messages.MessageField('AdditionalProperty', 1, repeated=True) + + class OwnerValue(messages.Message): + """The owner of the object. This will always be the uploader of the + object. + + Fields: + entity: The entity, in the form user-userId. + entityId: The ID for the entity. + """ + + entity = messages.StringField(1) + entityId = messages.StringField(2) + + acl = messages.MessageField('ObjectAccessControl', 1, repeated=True) + bucket = messages.StringField(2) + cacheControl = messages.StringField(3) + componentCount = messages.IntegerField(4, variant=messages.Variant.INT32) + contentDisposition = messages.StringField(5) + contentEncoding = messages.StringField(6) + contentLanguage = messages.StringField(7) + contentType = messages.StringField(8) + crc32c = messages.StringField(9) + etag = messages.StringField(10) + generation = messages.IntegerField(11) + id = messages.StringField(12) + kind = messages.StringField(13, default=u'storage#object') + md5Hash = messages.StringField(14) + mediaLink = messages.StringField(15) + metadata = messages.MessageField('MetadataValue', 16) + metageneration = messages.IntegerField(17) + name = messages.StringField(18) + owner = messages.MessageField('OwnerValue', 19) + selfLink = messages.StringField(20) + size = messages.IntegerField(21, variant=messages.Variant.UINT64) + storageClass = messages.StringField(22) + timeDeleted = message_types.DateTimeField(23) + updated = message_types.DateTimeField(24) + + +class ObjectAccessControl(messages.Message): + """An access-control entry. + + Messages: + ProjectTeamValue: The project team associated with the entity, if any. + + Fields: + bucket: The name of the bucket. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following forms: + - user-userId - user-email - group-groupId - group-email - domain- + domain - project-team-projectId - allUsers - allAuthenticatedUsers + Examples: - The user liz@example.com would be user-liz@example.com. - + The group example@googlegroups.com would be group- + example@googlegroups.com. - To refer to all members of the Google Apps + for Business domain example.com, the entity would be domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this is + always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. + """ + + class ProjectTeamValue(messages.Message): + """The project team associated with the entity, if any. + + Fields: + projectNumber: The project number. + team: The team. Can be owners, editors, or viewers. + """ + + projectNumber = messages.StringField(1) + team = messages.StringField(2) + + bucket = messages.StringField(1) + domain = messages.StringField(2) + email = messages.StringField(3) + entity = messages.StringField(4) + entityId = messages.StringField(5) + etag = messages.StringField(6) + generation = messages.IntegerField(7) + id = messages.StringField(8) + kind = messages.StringField(9, default=u'storage#objectAccessControl') + object = messages.StringField(10) + projectTeam = messages.MessageField('ProjectTeamValue', 11) + role = messages.StringField(12) + selfLink = messages.StringField(13) + + +class ObjectAccessControls(messages.Message): + """An access-control list. + + Fields: + items: The list of items. + kind: The kind of item this is. For lists of object access control + entries, this is always storage#objectAccessControls. + """ + + items = messages.MessageField('extra_types.JsonValue', 1, repeated=True) + kind = messages.StringField(2, default=u'storage#objectAccessControls') + + +class Objects(messages.Message): + """A list of objects. + + Fields: + items: The list of items. + kind: The kind of item this is. For lists of objects, this is always + storage#objects. + nextPageToken: The continuation token, used to page through large result + sets. Provide this value in a subsequent request to return the next page + of results. + prefixes: The list of prefixes of objects matching-but-not-listed up to + and including the requested delimiter. + """ + + items = messages.MessageField('Object', 1, repeated=True) + kind = messages.StringField(2, default=u'storage#objects') + nextPageToken = messages.StringField(3) + prefixes = messages.StringField(4, repeated=True) + + +class StandardQueryParameters(messages.Message): + """Query parameters accepted by all methods. + + Enums: + AltValueValuesEnum: Data format for the response. + + Fields: + alt: Data format for the response. + fields: Selector specifying which fields to include in a partial response. + key: API key. Your API key identifies your project and provides you with + API access, quota, and reports. Required unless you provide an OAuth 2.0 + token. + oauth_token: OAuth 2.0 token for the current user. + prettyPrint: Returns response with indentations and line breaks. + quotaUser: Available to use for quota purposes for server-side + applications. Can be any arbitrary string assigned to a user, but should + not exceed 40 characters. Overrides userIp if both are provided. + trace: A tracing token of the form "token:" to include in api + requests. + userIp: IP address of the site where the request originates. Use this if + you want to enforce per-user limits. + """ + + class AltValueValuesEnum(messages.Enum): + """Data format for the response. + + Values: + json: Responses with Content-Type of application/json + """ + json = 0 + + alt = messages.EnumField('AltValueValuesEnum', 1, default=u'json') + fields = messages.StringField(2) + key = messages.StringField(3) + oauth_token = messages.StringField(4) + prettyPrint = messages.BooleanField(5, default=True) + quotaUser = messages.StringField(6) + trace = messages.StringField(7) + userIp = messages.StringField(8) + + +class StorageBucketAccessControlsDeleteRequest(messages.Message): + """A StorageBucketAccessControlsDeleteRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + + +class StorageBucketAccessControlsDeleteResponse(messages.Message): + """An empty StorageBucketAccessControlsDelete response.""" + + +class StorageBucketAccessControlsGetRequest(messages.Message): + """A StorageBucketAccessControlsGetRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + + +class StorageBucketAccessControlsListRequest(messages.Message): + """A StorageBucketAccessControlsListRequest object. + + Fields: + bucket: Name of a bucket. + """ + + bucket = messages.StringField(1, required=True) + + +class StorageBucketsDeleteRequest(messages.Message): + """A StorageBucketsDeleteRequest object. + + Fields: + bucket: Name of a bucket. + ifMetagenerationMatch: If set, only deletes the bucket if its + metageneration matches this value. + ifMetagenerationNotMatch: If set, only deletes the bucket if its + metageneration does not match this value. + """ + + bucket = messages.StringField(1, required=True) + ifMetagenerationMatch = messages.IntegerField(2) + ifMetagenerationNotMatch = messages.IntegerField(3) + + +class StorageBucketsDeleteResponse(messages.Message): + """An empty StorageBucketsDelete response.""" + + +class StorageBucketsGetRequest(messages.Message): + """A StorageBucketsGetRequest object. + + Enums: + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. + + Fields: + bucket: Name of a bucket. + ifMetagenerationMatch: Makes the return of the bucket metadata conditional + on whether the bucket's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + projection: Set of properties to return. Defaults to noAcl. + """ + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl. + + Values: + full: Include all properties. + noAcl: Omit acl and defaultObjectAcl properties. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + ifMetagenerationMatch = messages.IntegerField(2) + ifMetagenerationNotMatch = messages.IntegerField(3) + projection = messages.EnumField('ProjectionValueValuesEnum', 4) + + +class StorageBucketsInsertRequest(messages.Message): + """A StorageBucketsInsertRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this bucket. + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, + unless the bucket resource specifies acl or defaultObjectAcl properties, + when it defaults to full. + + Fields: + bucket: A Bucket resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this bucket. + project: A valid API project identifier. + projection: Set of properties to return. Defaults to noAcl, unless the + bucket resource specifies acl or defaultObjectAcl properties, when it + defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this bucket. + + Values: + authenticatedRead: Project team owners get OWNER access, and + allAuthenticatedUsers get READER access. + private: Project team owners get OWNER access. + projectPrivate: Project team members get access according to their + roles. + publicRead: Project team owners get OWNER access, and allUsers get + READER access. + publicReadWrite: Project team owners get OWNER access, and allUsers get + WRITER access. + """ + authenticatedRead = 0 + private = 1 + projectPrivate = 2 + publicRead = 3 + publicReadWrite = 4 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl, unless the bucket + resource specifies acl or defaultObjectAcl properties, when it defaults to + full. + + Values: + full: Include all properties. + noAcl: Omit acl and defaultObjectAcl properties. + """ + full = 0 + noAcl = 1 + + bucket = messages.MessageField('Bucket', 1) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 2) + project = messages.StringField(3, required=True) + projection = messages.EnumField('ProjectionValueValuesEnum', 4) + + +class StorageBucketsListRequest(messages.Message): + """A StorageBucketsListRequest object. + + Enums: + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. + + Fields: + maxResults: Maximum number of buckets to return. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + project: A valid API project identifier. + projection: Set of properties to return. Defaults to noAcl. + """ + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl. + + Values: + full: Include all properties. + noAcl: Omit acl and defaultObjectAcl properties. + """ + full = 0 + noAcl = 1 + + maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) + pageToken = messages.StringField(2) + project = messages.StringField(3, required=True) + projection = messages.EnumField('ProjectionValueValuesEnum', 4) + + +class StorageBucketsPatchRequest(messages.Message): + """A StorageBucketsPatchRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this bucket. + ProjectionValueValuesEnum: Set of properties to return. Defaults to full. + + Fields: + bucket: Name of a bucket. + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata conditional + on whether the bucket's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + projection: Set of properties to return. Defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this bucket. + + Values: + authenticatedRead: Project team owners get OWNER access, and + allAuthenticatedUsers get READER access. + private: Project team owners get OWNER access. + projectPrivate: Project team members get access according to their + roles. + publicRead: Project team owners get OWNER access, and allUsers get + READER access. + publicReadWrite: Project team owners get OWNER access, and allUsers get + WRITER access. + """ + authenticatedRead = 0 + private = 1 + projectPrivate = 2 + publicRead = 3 + publicReadWrite = 4 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to full. + + Values: + full: Include all properties. + noAcl: Omit acl and defaultObjectAcl properties. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + bucketResource = messages.MessageField('Bucket', 2) + ifMetagenerationMatch = messages.IntegerField(3) + ifMetagenerationNotMatch = messages.IntegerField(4) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) + projection = messages.EnumField('ProjectionValueValuesEnum', 6) + + +class StorageBucketsUpdateRequest(messages.Message): + """A StorageBucketsUpdateRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this bucket. + ProjectionValueValuesEnum: Set of properties to return. Defaults to full. + + Fields: + bucket: Name of a bucket. + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata conditional + on whether the bucket's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + projection: Set of properties to return. Defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this bucket. + + Values: + authenticatedRead: Project team owners get OWNER access, and + allAuthenticatedUsers get READER access. + private: Project team owners get OWNER access. + projectPrivate: Project team members get access according to their + roles. + publicRead: Project team owners get OWNER access, and allUsers get + READER access. + publicReadWrite: Project team owners get OWNER access, and allUsers get + WRITER access. + """ + authenticatedRead = 0 + private = 1 + projectPrivate = 2 + publicRead = 3 + publicReadWrite = 4 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to full. + + Values: + full: Include all properties. + noAcl: Omit acl and defaultObjectAcl properties. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + bucketResource = messages.MessageField('Bucket', 2) + ifMetagenerationMatch = messages.IntegerField(3) + ifMetagenerationNotMatch = messages.IntegerField(4) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) + projection = messages.EnumField('ProjectionValueValuesEnum', 6) + + +class StorageChannelsStopResponse(messages.Message): + """An empty StorageChannelsStop response.""" + + +class StorageDefaultObjectAccessControlsDeleteRequest(messages.Message): + """A StorageDefaultObjectAccessControlsDeleteRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + + +class StorageDefaultObjectAccessControlsDeleteResponse(messages.Message): + """An empty StorageDefaultObjectAccessControlsDelete response.""" + + +class StorageDefaultObjectAccessControlsGetRequest(messages.Message): + """A StorageDefaultObjectAccessControlsGetRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + + +class StorageDefaultObjectAccessControlsListRequest(messages.Message): + """A StorageDefaultObjectAccessControlsListRequest object. + + Fields: + bucket: Name of a bucket. + ifMetagenerationMatch: If present, only return default ACL listing if the + bucket's current metageneration matches this value. + ifMetagenerationNotMatch: If present, only return default ACL listing if + the bucket's current metageneration does not match the given value. + """ + + bucket = messages.StringField(1, required=True) + ifMetagenerationMatch = messages.IntegerField(2) + ifMetagenerationNotMatch = messages.IntegerField(3) + + +class StorageObjectAccessControlsDeleteRequest(messages.Message): + """A StorageObjectAccessControlsDeleteRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + generation = messages.IntegerField(3) + object = messages.StringField(4, required=True) + + +class StorageObjectAccessControlsDeleteResponse(messages.Message): + """An empty StorageObjectAccessControlsDelete response.""" + + +class StorageObjectAccessControlsGetRequest(messages.Message): + """A StorageObjectAccessControlsGetRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + generation = messages.IntegerField(3) + object = messages.StringField(4, required=True) + + +class StorageObjectAccessControlsInsertRequest(messages.Message): + """A StorageObjectAccessControlsInsertRequest object. + + Fields: + bucket: Name of a bucket. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + object = messages.StringField(3, required=True) + objectAccessControl = messages.MessageField('ObjectAccessControl', 4) + + +class StorageObjectAccessControlsListRequest(messages.Message): + """A StorageObjectAccessControlsListRequest object. + + Fields: + bucket: Name of a bucket. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + """ + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + object = messages.StringField(3, required=True) + + +class StorageObjectAccessControlsPatchRequest(messages.Message): + """A StorageObjectAccessControlsPatchRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + generation = messages.IntegerField(3) + object = messages.StringField(4, required=True) + objectAccessControl = messages.MessageField('ObjectAccessControl', 5) + + +class StorageObjectAccessControlsUpdateRequest(messages.Message): + """A StorageObjectAccessControlsUpdateRequest object. + + Fields: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + + bucket = messages.StringField(1, required=True) + entity = messages.StringField(2, required=True) + generation = messages.IntegerField(3) + object = messages.StringField(4, required=True) + objectAccessControl = messages.MessageField('ObjectAccessControl', 5) + + +class StorageObjectsComposeRequest(messages.Message): + """A StorageObjectsComposeRequest object. + + Enums: + DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access + controls to the destination object. + + Fields: + composeRequest: A ComposeRequest resource to be passed as the request + body. + destinationBucket: Name of the bucket in which to store the new object. + destinationObject: Name of the new object. + destinationPredefinedAcl: Apply a predefined set of access controls to the + destination object. + ifGenerationMatch: Makes the operation conditional on whether the object's + current generation matches the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + """ + + class DestinationPredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to the destination object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + composeRequest = messages.MessageField('ComposeRequest', 1) + destinationBucket = messages.StringField(2, required=True) + destinationObject = messages.StringField(3, required=True) + destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 4) + ifGenerationMatch = messages.IntegerField(5) + ifMetagenerationMatch = messages.IntegerField(6) + + +class StorageObjectsCopyRequest(messages.Message): + """A StorageObjectsCopyRequest object. + + Enums: + DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access + controls to the destination object. + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, + unless the object resource specifies the acl property, when it defaults + to full. + + Fields: + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's name + value, if any. + destinationPredefinedAcl: Apply a predefined set of access controls to the + destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether the + source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on whether + the source object's current metageneration does not match the given + value. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + sourceBucket: Name of the bucket in which to find the source object. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). + sourceObject: Name of the source object. + """ + + class DestinationPredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to the destination object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl, unless the object + resource specifies the acl property, when it defaults to full. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + destinationBucket = messages.StringField(1, required=True) + destinationObject = messages.StringField(2, required=True) + destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) + ifGenerationMatch = messages.IntegerField(4) + ifGenerationNotMatch = messages.IntegerField(5) + ifMetagenerationMatch = messages.IntegerField(6) + ifMetagenerationNotMatch = messages.IntegerField(7) + ifSourceGenerationMatch = messages.IntegerField(8) + ifSourceGenerationNotMatch = messages.IntegerField(9) + ifSourceMetagenerationMatch = messages.IntegerField(10) + ifSourceMetagenerationNotMatch = messages.IntegerField(11) + object = messages.MessageField('Object', 12) + projection = messages.EnumField('ProjectionValueValuesEnum', 13) + sourceBucket = messages.StringField(14, required=True) + sourceGeneration = messages.IntegerField(15) + sourceObject = messages.StringField(16, required=True) + + +class StorageObjectsDeleteRequest(messages.Message): + """A StorageObjectsDeleteRequest object. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, permanently deletes a specific revision of this + object (as opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the object's + current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + object: Name of the object. + """ + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + ifGenerationMatch = messages.IntegerField(3) + ifGenerationNotMatch = messages.IntegerField(4) + ifMetagenerationMatch = messages.IntegerField(5) + ifMetagenerationNotMatch = messages.IntegerField(6) + object = messages.StringField(7, required=True) + + +class StorageObjectsDeleteResponse(messages.Message): + """An empty StorageObjectsDelete response.""" + + +class StorageObjectsGetRequest(messages.Message): + """A StorageObjectsGetRequest object. + + Enums: + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the object's + generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + object: Name of the object. + projection: Set of properties to return. Defaults to noAcl. + """ + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + ifGenerationMatch = messages.IntegerField(3) + ifGenerationNotMatch = messages.IntegerField(4) + ifMetagenerationMatch = messages.IntegerField(5) + ifMetagenerationNotMatch = messages.IntegerField(6) + object = messages.StringField(7, required=True) + projection = messages.EnumField('ProjectionValueValuesEnum', 8) + + +class StorageObjectsInsertRequest(messages.Message): + """A StorageObjectsInsertRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this object. + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, + unless the object resource specifies the acl property, when it defaults + to full. + + Fields: + bucket: Name of the bucket in which to store the new object. Overrides the + provided object metadata's bucket value, if any. + contentEncoding: If set, sets the contentEncoding property of the final + object to this value. Setting this parameter is equivalent to setting + the contentEncoding metadata property. This can be useful when uploading + an object with uploadType=media to indicate the encoding of the content + being uploaded. + ifGenerationMatch: Makes the operation conditional on whether the object's + current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + name: Name of the object. Required when the object metadata is not + otherwise provided. Overrides the object metadata's name value, if any. + object: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl, unless the object + resource specifies the acl property, when it defaults to full. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + contentEncoding = messages.StringField(2) + ifGenerationMatch = messages.IntegerField(3) + ifGenerationNotMatch = messages.IntegerField(4) + ifMetagenerationMatch = messages.IntegerField(5) + ifMetagenerationNotMatch = messages.IntegerField(6) + name = messages.StringField(7) + object = messages.MessageField('Object', 8) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = messages.EnumField('ProjectionValueValuesEnum', 10) + + +class StorageObjectsListRequest(messages.Message): + """A StorageObjectsListRequest object. + + Enums: + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. + + Fields: + bucket: Name of the bucket in which to look for objects. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain delimiter + will have their name, truncated after the delimiter, returned in + prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As duplicate + prefixes are omitted, fewer total results may be returned than + requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. + """ + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + delimiter = messages.StringField(2) + maxResults = messages.IntegerField(3, variant=messages.Variant.UINT32) + pageToken = messages.StringField(4) + prefix = messages.StringField(5) + projection = messages.EnumField('ProjectionValueValuesEnum', 6) + versions = messages.BooleanField(7) + + +class StorageObjectsPatchRequest(messages.Message): + """A StorageObjectsPatchRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this object. + ProjectionValueValuesEnum: Set of properties to return. Defaults to full. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the object's + current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + object: Name of the object. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to full. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + ifGenerationMatch = messages.IntegerField(3) + ifGenerationNotMatch = messages.IntegerField(4) + ifMetagenerationMatch = messages.IntegerField(5) + ifMetagenerationNotMatch = messages.IntegerField(6) + object = messages.StringField(7, required=True) + objectResource = messages.MessageField('Object', 8) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = messages.EnumField('ProjectionValueValuesEnum', 10) + + +class StorageObjectsUpdateRequest(messages.Message): + """A StorageObjectsUpdateRequest object. + + Enums: + PredefinedAclValueValuesEnum: Apply a predefined set of access controls to + this object. + ProjectionValueValuesEnum: Set of properties to return. Defaults to full. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the object's + current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + object: Name of the object. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. + """ + + class PredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to this object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to full. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + generation = messages.IntegerField(2) + ifGenerationMatch = messages.IntegerField(3) + ifGenerationNotMatch = messages.IntegerField(4) + ifMetagenerationMatch = messages.IntegerField(5) + ifMetagenerationNotMatch = messages.IntegerField(6) + object = messages.StringField(7, required=True) + objectResource = messages.MessageField('Object', 8) + predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = messages.EnumField('ProjectionValueValuesEnum', 10) + + +class StorageObjectsWatchAllRequest(messages.Message): + """A StorageObjectsWatchAllRequest object. + + Enums: + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. + + Fields: + bucket: Name of the bucket in which to look for objects. + channel: A Channel resource to be passed as the request body. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain delimiter + will have their name, truncated after the delimiter, returned in + prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As duplicate + prefixes are omitted, fewer total results may be returned than + requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. + """ + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + bucket = messages.StringField(1, required=True) + channel = messages.MessageField('Channel', 2) + delimiter = messages.StringField(3) + maxResults = messages.IntegerField(4, variant=messages.Variant.UINT32) + pageToken = messages.StringField(5) + prefix = messages.StringField(6) + projection = messages.EnumField('ProjectionValueValuesEnum', 7) + versions = messages.BooleanField(8) + + diff --git a/samples/storage_sample/testdata/fifteen_byte_file b/samples/storage_sample/testdata/fifteen_byte_file new file mode 100644 index 0000000..6b665aa --- /dev/null +++ b/samples/storage_sample/testdata/fifteen_byte_file @@ -0,0 +1,4 @@ +a +ab +abc +abcde diff --git a/samples/storage_sample/testdata/filename_with_spaces b/samples/storage_sample/testdata/filename_with_spaces new file mode 100644 index 0000000..6b665aa --- /dev/null +++ b/samples/storage_sample/testdata/filename_with_spaces @@ -0,0 +1,4 @@ +a +ab +abc +abcde diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py new file mode 100644 index 0000000..6d01828 --- /dev/null +++ b/samples/storage_sample/uploads_test.py @@ -0,0 +1,141 @@ +"""Integration tests for uploading and downloading to GCS. + +These tests exercise most of the corner cases for upload/download of +files in apitools, via GCS. There are no performance tests here yet. +""" + +import json +import os +import random +import string +import StringIO +import unittest + +import apitools.base.py as apitools_base +import storage + +_CLIENT = None +def _GetClient(): + global _CLIENT + if _CLIENT is None: + _CLIENT = storage.StorageV1() + return _CLIENT + +class UploadsTest(unittest.TestCase): + _DEFAULT_BUCKET = 'apitools' + _TESTDATA_PREFIX = 'uploads' + + def setUp(self): + self.__client = _GetClient() + self.__files = [] + self.__content = '' + self.__buffer = None + self.__upload = None + + def tearDown(self): + self.__DeleteFiles() + + def __ResetUpload(self, size, auto_transfer=True): + self.__content = ''.join( + random.choice(string.letters) for _ in xrange(size)) + self.__buffer = StringIO.StringIO(self.__content) + self.__upload = storage.Upload.FromStream( + self.__buffer, 'text/plain', auto_transfer=auto_transfer) + + def __DeleteFiles(self): + for filename in self.__files: + self.__DeleteFile(filename) + + def __DeleteFile(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + req = storage.StorageObjectsDeleteRequest( + bucket=self._DEFAULT_BUCKET, object=object_name) + self.__client.objects.Delete(req) + + def __InsertRequest(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + return storage.StorageObjectsInsertRequest( + name=object_name, bucket=self._DEFAULT_BUCKET) + + def __GetRequest(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + return storage.StorageObjectsGetRequest( + object=object_name, bucket=self._DEFAULT_BUCKET) + + def __InsertFile(self, filename, request=None): + if request is None: + request = self.__InsertRequest(filename) + response = self.__client.objects.Insert(request, upload=self.__upload) + self.assertIsNotNone(response) + self.__files.append(filename) + return response + + def testZeroBytes(self): + filename = 'zero_byte_file' + self.__ResetUpload(0) + response = self.__InsertFile(filename) + self.assertEqual(0, response.size) + + def testSimpleUpload(self): + filename = 'fifteen_byte_file' + self.__ResetUpload(15) + response = self.__InsertFile(filename) + self.assertEqual(15, response.size) + + def testMultipartUpload(self): + filename = 'fifteen_byte_file' + self.__ResetUpload(15) + request = self.__InsertRequest(filename) + request.object = storage.Object(contentLanguage='en') + response = self.__InsertFile(filename, request=request) + self.assertEqual(15, response.size) + self.assertEqual('en', response.contentLanguage) + + def testAutoUpload(self): + filename = 'ten_meg_file' + size = 10 << 20 + self.__ResetUpload(size) + request = self.__InsertRequest(filename) + response = self.__InsertFile(filename, request=request) + self.assertEqual(size, response.size) + + def testBreakAndResumeUpload(self): + filename = 'ten_meg_file_' + ''.join(random.sample(string.letters, 5)) + size = 10 << 20 + self.__ResetUpload(size, auto_transfer=False) + self.__upload.strategy = 'resumable' + self.__upload.total_size = size + # Start the upload + request = self.__InsertRequest(filename) + initial_response = self.__client.objects.Insert( + request, upload=self.__upload) + self.assertIsNotNone(initial_response) + self.assertEqual(0, self.__buffer.tell()) + # Pretend the process died, and resume with a new attempt at the + # same upload. + upload_data = json.dumps(self.__upload.serialization_data) + second_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + second_upload_attempt._Upload__SendChunk(0) + self.assertEqual(second_upload_attempt.chunksize, self.__buffer.tell()) + # Simulate a third try, and stream from there. + final_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + final_upload_attempt.StreamInChunks() + self.assertEqual(size, self.__buffer.tell()) + # Verify the upload + object_info = self.__client.objects.Get(self.__GetRequest(filename)) + self.assertEqual(size, object_info.size) + # Confirm that a new attempt successfully does nothing. + completed_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + self.assertTrue(completed_upload_attempt.complete) + completed_upload_attempt.StreamInChunks() + # Verify the upload didn't pick up extra bytes. + object_info = self.__client.objects.Get(self.__GetRequest(filename)) + self.assertEqual(size, object_info.size) + # TODO(craigcitro): Add tests for callbacks (especially around + # finish callback). + +if __name__ == '__main__': + unittest.main() -- GitLab From 6f33bb715909fe80d6f896d6e4e1deffab129137 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 17 Oct 2014 16:45:38 -0700 Subject: [PATCH 036/295] Drop python 2.6 testing for now. This is due to apputils causing grief; we'll switch off apputils soon. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a4e70e7..f25e289 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27 +envlist = py27 [testenv] deps = nose -- GitLab From c0642cfb6fa3ad34402058791fb61d37365101a4 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 17 Oct 2014 16:58:03 -0700 Subject: [PATCH 037/295] Drop python 2.6 in travis (not just tox). --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a593c03..b9a9a4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python env: - - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=pypy install: -- GitLab From ba319199c1fbe81f6846abca98b6989891a2a34f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 5 Nov 2014 18:34:39 -0800 Subject: [PATCH 038/295] Update one test to use unittest. I'm likely to do this for all tests, but this one's a start. --- apitools/gen/client_generation_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 141eb4f..83a528f 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -7,8 +7,7 @@ import shutil import subprocess import sys import tempfile - -from google.apputils import basetest as googletest +import unittest from apitools.gen import util @@ -31,7 +30,7 @@ def TempDir(): shutil.rmtree(path) -class ClientGenerationTest(googletest.TestCase): +class ClientGenerationTest(unittest.TestCase): def setUp(self): super(ClientGenerationTest, self).setUp() @@ -73,4 +72,4 @@ class ClientGenerationTest(googletest.TestCase): if __name__ == '__main__': - googletest.main() + unittest.main() -- GitLab From f5490d4cef245439e8f4b5f084f549c2a1aa45eb Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 5 Nov 2014 18:38:45 -0800 Subject: [PATCH 039/295] Small test tweaks. --- apitools/gen/client_generation_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 83a528f..06e10d0 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -9,8 +9,6 @@ import sys import tempfile import unittest -from apitools.gen import util - _API_LIST = [ 'drive.v2', 'bigquery.v2', @@ -18,6 +16,7 @@ _API_LIST = [ 'storage.v1', ] + @contextlib.contextmanager def TempDir(): original_dir = os.getcwd() @@ -44,7 +43,7 @@ class ClientGenerationTest(unittest.TestCase): # unittest in 2.6 doesn't have skipIf. return for api in _API_LIST: - with TempDir() as path: + with TempDir(): args = [ self.gen_client_binary, '--client_id=12345', @@ -54,7 +53,8 @@ class ClientGenerationTest(unittest.TestCase): '--overwrite', 'client', ] - logging.info('Testing API %s with command line: %s', api, ' '.join(args)) + logging.info( + 'Testing API %s with command line: %s', api, ' '.join(args)) retcode = subprocess.call(args) if retcode == 128: logging.error('Failed to fetch discovery doc, continuing.') -- GitLab From c5825cc0b47b4036fbddf14117e4edcc19267802 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 6 Nov 2014 14:55:51 -0800 Subject: [PATCH 040/295] Code drop from internal. --- apitools/__init__.py | 1 + apitools/base/__init__.py | 4 + apitools/base/py/__init__.py | 10 +- apitools/base/py/app2.py | 5 +- apitools/base/py/base_api.py | 110 ++++++++------ apitools/base/py/base_api_test.py | 68 ++++++++- apitools/base/py/base_cli.py | 8 +- apitools/base/py/batch.py | 135 ++++++++++------- apitools/base/py/cli.py | 13 ++ apitools/base/py/credentials_lib.py | 4 +- apitools/base/py/encoding.py | 197 ++++++++++++++++++++++++- apitools/base/py/encoding_test.py | 129 +++++++++++++++- apitools/base/py/exceptions.py | 4 + apitools/base/py/extra_types.py | 57 ++++++- apitools/base/py/extra_types_test.py | 40 ++++- apitools/base/py/http_wrapper.py | 42 ++++-- apitools/base/py/list_pager.py | 49 ++++++ apitools/base/py/transfer.py | 26 ++-- apitools/base/py/util.py | 106 ++++++++++++- apitools/gen/__init__.py | 4 + apitools/gen/client_generation_test.py | 1 + apitools/gen/command_registry.py | 53 ++++--- apitools/gen/extended_descriptor.py | 57 ++++--- apitools/gen/gen_client.py | 47 +++++- apitools/gen/gen_client_lib.py | 8 +- apitools/gen/message_registry.py | 63 ++++---- apitools/gen/service_registry.py | 46 +++--- apitools/gen/util.py | 22 ++- apitools_public.tar | Bin 0 -> 307200 bytes setup.py | 6 +- 30 files changed, 1052 insertions(+), 263 deletions(-) create mode 100644 apitools/base/py/cli.py create mode 100644 apitools/base/py/list_pager.py create mode 100755 apitools_public.tar diff --git a/apitools/__init__.py b/apitools/__init__.py index 1847842..54fa3d5 100644 --- a/apitools/__init__.py +++ b/apitools/__init__.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +"""Shared __init__.py for apitools.""" from pkgutil import extend_path __path__ = extend_path(__path__, __name__) diff --git a/apitools/base/__init__.py b/apitools/base/__init__.py index 4265cc3..54fa3d5 100644 --- a/apitools/base/__init__.py +++ b/apitools/base/__init__.py @@ -1 +1,5 @@ #!/usr/bin/env python +"""Shared __init__.py for apitools.""" + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py index a0f920e..cbf7f86 100644 --- a/apitools/base/py/__init__.py +++ b/apitools/base/py/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Top-level imports for apitools base files.""" +# pylint:disable=wildcard-import from apitools.base.py.base_api import * from apitools.base.py.batch import * from apitools.base.py.credentials_lib import * @@ -8,14 +9,7 @@ from apitools.base.py.encoding import * from apitools.base.py.exceptions import * from apitools.base.py.extra_types import * from apitools.base.py.http_wrapper import * +from apitools.base.py.list_pager import * from apitools.base.py.transfer import * from apitools.base.py.util import * -try: - from apitools.base.py.app2 import * - from apitools.base.py.base_cli import * - # pylint: enable=g-import-not-at-top -except ImportError: - # We want to allow this to fail in some cases, such as importing on - # GAE. - pass diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 80b36ed..2a90d55 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -16,7 +16,7 @@ import gflags as flags __all__ = [ 'NewCmd', 'Repl', - ] +] flags.DEFINE_boolean( 'debug_mode', False, @@ -48,7 +48,8 @@ class NewCmd(appcommands.Cmd): argspec = inspect.getargspec(func) if argspec.args and argspec.args[0] == 'self': - argspec = argspec._replace(args=argspec.args[1:]) # pylint: disable=protected-access,g-line-too-long + argspec = argspec._replace( # pylint: disable=protected-access + args=argspec.args[1:]) self._argspec = argspec # TODO(craigcitro): Do we really want to support all this # nonsense? diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index ad1eafd..2ee5fcf 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -2,6 +2,7 @@ """Base class for api services.""" import contextlib +import datetime import httplib import logging import pprint @@ -25,7 +26,7 @@ __all__ = [ 'BaseApiClient', 'BaseApiService', 'NormalizeApiEndpoint', - ] +] # TODO(craigcitro): Remove this once we quiet the spurious logging in # oauth2client (or drop oauth2client). @@ -137,7 +138,7 @@ class _UrlBuilder(object): self.query_params.update(query_params) self.__scheme = components.scheme self.__netloc = components.netloc - self.relative_path = components.path + self.relative_path = components.path or '' @classmethod def FromUrl(cls, url): @@ -145,7 +146,7 @@ class _UrlBuilder(object): query_params = urlparse.parse_qs(urlparts.query) base_url = urlparse.urlunsplit(( urlparts.scheme, urlparts.netloc, '', None, None)) - relative_path = urlparts.path + relative_path = urlparts.path or '' return cls(base_url, relative_path=relative_path, query_params=query_params) @property @@ -190,10 +191,9 @@ class BaseApiClient(object): def __init__(self, url, credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, num_retries=5, - credentials_args=None, default_global_params=None): - _RequireClassAttrs(self, ( - '_package', '_scopes', '_client_id', '_client_secret', - 'messages_module')) + credentials_args=None, default_global_params=None, + additional_http_headers=None): + _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) if default_global_params is not None: util.Typecheck(default_global_params, self.params_type) self.__default_global_params = default_global_params @@ -202,11 +202,11 @@ class BaseApiClient(object): self.__num_retries = 5 # We let the @property machinery below do our validation. self.num_retries = num_retries - self._url = url self._credentials = credentials if get_credentials and not credentials: credentials_args = credentials_args or {} self._SetCredentials(**credentials_args) + self._url = NormalizeApiEndpoint(url) self._http = http or http_wrapper.GetHttp() # Note that "no credentials" is totally possible. if self._credentials is not None: @@ -214,9 +214,13 @@ class BaseApiClient(object): # TODO(craigcitro): Remove this field when we switch to proto2. self.__include_fields = None + self.additional_http_headers = additional_http_headers or {} + # TODO(craigcitro): Finish deprecating these fields. _ = model + self.__response_type_model = 'proto' + def _SetCredentials(self, **kwds): """Fetch credentials, and set them for this client. @@ -253,7 +257,7 @@ class BaseApiClient(object): 'client_secret': cls._CLIENT_SECRET, 'scope': ' '.join(sorted(util.NormalizeScopes(cls._SCOPES))), 'user_agent': cls._USER_AGENT, - } + } @property def base_model_class(self): @@ -299,6 +303,18 @@ class BaseApiClient(object): yield self.__include_fields = None + @property + def response_type_model(self): + return self.__response_type_model + + @contextlib.contextmanager + def JsonResponseModel(self): + """In this context, return raw JSON instead of proto.""" + old_model = self.response_type_model + self.__response_type_model = 'json' + yield + self.__response_type_model = old_model + @property def num_retries(self): return self.__num_retries @@ -328,6 +344,7 @@ class BaseApiClient(object): def ProcessHttpRequest(self, http_request): """Hook for pre-processing of http requests.""" + http_request.headers.update(self.additional_http_headers) if self.log_request: logging.info('Making http %s to %s', http_request.http_method, http_request.url) @@ -374,11 +391,33 @@ class BaseApiService(object): def __init__(self, client): self.__client = client + self._method_configs = {} + self._upload_configs = {} @property def _client(self): return self.__client + @property + def client(self): + return self.__client + + def GetMethodConfig(self, method): + return self._method_configs[method] + + def GetUploadConfig(self, method): + return self._upload_configs.get(method) + + def GetRequestType(self, method): + method_config = self.GetMethodConfig(method) + return getattr(self.client.MESSAGES_MODULE, + method_config.request_type_name) + + def GetResponseType(self, method): + method_config = self.GetMethodConfig(method) + return getattr(self.client.MESSAGES_MODULE, + method_config.response_type_name) + def __CombineGlobalParams(self, global_params, default_params): util.Typecheck(global_params, (types.NoneType, self.__client.params_type)) result = self.__client.params_type() @@ -405,42 +444,16 @@ class BaseApiService(object): query_info[k] = v.encode('utf8') elif isinstance(v, str): query_info[k] = v.decode('utf8') + elif isinstance(v, datetime.datetime): + query_info[k] = v.isoformat() return query_info def __ConstructRelativePath(self, method_config, request, relative_path=None): """Determine the relative path for request.""" - path = relative_path or method_config.relative_path - # TODO(user): Why does the discovery document have pluses? - # Figure this out. - path = path.replace('+', '') - - for param in method_config.path_params: - param_template = '{%s}' % param - if param_template not in path: - raise exceptions.InvalidUserInputError( - 'Missing path parameter %s' % param) - try: - # TODO(craigcitro): Do we want to support some sophisticated - # mapping here? - value = getattr(request, param) - except AttributeError: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - if value is None: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - try: - # TODO(user): this isn't likely to break anything, but if you notice - # that it does please contact me and I'll get a concrete fix in ASAP - if not isinstance(value, basestring): - value = str(value) - path = path.replace(param_template, - urllib.quote(value.encode('utf_8'), '')) - except TypeError as e: - raise exceptions.InvalidUserInputError( - 'Error setting required parameter %s to value %s: %s' % ( - param, value, e)) - return path + params = dict([(param, getattr(request, param, None)) + for param in method_config.path_params]) + return util.ExpandRelativePath(method_config, params, + relative_path=relative_path) def __FinalizeRequest(self, http_request, url_builder): """Make any final general adjustments to the request.""" @@ -462,10 +475,13 @@ class BaseApiService(object): http_response = http_wrapper.Response( info=http_response.info, content='{}', request_url=http_response.request_url) - response_type = _LoadClass( - method_config.response_type_name, self.__client.MESSAGES_MODULE) - return self.__client.DeserializeMessage( - response_type, http_response.content) + if self.__client.response_type_model == 'json': + return http_response.content + else: + response_type = _LoadClass( + method_config.response_type_name, self.__client.MESSAGES_MODULE) + return self.__client.DeserializeMessage( + response_type, http_response.content) def __SetBaseHeaders(self, http_request, client): """Fill in the basic headers on http_request.""" @@ -548,12 +564,12 @@ class BaseApiService(object): # objects, and pass in self.__client.num_retries when initializing # an upload or download. if download is not None: - download.InitializeDownload(http_request, client=self._client) + download.InitializeDownload(http_request, client=self.client) return http_response = None if upload is not None: - http_response = upload.InitializeUpload(http_request, client=self._client) + http_response = upload.InitializeUpload(http_request, client=self.client) if http_response is None: http_response = http_wrapper.MakeRequest( self.__client.http, http_request, retries=self.__client.num_retries) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 9f971c9..4ea978f 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -1,34 +1,61 @@ #!/usr/bin/env python +import datetime +import sys +import urllib + +from protorpc import message_types from protorpc import messages from google.apputils import basetest as googletest from apitools.base.py import base_api +from apitools.base.py import http_wrapper class SimpleMessage(messages.Message): field = messages.StringField(1) +class MessageWithTime(messages.Message): + timestamp = message_types.DateTimeField(1) + + +class StandardQueryParameters(messages.Message): + field = messages.StringField(1) + + class FakeCredentials(object): + def authorize(self, _): # pylint: disable=invalid-name return None class FakeClient(base_api.BaseApiClient): - MESSAGES_MODULE = 'message_module' + MESSAGES_MODULE = sys.modules[__name__] _PACKAGE = 'package' _SCOPES = ['scope1'] _CLIENT_ID = 'client_id' _CLIENT_SECRET = 'client_secret' +class FakeService(base_api.BaseApiService): + + def __init__(self, client=None): + client = client or FakeClient( + 'http://www.example.com/', credentials=FakeCredentials()) + super(FakeService, self).__init__(client) + + class BaseApiTest(googletest.TestCase): def __GetFakeClient(self): return FakeClient('', credentials=FakeCredentials()) + def testUrlNormalization(self): + client = FakeClient('http://www.googleapis.com', get_credentials=False) + self.assertTrue(client.url.endswith('/')) + def testNoCredentials(self): client = FakeClient('', get_credentials=False) self.assertIsNotNone(client) @@ -41,6 +68,45 @@ class BaseApiTest(googletest.TestCase): with client.IncludeFields(('field',)): self.assertEqual('{"field": null}', client.SerializeMessage(msg)) + def testJsonResponse(self): + method_config = base_api.ApiMethodInfo(response_type_name='SimpleMessage') + service = FakeService() + http_response = http_wrapper.Response( + info={'status': '200'}, content='{"field": "abc"}', + request_url='http://www.google.com') + response_message = SimpleMessage(field='abc') + self.assertEqual(response_message, service.ProcessHttpResponse( + method_config, http_response)) + with service.client.JsonResponseModel(): + self.assertEqual(http_response.content, service.ProcessHttpResponse( + method_config, http_response)) + + def testAdditionalHeaders(self): + additional_headers = {'Request-Is-Awesome': '1'} + client = self.__GetFakeClient() + + # No headers to start + http_request = http_wrapper.Request('http://www.example.com') + new_request = client.ProcessHttpRequest(http_request) + self.assertFalse('Request-Is-Awesome' in new_request.headers) + + # Add a new header and ensure it's added to the request. + client.additional_http_headers = additional_headers + http_request = http_wrapper.Request('http://www.example.com') + new_request = client.ProcessHttpRequest(http_request) + self.assertTrue('Request-Is-Awesome' in new_request.headers) + + def testQueryEncoding(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithTime', query_params=['timestamp']) + service = FakeService() + request = MessageWithTime( + timestamp=datetime.datetime(2014, 10, 07, 12, 53, 13)) + http_request = service.PrepareHttpRequest(method_config, request) + + url_timestamp = urllib.quote(request.timestamp.isoformat()) + self.assertTrue(http_request.url.endswith(url_timestamp)) + if __name__ == '__main__': googletest.main() diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index ee5192b..f9d7d1a 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -22,7 +22,7 @@ __all__ = [ 'FormatOutput', 'SetupLogger', 'run_main', - ] +] # TODO(craigcitro): We should move all the flags for the @@ -33,7 +33,7 @@ _BASE_FLAGS_DECLARED = False _OUTPUT_FORMATTER_MAP = { 'protorpc': lambda x: x, 'json': encoding.MessageToJson, - } +} def DeclareBaseFlags(): @@ -56,6 +56,7 @@ def DeclareBaseFlags(): 'protorpc', _OUTPUT_FORMATTER_MAP.viewkeys(), 'Display format for results.') + _BASE_FLAGS_DECLARED = True # NOTE: This is specified here so that it can be read by other files @@ -82,6 +83,7 @@ def FormatOutput(message, output_format=None): class _SmartCompleter(rlcompleter.Completer): + def _callable_postfix(self, val, word): if ('(' in readline.get_line_buffer() or not callable(val)): @@ -107,7 +109,7 @@ class ConsoleWithReadline(code.InteractiveConsole): '_SmartCompleter': _SmartCompleter, 'readline': readline, 'rlcompleter': rlcompleter, - }) + }) code.InteractiveConsole.__init__(self, new_locals, filename) readline.parse_and_bind('tab: complete') readline.set_completer(_SmartCompleter(new_locals).complete) diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index c5f8567..eaf5eba 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -1,16 +1,12 @@ #!/usr/bin/env python -"""HTTP batch requests for apitools. - -This library is copied heavily from apiclient's BatchHttpRequest. -Some unneeded parts are removed, and apitools' BatchHttpRequest is -modified to work with the classes in http_wrapper instead of httplib2. -""" +"""Library for handling batch HTTP requests for apitools.""" import collections import email.generator as generator import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart import email.parser as email_parser +import httplib import itertools import StringIO import time @@ -23,29 +19,54 @@ from apitools.base.py import http_wrapper __all__ = [ 'BatchApiRequest', - ] +] + + +class RequestResponseAndHandler(collections.namedtuple( + 'RequestResponseAndHandler', ['request', 'response', 'handler'])): + """Container for data related to completing an HTTP request. + This contains an HTTP request, its response, and a callback for handling + the response from the server. -RequestResponseHandler = collections.namedtuple( - 'RequestResponseHandler', ['request', 'response', 'handler']) + Attributes: + request: An http_wrapper.Request object representing the HTTP request. + response: The http_wrapper.Response object returned from the server. + handler: A callback function accepting two arguments, response + and exception. Response is an http_wrapper.Response object, and + exception is an apiclient.errors.HttpError object if an error + occurred, or otherwise None. + """ class BatchApiRequest(object): - """Friendly interface for batching API requests.""" - class ApiRequestResponse(object): - """Each individual request is stored in its own object.""" + class ApiCall(object): + """Holds request and response information for each request. + + ApiCalls are ultimately exposed to the client once the HTTP batch request + has been completed. + + Attributes: + http_request: A client-supplied http_wrapper.Request to be + submitted to the server. + response: A http_wrapper.Response object given by the server as a + response to the user request, or None if an error occurred. + exception: An apiclient.errors.HttpError object if an error + occurred, or None. + """ def __init__(self, request, retryable_codes, service, method_config): """Initialize an individual API request. Args: request: An http_wrapper.Request object. - retryable_codes: A list of HTTP codes that can be retried. + retryable_codes: A list of integer HTTP codes that can be retried. service: A service inheriting from base_api.BaseApiService. method_config: Method config for the desired API request. """ - self.__retryable_codes = retryable_codes + self.__retryable_codes = list( + set(retryable_codes + [httplib.UNAUTHORIZED])) self.__http_response = None self.__service = service self.__method_config = method_config @@ -67,13 +88,21 @@ class BatchApiRequest(object): def exception(self): return self.__exception + @property + def authorization_failed(self): + return (self.__http_response and ( + self.__http_response.status_code == httplib.UNAUTHORIZED)) + @property def terminal_state(self): return (self.__http_response and ( self.__http_response.status_code not in self.__retryable_codes)) def HandleResponse(self, http_response, exception): - """Callback used with BatchApiRequest. + """Handles an incoming http response to the request in http_request. + + This is intended to be used as a callback function for + BatchHttpRequest.Add. Args: http_response: Deserialized http_wrapper.Response object. @@ -81,7 +110,7 @@ class BatchApiRequest(object): """ self.__http_response = http_response self.__exception = exception - if self.terminal_state: + if self.terminal_state and not self.__exception: self.__response = self.__service.ProcessHttpResponse( self.__method_config, self.__http_response) @@ -90,7 +119,7 @@ class BatchApiRequest(object): Args: batch_url: Base URL for batch API calls. - retryable_codes: A list of HTTP codes that can be retried. + retryable_codes: A list of integer HTTP codes that can be retried. """ self.api_requests = [] self.retryable_codes = retryable_codes or [] @@ -101,17 +130,18 @@ class BatchApiRequest(object): Args: service: A class inheriting base_api.BaseApiService. - method: The desired method from the service. + method: A string indicated desired method from the service. See + the example in the class docstring. request: An input message appropriate for the specified service.method. global_params: Optional additional parameters to pass into - method.PrepareHttpRequest. + method.PrepareHttpRequest. Returns: None """ # Retrieve the configs for the desired method and service. method_config = service.GetMethodConfig(method) - upload_config = service.GetMethodUploadConfig(method) + upload_config = service.GetUploadConfig(method) # Prepare the HTTP Request. http_request = service.PrepareHttpRequest( @@ -119,7 +149,7 @@ class BatchApiRequest(object): upload_config=upload_config) # Create the request and add it to our master list. - api_request = self.ApiRequestResponse( + api_request = self.ApiCall( http_request, self.retryable_codes, service, method_config) self.api_requests.append(api_request) @@ -128,13 +158,13 @@ class BatchApiRequest(object): Args: http: httplib2.Http object for use in the request. - sleep_between_polls: How long to sleep between polls, in seconds. + sleep_between_polls: Integer number of seconds to sleep between polls. max_retries: Max retries. Any requests that have not succeeded by - this number of retries simply report the last response or - exception, whatever it happened to be. + this number of retries simply report the last response or + exception, whatever it happened to be. Returns: - List of ApiRequestResponses. + List of ApiCalls. """ requests = [request for request in self.api_requests if not request.terminal_state] @@ -154,6 +184,10 @@ class BatchApiRequest(object): requests = [request for request in self.api_requests if not request.terminal_state] + if (any(request.authorization_failed for request in requests) + and hasattr(http.request, 'credentials')): + http.request.credentials.refresh(http) + if not requests: break @@ -167,8 +201,8 @@ class BatchHttpRequest(object): """Constructor for a BatchHttpRequest. Args: - batch_url: string, URL to send batch requests to. - callback: callable, A callback to be called for each response, of the + batch_url: URL to send batch requests to. + callback: A callback to be called for each response, of the form callback(response, exception). The first parameter is the deserialized Response object. The second is an apiclient.errors.HttpError exception object if an HTTP error @@ -189,11 +223,11 @@ class BatchHttpRequest(object): # Unique ID on which to base the Content-ID headers. self.__base_id = uuid.uuid4() - def _IdToHeader(self, request_id): + def _ConvertIdToHeader(self, request_id): """Convert an id to a Content-ID header value. Args: - request_id: string, identifier of individual request. + request_id: String identifier for a individual request. Returns: A Content-ID header with the id_ encoded into it. A UUID is prepended to @@ -202,14 +236,15 @@ class BatchHttpRequest(object): """ return '<%s+%s>' % (self.__base_id, urllib.quote(request_id)) - def _HeaderToId(self, header): + @staticmethod + def _ConvertHeaderToId(header): """Convert a Content-ID header value to an id. - Presumes the Content-ID header conforms to the format that _IdToHeader() - returns. + Presumes the Content-ID header conforms to the format that + _ConvertIdToHeader() returns. Args: - header: string, Content-ID header value. + header: A string indicating the Content-ID header value. Returns: The extracted id value. @@ -229,7 +264,7 @@ class BatchHttpRequest(object): """Convert a http_wrapper.Request object into a string. Args: - request: http_wrapper.Request, the request to serialize. + request: A http_wrapper.Request to serialize. Returns: The request as a string in application/http format. @@ -242,11 +277,10 @@ class BatchHttpRequest(object): major, minor = request.headers.get( 'content-type', 'application/json').split('/') msg = mime_nonmultipart.MIMENonMultipart(major, minor) - headers = request.headers.copy() # MIMENonMultipart adds its own Content-Type header. # Keep all of the other headers in headers. - for key, value in headers.iteritems(): + for key, value in request.headers.iteritems(): if key == 'content-type': continue msg[key] = value @@ -258,11 +292,11 @@ class BatchHttpRequest(object): msg.set_payload(request.body) # Serialize the mime message. - fp = StringIO.StringIO() + str_io = StringIO.StringIO() # maxheaderlen=0 means don't line wrap headers. - g = generator.Generator(fp, maxheaderlen=0) - g.flatten(msg, unixfrom=False) - body = fp.getvalue() + gen = generator.Generator(str_io, maxheaderlen=0) + gen.flatten(msg, unixfrom=False) + body = str_io.getvalue() # Strip off the \n\n that the MIME lib tacks onto the end of the payload. if request.body is None: @@ -274,7 +308,7 @@ class BatchHttpRequest(object): """Convert string into Response and content. Args: - payload: string, headers and body as a string. + payload: Header and body string to be deserialized. Returns: A Response object @@ -302,7 +336,7 @@ class BatchHttpRequest(object): Auto incrementing number that avoids conflicts with ids already used. Returns: - string, a new unique id. + A new unique id string. """ return str(self.__last_auto_id.next()) @@ -310,8 +344,8 @@ class BatchHttpRequest(object): """Add a new request. Args: - request: http_wrapper.Request, http_wrapper.Request to add to the batch. - callback: callable, A callback to be called for this response, of the + request: A http_wrapper.Request to add to the batch. + callback: A callback to be called for this response, of the form callback(response, exception). The first parameter is the deserialized response object. The second is an apiclient.errors.HttpError exception object if an HTTP error @@ -320,14 +354,14 @@ class BatchHttpRequest(object): Returns: None """ - self.__request_response_handlers[self._NewId()] = RequestResponseHandler( + self.__request_response_handlers[self._NewId()] = RequestResponseAndHandler( request, None, callback) def _Execute(self, http): """Serialize batch request, send to server, process response. Args: - http: httplib2.Http, an http object to be used to make the request with. + http: A httplib2.Http object to be used to make the request with. Raises: httplib2.HttpLib2Error if a transport error has occured. @@ -341,7 +375,7 @@ class BatchHttpRequest(object): for key in self.__request_response_handlers: msg = mime_nonmultipart.MIMENonMultipart('application', 'http') msg['Content-Transfer-Encoding'] = 'binary' - msg['Content-ID'] = self._IdToHeader(key) + msg['Content-ID'] = self._ConvertIdToHeader(key) body = self._SerializeRequest( self.__request_response_handlers[key].request) @@ -368,9 +402,11 @@ class BatchHttpRequest(object): raise exceptions.BatchError('Response not in multipart/mixed format.') for part in mime_response.get_payload(): - request_id = self._HeaderToId(part['Content-ID']) + request_id = self._ConvertHeaderToId(part['Content-ID']) response = self._DeserializeResponse(part.get_payload()) + # Disable protected access because namedtuple._replace(...) + # is not actually meant to be protected. self.__request_response_handlers[request_id] = ( self.__request_response_handlers[request_id]._replace( # pylint: disable=protected-access response=response)) @@ -379,7 +415,7 @@ class BatchHttpRequest(object): """Execute all the requests as a single batched HTTP request. Args: - http: httplib2.Http object to be used with the request. + http: A httplib2.Http object to be used with the request. Returns: None @@ -398,7 +434,6 @@ class BatchHttpRequest(object): if response.status_code >= 300: exception = exceptions.HttpError.FromResponse(response) - response = None if callback is not None: callback(response, exception) diff --git a/apitools/base/py/cli.py b/apitools/base/py/cli.py new file mode 100644 index 0000000..b24470b --- /dev/null +++ b/apitools/base/py/cli.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +"""Top-level import for all CLI-related functionality in apitools. + +Note that importing this file will ultimately have side-effects, and +may require imports not available in all environments (such as App +Engine). In particular, picking up some readline-related imports can +cause pain. +""" + +# pylint:disable=wildcard-import + +from apitools.base.py.app2 import * +from apitools.base.py.base_cli import * diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 6c673b0..e467a39 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -27,7 +27,7 @@ __all__ = [ 'GetCredentials', 'ServiceAccountCredentials', 'ServiceAccountCredentialsFromFile', - ] +] @@ -45,7 +45,7 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, 'client_secret': client_secret, 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), 'user_agent': user_agent or '%s-generated/0.1' % package_name, - } + } if service_account_name is not None: credentials = ServiceAccountCredentialsFromFile( service_account_name, service_account_keyfile, scopes) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 6853cc8..c44897f 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -3,9 +3,12 @@ import base64 import collections +import datetime import json +import logging +from protorpc import message_types from protorpc import messages from protorpc import protojson @@ -19,7 +22,8 @@ __all__ = [ 'MessageToDict', 'PyValueToMessage', 'MessageToPyValue', - ] + 'MessageToRepr', +] _Codec = collections.namedtuple('_Codec', ['encoder', 'decoder']) @@ -103,6 +107,110 @@ def MessageToPyValue(message): return json.loads(MessageToJson(message)) +def MessageToRepr(msg, multiline=False, **kwargs): + """Return a repr-style string for a protorpc message. + + protorpc.Message.__repr__ does not return anything that could be considered + python code. Adding this function lets us print a protorpc message in such + a way that it could be pasted into code later, and used to compare against + other things. + + Args: + msg: protorpc.Message, the message to be repr'd. + multiline: bool, True if the returned string should have each field + assignment on its own line. + **kwargs: {str:str}, Additional flags for how to format the string. + + Known **kwargs: + shortstrings: bool, True if all string values should be truncated at + 100 characters, since when mocking the contents typically don't matter + except for IDs, and IDs are usually less than 100 characters. + no_modules: bool, True if the long module name should not be printed with + each type. + + Returns: + str, A string of valid python (assuming the right imports have been made) + that recreates the message passed into this function. + """ + + # TODO(user): craigcitro suggests a pretty-printer from apitools/gen. + + indent = kwargs.get('indent', 0) + + def IndentKwargs(kwargs): + kwargs = dict(kwargs) + kwargs['indent'] = kwargs.get('indent', 0) + 4 + return kwargs + + if isinstance(msg, list): + s = '[' + for item in msg: + if multiline: + s += '\n' + ' '*(indent + 4) + s += MessageToRepr( + item, multiline=multiline, **IndentKwargs(kwargs)) + ',' + if multiline: + s += '\n' + ' '*indent + s += ']' + return s + + if isinstance(msg, messages.Message): + s = type(msg).__name__ + '(' + if not kwargs.get('no_modules'): + s = msg.__module__ + '.' + s + names = sorted([field.name for field in msg.all_fields()]) + for name in names: + field = msg.field_by_name(name) + if multiline: + s += '\n' + ' '*(indent + 4) + value = getattr(msg, field.name) + s += field.name + '=' + MessageToRepr( + value, multiline=multiline, **IndentKwargs(kwargs)) + ',' + if multiline: + s += '\n'+' '*indent + s += ')' + return s + + if isinstance(msg, basestring): + if kwargs.get('shortstrings') and len(msg) > 100: + msg = msg[:100] + + if isinstance(msg, datetime.datetime): + + class SpecialTZInfo(datetime.tzinfo): + + def __init__(self, offset): + super(SpecialTZInfo, self).__init__() + self.offset = offset + + def __repr__(self): + s = 'TimeZoneOffset(' + repr(self.offset) + ')' + if not kwargs.get('no_modules'): + s = 'protorpc.util.' + s + return s + + msg = datetime.datetime( + msg.year, msg.month, msg.day, msg.hour, msg.minute, msg.second, + msg.microsecond, SpecialTZInfo(msg.tzinfo.utcoffset(0))) + + return repr(msg) + + +def _GetField(message, field_path): + for field in field_path: + if field not in dir(message): + raise KeyError('no field "%s"' % field) + message = getattr(message, field) + return message + + +def _SetField(dictblob, field_path, value): + for field in field_path[:-1]: + dictblob[field] = {} + dictblob = dictblob[field] + dictblob[field_path[-1]] = value + + def _IncludeFields(encoded_message, message, include_fields): """Add the requested fields to the encoded message.""" if include_fields is None: @@ -110,12 +218,15 @@ def _IncludeFields(encoded_message, message, include_fields): result = json.loads(encoded_message) for field_name in include_fields: try: - message.field_by_name(field_name) + value = _GetField(message, field_name.split('.')) + nullvalue = None + if isinstance(value, list): + nullvalue = [] except KeyError: raise exceptions.InvalidDataError( 'No field named %s in message of type %s' % ( field_name, type(message))) - result[field_name] = None + _SetField(result, field_name.split('.'), nullvalue) return json.dumps(result) @@ -123,7 +234,7 @@ def _GetFieldCodecs(field, attr): result = [ getattr(_CUSTOM_FIELD_CODECS.get(field), attr, None), getattr(_FIELD_TYPE_CODECS.get(type(field)), attr, None), - ] + ] return [x for x in result if x is not None] @@ -137,11 +248,18 @@ class _ProtoJsonApiTools(protojson.ProtoJson): cls._INSTANCE = cls() return cls._INSTANCE - def decode_message(self, message_type, encoded_message): # pylint: disable=invalid-name + def decode_message(self, message_type, encoded_message): if message_type in _CUSTOM_MESSAGE_CODECS: return _CUSTOM_MESSAGE_CODECS[message_type].decoder(encoded_message) + # We turn off the default logging in protorpc. We may want to + # remove this later. + old_level = logging.getLogger().level + logging.getLogger().setLevel(logging.ERROR) result = super(_ProtoJsonApiTools, self).decode_message( message_type, encoded_message) + logging.getLogger().setLevel(old_level) + result = _ProcessUnknownEnums(result, encoded_message) + result = _ProcessUnknownMessages(result, encoded_message) return _DecodeUnknownFields(result, encoded_message) def decode_field(self, field, value): @@ -161,11 +279,18 @@ class _ProtoJsonApiTools(protojson.ProtoJson): return value if isinstance(field, messages.MessageField): field_value = self.decode_message(field.message_type, json.dumps(value)) + elif isinstance(field, messages.EnumField): + try: + field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) + except messages.DecodeError: + if not isinstance(value, basestring): + raise + field_value = None else: field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) return field_value - def encode_message(self, message): # pylint: disable=invalid-name + def encode_message(self, message): if isinstance(message, messages.FieldList): return '[%s]' % (', '.join(self.encode_message(x) for x in message)) if type(message) in _CUSTOM_MESSAGE_CODECS: @@ -188,7 +313,8 @@ class _ProtoJsonApiTools(protojson.ProtoJson): value = result.value if result.complete: return value - if isinstance(field, messages.MessageField): + if (isinstance(field, messages.MessageField) and + not isinstance(field, message_types.DateTimeField)): value = json.loads(self.encode_message(value)) return super(_ProtoJsonApiTools, self).encode_field(field, value) @@ -300,4 +426,61 @@ def _SafeDecodeBytes(unused_field, value): return CodecResult(value=result, complete=complete) +def _ProcessUnknownEnums(message, encoded_message): + """Add unknown enum values from encoded_message as unknown fields. + + ProtoRPC diverges from the usual protocol buffer behavior here and + doesn't allow unknown fields. Throwing on unknown fields makes it + impossible to let servers add new enum values and stay compatible + with older clients, which isn't reasonable for us. We simply store + unrecognized enum values as unknown fields, and all is well. + + Args: + message: Proto message we've decoded thus far. + encoded_message: JSON string we're decoding. + + Returns: + message, with any unknown enums stored as unrecognized fields. + """ + if not encoded_message: + return message + decoded_message = json.loads(encoded_message) + for field in message.all_fields(): + if (isinstance(field, messages.EnumField) and + field.name in decoded_message and + message.get_assigned_value(field.name) is None): + message.set_unrecognized_field(field.name, decoded_message[field.name], + messages.Variant.ENUM) + return message + + +def _ProcessUnknownMessages(message, encoded_message): + """Store any remaining unknown fields as strings. + + ProtoRPC currently ignores unknown values for which no type can be + determined (and logs a "No variant found" message). For the purposes + of reserializing, this is quite harmful (since it throws away + information). Here we simply add those as unknown fields of type + string (so that they can easily be reserialized). + + Args: + message: Proto message we've decoded thus far. + encoded_message: JSON string we're decoding. + + Returns: + message, with any remaining unrecognized fields saved. + """ + if not encoded_message: + return message + decoded_message = json.loads(encoded_message) + message_fields = [x.name for x in message.all_fields()] + list( + message.all_unrecognized_fields()) + missing_fields = [x for x in decoded_message.iterkeys() + if x not in message_fields] + for field_name in missing_fields: + message.set_unrecognized_field(field_name, decoded_message[field_name], + messages.Variant.STRING) + return message + + RegisterFieldTypeCodec(_SafeEncodeBytes, _SafeDecodeBytes)(messages.BytesField) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 5c28d60..0309438 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -2,9 +2,12 @@ import base64 +import datetime import json +from protorpc import message_types from protorpc import messages +from protorpc import util from google.apputils import basetest as googletest from apitools.base.py import encoding @@ -20,6 +23,10 @@ class BytesMessage(messages.Message): repfield = messages.BytesField(2, repeated=True) +class TimeMessage(messages.Message): + timefield = message_types.DateTimeField(3) + + @encoding.MapUnrecognizedFields('additional_properties') class AdditionalPropertiesMessage(messages.Message): @@ -36,8 +43,20 @@ class CompoundPropertyType(messages.Message): name = messages.StringField(2) +class MessageWithEnum(messages.Message): + + class ThisEnum(messages.Enum): + VALUE_ONE = 1 + VALUE_TWO = 2 + + field_one = messages.EnumField(ThisEnum, 1) + field_two = messages.EnumField(ThisEnum, 2, default=ThisEnum.VALUE_TWO) + ignored_field = messages.EnumField(ThisEnum, 3) + + @encoding.MapUnrecognizedFields('additional_properties') class AdditionalMessagePropertiesMessage(messages.Message): + class AdditionalProperty(messages.Message): key = messages.StringField(1) value = messages.MessageField(CompoundPropertyType, 2) @@ -48,6 +67,11 @@ class AdditionalMessagePropertiesMessage(messages.Message): class HasNestedMessage(messages.Message): nested = messages.MessageField(AdditionalPropertiesMessage, 1) + nested_list = messages.StringField(2, repeated=True) + + +class ExtraNestedMessage(messages.Message): + nested = messages.MessageField(HasNestedMessage, 1) class EncodingTest(googletest.TestCase): @@ -72,7 +96,7 @@ class EncodingTest(googletest.TestCase): enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % { 'b': urlsafe_b64_str, - } + } rep_msg = BytesMessage(repfield=[data, data]) self.assertEqual(rep_msg, encoding.JsonToMessage(BytesMessage, enc_rep_msg)) self.assertEqual(enc_rep_msg, encoding.MessageToJson(rep_msg)) @@ -84,9 +108,32 @@ class EncodingTest(googletest.TestCase): '{"field": null}', encoding.MessageToJson(msg, include_fields=['field'])) self.assertEqual( - '{"repfield": null}', + '{"repfield": []}', encoding.MessageToJson(msg, include_fields=['repfield'])) + def testNestedIncludeFields(self): + msg = HasNestedMessage( + nested=AdditionalPropertiesMessage( + additional_properties=[])) + self.assertEqual( + '{"nested": null}', + encoding.MessageToJson(msg, include_fields=['nested'])) + self.assertEqual( + '{"nested": {"additional_properties": []}}', + encoding.MessageToJson( + msg, include_fields=['nested.additional_properties'])) + msg = ExtraNestedMessage(nested=msg) + self.assertEqual( + '{"nested": {"nested": null}}', + encoding.MessageToJson(msg, include_fields=['nested.nested'])) + self.assertEqual( + '{"nested": {"nested_list": []}}', + encoding.MessageToJson(msg, include_fields=['nested.nested_list'])) + self.assertEqual( + '{"nested": {"nested": {"additional_properties": []}}}', + encoding.MessageToJson( + msg, include_fields=['nested.nested.additional_properties'])) + def testAdditionalPropertyMapping(self): msg = AdditionalPropertiesMessage() msg.additional_properties = [ @@ -94,7 +141,7 @@ class EncodingTest(googletest.TestCase): key='key_one', value='value_one'), AdditionalPropertiesMessage.AdditionalProperty( key='key_two', value='value_two'), - ] + ] encoded_msg = encoding.MessageToJson(msg) self.assertEqual( @@ -125,7 +172,7 @@ class EncodingTest(googletest.TestCase): key='key_one', value='value_one'), AdditionalPropertiesMessage.AdditionalProperty( key='key_two', value='value_two'), - ] + ] msg = HasNestedMessage(nested=nested_msg) encoded_msg = encoding.MessageToJson(msg) @@ -142,6 +189,80 @@ class EncodingTest(googletest.TestCase): self.assertEqual(1, len(new_msg.nested.additional_properties)) self.assertEqual(2, len(msg.nested.additional_properties)) + def testValidEnums(self): + message_json = '{"field_one": "VALUE_ONE"}' + message = encoding.JsonToMessage(MessageWithEnum, message_json) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_ONE, message.field_one) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) + self.assertEqual(json.loads(message_json), + json.loads(encoding.MessageToJson(message))) + + def testIgnoredEnums(self): + json_with_typo = '{"field_one": "VALUE_OEN"}' + message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) + self.assertEqual(None, message.field_one) + self.assertEqual(('VALUE_OEN', messages.Variant.ENUM), + message.get_unrecognized_field_info('field_one')) + self.assertEqual(json.loads(json_with_typo), + json.loads(encoding.MessageToJson(message))) + + empty_json = '' + message = encoding.JsonToMessage(MessageWithEnum, empty_json) + self.assertEqual(None, message.field_one) + + def testIgnoredEnumsWithDefaults(self): + json_with_typo = '{"field_two": "VALUE_OEN"}' + message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) + self.assertEqual(json.loads(json_with_typo), + json.loads(encoding.MessageToJson(message))) + + def testUnknownNestedRoundtrip(self): + json_message = '{"field": "abc", "submessage": {"a": 1, "b": "foo"}}' + message = encoding.JsonToMessage(SimpleMessage, json_message) + self.assertEqual(json.loads(json_message), + json.loads(encoding.MessageToJson(message))) + + def testJsonDatetime(self): + msg = TimeMessage(timefield=datetime.datetime( + 2014, 7, 2, 23, 33, 25, 541000, + tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) + self.assertEqual( + '{"timefield": "2014-07-02T23:33:25.541000+00:00"}', + encoding.MessageToJson(msg)) + + def testMessageToRepr(self): + # pylint:disable=bad-whitespace, Using the same string returned by + # MessageToRepr, with the module names fixed. + msg = SimpleMessage(field='field',repfield=['field','field',],) + self.assertEqual( + encoding.MessageToRepr(msg), + r"__main__.SimpleMessage(field='field',repfield=['field','field',],)") + self.assertEqual( + encoding.MessageToRepr(msg, no_modules=True), + r"SimpleMessage(field='field',repfield=['field','field',],)") + + def testMessageToReprWithTime(self): + msg = TimeMessage(timefield=datetime.datetime( + 2014, 7, 2, 23, 33, 25, 541000, + tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) + self.assertEqual( + encoding.MessageToRepr(msg, multiline=True), + # pylint:disable=line-too-long, Too much effort to make MessageToRepr + # wrap lines properly. + """\ +__main__.TimeMessage( + timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=protorpc.util.TimeZoneOffset(datetime.timedelta(0))), +)""") + self.assertEqual( + encoding.MessageToRepr(msg, multiline=True, no_modules=True), + # pylint:disable=line-too-long, Too much effort to make MessageToRepr + # wrap lines properly. + """\ +TimeMessage( + timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=TimeZoneOffset(datetime.timedelta(0))), +)""") + if __name__ == '__main__': googletest.main() diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index 04a00a4..55faa49 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -94,3 +94,7 @@ class TransferInvalidError(TransferError): class NotYetImplementedError(GeneratedClientError): """This functionality is not yet implemented.""" + + +class StreamExhausted(Error): + """Attempted to read more bytes from a stream than were available.""" diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 15783ac..4b15683 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -6,6 +6,7 @@ from protorpc. """ import collections +import datetime import json import numbers @@ -18,13 +19,14 @@ from apitools.base.py import exceptions from apitools.base.py import util __all__ = [ + 'DateField', 'DateTimeMessage', 'JsonArray', 'JsonObject', 'JsonValue', 'JsonProtoEncoder', 'JsonProtoDecoder', - ] +] # We import from protorpc. # pylint:disable=invalid-name @@ -32,6 +34,26 @@ DateTimeMessage = message_types.DateTimeMessage # pylint:enable=invalid-name +class DateField(messages.Field): + """Field definition for Date values.""" + + # We insert our own metaclass here to avoid letting ProtoRPC + # register this as the default field type for strings. + # * since ProtoRPC does this via metaclasses, we don't have any + # choice but to use one ourselves + # * since a subclass's metaclass must inherit from its superclass's + # metaclass, we're forced to have this hard-to-read inheritance. + # + class __metaclass__(messages.Field.__metaclass__): # pylint: disable=invalid-name + + def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument + super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) + + VARIANTS = frozenset([messages.Variant.STRING]) + DEFAULT_VARIANT = messages.Variant.STRING + type = datetime.date + + def _ValidateJsonValue(json_value): entries = [(f, json_value.get_assigned_value(f.name)) for f in json_value.all_fields()] @@ -155,7 +177,7 @@ _JSON_PROTO_TO_PYTHON_MAP = { JsonArray: _JsonArrayToPythonValue, JsonObject: _JsonObjectToPythonValue, JsonValue: _JsonValueToPythonValue, - } +} _JSON_PROTO_TYPES = tuple(_JSON_PROTO_TO_PYTHON_MAP.keys()) @@ -181,12 +203,25 @@ def _JsonToJsonProto(json_data, unused_decoder=None): return _PythonValueToJsonProto(json.loads(json_data)) +def _JsonToJsonValue(json_data, unused_decoder=None): + result = _PythonValueToJsonProto(json.loads(json_data)) + if isinstance(result, JsonValue): + return result + elif isinstance(result, JsonObject): + return JsonValue(object_value=result) + elif isinstance(result, JsonArray): + return JsonValue(array_value=result) + else: + raise exceptions.InvalidDataError( + 'Malformed JsonValue: %s' % json_data) + + # pylint:disable=invalid-name JsonProtoEncoder = _JsonProtoToJson JsonProtoDecoder = _JsonToJsonProto # pylint:enable=invalid-name encoding.RegisterCustomMessageCodec( - encoder=JsonProtoEncoder, decoder=JsonProtoDecoder)(JsonValue) + encoder=JsonProtoEncoder, decoder=_JsonToJsonValue)(JsonValue) encoding.RegisterCustomMessageCodec( encoder=JsonProtoEncoder, decoder=JsonProtoDecoder)(JsonObject) encoding.RegisterCustomMessageCodec( @@ -230,3 +265,19 @@ def _DecodeInt64Field(unused_field, value): encoding.RegisterFieldTypeCodec(_EncodeInt64Field, _DecodeInt64Field)( messages.IntegerField) + + +def _EncodeDateField(field, value): + """Encoder for datetime.date objects.""" + if field.repeated: + result = [d.isoformat() for d in value] + else: + result = value.isoformat() + return encoding.CodecResult(value=result, complete=True) + + +def _DecodeDateField(unused_field, value): + date = datetime.datetime.strptime(value, '%Y-%m-%d').date() + return encoding.CodecResult(value=date, complete=True) + +encoding.RegisterFieldTypeCodec(_EncodeDateField, _DecodeDateField)(DateField) diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 7275671..8dd816f 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python +import datetime import json import math @@ -60,11 +61,25 @@ class ExtraTypesTest(googletest.TestCase): extra_types.JsonValue(integer_value=3), extra_types.JsonValue(string_value='four'), extra_types.JsonValue(boolean_value=False), - ]) + ]) self.assertRoundTrip(array) self.assertRoundTrip(json_array) self.assertTranslations(array, json_array) + def testArrayAsValue(self): + array_json = '[3, "four", false]' + array = [3, 'four', False] + value = encoding.JsonToMessage(extra_types.JsonValue, array_json) + self.assertTrue(isinstance(value, extra_types.JsonValue)) + self.assertEqual(array, encoding.MessageToPyValue(value)) + + def testObjectAsValue(self): + obj_json = '{"works": true}' + obj = {'works': True} + value = encoding.JsonToMessage(extra_types.JsonValue, obj_json) + self.assertTrue(isinstance(value, extra_types.JsonValue)) + self.assertEqual(obj, encoding.MessageToPyValue(value)) + def testDictEncoding(self): d = {'a': 6, 'b': 'eleventeen'} json_d = extra_types.JsonObject(properties=[ @@ -72,7 +87,7 @@ class ExtraTypesTest(googletest.TestCase): key='a', value=extra_types.JsonValue(integer_value=6)), extra_types.JsonObject.Property( key='b', value=extra_types.JsonValue(string_value='eleventeen')), - ]) + ]) self.assertRoundTrip(d) # We don't know json_d will round-trip, because of randomness in # python dictionary iteration ordering. We also need to force @@ -98,6 +113,25 @@ class ExtraTypesTest(googletest.TestCase): self.assertEqual(json_value, encoding.MessageToJson(value)) self.assertEqual(json_obj, encoding.MessageToJson(obj)) + def testDateField(self): + + class DateMsg(messages.Message): + start_date = extra_types.DateField(1) + all_dates = extra_types.DateField(2, repeated=True) + + msg = DateMsg( + start_date=datetime.date(1752, 9, 9), all_dates=[ + datetime.date(1979, 5, 6), + datetime.date(1980, 10, 24), + datetime.date(1981, 1, 19), + ]) + json_msg = json.dumps({ + 'start_date': '1752-09-09', 'all_dates': [ + '1979-05-06', '1980-10-24', '1981-01-19', + ]}) + self.assertEqual(json_msg, encoding.MessageToJson(msg)) + self.assertEqual(msg, encoding.JsonToMessage(DateMsg, json_msg)) + def testInt64(self): # Testing roundtrip of type 'long' @@ -123,7 +157,7 @@ class ExtraTypesTest(googletest.TestCase): result = json_msg if json_msg else message return result return DoRoundtrip(class_type=class_type, json_msg=json_msg, - message=message, times=times-1) + message=message, times=times - 1) # Single json_msg = ('{"such_string": "poot", "wow": "-1234",' diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 5b911c7..8c3ee28 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -15,11 +15,13 @@ import urlparse import httplib2 from apitools.base.py import exceptions +from apitools.base.py import util __all__ = [ 'GetHttp', 'MakeRequest', - ] + 'Request', +] # 308 and 429 don't have names in httplib. @@ -31,7 +33,7 @@ _REDIRECT_STATUS_CODES = ( httplib.SEE_OTHER, httplib.TEMPORARY_REDIRECT, RESUME_INCOMPLETE, - ) +) class Request(object): @@ -65,15 +67,22 @@ class Response(collections.namedtuple( __slots__ = () def __len__(self): + def ProcessContentRange(content_range): + _, _, range_spec = content_range.partition(' ') + byte_range, _, _ = range_spec.partition('/') + start, _, end = byte_range.partition('-') + return int(end) - int(start) + 1 + if '-content-encoding' in self.info and 'content-range' in self.info: # httplib2 rewrites content-length in the case of a compressed # transfer; we can't trust the content-length header in that # case, but we *can* trust content-range, if it's present. - _, _, range_spec = self.info['content-range'].partition(' ') - byte_range, _, _ = range_spec.partition('/') - start, _, end = byte_range.partition('-') - return int(end) - int(start) + 1 - return int(self.info.get('content-length', len(self.content))) + return ProcessContentRange(self.info['content-range']) + elif 'content-length' in self.info: + return int(self.info.get('content-length')) + elif 'content-range' in self.info: + return ProcessContentRange(self.info['content-range']) + return len(self.content) @property def status_code(self): @@ -139,21 +148,30 @@ def MakeRequest(http, http_request, retries=5, redirections=5): raise logging.error('Caught socket error, retrying: %s', e) exc = e + except httplib.IncompleteRead as e: + if http_request.http_method != 'GET': + raise + logging.error('Caught IncompleteRead error, retrying: %s', e) + exc = e if info is not None: response = Response(info, content, http_request.url) - if (response.status_code < 500 and response.status_code != 429 and + if (response.status_code < 500 and + response.status_code != TOO_MANY_REQUESTS and not response.retry_after): break - logging.info('Retrying request to url <%s> after status code %s', + logging.info('Retrying request to url <%s> after status code %s.', response.request_url, response.status_code) + elif isinstance(exc, httplib.IncompleteRead): + logging.info('Retrying request to url <%s> after incomplete read.', + str(http_request.url)) else: - logging.info('Retrying request to url <%s> after connection break', + logging.info('Retrying request to url <%s> after connection break.', str(http_request.url)) # TODO(craigcitro): Make this timeout configurable. if response: - time.sleep(response.retry_after or 2 ** retry) + time.sleep(response.retry_after or util.CalculateWaitForRetry(retry)) else: - time.sleep(2 ** retry) + time.sleep(util.CalculateWaitForRetry(retry)) if response is None: raise exceptions.InvalidDataFromServerError( 'HTTP error on final retry: %s' % exc) diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py new file mode 100644 index 0000000..d8f5971 --- /dev/null +++ b/apitools/base/py/list_pager.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +"""A helper function that executes a series of List queries for many APIs.""" + +import copy + +__all__ = [ + 'YieldFromList', +] + + +def YieldFromList( + service, request, limit=None, batch_size=100, + method='List', field='items', predicate=None): + """Make a series of List requests, keeping track of page tokens. + + Args: + service: apitools_base.BaseApiService, A service with a .List() method. + request: protorpc.messages.Message, The request message corresponding to the + service's .List() method, with all the attributes populated except + the .maxResults and .pageToken attributes. + limit: int, The maximum number of records to yield. None if all available + records should be yielded. + batch_size: int, The number of items to retrieve per request. + method: str, The name of the method used to fetch resources. + field: str, The field in the response that will be a list of items. + predicate: lambda, A function that returns true for items to be yielded. + + Yields: + protorpc.message.Message, The resources listed by the service. + + """ + request = copy.deepcopy(request) + request.maxResults = batch_size + request.pageToken = None + while limit is None or limit: + response = getattr(service, method)(request) + items = getattr(response, field) + if predicate: + items = filter(predicate, items) + for item in items: + yield item + if limit is None: + continue + limit -= 1 + if not limit: + return + request.pageToken = response.nextPageToken + if not request.pageToken: + return diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 37f6526..610ef2d 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Upload and download support for apitools.""" +import email.generator as email_generator import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart import httplib @@ -8,17 +9,17 @@ import io import json import mimetypes import os +import StringIO import threading -import mimeparse - from apitools.base.py import exceptions from apitools.base.py import http_wrapper +from apitools.base.py import util __all__ = [ 'Download', 'Upload', - ] +] _RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 _SIMPLE_UPLOAD = 'simple' @@ -124,7 +125,7 @@ class Download(_Transfer): httplib.NO_CONTENT, httplib.PARTIAL_CONTENT, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, - )) + )) _REQUIRED_SERIALIZATION_KEYS = set(( 'auto_transfer', 'progress', 'total_size', 'url')) @@ -179,7 +180,7 @@ class Download(_Transfer): 'progress': self.progress, 'total_size': self.total_size, 'url': self.url, - } + } @property def total_size(self): @@ -446,7 +447,7 @@ class Upload(_Transfer): 'mime_type': self.mime_type, 'total_size': self.total_size, 'url': self.url, - } + } @property def complete(self): @@ -520,7 +521,7 @@ class Upload(_Transfer): 'Upload too big: %s larger than max size %s' % ( self.total_size, upload_config.max_size)) # Validate mime type - if not mimeparse.best_match(upload_config.accept, self.mime_type): + if not util.AcceptableMimeType(upload_config.accept, self.mime_type): raise exceptions.InvalidUserInputError( 'MIME type %s does not match any accepted MIME ranges %s' % ( self.mime_type, upload_config.accept)) @@ -563,7 +564,13 @@ class Upload(_Transfer): msg.set_payload(self.stream.read()) msg_root.attach(msg) - http_request.body = msg_root.as_string() + # encode the body: note that we can't use `as_string`, because + # it plays games with `From ` lines. + fp = StringIO.StringIO() + g = email_generator.Generator(fp, mangle_from_=False) + g.flatten(msg_root, unixfrom=False) + http_request.body = fp.getvalue() + multipart_boundary = msg_root.get_boundary() http_request.headers['content-type'] = ( 'multipart/related; boundary=%r' % multipart_boundary) @@ -704,5 +711,6 @@ class Upload(_Transfer): # TODO(craigcitro): Add retries on no progress? last_byte = self.__GetLastByte(response.info['range']) if last_byte + 1 != end: - response = self.__SendChunk(last_byte + 1, data[last_byte + 1 - start:]) + new_start = last_byte + 1 - start + response = self.__SendChunk(last_byte + 1, data=data[new_start:]) return response diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 8246231..cd882a7 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -4,7 +4,9 @@ import collections import httplib import os +import random import types +import urllib import urllib2 from apitools.base.py import exceptions @@ -12,7 +14,9 @@ from apitools.base.py import exceptions __all__ = [ 'DetectGae', 'DetectGce', - ] +] + +_RESERVED_URI_CHARS = r":/?#[]@!$&'()*+,;=" def DetectGae(): @@ -33,7 +37,7 @@ def DetectGce(): """Determine whether or not we're running on GCE. This is based on: - https://developers.google.com/compute/docs/instances#dmi + https://cloud.google.com/compute/docs/metadata#runninggce Returns: True iff we're running on a GCE instance. @@ -42,7 +46,8 @@ def DetectGce(): o = urllib2.urlopen('http://metadata.google.internal') except urllib2.URLError: return False - return o.getcode() == httplib.OK + return (o.getcode() == httplib.OK and + o.headers.get('metadata-flavor') == 'Google') def NormalizeScopes(scope_spec): @@ -63,5 +68,98 @@ def Typecheck(arg, arg_type, msg=None): msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) else: msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) - raise exceptions.TypecheckError(msg) + raise exceptions.TypecheckError(msg) return arg + + +def ExpandRelativePath(method_config, params, relative_path=None): + """Determine the relative path for request.""" + path = relative_path or method_config.relative_path or '' + + for param in method_config.path_params: + param_template = '{%s}' % param + # For more details about "reserved word expansion", see: + # http://tools.ietf.org/html/rfc6570#section-3.2.2 + reserved_chars = '' + reserved_template = '{+%s}' % param + if reserved_template in path: + reserved_chars = _RESERVED_URI_CHARS + path = path.replace(reserved_template, param_template) + if param_template not in path: + raise exceptions.InvalidUserInputError( + 'Missing path parameter %s' % param) + try: + # TODO(craigcitro): Do we want to support some sophisticated + # mapping here? + value = params[param] + except KeyError: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + if value is None: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + try: + if not isinstance(value, basestring): + value = str(value) + path = path.replace(param_template, + urllib.quote(value.encode('utf_8'), reserved_chars)) + except TypeError as e: + raise exceptions.InvalidUserInputError( + 'Error setting required parameter %s to value %s: %s' % ( + param, value, e)) + return path + + +def CalculateWaitForRetry(retry_attempt, max_wait=60): + """Calculates amount of time to wait before a retry attempt. + + Wait time grows exponentially with the number of attempts. + A random amount of jitter is added to spread out retry attempts from different + clients. + + Args: + retry_attempt: Retry attempt counter. + max_wait: Upper bound for wait time. + + Returns: + Amount of time to wait before retrying request. + """ + + wait_time = 2 ** retry_attempt + # randrange requires a nonzero interval, so we want to drop it if + # the range is too small for jitter. + if retry_attempt: + max_jitter = (2 ** retry_attempt) / 2 + wait_time += random.randrange(-max_jitter, max_jitter) + return min(wait_time, max_wait) + + +def AcceptableMimeType(accept_patterns, mime_type): + """Return True iff mime_type is acceptable for one of accept_patterns. + + Note that this function assumes that all patterns in accept_patterns + will be simple types of the form "type/subtype", where one or both + of these can be "*". We do not support parameters (i.e. "; q=") in + patterns. + + Args: + accept_patterns: list of acceptable MIME types. + mime_type: the mime type we would like to match. + + Returns: + Whether or not mime_type matches (at least) one of these patterns. + """ + unsupported_patterns = [p for p in accept_patterns if ';' in p] + if unsupported_patterns: + raise exceptions.GeneratedClientError( + 'MIME patterns with parameter unsupported: "%s"' % ', '.join( + unsupported_patterns)) + def MimeTypeMatches(pattern, mime_type): + """Return True iff mime_type is acceptable for pattern.""" + # Some systems use a single '*' instead of '*/*'. + if pattern == '*': + pattern = '*/*' + return all(accept in ('*', provided) for accept, provided + in zip(pattern.split('/'), mime_type.split('/'))) + + return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) diff --git a/apitools/gen/__init__.py b/apitools/gen/__init__.py index 4265cc3..54fa3d5 100644 --- a/apitools/gen/__init__.py +++ b/apitools/gen/__init__.py @@ -1 +1,5 @@ #!/usr/bin/env python +"""Shared __init__.py for apitools.""" + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 06e10d0..292d443 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python """Test gen_client against all the APIs we use regularly.""" import contextlib diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 37c6183..eac21e1 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -9,6 +9,8 @@ from protorpc import messages from apitools.gen import extended_descriptor +# This is a code generator; we're purposely verbose. +# pylint:disable=too-many-statements _VARIANT_TO_FLAG_TYPE_MAP = { messages.Variant.DOUBLE: 'float', @@ -24,7 +26,7 @@ _VARIANT_TO_FLAG_TYPE_MAP = { messages.Variant.ENUM: 'enum', messages.Variant.SINT32: 'integer', messages.Variant.SINT64: 'integer', - } +} class FlagInfo(messages.Message): @@ -134,7 +136,7 @@ class CommandRegistry(object): name=name, description=extended_field.description, conversion=self.__GetConversion(extended_field, request_type), - )) + )) arg_names.append(name) flags = [] for extended_field in sorted(request_type.fields, key=lambda x: x.name): @@ -183,7 +185,7 @@ class CommandRegistry(object): client_method_path=calling_path, has_upload=bool(method_info.upload_config), has_download=bool(method_info.supports_download) - ) + ) self.__command_list.append(command_info) def __LookupMessage(self, message, field): @@ -286,12 +288,17 @@ class CommandRegistry(object): printer("'history_file',") printer('%r,', '~/.%s.%s.history' % (self.__package, self.__version)) printer("'File with interactive shell history.')") + printer('flags.DEFINE_multistring(') + with printer.Indent(' '): + printer("'add_header', [],") + printer("'Additional http headers (as key=value strings). Can be '") + printer("'specified multiple times.')") for flag_info in self.__global_flags: self.__PrintFlag(printer, flag_info) printer() printer() printer('FLAGS = flags.FLAGS') - printer('apitools_base.DeclareBaseFlags()') + printer('apitools_base_cli.DeclareBaseFlags()') printer('%s()', function_name) def __PrintGetGlobalParams(self, printer): @@ -319,12 +326,15 @@ class CommandRegistry(object): printer('log_response = FLAGS.log_response or FLAGS.log_request_response') printer('api_endpoint = apitools_base.NormalizeApiEndpoint(' 'FLAGS.api_endpoint)') + printer("additional_http_headers = dict(x.split('=', 1) for x in " + "FLAGS.add_header)") printer('try:') with printer.Indent(): printer('client = client_lib.%s(', self.__client_info.client_class_name) with printer.Indent(indent=' '): printer('api_endpoint, log_request=log_request,') - printer('log_response=log_response)') + printer('log_response=log_response,') + printer('additional_http_headers=additional_http_headers)') printer('except apitools_base.CredentialsError as e:') with printer.Indent(): printer("print 'Error creating credentials: %%s' %% e") @@ -334,14 +344,15 @@ class CommandRegistry(object): printer() def __PrintCommandDocstring(self, printer, command_info): - for line in textwrap.wrap('"""%s' % command_info.description, - printer.CalculateWidth()): - printer(line) - extended_descriptor.PrintIndentedDescriptions( - printer, command_info.args, 'Args') - extended_descriptor.PrintIndentedDescriptions( - printer, command_info.flags, 'Flags') - printer('"""') + with printer.CommentContext(): + for line in textwrap.wrap('"""%s' % command_info.description, + printer.CalculateWidth()): + printer(line) + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.args, 'Args') + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.flags, 'Flags') + printer('"""') def __PrintFlag(self, printer, flag_info): printer('flags.DEFINE_%s(', flag_info.type) @@ -357,7 +368,8 @@ class CommandRegistry(object): drop_whitespace=False) for line in description_lines[:-1]: printer('%r', line) - printer('%r%s', description_lines[-1], ',' if flag_info.fv else ')') + last_line = description_lines[-1] if description_lines else '' + printer('%r%s', last_line, ',' if flag_info.fv else ')') if flag_info.fv: printer('flag_values=%s)', flag_info.fv) if flag_info.required: @@ -365,6 +377,7 @@ class CommandRegistry(object): def __PrintPyShell(self, printer): printer('class PyShell(appcommands.Cmd):') + printer() with printer.Indent(): printer('def Run(self, _):') with printer.Indent(): @@ -393,7 +406,7 @@ class CommandRegistry(object): printer('}') printer("if platform.system() == 'Linux':") with printer.Indent(): - printer('console = apitools_base.ConsoleWithReadline(') + printer('console = apitools_base_cli.ConsoleWithReadline(') with printer.Indent(indent=' '): printer('local_vars, histfile=FLAGS.history_file)') printer('else:') @@ -430,6 +443,8 @@ class CommandRegistry(object): printer(flags_import) printer() printer('import %s as apitools_base', self.__base_files_package) + printer('from %s import cli as apitools_base_cli', + self.__base_files_package) import_prefix = '' if self.__root_package: import_prefix = 'from %s ' % self.__root_package @@ -451,7 +466,7 @@ class CommandRegistry(object): printer("appcommands.AddCmd('%s', %s)", command_info.name, command_info.class_name) printer() - printer('apitools_base.SetupLogger()') + printer('apitools_base_cli.SetupLogger()') # TODO(craigcitro): Just call SetDefaultCommand as soon as # another appcommands release happens and this exists # externally. @@ -460,7 +475,7 @@ class CommandRegistry(object): printer("appcommands.SetDefaultCommand('pyshell')") printer() printer() - printer('run_main = apitools_base.run_main') + printer('run_main = apitools_base_cli.run_main') printer() printer("if __name__ == '__main__':") with printer.Indent(): @@ -470,7 +485,7 @@ class CommandRegistry(object): """Print all commands in this registry using printer.""" for command_info in self.__command_list: arg_list = [arg_info.name for arg_info in command_info.args] - printer('class %s(apitools_base.NewCmd):', command_info.class_name) + printer('class %s(apitools_base_cli.NewCmd):', command_info.class_name) with printer.Indent(): printer('"""Command wrapping %s."""', command_info.client_method_path) printer() @@ -525,6 +540,6 @@ class CommandRegistry(object): printer('result = client.%s(', command_info.client_method_path) with printer.Indent(indent=' '): printer('%s)', ', '.join(call_args)) - printer('print apitools_base.FormatOutput(result)') + printer('print apitools_base_cli.FormatOutput(result)') printer() printer() diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 69e3787..9ddf932 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -20,6 +20,8 @@ from protorpc import descriptor from protorpc import message_types from protorpc import messages +import apitools.base.py as apitools_base + class ExtendedEnumValueDescriptor(messages.Message): """Enum value descriptor with additional fields. @@ -134,14 +136,16 @@ def WritePythonFile(file_descriptor, package, version, printer): def PrintIndentedDescriptions(printer, ls, name, prefix=''): if ls: - width = printer.CalculateWidth() - len(prefix) - printer(prefix) - printer('%s%s:', prefix, name) - for x in ls: - description = '%s: %s' % (x.name, x.description) - for line in textwrap.wrap(description, width, initial_indent=' ', - subsequent_indent=' '): - printer('%s%s', prefix, line) + with printer.Indent(indent=prefix): + with printer.CommentContext(): + width = printer.CalculateWidth() - len(prefix) + printer() + printer(name + ':') + for x in ls: + description = '%s: %s' % (x.name, x.description) + for line in textwrap.wrap(description, width, initial_indent=' ', + subsequent_indent=' '): + printer(line) def _EmptyMessage(message_type): @@ -333,19 +337,23 @@ class _ProtoRpcPrinter(ProtoPrinter): short_description = ( _EmptyMessage(message_type) and len(description) < (self.__printer.CalculateWidth() - 6)) - if short_description: - self.__printer('"""%s"""', description) - return - for line in textwrap.wrap('"""%s' % description, - self.__printer.CalculateWidth()): - self.__printer(line) - - PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums') - PrintIndentedDescriptions( - self.__printer, message_type.message_types, 'Messages') - PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields') - self.__printer('"""') - self.__printer() + with self.__printer.CommentContext(): + if short_description: + # Note that we use explicit string interpolation here since + # we're in comment context. + self.__printer('"""%s"""' % description) + return + for line in textwrap.wrap('"""%s' % description, + self.__printer.CalculateWidth()): + self.__printer(line) + + PrintIndentedDescriptions(self.__printer, message_type.enum_types, + 'Enums') + PrintIndentedDescriptions( + self.__printer, message_type.message_types, 'Messages') + PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields') + self.__printer('"""') + self.__printer() def PrintMessage(self, message_type): if message_type.alias_for: @@ -378,7 +386,7 @@ def _PrintMessages(proto_printer, message_list): _MESSAGE_FIELD_MAP = { message_types.DateTimeMessage.definition_name(): message_types.DateTimeField, - } +} def _PrintFields(fields, printer): @@ -393,12 +401,15 @@ def _PrintFields(fields, printer): 'label_format': '', 'variant_format': '', 'default_format': '', - } + } message_field = _MESSAGE_FIELD_MAP.get(field.type_name) if message_field: printed_field_info['module'] = 'message_types' field_type = message_field + elif field.type_name == 'extra_types.DateField': + printed_field_info['module'] = 'extra_types' + field_type = apitools_base.DateField else: field_type = messages.Field.lookup_field_type_by_variant(field.variant) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 2ef4d93..3d654ef 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -21,14 +21,15 @@ flags.DEFINE_string( '--discovery_url.') flags.DEFINE_string( 'discovery_url', '', - 'URL of the discovery document to use. Mutually exclusive with --infile.') + 'URL (or "name/version") of the discovery document to use. ' + 'Mutually exclusive with --infile.') flags.DEFINE_string( 'base_package', 'apitools.base.py', 'Base package path of apitools (defaults to ' 'apitools.base.py)' - ) +) flags.DEFINE_string( 'outdir', '', 'Directory name for output files. (Defaults to the API name.)') @@ -39,7 +40,7 @@ flags.DEFINE_string( 'root_package_dir', '', 'Ultimate destination for generated code (used for generating ' 'correct import lines). Defaults to the value of FLAGS.outdir.' - ) +) flags.DEFINE_string( 'root_package', '', 'Python import path for where these modules should be imported from.') @@ -52,6 +53,10 @@ flags.DEFINE_multistring( flags.DEFINE_string( 'api_key', None, 'API key to use for API access.') +flags.DEFINE_string( + 'client_json', None, + 'Use the given file downloaded from the dev. console for client_id ' + 'and client_secret.') flags.DEFINE_string( 'client_id', None, 'Client ID to use for the generated client.') @@ -81,8 +86,6 @@ flags.DEFINE_boolean( FLAGS = flags.FLAGS -flags.MarkFlagAsRequired('client_id') -flags.MarkFlagAsRequired('client_secret') flags.RegisterValidator( 'infile', lambda i: not (i and FLAGS.discovery_url), 'Cannot specify both --infile and --discovery_url') @@ -114,15 +117,45 @@ def _GetCodegenFromFlags(): FLAGS.strip_prefix, FLAGS.experimental_name_convention, FLAGS.experimental_capitalize_enums) + + if FLAGS.client_json: + try: + with open(FLAGS.client_json) as client_json: + f = json.loads(client_json.read()) + web = f.get('web', {}) + client_id = web.get('client_id') + client_secret = web.get('client_secret') + except IOError: + raise exceptions.NotFoundError( + 'Failed to open client json file: %s' % FLAGS.client_json) + else: + client_id = FLAGS.client_id + client_secret = FLAGS.client_secret + + if client_id is None: + logging.warning('No client ID supplied') + client_id = '' + + if client_secret is None: + logging.warning('No client secret supplied') + client_secret = '' + client_info = util.ClientInfo.Create( - discovery_doc, FLAGS.scope, FLAGS.client_id, FLAGS.client_secret, + discovery_doc, FLAGS.scope, client_id, client_secret, FLAGS.user_agent, names, FLAGS.api_key) outdir = os.path.expanduser(FLAGS.outdir) or client_info.default_directory if os.path.exists(outdir) and not FLAGS.overwrite: raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') - root_package = FLAGS.root_package + if FLAGS.root_package: + root_package = FLAGS.root_package + else: + if not FLAGS.root_package_dir: + FLAGS.root_package_dir = outdir + FLAGS.root_package_dir = os.path.abspath(FLAGS.root_package_dir) + root_package = ( + util.GetPackage(FLAGS.root_package_dir)) base_package = FLAGS.base_package return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 58e8fc3..1c5df9b 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -23,13 +23,13 @@ def _StandardQueryParametersSchema(discovery_doc): 'type': 'object', 'description': 'Query parameters accepted by all methods.', 'properties': discovery_doc.get('parameters', {}), - } + } # We add an entry for the trace, since Discovery doesn't. standard_query_schema['properties']['trace'] = { 'type': 'string', 'description': base_cli.TRACE_HELP, 'location': 'query', - } + } return standard_query_schema @@ -106,7 +106,8 @@ class DescriptorGenerator(object): if api_methods: self.__services_registry.AddServiceFromResource( 'api', {'methods': api_methods}) - self.__client_info = self.__client_info._replace(scopes=self.__services_registry.scopes) # pylint:disable=protected-access,g-line-too-long + self.__client_info = self.__client_info._replace( # pylint:disable=protected-access + scopes=self.__services_registry.scopes) @property def client_info(self): @@ -137,6 +138,7 @@ class DescriptorGenerator(object): printer = self._GetPrinter(out) printer('"""Common imports for generated %s client library."""', self.__client_info.package) + printer('# pylint:disable=wildcard-import') printer() printer('import pkgutil') printer() diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 757adb4..1ddcb81 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -34,7 +34,7 @@ class MessageRegistry(object): variant=messages.FloatField.DEFAULT_VARIANT), 'any': TypeInfo(type_name='extra_types.JsonValue', variant=messages.Variant.MESSAGE), - } + } PRIMITIVE_FORMAT_MAP = { 'int32': TypeInfo(type_name='integer', @@ -51,11 +51,11 @@ class MessageRegistry(object): variant=messages.Variant.FLOAT), 'byte': TypeInfo(type_name='byte', variant=messages.BytesField.DEFAULT_VARIANT), - 'date': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', - variant=messages.Variant.MESSAGE), + 'date': TypeInfo(type_name='extra_types.DateField', + variant=messages.Variant.STRING), 'date-time': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', variant=messages.Variant.MESSAGE), - } + } def __init__(self, client_info, names, description, root_package_dir, base_files_package): @@ -70,7 +70,7 @@ class MessageRegistry(object): # Add required imports self.__file_descriptor.additional_imports = [ 'from protorpc import messages', - ] + ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() # A set of types that we're currently adding (for cycle detection). @@ -207,10 +207,10 @@ class MessageRegistry(object): attrs = { 'items': { '$ref': entries_type_name, - }, + }, 'description': description, 'type': 'array', - } + } field_name = 'additionalProperties' message.fields.append(self.__FieldDescriptorFromProperties( field_name, len(properties) + 1, attrs)) @@ -223,6 +223,9 @@ class MessageRegistry(object): # TODO(craigcitro): Is schema_name redundant? if self.__GetDescriptor(schema_name): return + if schema.get('enum'): + self.__DeclareEnum(schema_name, schema) + return if schema.get('type') == 'any': self.__DeclareMessageAlias(schema, 'extra_types.JsonValue') return @@ -259,10 +262,10 @@ class MessageRegistry(object): 'key': { 'type': 'string', 'description': 'Name of the additional property.', - }, - 'value': property_schema, }, - } + 'value': property_schema, + }, + } self.AddDescriptorFromSchema(new_type_name, schema) return new_type_name @@ -278,9 +281,9 @@ class MessageRegistry(object): 'entry': { 'type': 'array', 'items': entry_schema, - }, }, - } + }, + } self.AddDescriptorFromSchema(entry_type_name, schema) return entry_type_name @@ -316,40 +319,48 @@ class MessageRegistry(object): return descriptor.FieldDescriptor.Label.REQUIRED elif attrs.get('type') == 'array': return descriptor.FieldDescriptor.Label.REPEATED + elif attrs.get('repeated'): + return descriptor.FieldDescriptor.Label.REPEATED return descriptor.FieldDescriptor.Label.OPTIONAL + def __DeclareEnum(self, enum_name, attrs): + description = attrs.get('description', '') + self.AddEnumDescriptor(enum_name, description, + attrs['enum'], attrs['enumDescriptions']) + self.__AddIfUnknown(enum_name) + return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) + + def __AddIfUnknown(self, type_name): + type_name = self.__names.ClassName(type_name) + full_type_name = self.__ComputeFullName(type_name) + if (full_type_name not in self.__message_registry.viewkeys() and + type_name not in self.__message_registry.viewkeys()): + self.__unknown_types.add(type_name) + def __GetTypeInfo(self, attrs, name_hint): """Return a TypeInfo object for attrs, creating one if needed.""" - def AddIfUnknown(type_name): - type_name = self.__names.ClassName(type_name) - full_type_name = self.__ComputeFullName(type_name) - if (full_type_name not in self.__message_registry.viewkeys() and - type_name not in self.__message_registry.viewkeys()): - self.__unknown_types.add(type_name) - type_ref = self.__names.ClassName(attrs.get('$ref')) type_name = attrs.get('type') if not (type_ref or type_name): raise ValueError('No type found for %s' % attrs) if type_ref: - AddIfUnknown(type_ref) + self.__AddIfUnknown(type_ref) return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) if 'enum' in attrs: enum_name = '%sValuesEnum' % name_hint - description = attrs.get('description', '') - self.AddEnumDescriptor(enum_name, description, - attrs['enum'], attrs['enumDescriptions']) - AddIfUnknown(enum_name) - return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) + return self.__DeclareEnum(enum_name, attrs) if 'format' in attrs: type_info = self.PRIMITIVE_FORMAT_MAP.get(attrs['format']) if (type_info.type_name.startswith('protorpc.message_types.') or type_info.type_name.startswith('message_types.')): self.__AddImport('from protorpc import message_types') + if type_info.type_name.startswith('extra_types.'): + self.__AddImport( + 'from %s import extra_types' % self.__base_files_package) if type_info is None: raise ValueError('Unknown format %s for type %s' % ( attrs['format'], type_name)) @@ -384,7 +395,7 @@ class MessageRegistry(object): schema = dict(attrs) schema['id'] = name_hint self.AddDescriptorFromSchema(name_hint, schema) - AddIfUnknown(name_hint) + self.__AddIfUnknown(name_hint) return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) raise ValueError('Unknown type: %s' % type_name) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index de6a652..2b3a832 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -9,6 +9,11 @@ import textwrap from apitools.base.py import base_api +# We're a code generator. I don't care. +# pylint:disable=too-many-statements + +_MIME_PATTERN_RE = re.compile(r'(?i)[a-z0-9_*-]+/[a-z0-9_*-]+') + class ServiceRegistry(object): """Registry for service types.""" @@ -50,7 +55,8 @@ class ServiceRegistry(object): description = '%s%s%s' % (first_line, newline, remaining) else: description = '%s method for the %s service.' % (method_name, name) - printer('"""%s', description) + with printer.CommentContext(): + printer('"""%s' % description) printer() printer('Args:') printer(' request: (%s) input message', method_info.request_type_name) @@ -73,6 +79,8 @@ class ServiceRegistry(object): printer('class %s(base_api.BaseApiService):', class_name) with printer.Indent(): printer('"""Service class for the %s resource."""', name) + printer() + printer('_NAME = %s', repr(name)) # Print the configs for the methods first. printer() @@ -80,7 +88,7 @@ class ServiceRegistry(object): with printer.Indent(): printer('super(%s.%s, self).__init__(client)', client_class_name, class_name) - printer('self.__configs = {') + printer('self._method_configs = {') with printer.Indent(indent=' '): for method_name, method_info in method_info_map.iteritems(): printer("'%s': base_api.ApiMethodInfo(", method_name) @@ -93,7 +101,7 @@ class ServiceRegistry(object): printer('),') printer('}') printer() - printer('self.__upload_configs = {') + printer('self._upload_configs = {') with printer.Indent(indent=' '): for method_name, method_info in method_info_map.iteritems(): upload_config = method_info.upload_config @@ -105,16 +113,6 @@ class ServiceRegistry(object): printer('%s=%r,', attr, getattr(upload_config, attr)) printer('),') printer('}') - printer() - - printer('def GetMethodConfig(self, method):') - with printer.Indent(): - printer('return self.__configs.get(method)') - printer() - - printer('def GetMethodUploadConfig(self, method):') - with printer.Indent(): - printer('return self.__upload_configs.get(method)') # Now write each method in turn. for method_name, method_info in method_info_map.iteritems(): @@ -130,8 +128,7 @@ class ServiceRegistry(object): printer("config = self.GetMethodConfig('%s')", method_name) upload_config = method_info.upload_config if upload_config is not None: - printer("upload_config = self.GetMethodUploadConfig('%s')", - method_name) + printer("upload_config = self.GetUploadConfig('%s')", method_name) arg_lines = ['config, request, global_params=global_params'] if method_info.upload_config: arg_lines.append('upload=upload, upload_config=upload_config') @@ -195,18 +192,16 @@ class ServiceRegistry(object): printer() client_info_items = client_info._asdict().iteritems() # pylint:disable=protected-access for attr, val in client_info_items: - if attr == 'scopes': - # We want to drop one scope as a special case. - extra_scope = 'https://www.googleapis.com/auth/cloud-platform' - if extra_scope in val: - val.remove(extra_scope) + if attr == 'scopes' and not val: + val = ['https://www.googleapis.com/auth/userinfo.email'] printer('_%s = %r' % (attr.upper(), val)) printer() printer("def __init__(self, url='', credentials=None,") with printer.Indent(indent=' '): printer('get_credentials=True, http=None, model=None,') printer('log_request=False, log_response=False,') - printer('credentials_args=None, default_global_params=None):') + printer('credentials_args=None, default_global_params=None,') + printer('additional_http_headers=None):') with printer.Indent(): printer('"""Create a new %s handle."""', client_info.package) printer('url = url or %r', self.__base_url) @@ -215,7 +210,8 @@ class ServiceRegistry(object): printer(' get_credentials=get_credentials, http=http, model=model,') printer(' log_request=log_request, log_response=log_response,') printer(' credentials_args=credentials_args,') - printer(' default_global_params=default_global_params)') + printer(' default_global_params=default_global_params,') + printer(' additional_http_headers=additional_http_headers)') for name in self.__service_method_info_map.iterkeys(): printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) @@ -314,6 +310,10 @@ class ServiceRegistry(object): 'method %s, using */*', method_id) config.accept.extend([ str(a) for a in media_upload_config.get('accept', '*/*')]) + + for accept_pattern in config.accept: + if not _MIME_PATTERN_RE.match(accept_pattern): + logging.warn('Unexpected MIME type: %s', accept_pattern) protocols = media_upload_config.get('protocols', {}) for protocol in ('simple', 'resumable'): media = protocols.get(protocol, {}) @@ -339,7 +339,7 @@ class ServiceRegistry(object): request_type_name=self.__names.ClassName(request), response_type_name=self.__names.ClassName(response), request_field=request_field, - ) + ) if method_description.get('supportsMediaUpload', False): method_info.upload_config = self.__ComputeUploadConfig( method_description.get('mediaUpload'), method_id) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index a517176..e07cd16 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -44,7 +44,9 @@ class Names(object): @staticmethod def __ToCamel(name, separator='_'): - return ''.join(s[0].upper() + s[1:] for s in name.split(separator)) + # TODO(craigcitro): Consider what to do about leading or trailing + # underscores (such as `_refValue` in discovery). + return ''.join(s[0:1].upper() + s[1:] for s in name.split(separator)) @staticmethod def __ToLowerCamel(name, separator='_'): @@ -160,7 +162,7 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( 'client_secret': client_secret, 'user_agent': user_agent, 'api_key': api_key, - } + } client_class_name = ''.join( map(names.ClassName, (client_info['package'], client_info['version']))) client_info['client_class_name'] = client_class_name @@ -219,6 +221,7 @@ class SimplePrettyPrinter(object): self.__out = out self.__indent = '' self.__skip = False + self.__comment_context = False @property def indent(self): @@ -234,10 +237,23 @@ class SimplePrettyPrinter(object): yield self.__indent = previous_indent + @contextlib.contextmanager + def CommentContext(self): + """Print without any argument formatting.""" + old_context = self.__comment_context + self.__comment_context = True + yield + self.__comment_context = old_context + def __call__(self, *args): + if self.__comment_context and args[1:]: + raise Error('Cannot do string interpolation in comment context') if args and args[0]: - line = (args[0] % args[1:]).rstrip() + if not self.__comment_context: + line = (args[0] % args[1:]).rstrip() + else: + line = args[0].rstrip() line = line.encode('ascii', 'backslashreplace') print >>self.__out, '%s%s' % (self.__indent, line) else: diff --git a/apitools_public.tar b/apitools_public.tar new file mode 100755 index 0000000000000000000000000000000000000000..f53b90bca7c593c342d5d6359e3802bf2ced015d GIT binary patch literal 307200 zcmdPXXP`MSFfcGMH#KDd(FO(v=rl-}!NADK%-GD(*a$3dZf;=ApkPXKmoYFfloppH z7AYt&6y@ia;IogAJU+dHSWZvBB)>v0GcS{dZUjZJnTZK{yu5y@o#8oWMsgg zU_e7R!Hpi_@gH51UlGfdnpc*SSzMxEt58sBWX{DEU6NW{lA2c*%axK^P^@69ke6Sa z%9Wg-o12)IQmkOhrJ$goP>@-mkeOFpl9-dDkXVpelAoVb4Az*LmmUjN0n!CBshA5M z+@p-a6#{ztL9UK2zOH(?DKv8!f&6c1VuX?Z4b99(^FPfZ1j%4!c-H0OidKwvD$UGE zQ3x(cEGaFH)yOC*DJZtm*DonbEXyp`P0rNIFG|->E=tTyPtGhU%GZZwd%fbabo->D z#JuDT+uX$BlGGwi5_D^FaV5Y^N>IowR!CGx&dy99JyB^jv-&_Yr-Co?Zq0qhQ(`lFL`GGnnC0xC2?&PYiuNX<)0%}dTqEyisO zR5jEn=hC90)VvafVsIpKafM{07NsgAg0MnbYPmvYZf<5CNI_0wQF^LEazx5hzfLQVUB{i%W{Rv=ow3OG;9U6pB(yiYgU~OA89}i%Ph(6oM2WF{J=Ovbu7NsiW=fTX(O)bgD zPbt>nQcy_A2ZbO=A=v#2$@zI@sYS)0B&Csb zHQ)*+2VzyRJ{L%1h-;8RJ4Ig7EnL?VY78K>Dl!EIfP%#5C45T(Cv$&)vGpQ7038;ih0a=-v zqL5Un5DclwiWLkMN-~P_OVcwHEJ4*;Nk(R|LP~yeX>Mv>i5}Rk3JMBPcjXshI4n88 zpt2}4J)=Y+zdSFsNFl#SAvLcgv!qfXv9u&3zbLZ`Y#%7l6coT-LDvDU|1yge(u)%F zN?^q!G#ro|keZ&Dqu>fQ8p9E#d7u(BKMibtVlvnim@D#96cTfCp!VmNWTZkJ2(Gz7 zRc~^BUP)1Yjt;mwf^u@ej@AMB22@EwN@7qgnx6+X393+`JhKE;IVF}ry{)IIQ#3CIzdc@VY^ zs5(ndECn|kVCfzr1rC8Ch1|rv#Pn29Diov^C4;JI4O0VEO_0NK^FdV|IN)Ih zm6jBjB<7`nVzM|Ru_(0|9E=*7AS07f^HS3?LCqgj(~;bXi1Nt%QY8hA{2~RV$o$eG zB~9c!k(j3d3X`(Tl+wf;kVAZs(g)NfsTHY3$(h9RD(y1IModII&|S zwcz~HqU2O1Q1;42wi8lqr>3PA6{V&?WI+vq+{A2<6}kB-nQ58H;L00Z*Ml-)W?phm zDL5#SN=p>-@=FwQGIKLaQd2MjqY!K8Z5CuB0{GXPY zUJ7ZPD5PcPq#_j`{z+M>$t9RhPt2=?IJhXaxHJcxKftY|+*D9gD>FF}mO;SnskHo} zT(I?^fCY=>K)GoOi3*V50UM{IfZ{=@Ytg)!oS$2e3Cd9UV8=nd32k7drl5KlmNP)5 z7ou?lZexQ}s6uXPN@k)$No4`Dx5M*`vM~yU^8BJ~P|AgrL!fkw$O4&pFmL1+L0t_A zI;gjD6H`(Z63Y@Za}twsQjuC>(8QtxDosHNFgXzvnMsujiBMw@rCVNpNoF#%d7oJf z4XM-=h0HuquqKw2fT}QXLI5Q!kZYkvXe8z-q*f&67UZOYIvxc@`DK|Y5RD*}js>9h zcV>k`Qff|qxh5jNxTF?kmL-;CmZd6y0;CuvL_jG3WC!*zP=NRz>Htvu>p=P?nZ*i8 ziN&cYpyn(nE2e<#0Hyw-{9H&`1+og9FhO~tJR>t1R2;&Jrqq#2f{X4^iVBvpj~H2CgkKi(zdD9R>8@gJykDYKElpRIu^Ikbo&l%}vaNRR+bW z1&KxAlmg0C;PNgvwJ0^GQXwZZFB@EjC1vJ;>)pJ>+*D0a?gvF>W?o5ZQCea$xRQV+ zeUJn^P%#_@D#26p(-5)l4644Mbv)J>M-308EDH??kbUp~g!M+ip#w=r;K&6fACwXf zoQ6}NEfJVW`H-*yyFR}NGen_|N2D!K0OWyMC!qc?)Rf}Vq}ElEvLf|>=X??Iso%4{G_si3iiqWrwfWF3XF z)S{%s9B?`+F9L-{9=Ho%ng^|46hOHEYElU(&|yvCd~gFSv!oc@Bm`#`aDh;a6zvL_ zRT0!EPy+{HM}D3H!c7XfiJ3W|Y7R6cTdbo1sgb~CIixX>pQBJ*SzMBu3$Ob?MR;a$ zacL^3lb#H&BcUoFsRUH>L0T)Ii~;s+ex3q6&_Rg=R*XPOU1%x*MFX^L1q!#6%;MzI zVsI@FwlNo6Ugo8P8&aSIq66|U)Q71Ru=*I)udp^~a(-TML1uDkera(|r9y6EQ8uXP zD^dU#N}$#e)SQ%5NJk%1Dudz(6gJpXBB-iR^2;w#NK`=1t9nZC#x-0Rt$hvmHppY3 zv;wMe@HTmn0}xa`=Av1ukdaudkd&I5r%;rdoSIn%X$B$N1NC5WX;Lw0)FiJYrxKL! z@{0;UrF3cvxWStV>6Cy%7g7)z=_$B_n(v?pafbH@Am*m(DFl~7s(xr%#oC!c%4W!= za%y671|r<@ixfa3M@f~C-U7H?5Sd@9kO*p-7NnMxCg#A3(enJFoRsoRP?IVzKTj7N z^TnBEsX7XI`MSv&pm8Q0g`)h*#GH~!-L#_AR2_xPqN3EY{A5r;4h;rGc8B)=KqWV* z9s>2YQgsvxia;fQDrQNSp9XVBL1|J>W^zuYLP}5 zb(R8B3hB3jnh&r{019nzSqyG0V^j>tO&Vx*4k|QrArWMZ6hQ%rC7?F^(2pIB)QaTP zf)Y>$0}U)fG9$R~Er#@NH5CdVeuaiEDE2@@wK@tJiDjt@pjI9%n!x?&{IoRCh+Dow zacWME4wTHy1q~8HA_iU#fXcMIVzBBANHqyFs2r3)6~J)>vo*1xAP3al%+CW2`hv=9 z(6}$uX~{W>nYqOZ5Vc52F9|e0iI(=k#T2OA%u7v9EiO(h$^_@mw4%(sbVyb%$w%^q zA-WaiiN&DA45`2M6g<-uBJ)e3=76T*1LSlL%NLm3@$U!^q8jwa8 zs9c4&t&8%Di*>;P2J%L7ekrK^1mS1qDI_Z7B$gMKW|n}wnvlh~Jy3__ z7onGmnR&@L3ph~qR$Kyhcrmm`jxYz@e+M473iAvduS+-LxoY~XSRt_Iw=0w>@!P^}7eI@mpUT6v&G z8Yn(NJ+{(Z9dN|~iZ)Ps$}cT}nu5}(gk&;YB^ju=g?5F&(TYe~xv8loAeSkm<>%z& zmxCf6F-&F!t7kP5H6cDNEiO?=2X&J`&I4y@P<5DD0Gg|Tha}W{P-lWV^B6@gXiBJ{ z5|V!qb(S?K@ZfePA=wETxB*!T@inL)1?sP@0y6_;^gIt8}IV69>Ekq;;9nhGX0=#zA zfu?;OP_dYj3TiijqY}xG#1cr21{$V<`VBHs1InCe?g0sbf*)3-f_x9IzriCZkXbXB z8DPhy;)YKBEl8P z80;YoPE+Wf11Gk`5`~=9#NrZ#{5)7$1@aWMcZtZY1v#KjuobwB0GH%YHzL9s5kkn` zN-S1@#Q-QY(A{7SGY}++lE`3nB*-!b- zg6bS--U63cke)WQWQ>^VzB}wU?2rEtQ(#Rsc97yK*0j8Sy6%rUT%Wt!y$w9 zCHWw=MG)hl-F%QgVJQPtJ%f9;pn*w*lkEt(WV?aeLI0P{w6TDCXIywh+GcHts=7u^N02t@S0L8Bm$rZgxWK@9>aL)7+=AO(-IA~&NT5n>ANib7`sKt(8W`?Xk4 zAv7;1wYV4*5elgl1v#0?nV{}3sNzjUI0Ba8LHVc$6 zu#M2pF(h~87eSkPNywwm*di8D!y)?@(%i}~EdjSZk zYT!UdtD*C*3Sh;c6b14+EDB*M1<_3ftwhO$&K`j(^y2(HaFq^9{>i1qCHc9a>1mK_ zGV>HtQj3#|GLt~zUXlT;J|QEuFq=T-8>qPnZ@EJigJKs{8^F{-3rA2D0LcmFdI~O? z#o(?JXwo)KAw01NG=NhH&jfI%f+jaW9a!+V2&jvt0G~4f$0B$NBQs9{Itr$Ph%jj0 z2aT9Rdyn8nE}$VDXc)ueFuw>^=@x_9QYc-~VolI!Y+{~*l4G!fXRwljlVh-FFf4Gx zJwrVFLqin89fN`#{X#rlgBARPkf-JS-4q=CA{D$n{akbuQZqqg(4gF11j^}fpJe8O z%9RvQqXjxbh{!?UDLZgg32M|L~d6`{{c6xdnOp zxx4zh`i1By___u;dpP=qI68Uyc!orRN>l|m&k#S?V94?TN2nPAjzJ-w&Y?bzK?(t( zK>_~3u8`Um)b}sW&jHOW6r~myDwZWSp& z=UG!hi4A$86|*x7NqZK03O=A&P;iifH2GvECV>{yRq82t<|*XmE2NfzsxwfgfS8e& zuaE1Os;L9c44~O=aBzeBd!V6Q zm?@=@LLX0h*8rC{iNy*jsX3WRpmkxOkV#QU2aQMMR6;`tW@U*&VsZ&6%@#pQIgI=P zS=xb8eSoGZU^zS|6I>)f8U*0VBPB66F I4z5MOB_K#QWZoZ?WMM4m(n6TeGV_u% zQ&RIvpi>E;!EMNxE@(myYA7iDL56`wL=%%iYvU4&AamxRS{&*%$gD9)DRLd70Ge$m z1$D8&jbl)$m0zTgnFozbP{Rx)hA6whr4>>nVNOISz+44N(K-3W;NEz8ett@MW)AYm zZni>kNq#{=VtOhlc5@3#K@Lw#%*+9=yaTO*$Vn^BgRF-EITPeVP;x?F2?Cl004)+o z%z@jSU!(xCIJH<)2b>N;{Q+?NBb8OqV1W;&C#IBTf+tDA<0A?M`NhSVkQ5CHGq7%` z#~?YtQcuA#88mbW%B--G7vxSyM9U5-Q-o(^=A7*K4|2zBwqnM(g{l^ppXH@e>$j}nUy z+`P;Zczy(>2XOHUO2Xg_t&pFT10CZ7H%RqC#SUn-P%$V|7H5JM^q0W;SDCQBiw|V2 zm7an}etBvcXeA7&PY8AnD8RtsiDWRsf8Z4@dGH*Wp9k_eq+0@=R|0tzCJZV_72w4v z*u9|10Z?ZZIt>L4L^ezzBe5tK-24QkeNcjhha%J&nR&^eWwh|wGH~q;E|XJ>K+~nr zK~&Im0@}E75@={Iu^7I1DL+l2GQYG48pWV61=$Apc`JkMeRH=|ul$e~I zT2ic_6denjrY%YZk4!*oJFr9Z%kvb#rD|qcB`D;OdeV9d8ZP;H>LsvYBd9|_j)NJl zsHp(%t%JL%#TogfIVlQBsR(y~sK}~Nu z3P=k=6*NGOhOTW)RnoKuITF?%0rmPJ)faU595hr0nr+GiPsSJIWP*!Ea2Ew+4Ag3H zX9_f=2kNlrCxOQ!k;jSQLj($mC159mhBXK*Ifbr<)-}>I0BeDEQ}MJ|K%*p}9u{WX z1G?r6ntI&g((uNNbw1E{}7&@y_m)?N<=O?vvpsU@Wa zdIgoVa}ECW{|2Up28IkM`+p4$O$?1j_kYnYMi3?|E9#dP7wIQu=IN*Al_?ZdmSp7T zaVc{tDj#IFjlFWl&oQ z+B#1L%~K6su4h*Q~+;Y!m|Ydxjq1CphkV544SimOwo~D9VjbMtu|0rphjh& z3|Vds9%#T?7jP*lDFuVd8c=rxv8-B8NlA%|3$pM-0n~y7O_p$Rl@wK4fhtZ=r3D&q z0GCpb?R3x~K2Pvk5!a%k{33`-(0WjX)T(%}QkcHd;#7zLL@%fXfGVV+iNhLj7RUoT z6jD{f+W-ZL$=RSm0&tTKG;UZ>0$v>jT2_$=TFe63wg+mlg7T6=060^DRvLmwr9u57 z|6o0?AlJZ9&mdQq_y9*|Z%236U~RK~*= z2iwB)gdW7jFc}R^E-ujYDMG1@g1V8OxjHB_U?eR)(DY?qiiUb=9%zkgafuNqI85~n z)j`Q0xAMfI^n%2q;#5%Z7=cxD#XAOghWPvY1jmQD1_gWi`zhEes2k{kWVyJ&c@Lb` z^uSCFaC!#yBW%@STYRDZg!<4H!#qfuN9rosszbvNvSSo97!2ySXO!k9=7G{BI8ztp z*n+c-mA-y@W=TeAl3sFtF1EeOQ0qZCD!<59-5Io-S-}~!of$14Q&NjX<-Wv5J=dhW#*;C!z45`AwEvc zD=DgsFUZf#D=D_E1_vC-XX>D_#QdDpc&MrBR-h4gN@W z5Otu9!^JRFn2Hp%6;R3xZ3Xn~4>2aSqNFGhVGfeh3o`TaQd88e6woq1L?6h-pr{4; z2&r5`k_0(0GcR4;3Wv=g6QF*EO@P=!icpXYJjE$!s1|FgE2t`HU?w(b;Dgl1LTpx6 z2&fG3gvFZz@}L7aQj$S^`pmS<)S_Z!%b;x^1uH8B|6orAN6>5lXwk2gl>)R46b$VU z!a7~hR0OhGJtV&%GZ~~8uE-^|EHx*;0JO9XB<=&=jGmbaOYkJ>0vQ3VhkWx>Kq(c; zcBsdpbK17*Q1>Frz3kM=^8BKdVvJ0zIdXPNBihWviKVCS>*(j{<{BKLmzhT+j}hqq z85kNGVD$ft4M+QbG>Q%sW1-`Y3R-#vm0So;Nks_~FE@p&C^fmX2(+eO7p4eSokO&u z7&^)sG9jP`Kf{G~2f=~zKS}4m85)`zn~t9UMvp{<6#wz@nR%Hd@$pn23k1bKzVqLV z4AIB`jLl3&&wryw^fO=|{{s!xfX)+0Q9w!*;29=ZsSg>p0S~zpWT%6M4PYZKsTC!u z;NDG123LH1L1IZpe7u6K0+O%>Ojt)DJ|5IJjE^5}N75y>F)cyL|Det1G^qqY`Jc4% z-^j$oe6;+>j4kSmpvON_ey5scp!g@P{5LT*9i9K7TKMBLXteyt7dpdV4n6-DR8qeR z0Ofzu%70@6^U?C3E{P9I{HNw6=Ytl$#)C#*snHn##Xs)y-^kd&5dHi=Gh>6%_CH@*4Ryo{*LcuAiPR#{fxhaR(0D+s#ttaT2W@@L#2YSL z3JQ?;!_ZfWJs1@fvQsPZMxp}fMA*_)TxyIoLF)mrdIDJsG%mrOhD?4cC}6r#N5K$h zl5oz?Eyyp;OM!+%Wk_W~Dt14C7Cu)XTke@xlA4}c1hE{PGC{p-WEIFs7vv*o=tEr{ zo>`LNnpc{O%{q99hh$_HqbdUNz&S-hAkaCQXnHB&W0JUqX6~*M0H7dKC&;csW#G4NJ&jgEX^sgg&Cm-w;Pm6Gt=|( zi&9hK!6p(ijk-x18obaL9SkWO>LN&xV@q*53ZUen0NVNsK3FR;2PxMghaQSa;22j> zP{&O1>iCkpM`E!b=)!>%sA1UBpMpXjSP4$MG3-QyGc+wh42TCEtA%U|N}j-(EnPvA z+?2Z#-K$`y!73wHXv-@kwYWqBQKjjD_|A#NshZ&PHBwU36hJi{Xn|e;sM>{vT5)Pl z8Z;l~7N;xNqE&9-gm0^!n3N1&^Z^Rc@_2|sc-f6%W^QphDDx zEKSVO$SqFS0~@KMke3QJ2*QOZhs%Oihoq#Yse_XS$c0ct{PGE!13JVT>Q``u1`8ey zkTFn?CYhPU7nc-)Y;kl<)`nOE5`=h6y;=$EMkOl+CDmf3TF@c@kZf@Y@Pt8k)`9>3mfk;Y_ z3`GqesHKoFf@O72@Y%Mh;NrR%Ir&49C2AuUhvn+kwdy!+#S^jM&;+kGLFA>p(wv-H zqy;4K@&nZ`xy9+At!toNzNwIUx7apX9h}J3V{ru%vVE}F0wsu8vMq#}0=5&WP=S=@ z1i}`*z=k#IK*eWqNotBMo_YtN78HJ1+m5!;LQb^&peW=arhtP1J;#?Rd=5!;q^)&G6Z&mA2~Hsgdy8DnT`7_FHD8c_nbBA}HS zR6E#VxN}``YKevh*lGEBsp?3s)Px2P$Ymf!(G_~2kpj>H;|kERda$-F)}(@^?Dj17 z%P#@B58f<>ws!Cuq*st%08N(|elXNg$Vtu9z^fluWoiV`i!-f5LLJu3K%_!wJ&x$H zBIhtr1VYn~dbLt!UO{OIxXb{LM<`h-80aV{fd(Exjqd!?5|DIlEu={eJ>9?-k_bDV!j+1T^pvenf;b)N|0ZfKZOzA;Y$)0wn=Rg^$txf`o}JgaU;mIQYT6 z2`UwA5Tn(t6i6-OhI}Ch@j3Ri1}f{o*%MX};ZsaPd4|szT=^5JSR?K`{^(|vl;`6x3lu2Osv7EVh%X=;B|w=J1tTd3*X*G3 z9qKh$?;jG{nyB@TCuC?ABW^(E#e;VJgQguoBfzlX9FX^sV#n2w{MbRY14lUc<>#fM zhgyDKDh|~e>d;i<>ZcCZRje15Sd^KVSEA?Y7wU`D=s*cUXejHYrivQUM+Y1C8~jf_QqcxphPr(#XKjL`T6~N5M!(!N^!g!PrJ(L=6K?P~i(IbTB#^;MyG|qh759 z(g1cLD0)CH(KRsFH82V>GPW`{wlXr+gSbN5z`)ADK#7PE0nqRaxTOY5Q_!#o$qz~` zC_)Y{Wrc#uoXorutCY-Q(D7`xNr@@CXKnKBq&&L7XyO);^ zx@WYsC=Y!6FW4i>3Me)p9Y~Z5-T((al?QZMdj;t90+6>M@r07RG{9kC3n_Bdbzmcj z@KH=1m`tn=jf=jR%$Wmc+Y~=qMS;|24qm; zQ&Lij;esbEz23HC2*t1X-{NR#;S! ztOq(K2)_I@IX?$@~dVK=mmXSA2Y8PEI^%BOT;?3w2OO13bzM zorVVsfT{;*`v4{k>XbMZ6(v?83qYchi%+D1=Ha03KKzx-4TJNL) zI=>Zk7HMv3Ng`y~Dd=#1(0QtfW%-#Y3OT7I;Cp_-VHOnN%mvb3l$s7Ya~E_$btdTK zSn%}V3=``^Yp>TL6>DFD!^RJw*b)a>yVU9tbj(*_NqSA`;t5Wko9rkEwA%z$%1YKtD=MG+?6 z97K2pwBQ{alyEg6`H<6fu!g6Sl9F?NURi2U3AhkQ&n!#LLl_S(N5GSr3XpODTm&g7 zfC_#+(CY8xjMU_8q&U(+=mmB4fDS1^QX43E zax&8(CAS_ln}RDUP&x#q14yNc60Y$Ganu9?N#caf1lQb{MuLU0nh4F}xa>uw1F#nf zBr%AOKxwQ9l1@@GlS?$BH9-3}K&vEl6u=$?r+HM@fK@;|3CYMH0ciONE+0TdW*#`z zL&_0QQxUrFBNiolVeuEg~Q%{z1|o zq?7=~D_A$QjDVg12r>qg3_26Zon1MXe|AcQ_qu1j@MfUnO9^wNbb*AP2`DLo*BaPD z+bM_+f{TBslaDLNAt(hIvPmer2u<;r1+MWxMuE;jg=|^`_d7s$%Rr(WQeGe%4oyQ4 zhpK}T86-QY>nNCkj6y1A@M{C7Zgm|6b5QmG_ePO2C+;+hk|4oZ5_}U1)LF%#ETo_S z%i2~7jtbC%0DSduB9awgZD4OeH9-|-rWS*=fj61ugReJ*wrW5ZX`mXQpa4Gm8n%@c zW-E5bgB4>q2bA$3K@E0~f&#LOzCkMCUUw4Z`8!HeHI$1U?*w7Er9JTggOP%-+-r66g{vI!MAr1 zWG*-@U=KD>IR%MxaLPg{rJzM4&U6Mc8XP9r?T-f)d+`B5{vrPHA^!0Jks%)be(}DJ z0SdMXkbO!Z1Hf*xf;5!DY?OvFWO@~(4w4O^3LzXED#1>IDg?99)YWpKxGXXNe6n~+ zX+ch^28s-_n?cR_VhwQl5AWCpfHoJR7!K)#g2(s3wE$ZE3T`xmWkEfIV1GYY=z!b+ z%H7b~38yQf!Hc^Q#%RVO+Xx%Kz-aS;9g1jufHEp#W^^Yg3|+%Ql#!6IFKOy z52QfVQ7FwTEe4OlfOaSr*@C(d;JAg_0g4R0l+xUSVhwP=32F?IBauu)O+OIJz#1TG zAY1OB7DBc}Vp@n99|#kW7x91$!)6~O2tdw`2PuJj5Q}}F72XQA3MASH?m-~MX;CW3 z9OOY6l=KCWhIRFE=z}CVaJa)0Ca6?^cm`6!Ln>2Sh`r27O-{E&+%b5cuE zp~D%_Vh$2Ws1AgdHl(@}Qd&UW2~F+LMhbY)uNXc%1qla3O;AHQ2XwXzJOh*C54av! z+5sC68U+RihDJP=NYGJ;$KrEMNLt3@Xi&BUSN0GyJwcrguyv>r11b}gl$1OY^HOpk zH=%$p2}{gTNKPyUUswt0u7EBvMIFITPAq_2Pzq_}L1r#NlSReFp!-wcE26<22}n~2 zi)<)d8aj{*j!V5V$Z~2(4V$NcY!}GI#KZzP;UOg^a7PimND)5JTa;P=KDP&EB&etd z?Yb#0DblEbj2D8piGhs*xi2*bUW$Sfu7WKnOd!dZdTAUfu5cxAWd+CuiFv7zdAO8( z&=tdwyTzdqr~{ggs02;dL6UolLRM*Ui9%Xt9_WHx!nqObX)NJKWFkg*0-k~qLa52N zSP!{Zik5Ie86J1Kg(U7G@N^Pv@c?+WtpcQ4h0OSZD|JZ1!WHEydYQ%fpt+&M5)DYq zr+~(6z=41x=D~8v+7Lrd@Gt`BYqAPCh@WxAKgh2NXfuGIb$)uqB}D}w1|&`DD5$GO z>Z<1Is-~!Gg2M&8_l$7lgUm;Yd(fDPCKqOv1a>6W@(U~jk9l|`YHA>uqqj@YdJPbk z^{noTE{)ee4OjQLh=iAb3j+nK-WeUg9}jD zVkYp~3GliuNb!kic*C0a<(WAt$)E)vkd;B$S4`=FdcpCD1({Gw6tr-vD@o2Eq$s&4 z6?6rAW@1ipd`@N(A?2_Vn4miNLM_57ONtU9%|k*;LD!nagO+R;q!tlU3mQv|2VMS4 zP-RI`VqS3?VG}{)HV9isOIc7(7?~JL`JY%&U_`aD9~7zt*8iFqo1?G)H8D0GJ^vLH zbTr2}%YVm$g5><%+{C<;V%_BY+=9fC%%q%D1*jPG^i9y-T+q!9MTrV&nH8CN(8>T( zdF<<7Y&uqCiTsRcQZ`z*og z^;}%tJpEkbp)-Y$F1LD0YEo%BXde~mq!DnZ64JR<56;LhSAhGhSRp?Tb~z~IQd97n zG<8ibJZ5L4CZ^=178k4QC?K2cSX^A1o2rnPuTWYHD*uZW5+RGtlJoOQit=-EGV{_E zpf?HXscUk%`8c|RE~5}g$>p`2JcvdEhPm7BV<4o5?-)Xq7d74z^5l-EC@x`1Ri1n-2@8CQt^pJ>BS1R z3hAjOi6teF&5=3^>OrM>pzU&wMWEYQ!E5$F0}9Zi8RJ36G8C6o=73JHMIIC`&4cWF z2AQb?nV;43O)bgDPXYDHASPs{fnBJFFa$E*p`egfnwP9#tAOG%z0BNrkT_@+D`*`9 z$mD`lkTPgC2Q3qY3Tl89LM#Dyv7y3xAR07Hq5xtPq$Yy}q77ofyOcnyZoxa0K&C6e zlz^^KElLNKBYL2N_Hq)FQ#C-V0?-!#gC?L-lS@)lbQ6=4Q;R{f(GYV$B*;2j5CON- z&R}UQc5br?Mf#U{bQ;5HdzXsw$P)#cZmweEchN9HOoSaI9^2EFn(887C z(t-lej=jX390kzAXSn{n{Jdh&I$zKd2!wsbC5c5~OF=_LsX1wI7l4MiKu!RyVE`>& z1WnCA&4RcTtTVej1(a4GA|T};pTRC{#Hlqmu>xi%=x`xa-5^sG@{1He^MN3D!jf2S zW*$rvL<8IuUGUOdh#*Ke$PLi7?!~Ak!IK);@gTLZqz1PI;k4q)V!hnNip;zcSXzTv zfhb@VY!$$TmmWe0If#l&i_#L4Q{yx9;)^p){NCY%` z1@?11ic28s@VXV`Cy)uCVJ`&*#E>Uk6F8|t3(TO>JZNww7NwU#gG)(CDX26LoIPM| z3g`+3Pe@q=O1X)klVx=wg&Nd}p#233#i=C*jGy^n>4XO)~DqIE6G|(tA$Y_WGpflks6+i|;;u7Qou-l|K(`8jH+(>*cZ;E#g1}2?;VHoYRQVT|f=ejSzSX>{)S`TlSXyFcPH7R^ z>WfUU3n8w-ETr)y6}Zzui5Pns7M`jA8zU};Y$8cfC`v6Z$p>xY0A2m9keZmBp#T~( zR4C2|hmf*DNk(D`sOU>h%u^^xEdn(WL2X5FYAXgMEYSA+AlCpN$SM+0K8KXwWzmLK zkYoe$sDdrH9s?T+TfbkFUaYBL1NAGc$bkhnRFk@2eu+YAUVdqMhC)GpG4zOPP>LwF zR!9S_n^I6s(NRdPC;-<4pvF*6YGQGTf@%tAH6f&(1}WM=#UIFdnvhyrFCJzkq}D=l zl^x7g&>{-mRp33_iFuX8x+yoGNGCz2U<4OM;5<+Ynq`5h zQc!@XQx64;Lbo)4Go~I`A@l$PcosHRj+pj{fgioM5b_(%s`9-;j zC5T|tQ2-rh4sr%)ZV?u##idE$W_v1lEo5$fN@iN60;toJT2z!@qyU}FELKP?N`<87 z)D*aPpbm@&B`ie$1+sP&TrWaqFtIrSYAz^UfD$u`W1*)!L358QIE8@r4T9Gs!5nn2PT)Vbhn26mojF*paMgH~a~tpS@uOjs3XXBH@c z^JqFK%JcISl2eO7che(pcLJ?91O++NZcw8VsjHBx1M2tafeQ$5D1j3t$Zhc44NAb^ z99gWR0BU&RuFXMp0N6PSx%nxub-@Y>NI4PW2uQC97V=2dwuTlctLi9dX@M;Qr$I=i zt>BbcoC*&i9R+v>g-v-T6{RL-D`e)Cls%+Y}7EwI_()zEMu@K_3H zxGe>oz(Jd06T!o5Ir;eopq5BMPAVu1!fXJ=8rt9yv{O^816nPWTTlYE1?p(T24YYd z1F1DYEInAY0_yiyt7oL<6sTK)ECjW{i=!a|v5;#JAnHMbCYgyjFn!=lcR`w=YIPLU zK?7%?W076`-PEDY^ZX*{K53{cK_xnrS&Y0v1(Mvr#V5EU3hGON)Z()`F)sz0O+eWO zsSbi#paYE^O;FbkNhMTT54~Rx4LFct5Nn|diy_@=1zT9~z|AT~9a~GukB4mX1$#k9 z6TGqk6rOrPrFqcyC~^!y2RA^M;^Y>TK$g4Z7L>*~1T=02IZOf6afJ0pQy|^Vg8Y)yypl}NAe%0@A*fqW1lj`x>0m1q zBo={Bgv-n;&P+*F$j?g!Wgw6@6cmuV>d1bug06dpx(}^$?w4AYS_Iwx3-7;!7RAE5 z??}au9y}Bvg@%G|@YWWnS^$L|IOw5)4Jn}2-ShKP zk}6a6)Iq)j#WHAW2DA_o8iimjpwW5on(mU+iV_|0nP70Cq}24xlnOXEH7_MIr2=XP zw8jFRLk7#!(3BPpDH4#YW@sdXixLG}uqAp0iA5#guDFJ}g1RQ?lZ--ivH)yZK~a8g0W=)pM@Aw72HH;ppJWOOI&c&~Llhcoj%A6NIiO}etQ0D?g3c#F z6)33dW#wn)X@F+>!TJ$JJZOnKqzhG01WMOnX-K3(Em2ZZ3Mfh~D9Q&7c7rn_xc937 zKA0U``GAT=&^ShBUMhHe9aPA}f(7akkjufr09B_4ZdVn9DoPFY)QZd!bxqJXE=U=Q zRADJd3e7NekOg4V%JYlBqtj4*phH77p}lTMX$rC~HBSSqP7~xfLr{sY0Oo_H13(S( z%sd4|MU?}Z`BE=SEdou|sK=t)45|!aV;jhpLPs}{eG82^NO6H9SRrAW2s)4@9Wna} zD>O>;ib0+3%seDH9kBbsqi2v_IMhmTS)Z8)4i#{+ibpTF^%NYzagkPC+i_K;E`e&?qh`(u9>a;Oq%1Vxg%7stc{ua|CbxRzNgl z$}>_+GE$2`5f6$7P{|B+0=R64ZKN!LHJTt}eXvd@QdbYVhe5S6Wb{T4GZMg_gfvrO zz5uoLFbZ3+I{2U!d~hGC6Q!q!GMEDze1Vl)kYONbnFTcvnlr)uV9@jzxMou=wgL?w zg7l&&11*;YWip)pgxrM)O$y+7TUY@OI_Vi?9?EIOU;)tL%}mJTA0)3p!yeRd1GiGZ z%}$U~L~9GC1nN4Fn;;lewu7c8K=BAKWwAC3A$Ea`E!In|C`rvr(NITk9ii9^vlCfk zQEF~}St?i~s4Rd;LA?()8s`0!{CH57g=RmfQQ*=XxpV^!aDq}FNGaHEWQ7VEpi>Zv zQj1gbO2CCwdTL&3QDTk)NHw_p1BZsP0_OCcV+wr8P!}8+X`sEhpfPVqQU=wqkbx`z zyj17h6mYP^n}%=(*uAh}IRypKr4V@~@x_@{pt)G&sbu%m65qrMSR)`cFTEr~LsLOp z!5Hj91<<}_&~P-e?}{}Q>|n>&fT9v&IH(d;P*>1GwhL;80<4k&2dsiEWJq2wB( zyeP2%dFUIiR1fMT_tX-P)SLnhb#)!k>J5m$Y*C#9S;_!%3J61-V+)~BrP4Bsic3I+ zQ9MM}Rvo+`0HjMn0W1l2VLV8QjzT>6$_-FU8fG0h`jDCwF?ksJL0UmUp{lC|tFj?> zfdd~hIj^9tpl+qEgKi@vNrD^$O_u60dC*!|1L8Q4;SfhbhjUS9y1JoKRwpqL>(>mg|Z4R(WE49}gIr6Y3h6-g(lWhAr> z2{r+|%mvxsn6{z@A&TFSHNvK$pkioET+~@u za&8K=@GUOMFFBz-2mU-l8P2s6-(#4>b6cT9lYvl3A9jP@Gx}YSV#BYz2jYAb;P05KyMFQ-Jq& z(1uSy*61jt!DfdbQ{^D*AYq8P;08L)1FojghY&z32|#B}fTI&M!Ceen+y&`oA_`7* zNRI$CTmo&zgW^Cv0NhUiwX#4{Q=nuBI${CtJJ9k2JtXxYD@XjCd2sp@{ z;2;BcBZCPC8Yo4AIx8T*fEIYcI{NuVkih~-pn`n}8t_rDMQ&Q5tdVk0ErHevps8&Q zP1p#PEyVAT^aRbk;i);v`MIfz3K}7m1*za_7hE#rf@Y9GXVZgbelqj(^fVy_M0rML zPAYgE1}sY;H5EYxXF-l$a&8K^3k&I`H4j-puIswMWqEL zF!Mm+09rHx(g#h&Aph#2H@QbwK4V$`3z|s^2Au1EK|LqvDqm1h2HK-hoLW?tnVbrl9Dps$1+6blttiRKgsVFlY3g5N~Y!e=b2l^qK*A&Wab^V0I6wT-%C zL1t(H=*C54K~Mxc7Gyf-WTxhoz*Is7gCQ{u6ZHel-RERhr8*X5y5^-6{JDT`+kt6UHPDMY|B#>3QIFnA?uNg3rdSJ^Gk~rpy@y%GmlGIAwRLSB*Q2f>=p%$ z{33;vqWl6BQB6HAs1DHTQlI?v^wc5^b!2_un+Mc2^@>wVd_WtQG+^pogM$2nG$E(= z#fJv@#QV7Vxrcaw&hRrZv4FQzP(lgkX)?a4DVd21rQqOHK&srpH6HkI-i-Vd@c0UN zereS#Ju!W=mx9g+FzyVUg5B25L~#EQ(^(p&|QIA|gaHNkZ#q(T!N+@;VZw#AU^H9%&eT2>5N z!k3D}fe=Zo8i;io)CdyY1sNHGqy^+thfoev0(Chdd(NSU8z?Bi;ul#DsLhj}S_DpH z8sHCyat;hsUvKv8DfP#N|A*#-#O>! zrDdj<7J*mUfHNK>$K@$F26%$bp9bd!E(Hb0oE+G3YM@m@kj8s4Y#22!za%w9As;lX zS)N&(ijh%^QgaePy%DfitrUWwTm?{42U`eQ&<(c^ocX}3wc;~VtQ0(5uqy$V>X21k zRtg>=Apr_ysYOZ1c?QE&kbUt5iA9OI#VDl)NE|fbm6)4al3D~QGcYt4mZla}V$%(l zAgDdRC?&NhH3gdiP$>maj~-O2fL#c82|RmZxF{vHIJqdZ05tn&rGUf%nSr6WD7COO zwYUVlkrLdSwE|B>L)4}!K&8Rc@9+|-0JO9ow~k`4Jc<^G9pEkhR-l15(1-`o9sQ{|up&c%T#H9RENVN;m1h*cd8J3%%r78ICEJ(c#Nvsh2 zpt8mBDf#7jpjHHOvF(`#?#O`x4PGgMPDlrBfIw7QFrA>;VUTBFQ3yKD9>h;b%qg)2 z*LMjD;4}(%C_I6I+yW{;!6zsn+-y6DfP*QU@+IL3t0F zDR8MX!BTpIJw!mCDHeUuyhcE$85W($84SmnedeIj8X66VY=g^N77*2tTtry8B}6$= z4#Q=zp#kVjr8LkmDs*@ll2DaV; z&@r8O@TfOaV|-9*VQFSjDmdgEOG=80HS&|PbigM~LS{|U@{7Pc(4Z-3#~ZBGTvAd5 zt^hz=2TDqc^hyf~;L8a>;hbNhkdasn(g8719b!310IUx@VhJkPq5EYZsu0qUfKgC@ z+&HK1m#?5&tN@(|N`)-zf*cY9D(%4$rJ!1@2buyzj0P%zT@JPmvb_#u325jM6a?|{ zpb!N|AyHuox(yJT*syiFG!hFk<6(_Xa4Ld^0ImquA`0Eg2$Fhi*i7P7E&P&K6OwBQuIU1JFpPgi{M$Rg8V$txo-A~p&j zrw4VU#krs`ftv|km;hR{1Mv>rI!G!3jrZ!IRTps2fd@b03yU?7w1d?`eFOG%W*SN$ zf*KE?X=pdlL#uwl^9M{-Ymd~tF{YA$Fh6UkEhC%#@@SL8B8U3bqt> zb{-^+A`%c<$V0*%+;oP-5Nh6q1sg&?BsM|G3P}u}9l_}mwylSVEUpf{Ee3MoG0TM-J5M{saK;|SY|22aqQ%2de7eBeEL z#re75Gx|WyY_JKC?jLvy4cINppo7tM9fO@cJwZDl!9AGb%$(Fb&`Il{U1gwN5O|am zboxV4Q7Y&Nc0JIYUyyUup@x7v=cx*y{ziF5KB(X+N(HwU!N*2`D){_z@ERJBL!gza zt{rH{64U_5pi?nuo(L?0NP#(-NuVqZaRWF2An^(ulVl~bVghR`MU z(7|9xTMw-f0Fr|?_&{+B9l6cPOe#tQT_FT+0w^f>x&{Y3y1NF)`}(_t`nbY&dvYl# z#5)Ff#(TR)g4)@jrbxWAkEg3&NW7;Dx=66AbC4@|A{eYLz|q;;(H&%zI+!2q><>BM z8oWj|J~Y@hDBjWC)eouzTH2sBL%=N%q+xz=V?zhjzK%y01+6C2Q2=!hAqwDu42e_l z=9nCa6lgd*9@<~91$QZQphDnGkO~vaE6oM1{sx7SttrN2K9c3&jrLHpVQV$gbMliC zbD+r{B8Tpz#FP~93f#n;c#yl}K^sp}i;BUDq3It>a~zatbQCny;|mg#vlG)n$Cjwa z7boWzq=HvlLi-!At^!ia39HLMnIFB?0h+GR15FP@F0;a_51JIgUH)QlUkQ@#L9PSs z#ZLwI6XQ`r0klXCn>aXJz#4P%(_s+E|2)_IfJc0=t&VU|xfNCrf4{QP056Gz!R9zqr zIs>c8OhZq);6H3UMUFzWB_%Z+%*!m!P)JDyEiF$3@1iToNG(oyvSgwPXXh3@58#*B&4?cqjbhaN-XhWA!Lo!t^Vn`iQ z$$*FK6_64`IjFglr(U7}o}{RRE}uqDA3C7*+sUaQA-FGd!HEzwT$`C!mY)M!f0UV$ zs+*dYmYNI>36NuhHOW*QgDO^xdQaC70^Cz*m)G7?a7J7#h|H%JW%@yw44T! zd!WXmEx!iEww^+8YDqCp^;QWYZ3662wX9=;zm*itzckohScOp z@}QHf;uF(B)6+0Di0U1p4pK&e!UNWe1@C|a$1S!L?^&V_TC$&%S)z~vnshA!%|wH) zA_k4}gC-)NI*Us(b8>Vu5_8gY!S}?1i?I}N;DXkyf%X-{hr4wYK$onRrz+%Tre~CZ z&(eUJQj!lossuFN1U{(+eBf7R3FwN#{N&6;&`cC)g9W&1KuZ6R$Oc`i1K!Y7q>z{g zp0H9V2Td-3Zv}#k(re@J)U6cMVLNGxLAz>GG{DUcKiKkxU~r2D4W`#CaP+|qUW{~Sapnd>%ydn276jTO)E(L6Y5V;*$HK$dKq-3M-XIaB-Izxu0pl~yn7u|xI;CAM*2Xz_7#c} z%OMw8W#$!^q$Z|-rX)Z|-Q`33+6oH!IVtd#7}$~6oCrxEpi}1Ju}FhD-=I4?KwWDE z1&olyt^;l_*#Btx8L2;m>`~OPL{bDc1Bs71YKSBOHVrk0fJ=KsClfp@rvqLS1?r6F zr9;PE;TK5TC>X#pImS3YbgU0?Gc-7DfkF>4m;=7rJTEmJykZ1$wHUayisVY@mKAVx zp!Xob?gRT58rL|}4CvU$pj7bX%usJ5SpiL*pjbc>g_RRXLSP?&U4kSCnsr2sNr080 zMLViuG;2W42bEy~knN|SnZDHG5~w2~@^i+t=)K~XAbOE+i(D`<&8 zej02+1!S!$DAZwvHd1#U7I(09exTKOh(UR9G(pZig}4JWw+C)+sTP9=Zb9eqfR4e0 zx=#(zFCw3c8I1SqXH22h!;Q&F>*R1r9lAs)i~<3mcD;k^_ z$r*{DZ62vb&@MAbAxffDcgjzxw2H}71r4WRI1myK=)23X7@(j5y2DfxTBbk~8?td| zDGpZ6;L47WAuedjLrn(|DOk{w|s0IQp*j9jLD0nD=0sx93F%QdQs1*n-2SF@@ z1OwDpFohr=Ve81bq$Y#TdMM9GEdno^Do#}Z-KPpVZ5Fi1tQfX8IVZCWe3T-nj0#RI z$^>`hkee@{QUH7%5i}T}_QTseP_>Y|zF?~p(HkkY;A{pOWJVsr0XKm_7Q3VtW49XS zOK^dr1D-sGIzvfG2}u`t_jh_`S!y0wjY1-9{V&#Vg*OP0w$(s~a20Ifr^4xh8X_V2 zQ2%LQa|$>7hUi%wAC2@YA%?Xx+VDd2HWuqsWE z2jHi6;j4Plj7KTm!A2>7PmTtaOE3>8fbWk69f1ehj08=Z3aSdodO>LubTJ*M_=Y4t za14MZN)mHZf7W`O&fsPjdjmTi@LJ}yt7sTl$$b3;79F3#tfig3kqw!_;`pIXwU+@aRBT_h!jex0~$(& zj%Gt-QP!A2WwG?@q2_~2PpHMv5P%p1l?NpX^ez>|WyI(NB`46faPTT(XYiU%ur^S+ z1ZsLOPL1ezY&C;@7b0?N4xpppd?Hcyi}yY0azayC4a)#GD2qtp-Bdw&t$ppb!%(7^H(@_+`k zaRu5lkOSJE4~{Er3ZWw(C>DWEu?LTBK-~tKk|1ONsGNjMn`GvJP9iSNDbY*J$$?zP zSgZl{9^%4j4HO$e2{=Bn7<7tBY6@h7E#y8!Jy3H4v?35(PCye9h=NT1e}k>F)BuH~jzV;-4(Ql)SaAy)J%XI!4+%cdF$AeODPX6-`mSK-KnoU#Dp*bi zrGUKRlA_Y&62yoaED3;T?4h9ws>i^6D;))tuz&`Yl9G~hKGYzEL%cu&&`>z6qXDmnP>hCUa*!J_Qz_U_ps+>JgPzDxBq0F{c1=8}8vvTw z%}g%QK#Dx@3_WNx9W}<00~MNBK~|$;lthIy1)`)gbQi(;RB+E}fKv>@RnT%s2NIgF zwO-%=1*-&gDj*&NbBoa}fO|PxN1+UI#UO}<&;|+@9ETSm`xZ370UE`GO>#o)0fhj5 z<6x;4^|p^P&}m`dyEmZ831k(-8zA>YXM#%N=wo#pgj~&mj=Tu(78k~ixi-% z6_L#WH|SC|)CrjbTC)h+W(vNu5X8XiI;5)`!O;iJZwPN-$#byHKcEe6D3u~dW9}Xi!MZqZhl#6QBh_}syZmQse_ybbrwhw!V;Kf z@Q7kwi7u#?hiFMGD98crG0M!()34AiFE7^x4XEgr7Uh8Zx~VBxT>~E9057_R1wEwO z0JR*sD*(w$-~9I>Xi$>_${uVdW;a5fQh{ zycFp2%Sr{<1vv^S`KiU=wGpYQxuC%<@bR~xK@-RfFg)!++yz~U17CpycRe(^K@kcn zt8GEy3ZFj!kJ^Eb+sOkR(`l<-U5hd-2+b_p!#eel<}rAV6`VD} za}==9fEHb-Zh~e2u)~qhtwb>q)SX65UxL+wyolMqfCLZ36`*D$dP@ea&jRrc=qMBL z)D5bikj8ajgD*%nqqq-}@(=+H^%PRM8Jt=IT8arBF@@$9Xh=X?gph6(G{J(_+hpeC zfNBI#E=fu(&P-N-jv0dnA&|Ag^9#1*1}=9&MnGH73W*9inI$DTsR~J{B_*jv3ZO&= zUdaRYoHA@$4?L6wJN^K)4;HcF7cu<~@6zibiGlAUD99|y&(A5=g?LKeP|pBTY@xUt z>Yr%vguX6h1qrCwL>LBk6Lvk2?b(=RCOF9v)TawBOf&P+AqJ#ZWfp+DXE})_sfcw^ zpw)DsAz`fXg;v%p#zdDY1%ulJr8GX<_LMZ0K`^?~u7HUocHHAQF7o2{;X&y&D z!EPd`$^rMniOM9IY0%x&(83+nVW7;8wlfdrR#43Z>E40vCxdJm21kzqcu*eZZfNj< zxnODJ!T`k#*jNNq2W$WatGi$s9BcqM2oa6|TcrmUj8CcrhXiy$1+P)ymKUfi4p||K zFd5_!=r}60yZ}#^qc{N+<_fkDkArR21BpN)GBXXDbUG`0c)RKJA)xL;A0+hHw3AhMWX+TO`aByOCHrQw#h-*OZfH)OT;?fqV6gZL5Fk7NUE7Vqaz<{lXL@QJZ zBRZgsIan}b4`6V_z#Adp0td4V0v3iepdhB$LMUk1BDdQiPJ-6AXmulU`a>(4Ftp{T zR6;C-`VN!_kwzTh;Q%j{Az2NnISh&$#7I42@&W2LXqp9YL_p~iq4^C(4&qECOCW7= zXh#ka1MtBf=yQ2Wg<=-^xLYiQ(-Hup5+7`@lL1#h~Sskn2h-6+jn?fzE16OiqQbQ$&pAB`&Id__B{+yUG>yO;QsA_N+LZ)5Dl<(1?s_biz~nHkK}}UiaRUuB zBoRH7$ixy|C_QelvoPWdDT+``v&Cr-B+)>V9eOvLkn14@J!Fj=e8?A?vylr?WOKkC z2Ter82bJbQ4x@s$b5K$ql21?)8@O!5QpiH$6Cz7W;pz;#h7;P(fbF{irv!Kt7i$tj zP1IPD1-6DU_^JcY;DZ8mVR^AaQEFOZa!Gy>xNglz1+8|-Ooqm)A@n39;60&1V;FAaHu2+KYgDg%}NXjqCfO!_=6IppV?ZP5$Q2E`ZD4+RQ6lI}3O!H^!fGCH=z<&%n*9NTJM72A2DSk8Y%+KCdMlbIA>l3H9s(OIzg&wn*AHa0;&|JB^g*lhItS8VY`Z87ZU zzjDFOV+EB1NN2DXrNSg2M{Rlf!}yT1T8-cubMuq4p$E05=jW&Ar0OLW6o5LF#jsOW zLAz!^Nei?V459?Y!++>0TGGU?7-SptEFEX?uKUoU%&6;%7d-Km$GSdyFpauC!BpoR%}U z(ChQE%R##mLAx{|dlbPd^T6jPqu2myWPqBm$PEpM)@XHXE&>lLW0L>}H8u&5BO$&; zwHP#<4jsFP455S9#)I-LD1wtS^dOB)(1z`t%p@Zn1@+RRocw~+Jarv~#L|*{&@ow% zGdDmv7?S_vLE2%79Z5(Jk_I4yaSFByu+l>h#?VlQ-n{`1!(#X<4qR?3$;{DFPJj@ENePIW9UPAvkheFi0J$aQ(e8j#?G?70S|M_Ah%+A;-gQ7$U+ zEC!`M4WztthoPLjxWbpmRYW^R%F1+C4cHy3ErNl3gIV2wE^f+Gyz63}QES zIc%Y2tDajKk0t@?!Et4#Da3=bXM8+pen34wJ~uHlFFsx!6wydoA0(KWWdt? zgUw)3)aFCB6o32A*xb|rz5QouWMV$r{zDE!TCuUW|G*0db8|tLi07xIf)}tQL(T*T zHw-{qKtWwT$gXtIGLl5lFc9oWaj#&1Kk&dBtZ@ix>Y16qc**%WIiNEZ^7D#eLa?=5 zFg|G72gZi(J)y?2|KL;q!Fwi)3X);%P%s~CA^zqhV*dk{WA~l&QFq2d7eJ_^?6H6e zLTB+HTS{PpE}6+C@ZA_NLAXkgEKD$<65N2qtr)5RrXFrcP-;Pux(-(?7Z+E&bAC!H zV7^ z7$j9-v%oWhAd^6qCg|Q$kWpM**v3&oBPyW5$l_G+0cDUeVm+?-P`@BoXMcA;&nQ=y zcsEa1AD4Jv$AAD&KlflrYb)M4G&scHHy*ksBi`BH#nl;24x$f51|k#^8Q_YdgNqB) z9q>&o2+b=>P0mlx%dARG0Z+gdLsq7OYa(#P2fA@PH9fN!wE6)&2@942wT=?ePs&6x z3wqcJXao=1BnRgmNL?EQH4k)V0oV=&1$+Szot#q)TIhhV9o$%fw7wLQb3i9DLpd;$ zp*A)$Q;~95hf5E#*O@ zt|j>(A2=3dhUDkx6oby_($K`^G(_OXqaO4C+2#Q6+@re^c5pPvCU`Xm^Ab2(;aYL~ z1>^ycM?e>!KuVwdH1K#d>|}kgLC`b`jSQR~M2jD2IDt(-IE4#)MdkvU-p(vmNXdkp zoeA$@NzO0LNdaA!k(>`X>?##>bPLq$pdwTce9~H8I$Ft`13D(H zv{(VW6JG(-QK04&s6WD`ppXbYHw$cAW{Cpa!h*zN(DqZvygR6u176_J z(0T9N(wve^P;^*I$gL!h zo*k$W4s`+8$KH7$W8sE@tuM~VFDijvwS(>xQ0o991<8Bxz4@Re3Pq)PpnFeJ6cS6o z27&xxXkdVJqyp^lzH-ncHfR73l6F9c5rc=ki$R5YCTQhEr9w(RXsvrLc;Fgt9Hiq0 z_Lrv%bk5w<1$+RqLTPa+_|Qn`73+z43aFMt0xmBfbgoWmPHHi_-$1dHlb@FkItsQl zCl!2w18fBbXjnHX6_P?zQ^0KrgquLI2W}&Q=D#6X4tnW9aY>Pmf+IA*LG3B<5S#)u zOK2n(7nkP3f}jX|;v8f_Rxu=9B&DY2DdZ-mq(a)j;ACF}I<_yd1hgC@1CbcPYdlhr zj-SEh(wg1?NQ2 zO#nstxeBla-umgOdEi6?xt3YMRsk}&4ccU>p$-vL*HJKl&ro~jrKIMSc!LMeH6Uui zwTl8&BWzm~R0(ZTMdW3un1Vtv__BqVJay15)C%fa8W1mo+yRLns1j(Y z8Ilh!sNvcb6hJm17us+;Kx4Yt0tF|3?J3Mf#r2Guo)J1{_=*MJxcU5kz!KZx7~nSKKm0iXcSEl$^qhi1I^c#s)- zpkM^`VHLm`4WtNq&L5=D03Lb(XLN9H1R6sqdpe=J-9V8D+Q$J>0-~XL4|FeiY7Y3e zpyG72CEOrousFjOu+W$U-J%V0EB43+HxNL3qEjKeg24$5>{zhdVdjHl0wD}C&lco& zoXG@iE^X3@wz`747Dg(8Ugb+5m4F8RAVohkePpIV^Bg2=B3B#gkh3d6d+b2Bpnz_@ zH-I0{1-YOlw>Ul8%Fw_7eAEG!919ypMBWk#PTAmN85M#HQj;?ib3&rP2UNlpmsEka zG-+CaQaUK@fyR5#4xh+RODj$-f%*ZYrns~qwMYZSK**9}P@)H|w2F@hck)1Lp>r(| zGZbtUAPiL7Ky9t~cCdJwa~c^(=gARCFy zWFV!WJOnwlx2PZ)e1j%ZCIcxzj$znEIq;x^F_2d66sPM|rY08YfbP`M%gxU#$$&6Z z5-TB$jQmoVY-V0*Nh(CNI5jyx59#DBQ0C9gOfJfYN}xp^NE#AvdZi`FkWkPt&;&KT zLH$+mL@_v&KuHNY^Z}ap1rK|`+TqZBUp4z6rchHbS4dGRuF6gB$q==P~^fO72HpSIt!!< z8u(D9FnQE)+^@mZ8O#(ShNlD2u1-uv) zwn`Or9~0#e>OogWns9u;`zylS~K@Oze zD4HrzjKG5wGPauuy22jW@eo^(4qgHW2xv2aJ1o|qu?dSbP`#oDx!FWr4|cS^f&%C& zLYQ8V^TFj9I2C3VgZd(gdC94u+1b)m9q?_8@HB|3H98io5!wcZrA&w`v20#NI@t-5 zgu%K&Nn1eybUhY0{emms6wn!PnR&3#2Q^ed6+d*f2%e91kAkgpSuk?i~XM7euv!twJ=^ zo3Jfl*hXc+H5S-anh+zP=R3ewPs8;>hH{YxG{HJSfdiQ^(8SaaiX^CKqAS1+mnOdZQnYLgp9JB}lTLv-!Ie>Ja2@Ne^@VW_fv?$p5kU)gH z2jmja(V)Y+()uBod9Y1)?L1iFCCM@*8 zRv-?3f+*Gldk89(2|79!RFi|3Wr3!2V3`}9y%EYmZ3RSS3Jybf$pY`$f&&^{go6DA zYJ$Mmv4P4WXh{HTNK} z5>Q@2ouWa@wO~6SjzkMpXd40Q@?t$m3I#U<5Mc=IS|B+Z>>8b{Ry zZhXM3h=sbN2yQ$$exTBzrmAZmX2A!Qh84}=5)#r2#+~HQ0vT=uw5)_mgJTD8SqX6u zQdtR3bD)J4phAP<~b|KUnjM5RMR7EKy5xE{xbt9LJU|Uo1mW@y) zSjtAI4Axu>akVYz&>PUXd0;L$e!+bQXo(41eFIB9*vm!@q+AXT0;I$btw|xJ6KG2x z)D8tm_ZU~H3T`|V>$xPBq=LpZAljk%4N>D6=G0p%i+5 zGT3#+Ak(2bA!GAUj$TS?aY<$#_!M1eUl&OPlx^{>i~=)Gm9WyYS4vHkajiN5Gj&-pvHr`EJ&#cVFs?G1hpJIjsmHW2%I>K z=xM4Wuh4=_jTI~8mKK+QrWbQT+w@W(^U$EwpI@4nq5v6^2Gv07P=A8vl2ReJZiC$l zDmWEvk^GKQu_Go(5Hlm1;H#k&azUHlQ;Ul7ixj};d4OhYAX|9B#TdwNux_vgNvR;| z)QUvV)tUhaK^iH|QuTD%9z5TH}tt7}2-fEGtk zC&R|UAT6m-NH&L>Qw*zpF^UJ|&;(@%q=U^cOA55801d++)_;H!7qqAVmo4BJf+c6L zB)BMroE4*h6r`Xb5zv?f{M1$O&IT zDJZQWhlNIVYNaj6W~g#ou*nGZU~VyJ8(V5#iUuh8flL6&Ln9p|1W|+@@5Cf{kmpch z8leeRmLW#lpplBO5f)*f)(SL{K$0dX7@?^V97R|XFYG{FY^&lxCy8W&&K(C&Lc=f5 zQz%O;$^?y8WTt^50doETQl~H}H7CCuyz~P!AzzZ14Z5SR2z(K7W{IAHX9?)$J*d{i zoMP~@h`dx#CqJGPtZ($N-&%51QmG$Vr8a zIb?#DW`Ig{aBS!(fNwB_WMoi1h$RDq51iFN2{=&1LR%!5*#?rfV23-YgYALt)j=(n z!7^a|_={xFYF|)$0(xMvf~^8*eH9`xgUvxI7$C(wN@WCbFrw_mG!kqGIM6`}2b6j- z(tmMD5n3?7as#A44s{>IE|4i;GfB%8u6TME&|DCdnwwaFr(FcPWjY^pa2R;PM6Wo% zv?w_hnkgV0a@#{t-6#W>(EJU}M#ysoI2V9I+e^jJOlS)mCP!;9L2LtsCT71H6hBb? z_|pz#Qz>R+DYQ7X$TP2?v;^FRN7U%>Ys|qmKnfS=usv7=;%1PeAY4#I0Zk}S_n@RZ zE?6#sl?jmM667*8H2=bKiXK!gESRuOf+4~NRH%WR0xAnYG<32abjopJPHAd9)G*Lq zFlZcsZkzxYP_V<~LFo!qBEV`NaI#gfRe)B;A^D)}rU5b>lCwa*2c2GA44pPofD|pT z1I3}i0Z|LiQP7T6acT*c;ss)Gc4{S}t_M2+nzvvc09gua^gzr5os9%a+MsjAL9)=4 z2$6);Ja@?v-M^s z&};-BY06B4BoV!$)B@1mfyON?0wBu3 z5daM)@WmfyCeYDU@G?YDAi*YJ;NcH(2Iz3V#Nt#l6VR>ipx6S1GNeq_02SI`FGBmo zFn11k_&^Wo2!%9!L8Fsxuo*fXh4PHdIvi=pcWHYl^p*%Gw2R!VB zn2LZFuvQA-K!#OopxK17ROsXfXxyZNCs{>9LpaKr63EbEQ3&Jm~0G&6EG!_DuM5@_A*#j~j22FvW`U*N^4(W4&7M*~n zq>wT|9%#8VL?z7T^wbi3_g_GpHsJgQ(g@Rrqv`@X6Kp?h0~p+P$bt~CTr6xB9AqLM z47ChVKfC&c`hrSzv`B#Lx&w_YgZc!}0t8xF6D~u7LCIbLlvs*VL7Tuq85Ndn!2T%) zB^XEwL@P3qON)w9^Gb3m6*ANF@{3Xt`2$?S7K8h>ko=dI59xsCk{ z%)HbT1r1PU&dE;)t&dgm%ZK-u(m>-*O0aCBsi)wUUj(ic3QCI#@{3bJiyQO7Q&`Xo z!!q;IK}*Opi$SAvg{7e3!i>bC+_cgh1&!j&Jn$ij3MHV@s2Duh0V;L!AnUd>^Yb+I z6g*OkQWZd>NtwBzbKF1$Pf138aVjVT(82_|KL#}XQ35`zP6Oj)#?-{(%$!QlfLama zPO}uvL03p1$2&%`pirDx203?*4uu0W#X}ke#E+mTXoFYvz*_B4Q^2JPa$ycNRs&q~ z=R(esfr^0|W=LyEAZ-PB-UW}lWT#dZLlcEAfDOP=vVi4?D_NjcA9z27rP}%1<7^fBu7^v9X~k zgMpEenX#Fnv5|oRgMp!$p}8@Gf&t}L6E>l=xFoSiL4lztKfeT@YC`h(^bTSSOoPxxn;#By- zWKe^Qpof#eeFf@x>J?Oi+w4Up&>kgd?g{^?5TNsgu$%|so>~Gr=?Z=XLt-}UTw<6I z?4&Trc?`-5#s(IkcAtWYktK8+6L^~!Xh0e~B?xu59#@cSaHy|qyr-YDzi)t#YY1qC zp0R-iSBSrVysx8QB<$=h(1`~oMwVRhL9Q;IL9Wgr@xdXEA)&$G(+xn9kh%qCuCITX zt4n-zx4p0??uN(on7qTHPF|P!^cQ91}w78N>8PpF?N>u=N zmrL^DN9TeJgO0O+#~DD;3oe#m2@86|Dg2aDq!J3;|A8Kx0uDL!Gwwhe)6()mOM##v z1!6)C23?Ay9;*qCOB@LvOJau{x(Xeri;pkP$uB7eorR^K0pCX#AD@$&hq|K#w3Ze+ zBI67>EifoCFFjQQ>Z(1xp`ZX7!Gg$xwS#6}5Jn(t(T8gR)xRZR)v0-)omU7Wkk#wL)kAxVnRz7|sd*`y z3c8?G5*lDrK>J_}LCGIHi>R&(^%>$$7EntSoa#V3e!#^oJjsEbsg9xm(uxJG=7psP zg`!mOk`2(}5AfzwXrc#c1EnGGELDD*0_eP#+=8Oi;$rZ2W>9!4E5NS&wFY&^k`qCL zHzh@&W8phvl@&ndgD0**Mrb7`=4mNFjDYzR>!DiVMH19DaBe{_U?Hbag6{$Y7njf+0W|`}ThXv1D`T;m z1UlNeGCnb_B((_HB#?K(Qo3L%yq-m~4p|#^J2Q*pi&7ycO(5F|bv;-J<+iiTJOvyL zghZsFB~S`b&&f{)FFrzbFsKd(l`Np+nqaj!D7!$bT4*^3IyAkcC^NO#))d5qI1tpj zvNZ*Vuac5dFev+jbiyvPQvjWf0m>VY_0b?H@Hhjw^$BWq7NizIkDvz?>lul8DLJW- zTn1_@9vkb@fva#AZ&K_f92IjQN8UEPp91E5#}9qEqR4A+6Ur6LhDi=hK< zFZn{}%PUf$L$n}gz=GXM!BGL``I3Bv;?z9oQ3W86WP-N9Yh;3IR@jaPQ%wcG(%huf zBGB3ss2WgwfHvXh7LIYNdK5$Yj{ zOHV-ooT@U5L4BqasOuC!#~Y*;L6$2e=9NI|pQ8L61(0dr0Ys$E7m}HvftuXJyhPA8 z5wIaiiOJcZK}Kaz%F0MA&QM6n1a0Kc%+CY)F*6OcYzlnEE+|dtD1eH5sE0tSoItHW zaE4V-09`mzoSc!G3z~<6b+bTiWyoGOkak#-0*PwsK@@|VNZ@mgbK)T);4Fq@8Z?{g zA-n;NMbOM2dgy|zK~@=!FbFnK1&YE-P*qg{uF*6={LIv11#JaGs8^H~K&?v!*zyw4 z*$Wz);NXI`DM4p|z#Em|`~bn`}PPMV%VaB8ZKLTWms3#F_8 zxhly@Uq7idy;u*j^jI&yC|y6ZxVSXc(9put0_+xW`v#su6%^pJi3$oJDbS=Qxcvg^ zos^Umz%E^bbO}M_2x@=uK?YHWDA;e1cg03iou**aK{Sem|IZC0(R{q zxWLr|i6DAiV13Y9LJ!jHLAh5Jxu~#3=79YGO`71mW{XHLU_NN@4?Z;vvIcaQrc+`{ zF!*$7pUgb)7&v@_A3CI<2QIZW)SVMc(=$pGP}L)M4nXx-NfBtY!wR%|LPr65;s)4C z1zQEsi~yLOT2P_@8K8ie4H|ie+lsh!0uneNcfvZBina>iE-I{Lg}R#pyL%xPDS$me zk_$l=g96Ai4>o-ql$r>hSfiO+(X1lD&7ehAd1?8OBD(}!5W@lqUY*-2z^ZKxlq8SZ zFaf7(aFzl089>|0z{NjGBgsa=)W86gU!e*?83bkzib6$O1)P@wOFAdiVDvF>j zL}*b2GCHX!H8C4%6)cuPUeHhvN`<62*q&%mJqx;1*G9G24q9PBny=u+=?bdFdeGq} zkg-S@$pPSG2kM4`QVXPNL-rCh6Xk$H9I>cBwIUf3jF>3~?hle(o(WHR3Pq`jDL9;t zQSgB7V1~I8v0R5_+YyC_0yz9hw;tR2Nl>d8bXrug0%%YYv}`RIdQTZ>2m(~&f|DIA zK46I$q*$*wCpEPIksy)M3aIV|4Ls^OC+2|efhtK2Ps}WF%P#__4{!xn3GZDY!UV}w zqI5&{(t)!BD5rtyeux8ctvGW@EJ;LbUm+We2H zk)h$}{0}+tNva;4^FN?lr4@>ki!uvJzzyE?)V$Q9M9_(O&OV;tp-1?XN@7WBMJ9aq z1+;bsHv0lCg<;%$_<`S`0zM}*4}P6YQ4XxtQ3O+5Tv-gAqk$ah4_bK-Dz@Mk$RrjN zBNd&FAD2AM62i1>rjt4Z^3%U&r+^oZ?sv>BUplhN^8U#9`8+1B1xK#r`Z%ExrAty06DJ4;% z!b+h6(#KH;?K4uhf~>a%4Vd^sR_;U2wbkLOh1Lh?!3(Jvl$4a965!>PkYoT(zo5`@ z_VI)*uf!IfplE_$5L67>1_U{DTmhS3K?`ltu*yK|$W+j>Qqbx!P~_{mxVm}zxyC2u z=jWs*=D})3^_={4SPx2F2eKp;5)29o>OT4Duy$TCWJ?-2X7L*ZX~E$z3nEFpVUV^V ztRlu?C|D6>+zh4?n+>4ldT?*&gHwMzWStT;#VaTv(giFGu%;uuvdq+S$kG*EJaCPE4<$b1giDO?IzBMP=YmP=W|&p*V~3O@1zn(xR=%LL!XRFn$8pB8j_ zD0r93cRSe%&*xt|xh+6X3-o>~Ga^)xg=la)T8b4@g0lAeBU{;<(yUAh_3cKOQt53F^Vb7vvY0q-9osCl`9&$^2{9670otO?nA37SdD0j-4uZ3@*$%_{>XQe-K;(gM)Yf*RG}&;o_KI$8=< zw^Be8fz<^H3hJ=vRJTI-4q*~fsDl(BTAmOS!25VXL(M6vSVB_|?PN&M9yo}15P<;k z4a^asS?uD}_{6-F_@vCd6b zXdDq5+MvEVC?A1R8!U%_q7hV0feN)^(EV;1dZ`r!iFqlY;!p!73o#0uZS&!(Kvz5? zRKN@brRl^JP`c3rxji0aM1E0aJjfD+YG|NCItY5Ox=#br=tD$~9(Yn6hZ&$k3Y3>& z4Pr=MR8mrME6qy=B~(z!4sEJ}hE|I~ZRrxw`c;G%=???DnKsb1&2zdLRw-rs49RQ!7n+mrA>WW;5R5@t=GbcYE zG-9a?)tH(KGFw@ppfV>juf$3rC9@b*TiAj&-lQg%q^9UXF1+J{WO`7^0dJsyj*9Y4 z%*^uz8xL+U6jv7OL2N6I24y&qdeC8$$l{1w;9&DUAfJHRLy(FW)VMEps)U#d-tDIe zs){t?!K;kpMp< zLICfzf0)*gIubH6)ngg|qvp^bI8gDg|28l%GciLO|1&W%8omD+9CoxOaE||l33;WV?P=7fUGysoKN4r#jCI2UZu9>FV;6DQc1OEJPY+`JPe*dS5>FE9M zw2O0OlX2#MAL#ZDP_~CHUQ+-k4e$~&_y`lY#Rt#!;N~PG2Phz|k%o=wgLY%)=s{b` z`9+{C3g@7z$<547)yoAH5THB@QV0@>hl^lQotK{nmxgOV7DUwp-WCFO0@zjY5I#%| zbZQaC%`8bR0{1Ck00hKeS*A z$oh83I3b7)PCluiTSH-E5uo+wiJ5tz+hX#I6u>LoVO0wB)+NvwL|(c=BH9KI(7+CO zeH>^7FX&DtbQLE6-O* zEJ`m0SCNRt9InpdJ*Tv||&1D>uZDJejTC-9&=cpw0>7!Blk1%=|$q}^!N5L_`6Xtzr9HSod4JsADkp}iHTIho%pn(r@Kgc@p3L1p-K(Pk8xC%6Jl$lqO znx0w&DSJSp!=Q1f#1c@07P5mMv?3q6xEo{?C=N4|Q$a>RnF^VC8L35?;PL}JNe*5c zl30+b2b$J)EXV|{q|QuEMK}kt0u&l1Rtmo0AOfv>P0LINm%gB6kdj)QS(KWB8XO=O zfKNt;rXTR;5zsAyD8Uap^**z>1TtF&(NdgR0$K8e2n7Xgg=kRdftII*`ZhAr@$c<^8@*cIsE4;G}{X`qRL%sd5fY=VLxC3C?X2r3RC z%|Te154Ih$&xMM+8D|nj&Xh8sIQaT@eS0N}g;Egg^WdJQU`5MJ-(8c=j2moheco6`11ZeF!Y|Ro> zNoE=(#!=H5xZMLv5YW^BHvuvX4sr-o29}jUBebAx3GwmJVi9zwHRyyMNJvA15IL*B zU58Rmg6dGrETjp_zM!OsRuh9;OrRCY;AupR3K=X4O@L^1FSNgkRO`a4PjJ8>_F*YN zl!9!rQUKL$3ZX$h;Q9(|JGiC-HLjuimq7J2O3ne*ndDTvum}f-b7DazEE9kxN1<0Y zfu+!Dc?D4Yf#Lwr>I7sBU}uB813oyeBsl}j&M#6>2PLUuD}DX)@^U@M%DTjY%wo`V zqCQv?eDd5e1yV?YVhg3k1BwE0<)xzlSs(@KJ16EAqs0X1Bsb74!eFJaOb@mKn#GZW zADW&(;RxxVBDFjSG&&$j3#1b?SD%-z0BUeSE|rBfG$2Y)dw8H`3ut={I3;gB_GC^l-!j5!8o>&2=ACMu4OmX?GYusa ztrYyh%MBBAK&QlkcUULpfJQD8b0O>Y5c}3k5c~cV6re^16r~m<7NtTmBsl1yp^dU- zP66DX0(%{n1eHM-zZ7Mrg6|vx->#dMnGV^c2J!=1P=e}2m}kJoK&*#0e8A-dIL&}A zh=()+p$mW@>OfOh3JRqKphNz#>Inr)q3Hozq^tn-BDmiH?o+|ac(6PueZ#U5k~^T9 zkirE~PC?uRbqCJWjS@?cwF01IgW?HW6dt%l1MMGoE&`pL33f0v<$%jx&|zGeC7`U4 zUs|M)3%ZcH2z=}lBw-+x2Vjrnr0K!>vtXZsY)AGjD5K`2>7kVsAW?8s!1iuI{RxQz zaOr|%5&S4$Br&izKn1gFMQUOe$MtSh9tB{K(!5dA%eo}z?NdfF9m|;l%vQqF(tblJ2RdCF!MEC`~ zl@L6q0j^(5lao_ZK~0#XN_fnHiW^WX8twpSa4A5~ZY)YI$S;D-$>$^%mmr!?@HP%8 z{^4~t=y;vPlGHNLogyXR3ENcAL3H37$FY|HpsQ;@-9%`bf>v#?;s6?m;ED*!28T0@ z4QoUqrITV%Pd+anagjI3fglXkqleo30N>&YIVc{sS2#0I0lG;ODV(9H2fQo=su0mY z0gbytcglcc4K*c#Q!dB_$_mgdm7Ar)y9xYzxkFa4sYtLc#$WP@u{O%7l(Pg8Tr{ zfou(^QL6z}sD~^8FT?auoAEHauowl+ncyv-$ibGMlLHDHP~rrSg22-_EJr}?r%FNr zwX48ki#@4;lYmBIUM1Yq*m`#8$rR>5nCC%Z397y_5{uy*onaebK^vZnQd3g%K;t*X z>WJlskg+$IGQDJkBE6#2w4&7F4AhW=FDwLm2RRnOoBP0NsR*(x6Pz57vk`oiA4VF2 zBvf#QP*PF?OQ(YFILRr=1RZ}5P6E)A5zt1YLB(rHK4`^KW?p(uDl8W9tm}iMk zzJswrHD(g%MuD6h(4csJkpgs(7SaLH0hMZ?y$_(624T=5U$`gWJL-`8ub>lsK|?D= z#U+TQmO>_Ir##32P+1K+rM4J)Drag6VuTzLClCW5bqHke8yv+T&3Fba;iWgMCW8!z zgBvyAp%}O!un|nq*}pldpc9Ni-7d(SIy^Sv!<&$T80(NKtbA8iaLr4BtWhh;2br3j z0jhqB;f6s*w-Lw3fRkx_JUsr8I~Jgx8>pS*o|B)Hm;(=L9PtEdpF;;k;T=n8)eSZs z67^8~;BjsXHx_0atOA2&a8RiSNdchIhZM+}8>vOmavEwLR58309S@y80hM3XwJ@VY zGEx=5wQgc*NxlMfjxsexArm@W3N;L*Bp#$BJ`;3w5qy?VFFC(7uLN=$6DV9l^D+xd zQx!a2Ksz5Hr^kXS=EP#qGQU&>&~zAR9N*JL0lEVoB?!R%AFu_brI{&uAi@M1<)xDpsAAlV#Hul9_Wbh;>4UBSi`Io9959; zhvk(bNI|V`qgt%3T5Jct+7FrnpmC!EIt~Lg-d9+fUy=%|p5il8AVX>npmDv-WJo^) zwu~AYZ;&_($@fgrfJ8pHcn6P!Lscl?j3iJ714kt^kOM%QUUI?9S+S%l@GyLSQ7))> z1?6>6p$*zC18S^-3mPm*NfXq-EiKA}9FvSXbOv!Qs9=V4j=_-+IwL3#GUtgA1t=*C zEkS^CTxvxLXze;E)>BZ`BAw-;0I6jlvxu3H+jSu>0i_L4a~oQgK-R%RDu~=f=pk6J zD23L~&>#nm-WHVZRtTK*jBlngbYnGr)B**i9Pn`!yg55;TX1 z2npC84Ty^&O%l+7P}=I?bxokNPr!K?*21N#MW6%SQo)%LQeHtE5p8ItYZ$9n1Ua`w zU0Yp8!4TR4geH8*f(E_PJdD&0Dlfr>P%x}!g!aWz%OGq?8c$;dbfO!$tpYxV8k)9| z8xhcw2$b}&8Ut=s6hWropxUt{H>fG#NlVa~FQD2qGY>R^lamQb#`$^rAk(4Q1hq&| z29;mM;MR`EMxd*5|z&sTPY9->-38(;gM~njakV&v^kb=@YNQ{8Z z1T`cyK(j57GZ8^o0TmUerhu37!#RbesYR6#1<;8uh|^%ELoI`J6(EHXXuphtx`H;W z=><`$t)LE`HPkoMGmOcDT$7iZn3Z3oqmY}KmtO=Pe*`Ba*pV&ZwbXEzf=&%80o7X)AfIkGCTs6+$V z9nfT|tbncxG;~%BUJ8<5o(GC5uxmhvJ;U-f*saP6-l?et$fFV91qVqi-Y!>fEJ2CFEmq7$SqEf z&Q7fat#Ak11IlC&QFV{};u7!)o8VNEqL-Iil9Qhd_DgPYI%p4Fd}&^01!%4ooLZm( z1U|_D5?`RxcVGo2C_TYk0n=WPSP2@GgH>VR(|Vw$C@a8oIk@@&t;m9ndqTpexTGjP zGhe|L+C1>|*MqbZJpIAzzClf)+{6knZmIE3u$kWIzPS;U@g^Mlt;unY_ zaA4Si@_u|~J}3!;!yD4J0y!4s`T_;m9quuCF`xma#1gRYK}imD#IZsN=twPvd{8?D z)C>U~V+S4Df`%m|Y_P-ua`;1?0-~a=bd8{cBhUgEDI4oS?y%5MFD*&awNM8gc?r@3 zN+aNc7-{+nZSoH48qBg7UVE41Bf2%99x`Z=75Ml_Z~{XvfT3ntDL`5;;Gq+cGeAKJ zDn>!AThtB{_F@?0=r*uTuu2-VcfBYR5dn~302j27Fog(!%PdeK2u``+yP3c&SP>JG z2dUCPWoWz(w77;`3x{L?Otpf#j)D<9#Rq^6j|RtgQEG7sv||Y$i-WiVoEJf5BJx5) zJ+RIqXu5}#ZV-i#4Pl^jY@neCx6C~ilC&Th1ln%_tw&O@RRC|5%PmfaBwUan;`B+- z=rH(DD^Mx`rCaD=7c2oFCO*I}NI}hC&{zhQzQyUFOb`!s7c3t@6CtQBL|R+~>1m;# ztpsltL#Ld;egh{=ztnP1$b=G73WSV{CxY+70rjoG(eDWAb7kfw7o~#Ms)3Rdbd(m< zP*q4Q%g;;!AApyZlbKusJz)(bl$ZnBidG3eI1hVrQ*Z>^1Zm@gdO(n*4~a%ts)KqL z6hTnmAb0ch@=_~GG$74-kiQVKWYF3b=Wq#RJ}fV_9A0Taq6;}+LCwI|umMdPKo@Qy z^_CUjyVU=>=7>TxL;)n>Yl5nCXyX``FOgQ=f(-_R9n>^PD-MyC5cvo; zPzy2~vLXYKpAhvkIBCEV8fXd#v||u-nrBg2DroaVQ9fw=2&o_gI{-N;g0ed(ErDo^ z85c;p0^N27>VkqdErRMksN=yKi(xqsrR#=bm``Ss5xC0)8m>wN9ej{i44(4<4}F1e z_5=@mfzC<*1&spc>Iy`+3uFew36O#b)W!kv%Zu{!z$?!%x`5D4@St)J)arz<9S1i# z5$#R&+{}v96zD!(W!Q~@u#;Cn{Y~)hRnXx9@CGMJ0bZPnahEVyM?6%Ox(@6V6i}2} zfoEpH(G4oe!G%Un4k&wpyVs~=E6^$e+AIZ48-R?^11;rA%AiRFaX2@)V+ATw5PnyOr(k_>lC@Sy0^K&1SX61Nq*|<`uBo63 zZ>@vw2!qKYs(EP6u?5ev!mE1F8FsLM2E{&Tm=&D5py2>+*g~{|#!*U2i@_^p73^#k zj13H62@Yc*12lmP&i#6zBjaF2B7AfnYB*%JAJnD>4bFfMRtGHyE71i70JI$g+P(^D zZGo4zLI!jdGNB_rkf2kr1s`P#?goOig2rKDiemCW7&M*-tEKcHEh}i72yBZwv@H&9 zI}vE+feI3sQQ(GXQE_S!qQM7s3#hjZ4KS!hP~SpFr9d4kc*@nwEJkd5foy&U6^0n$ zi6{y|=R`t`hm9$rr3~1JhC0+*Q27DgW|NtxfND1=oxm~$w3>m>(Lu(LP%e?dF6 zAy+c!#TTU(G356ePVtJ&inJ3(5hY zK^oYq1|n)uScro)ytj>XCO0Y{1?RBE{L&~)xNT&gIvM^|u3iO<3 zsJX-y$Ef<@^N>h8bnq6-3ZTFTbt^zqv#{I*Y9PakdxZbsB|7qaE5faiC;=ZRgxf4M zMc9o*-qtwECNc!D?EeMN!hlcEE{-n&-Ks?C9dP*f{~DSYnVX~S|2H=hMK3uAoUs&`d#bNNRD32Et)_ zAii^AaVp{*w8Y}#)S{9gP%}HED6;_E<_DjGRti!Cs+&>I%?1xIgUZ2@qC`jz(~I{C z_VTsEh}z)d64R91l*d&|t>!2SP216L7d1lpmnx zK^m7B<{?y|#XAm9!R$x0-$0oHdVvWzh9JIzx*yz_Lb4RnSkgeM(4ksEXLEzQTTqQ? zNd|iEX)4H{;Dn=r?0tkNG%3)|TBLM<-36J&@p+{=Ikq81rJzG=!BfPL=(SZ(R38NH zgY3G5I>{I`-vGKy8Onz^5t?8@G3!^FlLH$t0Vh{z{2;|CN{R+GvQSeJB65&}1~vVF z)Ie_O!ERSsYF-KGrX*Vh9D$DFDMNFN#EvKf@k$n_LT$n@0L7Dr=Aa=1q~qb0MJ!8GV8(hd7O(9_XRQP)(5ORY=_wixGH;2mt#Jk32Z|u`5D} z0k9Hi_=DM)QQ%k%^;~LB8Z>)>RmOwt0F4Yq8|x@2f#N|)MZlaX9rY5LIG?jYVqI?UI33JnDUddz%B&^QneE3(xYM}E6|;TskM+0%1_Dyo$IX* zmQuG;0QsO6oF>86kQ_KL3uzKG`AJzM1`Oz+Kx`Qpw7RESJyG3C!AwU%JxSe4K|K|8 zS{AqzRfhxsC~?Ju6r$%eNN8&m6y+DB7M0-7#SpciZI&Rlm8fmF?9@t7od<1k+M?S7 ziEh*qz)TZQb|!3Dk~)6NP>Teo|zAtf=~dh%>i!#R{&oe3(6cItH5{n z<)uQ_gr?;eLAUrTgU@J9EXphf-PHwJ-c<}bm8>`ubesq1<}}d6G`J~QtdLm@y3I2q zHK!o0GzT(l2)^wTd`cRqeF%y?(4Awb^=mwiHeiaT9wKpoGZSchssJ><14&d!(y##+ zq;kMBPoqEwGM$F*07wYKO~dL)oF-z}jZ)Hr%05T}3VrLenYO%U}^u38<8ql&l0P z_rMYmH>p=Ef%cJrO;NH^04u5m@1li{|3dmDh?);<9wa7^VhtgmpOl417Hl&iBfyHl z=75tAk~NS@0xSXwIG?Wci30lyPoLmq}jX*P4 z1)v?7Dd1iN*qz`>Km}U`kTQ^4z|I4;tH3u4gH5yrIRG>oq6cDV7@C_J=_pw0C|GLh zAZ)cow8%iZK`N27SejevD46Ofm?4$I*pyiq=qMN(=qMPO5K?HUqX0sdXzGyDFJ$o% zD9ZIxN^=W}k&o3z_zqNufxM(^V5w_h37YCn%*g@!yI9>y0o7OPhL+}*x(24Y24*0I zhL#ovx`qb2MkY|6p{{|UuAwFTW(SZL6%=BjM{8qGjo?Lku;Cq83V?L?!IQ3#_{GqR zstuI(5DhSBTtXX%pv6<*sD?GqJ@ZP;Opr?^Wd+a>52#!R*E67M2E5{{vLIDKJtsde z9h$45sWv}76_iq7gIzE;BT{W?at3rd7cs?K3|$h5mTJrM%aPSUx2HlB8G-8DqRhm+ z5?i=kVToV?Jx{+7GZRSAC@7Sr7FEWV<`rkAgSLKx4x<52T^EDTF;7n|f*5Cv$GA`w z^KwBR0ByDa-}{5hJa|q~P%yz`G}vV9IS&*ez9soy;CO~rKhR}Zcv1vpCIqC}t0do7 z0~{;h`UD!`#U(}XxIi}zHKAbDiZI?KA6z(t(gUU~xy9+=eQ}`e)!^WU@<6%1*wzH@ zg3L5XOD4BC9p*)7PY;qm!2y8XBaoB}$wZlHut|J`DbQ4*V5@+kA8aaA9@=97&)tAs z2U?R1pP&cL>lT;hlqlH3-3glH0TmnZAQys0|6o?bqZ}Hnplv5PuzkN!G3d?LTD?5KTs>AtK^^RB z*gQVy2nR^LrT`j#0ACjm4hT^E*(yM-1fPxz4F_AWL7+Q3%5A|$VT%D2-`iRk80aXV zc-%JH(8yRv!Nk-ITI*o=0bEEUCkUuh5P1d?09;&n5BWwKi_cBW1f3;L(cH(VW(Ec< z^M9aoY~l+N(?JVYDyb1h_~-u&j13Ji?*B72A3gt@8bMFc4D9oNN=izO3ZU*RXkRDn za4qoa>r}`QAjP1a59(azCgxRw&P;;bwg#IpOwKQ;#Bxk+ zB&b&aS_1|$26{*>s8UA}hYWl`Pn-d@BET!qbQE$jb2CezH7 zgY0s$Rfm`jIXur+9lS$FT}PpyC^ZGN)eGr{8YLwq(1JHi|3O!G6zeEtr=}Kw8i}CI zirJtBA}F&cfN$l3Y?DN+ih(*9d@dMxrZOHh>4|oFqK<+i>~KNw%2|a(1wD{^H8f#| z`hW{@r~|CP%bq~9t$NTp5#C?YQ2_0tgsOwp)e6b^MUb^B;48@>OUJ?^EZiv2k{-R>#EPI)P%{Fu1zQgkG$Hxfsd)+rBOn7L z;Fz-lZS{jVJ2$Z+Gq*HX0lL;7G^&%DoL`gz-mO{*N+WvUrLmw!3F!D3_!01M2NXfo z7iZ*`=A?j*xCCp1EgwcC4?GXf-qFs0;8hNCl}yh~-Y8z=9+uE6^qvh@0~g zb5lVH0OWOuBzW@y$dt6ylH?4~`s)1BqU2QALS2wIko2d4_@H4OkO7cL0&fXVEP)Jn zC8a7Pf)4}&Sps$-B<$en7IuLmC<&#ZWK>8itQb623tI078vTOo{7p(#fQJj0f&w_3 zK@X?^4`&q>B*Rh`tV;v&ZBc45#4}(grKW&xzymoPnufsZxFBoeVM)+d0hH(UQc_b3 zKn&QXGEjd7ssuT~*g{T$0Ow#XP;(TV$B~i{bfJ?1=(vKMR7lysaf5F+CNWS;5Moi%~Ij!OVu5Ot6X>RWOJ{ z089H1>1>;L(2)j|F8?BM|AV2qv5_I#`Y$sx1B21^U-U>eINN{D`MJ6Ic?w900dk~z zDx_iu-SP+C{hnWh+Q7(7{edkdsug ziRs~yN>7GgE1i#I4CFQm&~1jP;9Kgj8Vhct!Ci%Zoiy}f3+M%<(6w*yyDz{kyMjvC z8aTuf9Q-TVKzCYWX%#yc!H=N=wT;~}b5fzhsp{^DsgB@S$@ryHOR#8y z9^~zq3_iJr6a`4Gi{;|t;({J!om`ZdnVy_kQk1W0rQixa7$3Ze2|OeZx`?0{a_1fB zoNLI5pPE9UKEztqB*oxv4(^KM-FHbt|ZEK+ys9IZPH3D+m=3J}hR`!A=29RVb+IW#wn)X%y!dm87O30Hz1tP}YFQoDsyv9>Q-5J~k2BpVj$R2y}RRYM~2Mrp7i%*h8M z6PRh>ducT^;f{gdc@EA+nAuQ+paK?7~P=mmr4 z>AcEIu+`=;ZxZh+9CdGaszQEQYEe;UN-Ahv0(5);sLoXYxf^tAALwEb@Etx1CHY*+ z3Z>wim%&waeol%4^l(d1qb3J@90Pa(jh=#QW(laWE+{Pq9UGVlI*%bG6*QOwG6UJO zprSRUD8B$w55YnJdV!BS_WZAbY=B;Ra;lzVK>@Zvf>(sF<`8lM2akHVJG&}m<`tKK zsy}E$6>`}H+9C6h=mgiUm?bT!c(+wgNli;E%>mtRrKOc!o>C0999-gpvKc5i^;}Xx zI|tp9Q}OK641$iLgyz9VupoIBoXbGPzk-4~$k|Bc2W%KBH3hWI9CS^hLZSjl4a{rc zA_=U~1z)NIT|@4kTB4DhQ>>$)l~|Mx zqL9;ONl_)tg|ONsd#G>N7e)U*LP1KQ|@g*P;IL6Hv46UqwU zT{N)$*`Oh$Ovq_2;2qjYsS0VJg?!*10K^Vu@Sbzfy#y(#WvMx!m665J>j}VD6M*i< zDlJLX2c5*PpPO2e2)Z;5>~u)b$Cnm?mWCr@RUOo&w9<#Gg&77~4q23!n4=Fh(HCl# zewm>@EFtS7CV|zV;zKEzJW>sAT5n zfz>18LqP$lO;Q9tuNM}Lu$~cUPd+GXrh=;O;`}^_xqJPGU)E5%m5n$k@38WF`p|v!Hbg z;8X-EpvXwjNvS1?#HVVo?}$p&;66N==^2vzbU-(8_1ft{w#~&>|HrE@g!Tl;&GvK|w0`3g!gxlwy8qi9%8;D11T7 zs$p^{tGP4s%ej<62k=97ycdHSpZP^4#d->EnH37)Y3QU%1@PgR;G5~dBDwh~r8)2x zsJo-9F7(!wWbom1Ac4%hl+=RMyp+_u5{0Bp&|+@TA$RGeiACUKO0yRS5K>%uv43b6>7H5F=D1%piL0V{_v2f6W+C*qA18Tyc z78M8;;9?(I_2C$f(E#a>R);p5)nnmnThVH_c&K$GR%-E93L2T|dHF@DDVhq7@T=Yx z$gR~tfuR6d!3DYn9a1KMMpmHN8TL!(Z0G$Vz3T^shD^0;pN-L?z5h!$FnKnlh9%oO!l z*tjiBC@(EP8hH>Fy8>_l0!q-}-Zku=4)8R9o|})Od$5jzI%sAmJ|{mpF(RJA--3)p3S9KG zT9TiiQ><51nx_G>N(U)OS0T~C{^PZeqP@;gf5(hcNwIVqqF)uw8 zTtMh31XmW9q~^L-WR`#jD&ajJP}F#&7NsI70F8eu<>e!}Lm{yQv@JvlI=GeryEzu5 z8N)8Hc4+!hQqa{^C@qGZOa;5opfbNyp(qt};vzIYK*l6O`U?AG3KB`0iyo8a!^jRS>p#E^Hi{R0AcYbB^}mKjM#kvtKTJ%FOh?y$AO#`K zm^jyeI2J?3L!t8opq0JF8Hu1<3YRF;z4`(!R3uQyo&@|E&&xt z1}(3M<>HDDat(G33UhUd4-N8+clK}$3I^SFpk$?QuN)ogps1pzuA!-=tz&Jg1f5ZW z8UX8bDJdy|L_ilhrz(_Zq=KvE{37sxba|?JQK~{wXQL;O^)O9#4YZbqrcV zlA@5G2WbI-0u1+%D7<4085D)=ch*-9$w-ZN%`3|+%FhEGpAG6Ep&YFT8eLS#OiNS1 z?ildJGhCpl1kg#r@x}RRCFO}lsi5k&ST7Y~F1T5w4i2WE_+WpxkZ{K!SJ2i!@H7Op zPzKMZr53?W0G$mAzhlJ(GRO;Zpg#Dz6>#K$Dgq3fQS?DZO_93H`k)hExWKJtP^c%D zz$afwiw9?7;vqRFzcd9lker;K3mZd*L;!3!QyCh+pmj)G3ZPq6LHFlCVib4aqt54m zYIIO;Qc!@msz6IApy?lUI|z76RUMQ<@CX(EsusFReaP@D=rgC8{Fo0kq5DgpNqkj~5icOJmT=_r(diW2CC%McJ7 zYy*6bxj3~1GL8;bj<^5-+{;VNK|Q+**<7T;P0zC=wJ0$u2X^2WGz3wN2DP^^X0t#J z0LNNu}tA?1_Pb^A@Rs^mU1&MhnL8+j_$I4Oz5=$~Pa=}w| z&>MF_S5y=w<`#ptuRzqqgZj|mNoR0~q7+%6#coAVvmnC+pq(|)2_a~Q9;6*qJ3*C! zjRPeR6l?Whrog4u!TXp&#uX$MCFX*rOHg%y%!OX+4GCDVx_HoPGth|=ptBIFRf}su z5d)S1mk7!VZuvzDx%ow)i$6*dGjoa+5|i>vOB9rfK=%QZrKTv9=NEytjT9v26=&w> zDS^iJpgjy_1;`DXR{Hvoo?T{YNt#}MQM!IcNp6mQQChN@skwo2acXi&W`3Tov7V8h z5!hCk+40F4iAA76M|DVV8ZM6FXl)EvLnaeai^@_{5dO=|1BW=o@1WuWJid?uw*yNf z0qhtBur~0)&m|dpu#<-|oB~Y0?H5SzL~|KgM1-Q14kF=9&zY2IG7KyA70addX^#nF8&%QYdBrsIr zdT9apsDAKLlH&Y=jLhN^(8OPA3N#>;6><{`z}uuUQj1dUp`zd;C&1wY$_UY5+aYIO zL2D`R)Jm`q;6YEif1xv^3b{~!gR)E}sNo8>4ZGhn(;!DGf=VDzAxLNIk%Atx7VM!xJ-0-13WpQcH?JR}GgGRmOuZ zlFBV80pB$hU!Is*Vrymqjt9`CnlOFE3W>R(MFODe474#FhZtc& zU7sBcD1gk22brm0t6-#{r3LXG%%NPMatze9ECP+_ftOHa7J;@eDCFhm zRizf?gO;nM7L_ID=zwkzC{Kk9e3s-ZfY+;KmMCPVfz1Hd#9%WrL7Nft6^e5~8xTSD zCnU5%eh1ehNGSzUf`G0uho}L~0Q$b$=Qja10qHPO}RXw=lHFOaML6RWEP*A}Q zO~<*Jc^YstAZkEdO=#M7Oa={ng4(IRnYpQ;{HXz&Hz_CqH3~{ni}H#=SNr9rf*XUd zgbj%$@Q^gP$CwMc-T>4P0~M1H?I07u@e6IOK}6w(f+O26zXWv8E%@Fv&;$r3}-lMX3s4hZQL#<(Fh|DS&fkMrtv5;u>U_l9rMl=n$}!d~j_Gz8^(1&vI- zR6PYHYlT8vB~1nJ@nQv_JP5lJ6_HKQoNNW&CJC|;A^{F6@U3wm_kr`T0_3^{5Z?-% z%|Rgy?zMpHTJRd)oXl+S!cFkJ8#p{6(E{B<7>+u^05Sj^hmd)?)M5pV#1e&^)WqTv zO-N9HZ%i#tg$Fh$@j|+qrFl^Ar>4Lo5HxU70FGer@naC5AY?&Tms+cX;;R5ujc2AQ zU^4`g7ciRD(5aEs6lZX$11{?kCAT^_s9-@2DG9;lIJgE!vc^gQ)}#jwK|po|Kx;ur z`H!R)?s9N_1zN`j%MQMv#X70Q8c++-a|5`{fdo8p`2lJUw4n|vih}cVQx%HAGYZ9^ znHJC%bMRRu3hG**AS^CPO-unL7j-RtEpBhVQ# zdQ_MQz(4+PU}kP;hQ9yP$ZT}{A0?h>#lb%Q58nX^JH8G!0AE~*J~9v4+z2~30JfF` zdNKvLCMw3Oga&s6z!EUTUij<;NEqpk0MM8w?2aq=;s3$lwgvR~LG0%dLe6YJIt&eT zNo-y^J`8h%OA_-^ z5{ps-L6|>pu%o+caJ;X- zOQ?@4sE=D&td|R3w$)V;)9+216)CqH_?z? zgNEv0)y_Viu6`l$o-Uxe6SnFg1*`)q73}I9QsinxWBvm<^g{(M!!sDF*Ff(@=*@H9>X{fgIFTqy;f}H1> zmz-0YlIogUP*MplM2ew-gyd4lSUGB;2eA+C2asw#^eBd?0ZT#L1UXe!y}A|@z2Ic5 z2fw5dS`dN{cZ6sJg(q|eupZQz5Ep7_sDn!^bsbG;SfcnGW{VP7k&=}H=pqQhRw5-R z(6Nv3s|G>thgyR&{H*{wh^9CdTrfEnWcq@~@ICX=^5OSRfy-!c5oD{55_;eXH;|jb zJFTEe0Gi{#3&|j(-0&;HFfWgUtdB=@gFx%aZL8HG7gd9HhZq?csMo^R&)ce3!$Sjl z@-Sjx4?1EJUs{x7is8 zGElh*H&YL~8lOn#DL_x)g3sWf7zXM1fO?Uj4Ce^n>I1n35#^p1%&j@#eL&E$YF*D_ zUB~iN&|o?!iVdN=e+Xqla40G(_~pZP0D%S^!6*8HWuR^apQ41B(7?hP$jJh!R)MDD zywq~o&KP+8gc_d^Q;=+07DJekOvw|#>nwXMWR6Ll%3|tGT zV@VzOf((`r!7W2v!>&g-{C=Y+=_b!^}mLe{l0qPu~U!YZw_Ann2FUHZa#wFf`IpFg4auFf@i19LT8> z)P952F{g$wPEL&ehnhs?x*7H(0a^UuIQ;QS{86Ejl-wEn-T znX&ok`afFr`H{`V-v3upQgW>TO&EYu0eGkmaqJGHWL3z?Oe#tQ9RLa*;pBpLdLd)D zaJ}Hxm6DPYs4oFIJ_>9YX!Zmtm4QZ|VFrL^CXg1zfX&lU0EvQ`FasU)6d+wx(0FNL zod1f)_kWBE1CyMj@@=M%68;`+eX@E(%i3*@qMww}ush~p-AO{_S z=C*PmTghQtmQjoYO}c{(K{X1^x8U3Cixt4DFd!3hpi^iRGC{NTD8@mjL*X~I;WZU> z!YZ`8tpM5x1X+rWVjjwpYAo(_%&P?5ae~;g3o%2XBrylPJEbIFAu$hhj4OD9ZfX&V z;h^FZVkV|FurLM7E9Anqw!q3th2Z@BJWx|5DOCY(48(9q?+UHU3~O4#8@)OTrA0Yl zOB56!9b33DpmrTfw-%{60y77^Vg}OA23K|90vg&b1T}LYn>-OxrA0ZA?b4vNl%PsB z9&~zkd^}2T6ru@S&E%xPTA6w&so+I)>WRh4nVF#J3Fw*-X!{B}JfaSFKJ+4M&~&Gb zYO$Ra=m1ghbrgDEo)v;ZU@)S3V}G$|E4;|$(;k_ZkpWd+zW4$xt(parn0 zpc%uwd+0n#-Ub6=b16Zek9o#gdwWVmsoZZPaoIY@Y(6K7ednf}aNnN)d^WgIvK=?w}ca zjAoXl=0PsGN`oY0sC!ZT?3Z5> znOfor8Y~5^HBU_eo1=j((I5;*TBDGd13DrJdWu?QY6)n519A=$T5bsjZ`*XO$Ve8_P?2tvAHpN``_4b^!!(9Ml(*caJK(L3qZ#?fR_6!q~w?9fjQ9S zA&@qFK_+COF1T=qoSmSMnwyxJqX#W)^NYZ%|3E_Va3R=4B19!4=xQdA7n&4vLH+ks7Ziy5N&2YOaipY9>xagE&*)}fC(TrB|)b3 zJpExZB^jWc3%Yt7|GXliV}oBE$bInnM{uo-Yo-#kArH$5u`bYPfo)`h#2DlP1<(TJ zP+vzUAJ_QM03Uxxm-rBmAlG0Ie;?4am#Kn{je?N@SA4LiZvb2gXu=D!wn`nm3kpR) zND_3*PcCQ`6JjPT9#%dhW(>hAWxxe&W-|Dg4R9X=(%^?U5pses%=yqt1#8P6vKbz< z-95iJ71ZcXP0WSN+vq4HXO!k;gD-mkO_pFZ+!9Mm^5bDe8TgtvNRtnF<_+9rj*kbI z!||Z_P=KE>4;r`1Nz;puPtHMD4eA^qalvg}s2Nb*APQL+;sH>v09-#HOF?=SAh(0M z6;S(7T&-XW+Wd|11wuF6?;sb!!T~gKXlP(!VQOyX12PiSzK@SDN-Zct?W#cA#h~*H zK#s@SFF^J+xOEP$%HR`vAbn_I1F{jlcL53w+`2)s*mNPH1Gip;8j!ES&Vd_;kkbP# z98N8Q&i^39!RA7j14BIqHUcdvfL9e`b0H*F!KOg9!@?BQ^8)t{QxkKs=`1bE!L6;d zCLF6gLFY0&97&?VDge}RGqT#|!%rA0a5 zWrPZ#b&rts{28gB1!+alJ~=3hgP8DRBf;9hh9p%gWF(e>+W5t(ko&$stzB?Hfkvi5 z9C%M4zsMRQmRJHhfFKooY60X#5%5|^SSJXatU;zjU4>Xs37IUjQgF-zpAH8jbbcZz=0TnSHS0l6f-TeoYmJAME?^b;MPTp13V+aP zx1a$4s9{hIARmBFeFG&4^o)h<1dw_xsSuDAU8oh35!c=y09o!7hxQJ*Tu)Dq~@TO*Dx2M z)rxbc*9q3#Xs9~Ul4~h;@YJ)ir-e%GZ0k@tsN)n6GQ%h_S4geim zX$zlMhs2tuCS=bq$RE(YA*`{W0dH`CT@2bVmspYrX^<*_b~)vMu8xK@SHammpfo8b zGa0nwq9`+|v?LXp?co)ul>+owS44XPRP>b=gN~>yg7+`MtFF9$?I>Jrh=`40kT$vlVO^;6co^lPMBtJRsbg=^i~Z_ z1*lNNG583&jRA6|7wB{)&}ugw@FC0PMVTe3um&nhTMpa;g<1#NG6Xr4I}tp&4X?MM zQyP%Y0Qlf0kkR1&Kd8t9ZB>Ho!w0eSQo-9zK_g8XFjqhZyED^}jx*0JE&(rU&(H){ z0iYZTs@K6Tg+vvWvs$1Nbxb%WG^Fec^I;ag`Ta+LLmvT5F2r?W4 zi6TgA4c#b^@i^iGmTF;BOwjm%m;;XuB%eTJ&}|^vv7iB3RHuTvsNj?0AniVIp9S4` zP&rGE2f%SvoLU5FU8aC1kKmpH4UOYHk`B755_H}>XtQl{PO%=|B1adrWnTkitX_6% zCHM?hPznOwVhh=x2{wsPLIi~nctj0W!h+0Dho(}{>=f8D;2~wOe?Yr8Ae&x6mg3qK z2rAeh+eMIlr3o%a6%eBywhAC$f=A%Nm8=FNx*_Q~6MR4yq8PGpea0XNn4A}vk2YDo`m`X?AQ2sgc^_upf)eKON!B|1l0v# zEzmkQvnn+Oo)c5SD|gjlsYgKrrHQQxS}&~tK0OYn4)7{Z)f9czVg+#58{U`#?ZpO{ zsh~az=GhfsC&7af5(5Y?L7Ebvbu-}dALKjOG)*dWy%acPASEYk^%}V69G_I0nUj(V zyA&L>+6F43hrEa>T0JqRL_HQX;+314l9>qcDyRhqGYz&-Em}PYv@H~(7VJk`16|b= zP>lz6gC4w@si3Q1r~`L!e0*?fNeHNc1y%;h8ql&Fw4)9>7p)84xC+`D2F|arfR5Ku zh}Tg7hlqkLSSDH>RY$B|5oq)jeDFAU-7+{mfp?QZv?`k))@f5t0Gg<(ODp0-j+8UsR!xkyxydl$x5SP?lN*Zizq*NX;$CEXo9RK`M0= zl1fWpMRsydYGP4Nr9w)6aw%l8NFgz=5^^{qBn)6ZG0ZT41tmD`BIiJG8x=YLg(wkW zxe6Nn$Y~3bZlN_kWU>;>fhG?E!#)^mEkGpz>S5)`mO%zfK!Xp^tOM)&lz=uz!$zP` zN0(q@Eg*9s`avu#V>9sa7wD`xOe4foRtlb=%QPYL;5zUlD`8Wm1x5LwMZ@5MqO{Ds z#2kgtARp*HdT=~Kis0hZJn-4gpw%#l6`imIR17}U5@H`HmSH1Kpe7KKAt3jIr1SDi zKts)-9q@W!v2y5+W6-IK;4!Dv#1x1xAYDOd#(~r>s3QvS1Oqk*no~gb!aN8Sf+ZY` z(GO7D8`9)a_bn|hfvsnPX@`y>=|LBgfNik_4=aJj#2{VoAh>}7KC>znHWCwBlmo8IkyS%H3R}B@zRVbOl`=GGfz!GkDAN{0 zM#G?9fFyXBkr3m+o9Q8oIY2`JC8fpSB`OM_8Y?poQYB#Phr_&t5t^VR4h~H{P&*F3 zb_C{Zh^^4f3oVO~d!R6N;J}0`gBRg2Nj*@b3Str@6j3A~XQsi*Y0$y7po47G5koMr zdjTMeOF+pRZ>&MAfJQX7hypE~d976H1T~n#SC5c5P zu6dAwSRGL9SX82;keZi*T&ZNHfkd#C4WL*9nV?{!U;qm!EaL^R0s`721Lrj4t{P&n z7JMa7YF>&0=rH~Y$li{;)bzxX%(7I7>mV}Fat+kPguB7c7LxN3)i3p&1l~0Pv05P= zJUCMXYJGr~DV8MWD1hb}5l+fW0d3dJ%+pBCOM$jW5pDuC^&lAw+~5NF)du1-%3KR> zYQk!^)D%e2rRJqXs}{#9Ky3iA!L3h-!$2_y?ad+vxj=&86o8R<;Q@^j*^rR21s(08 zVE|1*3fkx~0rPPYq~8G6gb_5L#t0;R2B(&Q{R7!41FjlCB_Vh4jMv%MG7${>4L%**{EXZ>1CkM1-l)uL*QKl zcocw(0Z52|E-?iae9*7}+kyxJ9nfh$;P!lC4r~((a@c^+e=jKlUt6CD8OnwnI}MEw z(8OF?X-*DkWf*wCA3ov(E_xv?7(^8To7hUtO99W0E7&T4&X)nD<@_Q@VumFFcozpF zwjoIZ8m{OK16ZQ~v^5Rv1hnc3)I&l}Ay8vb(gw_R5Q{(~G4LH&urR__ra_H@MU`GD z_{KX8MDr1*8DcIp1i`JO@*>cot3`Tvnl8_)IL&Qe4p7BXm#-9f0}wJb`Z?Mhon(;4Iz#0(9rWKdS{1no70EOCM!x({}Z0%*TpNd_odO7g*N9*A4P z(@~%tmz-D(>G^_&yYkZ%5*47mR#+Ps^HnGZ^{2tb7RUtX zO`af=K?DAfb`{j`5SyW)qz)LJg@_vc87Tq zG`x|VQ(Bx^mI|JdQ&0eBY=vA{~8$vQA6EK*7cqQtW_> zMCkO6m4aI;=voc1mEZys6rP~q0T)zY^B@BR;2=ac24r4pUJA?vkRFg9psO+AW~Jsq z`+Hgt+d)+iHWz^O=YU2ZbRk+HWTpL zhk2Hz>Ope20;s_V(X6A8UjXXLB<7$PlV3oLDVT+>m4Zf6etr(Jsi1u-2!(LpmgIxO z1C+Bt=OGmrLz+aOxPTS$pcIdG0yAvE2c)+F8=3^qY=eqr&}DMq`&U55V(H~$){%&j zN>HkUHc{iTwZM_8eURyph4V1&3O3*%g?b5GtHwhN1l7E7RnUePSRT|U1*b+>oIp}i zG3c;vuwTI2_QBmL4VVF7^|oN`Amb53&>$CK#fXrH)r&aq83Z+hAaxpO#T%$rijRlZ zY4Exa=0=#CVGhGyWP_W~pk6;r8>~1hPvkGcgBwqy`j# zP-V#da%`rfw7B(P>5zyVp@6M1jv1M-6b?(e#ODc!_t4kwz@i$dJUA3D+LWi)dil8 zQGgf-?(T!@S5RdFu?}Jh@(Kkj1r6xhAuGsrQsDXz)Pw`&BT%7Lo~lrs0Y1bLtPhlt zp)sZa+wW1Hk(vk1hOj<8$WDkmAa1UGRYRUO``d{EU^q5&J$L7RWpQBa3R0BBwv5y~JQXnGU0 zpB+++LNYFNIuhn?(3Cy&_)+NO7I=~gQhtJa3J@kd072eS0G*zMSf34Bc>}KAK(!-8 z3)&(QP-sC@H)NF(B*}qVOei}?;Wj}M0BC9e$trM+Lf5JwFGh#kjVMS^A`o>A3uN%y z7Sy?f6c>;j77x0d7&N2@K0zi2+KYh7!NVEcD~E;|*s)Lr@LY|yJ_@qqFCOf|`1GR0 zywaS+BGCRO%vJBjB}IuPsp*x_mNruPkBC?WTgW6NIG#W%=(f}f9s-2o7k#A_B&N~U z^?>?PO7z!IR`52`9*W?^wD zbSxJ%4~2aV7-&!zT)2S(0cU%ZQuoUT#}6d_5r-XsVhUn0>Zk!I79irdie^Mn zfEEMb6Mv``1)#NGcvr`vhdL-G2!whXtonrv9Y7ZlrKTvr7v3q9f>slQ>m5XV#_t5;*cQGEjqj_%4ovv`?UZ1Wl76ai9?&&_$o{oPxLr6q+5; zC`3xK1x=zt=1LKQprywk9jG_Uz!GO>zMiK)Xxlhqg)2l#uQ)X|8?>1PaX1(m*~S^N z%b+L~GBg6;1P5y?=I4RVlh=jW1x;DdyaXDQM&5`9%EqAL5|l<^WiKQwNn9L=LlY?U zurCdSYCx*kG4=?8>j|hjDlOlGnHH3qR+L(t5tLe73f@5osfdG15=&Awh+hl{H4Rr6 z6mZ9Kuy8F^byu3MhnO_9j;0A3U!unyLauMtrSgRk> zkI_SfHK;eLZf|d|4wV7Vv|-s90ZX5_0|=%96h00x4(cXLm?+o~)Ge0qJ9a6TZkk$Y|vIHBjN%aEKqNu zrd@;@un7n*YWO0=!A7BmFxW|uVhG!6G_W+<;zV#rfoz1N_u$kLm(;Yx(wq`ViVTMK z1EH2d0uuc&Oz_M%)Kj1xS)lW%b2IZ&!Sj@$8~Z?`4$z?zSgi?5C*b%2PZvSV1`qsz zccwy)49Ll;R46D)1s(ALi5F0`zz+b>Q7DBhU4q2G=B?Ps|p%+0WBKLPgBq^)C7&pz+48o;{|L&4rsTUj)H~}L;=(Rsd*^{`JlVK zQ}R=bLF-*%$0ER;1zV*78Fv94T3Mb7S+@$iAW;Y84Gm+kxuC-=ixt2NWkE}SVD5lf z0}T=6U9pg$f<}Xtf-}k)IiU7DD8M0#K$8k!*FwgUAUkSdi|ru8X^F+f`N^4yC8;Uk z<}S?fPEj0wxpu%v;uG#WmHtdN2yC!)2<7!}mY833_9O{swP@yD0Um+$p$6%rN~>OjK?mWE)yg09m-3NcV~1U#4m_AIpF0Xhpr56fjv zsJ@2WrUI?+LFRxv{gi|OVn|)JSOHYLs0kTTOT#9iU>O5+jSgzL1Iug3+d;r*HD;Ejg6^Zq z0G<5@?pDB}44Umh85JBg;4%<2gbT|<;4%YdF8EXeSccL=-Dm=Fen@3ODr64{yd*)i zNWeKfJ|3QtePL(gz*bNpTMY3tXwf8MWg1ELBlf$%V+z@BkTh<4ktf`U4^4O%1svGW z07cf0Kgi(PVL^r-VDLUCc85dvX5jKA$TYNc1&LUcG7&irKnr;i6(DH@UQpsHAdxk~ z7Nj9o8H0v%!3hK83O#t10f!NaPH-IsnxDx@gDqeH9aaQcLk(%!;Z8sx!^!a?Jh2k= zr81=Pm07Hir~ucc56)z%DbVr|(mu&8PLD6j&o6;ZZNOFQfyxzl+SgErYKC@al@;I` zp%ZSP>a`rS8ykG>VJ7IJ`tm&3(h+a~7NnP+T<|d_pZk^6a zvjXp#0FM=e)?y`=lq4o+fZL7Wz)vg&->wGQb_F&aa`blEjba?jt3q3kdRmmn$*k8OHa^I zNJ>pkEQLH(o4>N>Cj+qN2BUVzt* zf-cE`EaL)Qepn01AqdOiYtX<60^6c(xC=nR3*ILOTD1ZmqlP&iB#@F>1i5g$*wz4Q z324JFw1xnmBMs97u@k%p43yary#VMQVUTgS)T)Eq%Sf{v=w3nD1P2;A19=RzTG`(l z5^Ny0bC9cJh^q@UvB2BTuwhdLTkyq|V4p)*LZi6{lycBkM8nUh0(WCw<30VH{Xs_t zhPcAA0%)x_a+sj)YlJ%kHarR55DSWX^j0c3CgS0T6d+bhxu=%+Bo>!ARhFb`fCCB| z{F(~d3XmnSP)ndy0%#@%(w&6a35h!7<|#%m5a$L#)G&hQSwz~06fR&3A#s67K%iKH zE>VDN)B|&%5rkzYFQhx504mbKSsAjp8e|I0OnA>4R!+hDHE7|E;XzRU7BXTAP63b@ z0L=+QErlCYoLXY7kdm3AP?=u}J}C-5z62XYu!pv~pd%%q0UFSJ7PusW_!A}sOC)$) zL(J}049^l-R>Ab47Um$g;&d3{X>{m(ASAItOI7g7yB6p1&uavu*Jv2Mz6qYs6oS3(8drXGBE9mR)=rwg&fF&H+~_eK$8U7TX@0gXb|k- zEdsX@AbWN&=2+pqY~;cm6v#S|Z6mNE9g;{OCtN^Qzay9UdZ4}tco&?8x-K-PupLpBF*t0% zPJo*TT08+teIV08MFYq}#DO7TwV>1kSE33DTHKaG!Vb%1nKN|k7CcLgbc{XN4roOH zxm*C;lg}(lErx6%1Us!HA9PPTtmB>oZd$6QKo1>;=mt9-x5L2UjCHd!Vu3n*{W7Rm z1K!CEsh6OYC3MSkaB5x&bW{QP@~uNor1xh9+Xo1;~-uG3G7?Jez$$O@F8h zP^x`mJ1Fq_9Of_NE(*wfh>ZturLZPFj_v`>IgoWw;JO2L51j(&I3jQ`!OhTCFjP=P z9X5po6)b>3!($=-F8&(HMTwc|$(bcZ`I=S=kZTE{*WQ#TR)RX@MXAZ4^A8n3`{*D; znII2fw0kjc$v|`zK^oNE5;JqaQ`qp+vXB<^!*V@r&P1UAG(44}kXQm<@vi_GE(4hZ z!s^hi@1PDl%*&882MbMNcbI@fjg)O4AnT!ZEi8k8N@hr18=Q)qMnUWU6G3NEU`cwQ zmbwo3KtJT_7ql@7QZ}X{-Pr}sR`Ajg>|$KGDl-jiCVD3Ye2W%pY5z2-aNK}Lg1~1Afd*N?VGnl>%n+0U3RIvHD5QufETQ9ou<<2~ArffY1xt$z zN0$fYSp2Rh;Bthm;6^8C2@Lp{0dSaNYb}F{v!Ybc;Sr!Y4A4RpSZM)iG=nxa!%l$! zhc|dzDPl+lPnnyUrhw25Ep0*DuEB0m03CiC4^CGKwrJ{fAx(FvI#{?M3VlQwf?__j zumGj}Xwb#7V8gATM^mB~NI36A8A%dI_y44)=IPVK>jnk}2Ii)w;Cny~3=Gg|kTlx; zKSss|W(*3ZG;s}@p{1ZSqM*P4nsr2zLg5mUM^QXh?^GJ{jjta8VL93EL zSEJ<@l_;cElz4=X9&3Yn)HDJOz3a|?#G`Vm)aa4weA%K?uA(5Aw z5}%R^I)kL3B)^FAq5yyXH!(0aK+peX=7uJt`JaZ-4mTQS{&xlE*VGiy!J{SlMFq(U zh$K<0$HfI+PF<3ioeI4J0CZXm_-GoCQWtpYf!z9>3c2ztF(n10x}+$P3%Wu+GZnP* zxkLdn(TcKtB{L6VwL)eIsMVQbA5)W}XsKT!5}NNlea2O@W+c zTmm`>xi~d7w^$)3GaGa4#hFPtkfS{d zN{b5ei&Mepp@7!Hzf#1aKV z0|PxSP#pm|HcKHfDH(Q=UVcGp5ooy!j9-#kQ347zE@*X9ln+U7P@RZmhg}{TyxMF#Cl{Uo_Qsy>8V8!HAbLaDMguyc_p^+APNH=d6ie9=jj(>Yy=)BL2??hNsyIO z5Q7X23=r2WBgX{HC;3JABOy5_vACGxNC3wtDEdG%yr4}Zg{6r(nQ58H;0YB-jDQzc z6)ZorBmDhS3qTrDK(z+5DUAcqo49Kj6EFVX{h8WA`;pkf%5VUaD9xEO$0j5n%5nH_qG7c4PBU5{`l&P)tdi4@x4 zl@>_nFA-Oa!ptF%iNQf$Y^4AWMnonC)p*c^42eo82U6u&Dfp#=u6$O2Spv?2NM%5N z9xNL`bc2dqG_8m-0*^kV;Kp8XK%E7#1)jSgrG^5ILIXulN@{XGxN0s2SpZ{$rZ+P4 zl5e%{YFzcXhKv<}*qhO3tB!f?b=ydTC(B*^RUF7+Bpu0+P^7GOaO7cM|6D^@w!AqBv zR9Myn9Uhujk_c{RWP%P6%Yk0Xh+M#cVid)e?QUPy}_6APz&cVdEi9^kORo z*NT#&MCb@mG3YK?XwJ_pR)G2!q!*IipaQt!3{+vD`5sv*G}VLi8f@+qmVBXdh{(m3 zf{Dry(EJB87!-=grkYZgFEB$8*@vhJ*bvhyF7P~RIB0hZD3OA1|3hjh=)i(p2XymI zab|v=4y4B$4=$#m_a7-KDS=G@_dX%TGpw+I976?55&7W0GpOu@YKKm;f;;;#BlW=L zeE{eHx}=;`4QLc2IRKP=klY9=RpY^WKtWp!I-ay7A7&Wn5Y3!aB-MI|6phdbjckH? zp*0+&1O$}^pwI(*9cBXYftZx9E;0gGO62!4W3JT?!DJ7ttqR{Zub56`jF3m|S zNeu@{XlN?vf>#MZ4Tr}fR5^56DabyU2)I$At)OlNT~-B>12se|Ky9!bs5d}@pz0I3 zLRMET25nIWT~(k_p$9TXN1;LwmYXy4^FUb$qESHsWFlzq639^4IE)@>Fh&DOgAT-Z zpi`Z|S3<=@LrPsi0c9B=%ur0axHPFawXifbuLMc2f&wI35XxbG2e}BAQ{!E83rZ@X zSrt}UgEKQYilCFapc(zdyh;re1xUpPw&{H2fnDxPqMrUgTerT9lTU3_j8vQV_#ZUNJ1c#DmU-0GkA!D@jaB)^l`n_DwBG1a+w$ zK>Wlc&EMiy#9j#90DM z3J~8S#~W^|psFx!gZ50(%2DLl1RDwqtJETJjsaJFM##-7a702Q1vX^{4GCzX2Acz0 zcm=EJ6l`Hh0Tj=miYXpZD?$q!AJEo9s9ErO5NaAyCIwBgIH>1 zXp~tGR5HK~0v8bA;)uwi02J(cplv$f!WLUOXAE^5x@R@i_4PrEcXhyN3Q`JzvkavC zMJvrw?LkUVdXOP29R+ppfK)NTA_!9E+JbCH9OaD^&R}D31vc0+sE5E8$CT#AgT;`; z7?Pn7l>n;Sv4#;i$nllG;26TN30f3G9Rw;lQSCwxV`!2D2Q!}B2{i<+3?($cE&-(+ zNR&a8d6XJ@E|_K7^ubYz^#SW{%GRR0d&eL05mf}`2)HY zIW+|~OaYtc0vCI##jp;Af@-lI_I!t4`#`*fR8GZ%DtA!8pf)!^6%cZZA4!Lfg1ITg z7|_B^=*(hfejdamSl+|*4K(N=sR3jwC>jZP(hAKEh@;R1)r%|hN)js+Y!#Fs z)r68Y)FJ2!p+ThpNg?O~0F{NuG_=5i)_y1%6lAatEXEK`BeY5v$>;FlX=uF&uDw7h z7+DQ`BnE6Ea&ZT%QlPpaqiLX81=b%&?Y@IkJS41Otz?jk2~;BB0+Pr&32ZF9FhMmO zy+j0g6P&M!s&$}VgaiRN&w~o2VjTr_P-#kP-9v%h$PU5UJ_VT#!|KpfQml^X0_p23 zK+_?azK67VbrjUWLocLy9a<4VQ!&^Gq|O_(d4rVvps`700zi#&tQ9WQKtkCZi#v;< zu7d;s)Etn*VB;n6U>Q*90%pJ~7%&giE`u(mfT+_03nP_v;FPWanaBVqG~~e`a7gLp zgD8@rxw90(9ASs-+IfgVg1 zxT=QLHQ=ozV8bD{W#%bF!+i+ar(O*93z8R+#`IzlrhvNc;KLRnZ6DB$weKD-?FV=I)&(BFs%mdqx&mu(dp!g6yCP9`VA0>t9Vq6A+Iy9hI zMCbq=F8~i{g=na~3bqQW#jzkSLj@pV0rf0u#zxdH3JRb;jVkDfF^~Z%;5rnl2V^Ta zS%M0`{E~cdAc4jdK`98-E`_tf4KE!~KMr)_B_bujD_w9tfflCFQUDsyU_+sa70r>5 z%mNieHUXv5MllEM^2{^^0)rhOpFz!qw-BM`fLq|T>hL57b{|sB3)+L7aD}e?vRz1iD6vMPZeE<$!a9S@$9r*!gAE;`m zT6l2#2iZ!9&?EE`}R;>i5 z7J$Y@@{7Rf4yyss@)mkBAgVX@AY*i}A^=qqatILGaDz*JGGz2;;0GMkm6VhqJCKlC z1lR`UKp6stQN4)K27rbjM$HK77(h}LI582`DugFvc+V6z76CB{)hQYv?;u7Vz}A4O zL2$%EdxD^f4eCO4&ER1{1yy(ght#2(RYjs2d{AWp^ElW+V4p(+5h?&K)nQJDW)8TQpfLu^7Z}Mwy&$m|c1SvMFrnp1 zcntw<&^p3*y?N$>j$enBWROABVrbODh7pn1uRv8OfYyXU4{ZlGe2NN^!Q~1ld8g;+ zr{|=?W?L0(6{0J^9dhto0L%c;Rp$_e>Y%1*1>^t<@Rgz@=mZUrw{LX(9ED=UB&-&!enrh(SWrh?A7Do-s+ zRY=bV4NsTkgF+BK-3(b!0$S}0?KFcHrK*FjiwEuB1qCc5OBUEb5tKKG(klVtp;__tb#-w zC}n_(R8X=YWrW899%^XDf((NgjV1x|8nj1?CWE3J9GIBd6MeV^b4nh?6j<8DDyRfa zcA)mVYO%T!sC|h#e+^ZDRW}YnXj*`lFVF@OT0sr+H)@Ly8jA3O29&B(U==&K%m6P- zgtn>R1q@O(i|Bkpnrz5Y`9*O6>Jp1btwLkBs1vA%GfGKXp0XIgR4WUb z%pl1SUV#%F{8E68&B4M2w~bIIfr1W{l@RF&HB+HY4l5vr*9cpI*IY<11u>)sGY(Y~ zto;abCPvu88zNv|LW>D-u0Ztzv|s>TDF#`A0tr_53OZQoMznEh-!fz+$c0( zfE|X>e1doeDZzr0D|~4usP;gKez**La1Gs|4yvHhPPjUVBhVFLwxyvV25wHH*a#XZ zfEa?U;D97x!~zhgMPR=|jYKP}kqw8K3+w!X`U0@_BC;x^C&i#P8K`=K zECxW)4Xu|Dc^)+6h&Cx$tfPQ?IxsOe6*3tJN>y-UK&cAOfhHtWd%^JsiVwu(BYYhS zSPmZaDEbgNAt$rA1f11i@cG$h-iI0mv559&~qQ4x?= z;(c9%gB{&n6(H9@!OR8OQ>^EbSdto&nVSkV2s9K48rcBNyMeNX zhNhJQb`9WUro&asg`7mdg2j+A<6_Xb9Jqo6hZH!9P)5i>o`e*Q5buH6M2@3F8fq!f zp>ohMKyd$oiX3&2+tsZg!_$zyBfJt&&&^LM%}E8FEerJ-@+5~kC|JN_&maXzlb;ZY zwEUvn#1eEV$Ot3CK5*9*F_57S9;O5>V8)^sI-rk37B*OlVQ(#D;1%ivNH{3iDqzMt z_>@|3&?1J_AsHX3k^sv?VgWS14vID8u#JX=8TbGhn4O@IDu#`ugG~TeHb^Z?upl_# z!6(szxd>N)?q~q5d@o7_>w~04a0*f<)+wo=^~ez2SRI10*$GmV-Pg6(rJ~&$STwL88Lw!Qx!yJP=9sNQ;=YfGE9X6qc z&qlN?3iUHo6m#z*q|J@!_dw_TU^eN1+j5AxB~XzIo|gcRYr)(Bi#)LFK_w+wGf%MM z18g+-9P0wuj2?;=ETs;;h=m7kfXp{}Z`p<1k24C)JlwSYVK5Ef{{ z6|4tbshv6{tts;U}bmq8>!egSy}c`m@)&66f2618 z#X}Axjt8CEOr2AJ@Sp!-Xl!n1gm(UqshQE}`9I`ED2cjop8pY?3BHW5BtJg~T(m$B z&j1Z>B`QFYh(b%> zoc#23(7_szi3HG{s0E2d#i`KqBVbKCJ~CugMQCThR~FFqwd8Qg~gABkB4x_(a~B{La(Oi3}+2nF!62-)<25QjA>GxPI6(qNw;xdL%vTv8?Y zgj2}A2Bc-B>WCv^)U6ay{0#28sDtLmA>pE~qflK7KHs*M3p6|%o(i5HO3Z`oB?hNo z&`?83QDSl`>>L&sSP}$ZXP2iA9(ctXD$&SJiH%kV8w0-81=4X=P=JR5WCk5qD8Q1S zUPzFmvunJEt4{!|o<$rSZ>69PjuNmTApbxH$v_7pg3|;_guxOF=y>bWlGK32l8j<3 zyYEm!51eAa+rvR;bb)TSE6M?9AidI}9B|_TTmvhhhIzC)=mfmbq8#;D9dsFJA_v(H zIc+zwAQN)(7wBZ){5;U9-L?wq`p~uu76+CeKpG*< zQfO4bU7`b7F$URTpra74qflI$mX--x{R>y4SA;mV9@7etxlmIfrYLADV5)+d3qO_y zwrnrI2s*o$Uxc>p$T>eHRRLOP=R>xwBq~70+Tf`HwBx4~blMTPOn}cgfyQ~@D_G!) zL4}PDwB(Nm4Xi*m*%j+3fD%SLbc#y{bl+M^W)Uo1!-E){_Th@44od|U$dKEsbU^bc zpxa+c@{J%O;B*Qf@J4Zmf-Pzi1SeSNcqOE60=p1&{&!9$WY<|*KC+_x(vpji5Hu_h1|T_H4=Ral zX<2GfF=XZp)1!J&No4iNL9Ad4OA6o+M^=U$d7zG9etrooE|HaiqYpG`S`3p_uvGvH zVsR>19HO`cbcHhLIzL3WS6yE}IVZm~ML#Jsy*N1&R`G*c8pZk`t@;I(RRw# zG%u*f0P0PEX$&`l?vRJwlmIdeTn>Q-z)*@vur^TIgoY*56Yzm>D#WKkL9 zd=_Y#3cXYg>P+-@2VzhP-oStzMim5Y#2~^QY6gOo8tb&MyMp)(34wL5xDN0UnRxso>=}pc#t9loasAj=7+?f*KB<_=Yzs z(sS~Y5_6E6HjoKitSt#pI>gbUf;tNv0!RrHCB&ei3Mp@(jzL5`E}tTdgfn`p6{^ zw4DO4*kNkmF$798&=>&~Kta&{tp-#fs%4PW2R^VXH#0q>L?JP!I9~yL{9{Refo=|{ zJFNh97^I{DH5Z`*;IxpFriZmP0#&1~qYw=}5gppxhDt!1)uej^9x)0E>WKxJppsl2 zYNWaqxG@XmLBkoAj1Vm|gtL(nre1teYC%q7a;gTXOIJ{tlbKgym6BPUn3R)h3!3pv zO)g1I(FNTXUR(@!LveC`L29ur)DCC}7bBtqA`dRt9H1Rc&{BZU#zmjoTCyW!TIC=Lg*S!P40DLE5#rHA3XE=|nVpaqC8?0hs^_vv~K^ z67a-$DrB8HxcLtaIOq^As1;q3nWG08PzoqYEh(vlX~{1wfuwh6AVamlOb4G|0=g9k z>?CBf;YX7s!fx7z?Yakz_<(XUsJ{XsN@iQ*bejrDO%Iii6a|FwcX!Z?Iwm<|K8f!FniW z!!sx-G~i_nN&ydRwIgLQz2uzC_@dIBRB#!I=2^n_Ktr7vTR@#e+FJx|7!>336PgcU z=cI#nia-uo*MscQ19xh$9ixueW&qAR(8U|Y;M0FVp@E!-KzzOUc&M8YMI=}!T!$WV z76)YxaM=mU`(Sl28^HTw;F^fG22~pYyP$PwF{&mkc3~F#V7H>`BVd^`xO(z|jBsO= zKhUtlVG%qW2-|_nWb{Us9@q!?3?^~lAF3SZ{14^FzC&$b0N?X~`~C+*b2CE&wD}(s zbMw*j-{Ij$O9syQA5fK-n3tjpTD+?OTK5Av!>A-5G?D}=_fzvA$9=(OWuVKKb25`) zB6#L(@{3`Lpmi0DTU=QToyURB79|!GfM!PE^;BX(0W@#GjYv<+NlY&Wo#72;K=s2$ zkkIFW5Vs!Uo(V#o2F0xorf$EiEZc1fBGpT9KSnTAW#y3ORievVcZiR~N1+zO*PuPhAs_6Ht^Q zyCgKoM?oXMNI?nY9DQg%TuBphHbg37d=>64&~Qj;ajG8lnmKh`uGH0qcpKy{JPrf5 zGN8i&&=Gl1_W?W%2#YhAxKmw6dI}mYso=faU|T`0P0$W@L4w6t(;;#TC_W{#2-%~dIVG97i6x*zeTz#n^B_AGKrLukBQHfE z88nxm0lJkH91c(k&{Qxqw3G9Tic*tHpyQsPN#N9CO+5wl5Q41#$xl;o^Ko`NA;%iRYy=G|fZbLO8gK=LU2&=cWUWcDLU9IoW+*8Ysv5M-t|&hj6l7?b zH@7sW1T^)F$ibiqy@Ghi)C{=i42`USBGAc#pezAa0ll*wbo4MN2^52d3_xiDp8Mdr zNKZk-H?b0yhmw%5Fa+CPkdvxVl9`)YjG2r;Hsz&5QxT}wpPgE%uA>0HB@t5IfRbQ# zD)>T5&;<^lSOf_{`p6*v;!YyaRyU~3Mm00E7&0IYnL7fNX9_9#<#{>zi7C*41O>N3 zN@|%NXbVemeh%cMJE(z~DKNi+4wZun7N;f`rIrxopv)9h=Rn$J3Z5>Yu!ngP6x7gE znF{ahq31%B;6rf;RvRF01Scw@j0eXaiXGsw2INE&3?7^SIUIT)BP^Lcgche3DS$o}khTJhTN<3@*Y@>kCv0-hTkO^a5OX zC#5Qs78IlwC4(*lK~FUx)1g@o#Q~sf9-o|_SC*OwItN`{M*-9r&;woE0WO2Vi%R_* zeO=?7{r$pR{X#tb{UBurk~%c`V2A>BB-evdcp_+Wp(GP@o@kLmaY-dKgTVt1TtgKT zh!YgYgND!Yjp8BI3tBaVWH~r~Ac_L{p}t7~e4+PnFsC6M8l;-chu(5;t9BL(TH zC14>)kb{O@ur#m0u?V&vT!DZ_cwi=>Uty1aWjJ)Vft21-?sX+9+-CFdg0bt4Li;3f%Z=@w+! z5v2NsRBoVCL=`}HJ%dxC9!dm50t;z%GD0>bKN&Vs>6TiOoB>}RT9;kBG04Ex#R*-8!fe)G$L6kqRqyh0B zsP2L_bszrkP1FLD$&bm@W@dntn~-2fkEztgb-4+fb#*^{Sc=isRgHd zBtAqhD0!zT*eZZb1N**M14)e@_#S8I@??BGY%~?QaCqU9rt(hqb zwhHB`Nf2dlLFlk5l0s-x0aGzV0;&O;Ry_StQw4gy^2;xA%P-AK0jq+=5Xds5yTL(0 z0~>S(c^RCIVW&Q01{q3ngZo3l78<>95pWp*9wmqAK*J2&wnHq3N`W&W7NO<}P*#RC#NcbDAdv)Z#Dj7t zq%(xC6o9AzmjbX*134Dj`2}_BQj<&a!B?k%&h3B~CYi-0#Tqcpphh^T28GxU?`=Y2 z4$F1*d1;yHrI7QP!H1xtrAl>wNS_$dLrtv!*$29Vptx8;R~K#rr2Pe+(1W&M!D$-Q zv<7Pd^(H}81h^f8RDL7(|Ddi!TA~C`pUBEkk~{PO7>Iw7Re=HwVj!%TLze|b26&wq zIA35l0i*^R(uqmMpyC@^W}_Phiem+orE&_k3a})m0BJmerlBC$hu|^WG3bmL-%Ununbb~fEy=Z2SJ;mU>^s7&aEfY%ZN%E8d{*) zI?y~CDAmBuOelt|rHltr1gr@cvXP4<{*z`h^rNkh?vwgfbWCwD6J)NWOQ*ov!fv7&>=X zOp0G&!3**Ss9FFODNx6NnicTmja!OEfq*yB z(vfpcCc=LT3i-u)$r-u%DY!g{Y&ht;dQkZS?vH>o3V0O)G@XO9JUC(ql?q@*;80aZ zTAu(KjVV_LH)P=*I5f9|tpkS|?4&|i`hx_l2GTT$o^x&r*hxxCN-jm2ptG_-1AmY> z17|JJlnwM2QScH7=t}RP(md$&V?0ayzf>tuRr=;MmcDq2Cvc#jV!=cKynr= zRM2uIk_vE0LQMrlDdY$cm_BgM#1)t@MGD}nKEoXq0NVm*j$#nCYJp#8SU;_C6>z=)4m z2OXXVat6pJpg~$liv>Iu;ZzAR)g?2zL_-tYH;f0{9v=^0SqCZ2AR6Lf!%iTbFpU~8 z>vh0^04aY#mSv|_#-m#3omvUfSq#&P-%eON7H%t4yIXz{#K}lD!|Vdt4iDngJnW0> zz_H?&U#?J`TA~08BZb5Q&;q)m%)}DVmC@iE91}}GqOkc{NVF!Ug0*KP78F#17A)cj zaL`&Tq$q*~DJUY4%Q{e*1#&Y{rsx&q7r^}t%KV`69Gu)C1s<}&jwvb5xhWdzkYZC^ zM*)`mAq6hTPHf6SsWBf(H8?ARtOA|p3r!ZF6Alnb3zR;PO@jD{mUI00%tl-P0c}Dd z76VaZ2QdEqzb3|J2I%WQ%#2Kp859igIhk-W z5-Y&AxPpsw(4}4DriS?L1s>>Mv=OPeWqq~qHdLeu4TNIZmhOGG6%G{4tik}G;|=v zdr>9Y-aD8uILx3~3cP6q8gSqPC=_5VLTDQdQt*I>q~M3Z!1JLF#^z*Lj~Ucd0Ap~F zfj2XNS)ic^*k}{DIRsV&AwX#vwv7uk45910JhE*Ss|za0vd@1)5sFgumW9&mXV(Vwje$?u>fQuB9ZBVw^^m8Kq^CI z?a=NJJk+pkk3v=st*4NcqYEJ`2M23u5$OH{7!yMp6zYhLaU~hZ3eZCxl=hH@vOp1m ztQ>3yN|eA1MiE5Tl9-bd4<48V1#WR_2{>ix#e>FN;QexlJa{M`R2xEP=2F2*QR5sv zH1*(W!Mf3xzC#Q~HV3+>8+7n)YKjKvxH7P;UVJ>lp`bhTVA;VvwFH{4z{kyjrc%Kg zKns(>dpf}NDb$fL!yyZj!6``(p$!}=pv11OS`0N4bi4>Op`xpXB|DHfvV*|O)Zyp& zLEQpf=M2iDC9p&a=YULr5%?CP_3DgJzjktm?`N67P53y+( zY7)q1^_V<(e+)XXim*U0H7}(Y)Z^7q*F(P95upNn9!xR#0y=~s)Ob`+sjC))ASli> z5K6Im0n{)8l1SGaR+u2$Scqz zXNthndf0CMgr)}YE&<3qCz3}XZi6X7<*7Rsr5D34_k<}?2X%7`OH+$WtQ0^;;wWV1 zfmWPBb0YW%9 z1C8>6Drfj)0eEBuk`N)G3-c2sT_U7GT|G##jjhB76`>FvF#YOat)MFqz;z;M+#EDg z?^uutbuReyR{KwsjYn|v z;z3J#Kv&X$RwRKBB?J2h9NM6CsSLTZ6})6FIUh8;1{wSTB@swDRIC6l@IYNL9a_>lFoX zTtNa2Y$Zz41dZB2Uf{~zYo+TwkAQc+wkfIe@OoK+Ek?Jed(1mFStw+ktD+LYXz=R+) zEY#Fhi)~elbkub~Ywe136w*^mKrVsRtDxYANTE3%u3u9JDG4AYOVCX{5UUV^uzY|J zfu<`IkAVVq5Tq*6fS!jPNI}w1#3K*3^b%2EOzLjS0{NB;^X=6>18piOCtz zRyrsTf$C>i=?ogSh2~UHnE)>!KxrPN7Agngg6c}_B@iqhASG?kkzb(TjRq%fP&S3O zZ$L~)dkosFvjwFJ@OU*OV$d2sDB*%!=s_BVdWi)Esd*`|VgX_p!X0Ra;cP{LtVS~s z);dBr66RTO{iRwAYSe(PGJ*yaBoINKgdB4TF%+rXz*f#eyILSez;~9Sb={!0qQnoJ z1F;lDB7BZWElQA@3f5?GPc4B|3C`f!TLW~~t|so90VVv2ECGnm|4?sfD4|;eu4>&= zOCV(caZXJvN(XloA?Xg{YfxVi+#k_F?M&FBq(P+Q2VS`YjZ9Fg*F$j`aspHUbtoVf zqNhTL9#}U3S_;}iM4>#C3WJnnip^ShT7@xCOos)sdJ(v34&A04RGJ4#2O21fv87LZ z?Ep{#0_v253JZ7yL|f?^qFfURO4@K|s1}2&T978>Z-+>pt=|qozc36vEcS6 zq{RrH&IV6YKwHl)smVE@tL8HEVbj4-s~}BYcs2n=C#VpEW;c{A_@KT6tW&L!f-n#? zjhKiuWeRFagG*Sb72v5kB&Wm1L{y6vsu3H~ads0R0e~&cp^j7lH>jZL5tPGGd;y;NJ79h8AkcVn8MrT1Wh=!p-fN_zEg06xw-1#6qP=(lTGDeXu zDo6%h0|_cp6(DJ~7(9w#jiv1l^)@(2z!3s635KDPpfJH4Wx(_rs&WE-Pf(i z52= zh!3I33BBe4k78gOV}j2AgBq#GkpX3cwSh+;Ag1H3i(poRF5tqODxf(C6cW&l*$S%Q zxkHSS9O?{E9wQ=sAZl8OYY}{P=)Ew(@xJ~pp+2CAQCM)o+cz-h!}KEA9iLf}nhR<; zB8ln6Cl-TFBi7JF&iSBqLTE?afXZyN6+NJW8dTJPx|uqlt$Lv11<7oPeXz6#Zghf_ zf>&BVCZg08U`yT05_1p=6~KnsDnzTp4yZ0KFNdB?4LW@qG#9L&SXz>y4_dSj4%XD% z#LOI|CN3-()Z-zSV-<+qH)yp)_&Pz?Zz2E;@+EO)`;0Vz-*bHX4p zzeqt9w7DJ>nR@ZyvrECd7r_M`C|2MR0UL3}7;Z(g6`JPJiy0KNKpgPIKT;UmBC(N2 z*WeBSX+lp8XnY+7P#D{S2pwp`$jwhl&B19Na_T^0qoomu45XQe( z17(3*$eI{c07w(aMo3c9iwA9>gf^oP6XGBR@Dd5;MwHGFsEJ0TqrrteWXUMVa`>PN zxCw@JA_<}l>NOlKIn)L+mV^)OHpIi%H$r_04i}K`!Sgua${53E(EbFt9fMFt*uSuv z1F|&=X$zFPV@U~OU%4)1nN(_#0&F-PR1|=3XMx2&I0nE40XRBwHj|@~HGo#AB6Y7o zaRh7c#>YdJY6qo)2WUbn3m}U);9iHhBr`uxMlS^R3GDxw66hy`9vC*hzK+i)3w<95UIl#ORb`Q*P3JRFTfL}f=%arD& zfcyqGSpl*h1Z)D#k=RmWG*|`L&!FrM=7IA_W*Q>fVw(d3=V`EVux3OCbWbgTW;pP@ zTv$Q_sX%}^4Q3mXfth*m9FClQ@CAokVrEWi3TW6b4|F3U=ztCdSkVHTPY0h@0dL)b z@)}Y+!801zcO{htsR--fLec8TIzXWYTd@F2lnM&!j-aL-+`f{0g(T4VQN_^R44}az zn4>^uLY4u6!yKX?LPEkA|oK@>T{fdM5I9l5Q!f95$VWgLt+m+*Hc`an4StM z5q^JMT&1?MQ~ zY(SEBF)7-b8ukG;rFoepc6v^l_9~#oZZNyRE(EpdLFYh0&B1zZ4RXUBaZeEV)+3Pn zK%RkCNFZBv6hPL25-Y-7J+Sc_>L7X0{4_`fOiM;)8t94>1DJC_a-h_SzGwp^9uFE4 z1GS*ly`9vp6bub?6x4k|EF%!h9mFyQu|hyB69Y&)TS1{1Y6sj1a9RZMV1rJO>;Z1V zKop||1h&RYXkK<+et90myO8Dz$ViA$(3l5Z)Cuymrh=A&hM|Iu4a6--Gm6lgRZ!-d zVYN$2W@0?b#4Wta$%JfL1#dca%go6E&)|S+W{7goS#oKakhRj-x+0MIY1oo@=wdC1 z6TnMeAbtQfE^;A>RUMQOb5k)K1S>uu2I|4mF(~#S1vyG$2S+g^17R^MS{-H?v_%QB zA9VU&K?%HFgGC$63D9jb;47_=8#L;E`Oy2)z->*?{1d3L32IxT1Qzt54vsrZ7#l9JRSP*DJlWj%-lEF7Q}3;JdqXz2h^3o`^--hh0H z5`5~Rd7!f+KpRCBK%0KRAqPGpM+X{~a1+2@1PwyxC+FvY3L>0dgDXNhFbEV2FmX`0 zfQksv)rp`0C`v6Z%?0hLL=47)%?2IQ1)8V?*#{0Zm;vC6?7;4U1{i$Z9{6}c(4u^h zx!~#m$>&Hb{J`eJ0umG|#n9Dwkm%J>P**LEhh%jPgsq^_YfVtrOvy})263S27g`2D z^dicCXpV<0fB{$ z6+V}xhq5>moG!7I6Gf>xi6xn3si4@-KxAr2wdn`zqXa=!fv&Ja_H)g?o&_)vm`@3RukHdhvzQDsvo3qfiz;lF%46gmzIxKoFRoTSS2W&Q2b+y!b1cs z%*Yg5I13>O8qbA}MZ%;}90rny*sLCl(1oM~i!N{?M(LD+?>j?i2VZFj85$|JjYcvT z6r9+^QHBVLaasv(H-TERkZ}+24s3+2u;c`4zJeP-u!}(;(T^A&g(`#|?gt5HNNNQ) zYYFIus6^2W9+Qg)Cu3Wv;Se6gagY)h+?GX8N06X|Ew%FnRalT}4R%f=G+QXZhN8fY zMo2vXT^xl-=Fp4^F#x&@44#zWCLqNv#CC8Y0@0)nE*8L!gjE!zD}N()jFOH#3nWq=J=htB%xC}_YkAh=)# zEu;kP;wVlAmCaC>WB3=Iia@LOt7}mvEMa9SSPdd%VJQfu-3^IW&?y(mkaLu6!Q;Wv zh~_+`p8;;A<>bQ@*}|^;gxLj;8F1Vp=N;Ja8rU^3>r!(v(-csQ1LZ1p?52UFpcxK_ zVTj>Jd~HUs3qUy!VY4czWl{t>trT+CEOZ16QJ{h{7R;T{J`!4K2})DQE(EPFhECN( zBM&o3kkmlyDG-4fLhxihn;Gmfl z5@QOGF;XNW(8i6yApnkYaKQpff(qb95Nvo5G)M$?7{UyQ^OO}lOVo?OhkK@`fVM|f zDwO8sWM+eoyC}&>EKw*=RZju!c2EGF3j-<;6hPAonZ=;9wsaKAGcuDQ8-SG+a#IuY ziWM?J#wCFccLeo?K__CQ<|!nWfKNtIfE=X)c4K))W=<-Gzvvh=py0q6EYPG0QUj^d zz)=p%XO1Z;&}li)$R*NUXE|x02^_@S8!SD-#w3)KlpIr1pyN)^IWX`@F?`(}*dD}= zKgfaPkgN_~90LwJ2p@UW2GXJj)nLVtgalCnO-0C>Kt&5^ijClYbMOQxD3L?*$jL6C z^;$WVpe|fVVrHH~qCx_wAWDGkjQ}SQP-e&7Bv6NzF3^D#aJ7ighB<%%nd$^L$rNnC zn^nQlhr0h4H2Mf>enBk00tfetqn8gFn{ zfR_`324R9+14CVdL*hMyM*Ahn=|T55_0njzSW z%Hq-_s7|Oqic6CqVF_OU4V8oMb_AWN1r-CU2Q6Ji40IsNfbLR9F&ZVbaa3RE#)A!k zItLmw(8z+E{gPS)tpu@N@(G?7097=aTmyYH5Zd`q&{a3kQ+6`*^WsZVi%TfH3k?7H zPX@*&#-?cZf0>&Ynlg-@|3rtxf%E*QkksN5(7BP&!($W@(-SlEib2=W=72Y^I|g_b zgYGp4T~JY!nqHcdSd>!?kphZ`pb?Y$2p!o_=8s*}OcMR~1_wfu4QLt5j3|p(GfR4k|D>H(w znNv^7Ob2(Sk%W>Vg|}XrAxu$mNq$jcdMZRDmWzwa0pSxpC^I)PFEKr}2z+*MNNR3@ zOJ)&dJzRcKW_o5`Vh-puAIRCz>8T~j)ZyR1Jt>u){LSp(@(_O)e~sj# z#LV>M%#xyfO)CZ8#OzeWMPaZbHKBu63I&xV8Tok%MtWwjHmfq|;+>>aI2T%xD=WYP z2h>D2(lb*?$xkf?ohl8!n7KGRv%oVArl|a;Cg3w6aJ|!88ee`a8N5#Gy!{uyHZa*jaiy=-^jS8wI&l zqoAM;N|>Nii$Hf;sTMJZes^FT+p<&|XS zm4ZA4@92O`0hJ{v2P=S^5Q)XbsYNBOg{6r(8U~;S7}&kgMg~Y5cn1Ke+yE7W;0uRR zK*=b-D6yy#y#ERPWMPn|G%g3mt&IUapm%_{JF>>VLHMA9PU+Vp$Fy z&i^zpG%!SK|Ct#X8;rL9=n(x3INN{FW8x8eY@u~o2{Gq?f=V{jrXT2*8t9py;8|zT zUS(*D4%Ri!FX9648AO+drb*bnu<$cTLDxS)jey*Wotgq4PslIg;sULT1NFwyM)N=& z;gpinf}B(hb(j)$9R&^Wx?gbL5md;PB^G5S=9Q>}M%AHbenLY8(!s-j<|ow2;57%u z;A5;qKo?JduJZy79_MC)PCo~&v4biEO&#QC?u7ZC?uw&6oUxpjwdb!@cBBK zNu?#J#R?k5rO6qf9nVlBz}wxxqX*zAr=rXP@X6wuU~$mV+X|@_B}Iv#yKnMSAU8CE zPXmX{Q-Su%C08crfNlXTfgI)vYR-TTeFvZ33<@X(P>o!a3()|MsgjIT(B+Wj`3kv- z1;wBvN>hs=S8;)E;RIEf$_n5;M8#J6pnI!QbMgyPi;AJ!?Z8{@;B5{4GDCgPc%5z< zXimO_OFE$L(WrFw^9JP4;0rLAS1w$ zXbY1>ucAPe2^2#k*%lgv#d^UdMVWc&;28tZJ*wb4{K6cAJRSW);Iq5xpv#ccQ;Tre z3X{cStpbL8V8oEQVf(h{Ix-EJ+qSL2jlcU~yh)ZW2xpLnZN8i{V|joczQR z0(K_mRpRg{I30t>fQ$9Kiu3cpYx_{z9MI?mQy7+qL2|Ypbi*q+UTZ<)l!)}^<{#wi z7=oVIGV@A|jc~Y$aIE8Who@hNu@Tr6Aor@5Qf3Wi+JgBv6wRW{yb?1L9NvNEE8KC4 z&o^c!sJugUJ6Q+P0zKW1S|&X#UdI*h(Q3TuVIV0pj?!g zfyp8H6fy>-tN@x_2F<)=7Nw>@kCZ4z4h1wf>LG3Rg)}ok(FGY;1SK%Y85^*q3N;XR zJ8m&FaYA~(pi!j)NW%!cOeqCCuU)L5k*Sxer{I>KujiClqzRg#g(hu;=ZitjFzEPC zd_1(F0`H!I5(wUd(Ulb(!IyUAr$O4H3b1txMX3tOrA0-lc_lfO;5KGnx`GC%iI3I_ z)I_mAFR?fobdorzl?4h7&>7?K;6=3vWEOY@1Zb5+S*k*DDrl{PLQ!g3YEf!law@om zo0kuc0A`6G92}?H)^-4If1HjP(%7NIO4H98?m zL^%%KnhqXj1Jws$PlK%k*+Q~^!4n~HH=>0)PHzT)$D_fv(#4Z-cY*u}jmq50;*!*& z%v4DB1}9Y1T%TVAOD`x{4(wpif>xM*Q1=fM0+?H7eG_v)oxIc(1!!>x>r#WqZxvJ( z;BJ6NM|?bVC97L$P7Zj{3&_cksu}8VXj)d+gO2;=CKiCEG(aP`h$MiNo1?8_6|@zi zLG>?a=QT93$HzOSq_6CJ3ze^P$G?whn(GqG;#oQbYd=O`UVvKP>WC^5#%ngNYiyx^C$jR$C&LO2(Z$?<7(%`44C=+T6F2ipQz=pFQlDJk%g zmi!_GXbp>H$t$R`MDq)L>u_3WP7Zju0#PlXCv;G}BSJwB96hkihR_b$stuZDi^pgq zz!Ds0J|y-83%D(isy;JK0WN};{817aZi`(qi&B%38Xlk!Du&ecB?{2VIrt6HX#Pi7 zZHsUQI6R;U9a{as+ep!H9iSyUsHTDSWu_@)7UN2EU=^U`iNN?$5k}&Jrz5EAkXiz0 zy&LFByr>R@ggAJ8H*PoLZA+u2YgnXUaZYMpX)dJsgk0nxk{-k@P>m>MzaD55FR>^^ z10K{UYEe>!UQudpepxEAQfSeOz5^Im%pi&~Xb}%}f_rKSBEvdWf@=jx?!aN7f00Wj z?2Ht!LP%i15)8tpu*`_zBCsxSk^t3oklPoK%0N&OMp-cqzS|uo*@GzmQNnqgeFCx+y zv|zy2;0Fz6CugK4XWN6lQBagxmYH8#jM{rZDoDVppuqzZ26b>?*%;Guq*??`AEbwZ zsRL{WcpVCug{BK^8|bn{NSL9?Ld-<=1}G#!4H(Ci6x8ww8eE{@N9ooh>TpP;B1mwl zRhF1jnp&)*0Or8L7_8p#-L&?0F_s|92l=(xbrROIF5kh2Hy z4`85L0quuE0~H+huo4Dx%_V66eqwPkQV9m~A51-xe?Z6Eq{44ELXHW{h9FEkC@dk- z0_p*SjtZ>MQP2SS4czd+3^YwpZUf1Kt_)5^jJ{N57C^M&4sB>~D?lOwtd77?4=AL- z1tcQZK!Xd}5UgPa-@FMj0~~fBM}s{D@h_?g;5iHhTLo|kfP)T29yOG~CLs&u7b&RQ zL&>C)Vn$QFLw-}b|z(#du?cNWIOb#~ z7DICeBt)~zKuB{a^OCaMwP^S`zRH(a=ya)*bsBWyW>6`=00T3@jHprstgL^^27UD5b z0Dz+s8o1DKL$VmzN{C@lrHF1GWFSwq7&e55H0ll70|QkLUf~WJUP0WDh-@=a(dw9z z;s`&$GN1@F845a(3>vjir|Li>5H#ihQ-O$N@L*6Lc*G7gznhqYFb29st0W&b1O<<1 zq>)U-2}R(+ZBRLsoRONF7_E*_gfJ3%>S<~osOl^RHx9x13#1Lw+rZ|fN>BxcC^w+~ z0>`=zlFQ@K+yaRnS+C50Yx=(yGif{FaxT589zbL}r6IJ|#6dzbLUJzXSIG|)9527|Rg9SLi5gTo1=4eT9ACIZz|hyyC1@dYhPL3Jy#aj?<}l3DSY z3Qnq;;1ij_bNOJK;WHUfM6K(os-HDuIz^_=&bs(N+O=q!%e21V|SER(QfQ zV15x~S2M^zpwS&z2tdLPBnnz21WJ0))Hy(vBWlqGNdaItLIYkw0i}e|fgX|vIV%%- zkQt;MgJ?0LrUgXq(13asT&6?f1-vR1t{&1x%`XD=fS_vO2@z53gQ^j5oP*t|j*vh) zUIlUv3b@gxqo4tD8mxFhZKgu^79q?)*;WL%9JGEOw3r^gNf=x#Abf^a}N{0%vp9?g<3SFrNturcNc?n*^fd;ogIR;!Zp;o{! zJ&;BL?WF zgVlhJ<;f|@%u5G{8EAJlIN=v7Bo?JAL*wKFOqZF0~tl}I@dl-$5>f&@0wQa1$!^~@A?D^MMSXog{F7(fhz zwhTap5{fHvH7lUr109Zw(TK@T1ueuvZP|bg21O*OErN5g5SrEMpgtP7ag~vZ)Ubnx zh#r!wK^7pjjX+#=a9>K@3a%cSA`yWA)c|P^fzmK&qaR^3AyDGQp%&~;h!x<70=1}@tW-=vq4zbx<*| zD$tI1=%oYT4ewyw#|mF4lm07uxSbvKwZ$E$F&=NL5SNY~*x_In)QU3mOg(KR^ZJ z!8HXqCx8P3tOh=72Ubd;jgFRiq4@^Y2v`OJn+N7#u?P81IMARK=+0kAV;7d8GxPI6 z`whScR)f+AczHj#IgZxfMx;X6xzlLw0Xqa~mPo-CQg(oy1+CUlwSY=1hvJgNlFVet zUUulj8GKOyayEt}0O(1B(34bP%jw`}cR>`vx`jx6DD&>kdsyn}Sp16&Gk`_|PLIVq4CWB%a z>|tAw9bgt{nICxT4PsXYaxnxStwFO0WVsG_Y7cA*H00psBdP?DV~{HwxDtp9!86!! zF@!c~y#`-?1Xlvl20FqSO)Kc|U)-xQ^vW_*%ORJ)fC^!7`UeFf++4Jk70@ynzN!I9 zKh&@A&Lis7d}2xpVp9xK=pk2)kiY{+FQicdsTU#H4ZNZUe42M6e4C8|r2PkOia@o2 zM^8XAsrir-#q&~AQ=n&sgFOfKSW#*kB1q9=0&EZ@VS@)wq4fw@BS1@{2H10=9)bs8=_DX2+m=ok68EIH{GAUCzJ?VSNfB{))})u9GKPn89G3KCr)Ll8Ba zUU5lcQAu%mW=VzyXgH`Ozo;M?ZJms|Ch9rpaJS$v29|R`>WcN$5i7e;ViL0A4>A>m zV@)@>-(3t%b6_ukj6`c8qWA!L91~H^V6hh(N1y^gK>^|ja_6H^9fm&tfh`+A%W6<& z&P^-72cpF&a-oZs`awp3g9o0Vz#^cO2};Z0q=-m3D4YCX zNgS*Jlty611`hSnV1*!)Aq%wO=^ZQ%G60JkY;9o^DKP86^EjyG3|I&`(!u3A=(aIZnI$=? z;G6+E+R!JnxCA^*fl?5lT7|FQ1~wR`JTnb4ln@Wq0XfnXv=N420|^mANJ9^}y#}@x zTxzJ7Ap8Rl6PSZQ)Apbd7I4&pWDzEUTRpJh8KjVcX#uGQhp!GeI-r3Ewik5HEs`q{ zmZ7&

hAU!4?vMa6U1`J193{9$g2nYQcE}e3Kiz3IV$kHo64&2*jP>mNVRLTyrj< z8Vk9cBfp4&h6&h}xN@XAXx0ak8?iVK(tv;;QHyQrD#on5dAH3OX=ep**n? zwB;f>KQ~pOv;dm35ion85#6Ahhd=XKs)~fbPN}T zr@dkN1poPOMurAv#%Sk%7@HZJkDmX58Isf&!Fm3NV{tL~a9agXlE?%tQ&K3-NCa)2 zOiC>&2W{p9T}Dw1E}siBDLEAcx{)S3wX!_FCk#%o$|SwQEIkD)=xAa7S55Ny#a(I8_0BbQ7q}3p(Kl?0nE^cc3MYnfZC( zF?EHs1s?JV&MzwQNzF?y z$#BaoDlXAT)KN$Rn*yy*b(3=oG(Zb#5;Y+UYmziUogD>`Fjy_zXP~LHVhuGbgj8QUQEdFxY3wIjM}rkUH~7X37TH0NKH{FE~(4`UwX`? zpy1-_20r-0&(YU4-r3(T%+(JxbOq{U`uK;t2E{u&`nvjn=4sJX1cMHiQb$&!qX6ZG zdxm(#hx)m=27v}U{rvr0)nh>|a&WN3gSLLg$3vIFfrh3Fpm)8e7GsQ0ft(Jy=AbM! z53~Rde4QzVqGZq^F^Qnd2jfA#+hWMQ)Zl=E*Qu!XfZCAY^sa#_t^*2u+jz8?gVem> zDjm{7M{*IY?oS3?e+71<0&L41j*tacyimi?T@Kpei{=?nxM3f)1ub51PRvcsfv%G% zP6eHr4!$on9@hT^g&$~tcdA}-X%gtBt>{GEDg#~1SWS&+N8PAcO?4fGBJ~(U)#4Z< zQ1=0D24rLi;&W(v1(~InlV6@%qyZi-!S1S%d_r!**7|eK&nwPMNi9++2c5KDlCO}G zuaKAoYMSMwCW4lWfkL&UC^0h!F^^xGmy%jkoSa{Td_GPBXzT_&jg$b&q@b-TpkwJO zp*x$QUV&ZpQXFkyWf%*)Q%gZxp*Y&mDi)lZz!?VY<>G>zOwhe&(9nhKmc!+FA5i!Z z3IUL@pcAdZjoJ8kTtNnp42)=qHi(7Agn~A>@d2s#&Y*%2?0b+Cp(ZOSDFvh! zfkvhvC4XizINTIqr6%0NAU%jIqaGa>@2DFE%{8FnEFNXo4%9=S2+1r?$xP2Ift(Bq z@(9!`r5 zX!3zeg0c|EdZhLE;JGPKdIRmFhL(A@>hKyxNl6KMUMpz10{BodXd48i2XgEMhy_nR z*ou4i)DnfF%oI>_61+V)8PrAtuipZ-T{80&Ag8N?!U3`(3!2T)2RNbL0oe!k9yYtc zXR*QeIDkeJpfm2Etv_Ik^V1ZH3sRFa(=wBxrohL9A+0Nr`Nh_dqCx@e;L-vVcOjQ& zh>>i_2pvc>iah!dHCn`h)PtP?Vt^Vdkmv>%+Moad*Y=PkhN^FE*!oCPxpgVAbAd2 zm|#oX;E8rzp23#j!c!HBGxAGwQa~pLgC@p6X%1W+LedT-Fq5FePGHX}LvjM>+!XMY zDPZf06(BjHC>4AuO>uE%8R-0&%nIWju38W6C?Sj_gfuv0E7$7)_VQDeLO-AY`;4mJwjSDdwn(09? zh+Kk$9HvS5{BGwA&}rK#nMEM=I*?OfQf)yO-h$HrsG!L&QOGaW165FZsTG;UCB+(0 zHc)b-RsB`~Pq%+CYe1(T6jl$cxsKKvoEL?JgbJ)=Y+IT3V7 zd}*;lNfG#%Hc(Q8Mj6y#%oB#;JzdBdd&{c*oe}h8S7HKyYJULh)oDFhC zv^ps4k*~Rgrx7a!j9h@?foQnOSopzq3JU7TNdm(%SXx24{!$(6fns$l1;}ul2B^ue z0TI+hHW=nL&?0a+8<7Cip<78v#0c2NbZN!Wg=!Q%6AqsueUD3%V;cIXf{u6|}4fMI0VPuv;8aWT6~{ z?dqU2JupPU4uO_1$vMRuT3S#gAiKd0Z1i)pU@HgVMIpE)0L>Cmlfac1v|ItTi_pzT z&dH1~D$Pj+2MZScNEh!S8wF}b<)pz*q{8YtkjbE}r+7>ToyM-93OP{$q#W)L94-Z| zNr$V!=~(FHAE53LSPkriLE_vDvV$~dgLJ@Mh0W#AmU1x>0S!9{r${v_n za_oWohlFr~>me?rz&#hpVRN87364k9gsrYxtOr&F@&Pz$5@!X>s8}Hpbd7mRYF-KW z9=z0w66n=1;K~qMfukKR3cA+-bm}!+5VS1-yhnBX!By5PORU^5{mLdrI%19LKy&@a;gFD+N_ z%malc)EuYaM2I86BOtmlRy=eYB6MGp9Y$49q?%oc_pB9 zqo4q5A!9}Xnl@0-5IhJ5G;NfcSK^$XSCU!*8ZFLAgN77n;1QJG;EN3s^C}e*i_*c3 zD9})IZej^&cN#befm%-aIiO)IusI0-qD45e$I+w|Y!yJg8L$crKVqsyvW*MmQ&1v^ zj|W|m0-vMN0^gMaNyCT^Bf3$bwk=2@Xe1h<7PeFhwM&$e4<35XOIHAA!h(FzjqUlM zd=UZwNP0Qr3c=Jo~ogqSe%@h3F<8*B_?MV z=Oh+qK(AAQdIr>?18s+~vx7Q1zqCXLG|UT4vQSY-^40;n9Fkm-`-$M=D;3a;Q0L+T zm#FZDtP8C35L%R@foRQ^7Uh6brjnA9Yehj~9_ZEqg|yUig~X&J(Ah)aBUeGQA0_$V zBi9gZXob?E97xXv+y_Q65@m!7bf-Kl2SW>J6vbSia0YFe(Nl*ld`6Ln#2lnY4W7$^ zrXb`E{*bLP;A6BvnF!UD;J^m8Y(aIPjzT=7+XHQFg4gpDCxfSjKzAPLfu^>>ds#Kq z^}srzZdM2R19X`|d3ibX7K6ls%wo)&3_zm#s>L8wlv-Q@s#sy$IS`Hjv7jyh)k>gz z<_5Yy2p&x?`N^o!1is1!bU3H2DJ;goI>6&U@BmXt$%owR3c6bu)HR13XH;6011(YD zDnXtF4c)*-%i%U-CJ<2F0ZTrht`k@rcw{aovA85Y6*kF^5C;wL$Abp;D~b~H(o;1c z2XMiz`-R_krhtejsJTd6il9^cdZk66t0VGILqk)qC^a!f1G-LJK_RIqH8C4p%0XPC zpaIkF5fTyrPBA*56bo83pcfkC0~P^amW1jBNMb|WL{*+x1e!(HP=_9v2%13$Wd%@N zAsi11b;vDw3W;f;b^_!)H?Vs^g*YlkwGot|K?j$h1Tf0=w~(X{s#D-A3&76TP1acEJ*(1_RN`5jI^%tRxnmAw~ zfVTb%RADCOr69JM(Ek1>QzH`-^!q=}j7&!Ff1*p8z`6blRMdbPaIm8r!Br|~B}#ri zw91FAyFy#31#0b<7bOKq+Xn@p#!xq#%0p0wh!&M8PpK!}bOb5?Sz=H>RrV*@x29ku`ngvfVKcZQ-X=`15-f#F7T`*be#;S)oKNv1BA>AW)_3Z0L`Q*KsS|?R2HN{BvX;( zkhXfF9R7h#Wg2MbI7k(k0c}o$T5;eBZjhTH_N9Uj3I^}7F1CW-)&P%02h1U zphd_+RD%}?B<5HtXh5?G=)h8N$XP)~XEYT&)4&~XP*^}C40dlU+!@KBUH`=e`FSbe z1ds>b#}D2E4muzV)OZJZ9CQaa7kF-lO97G!kap68)&W2-Ez>Xrwc5bW#i76mqyQ;V zDBw_M3{eO64=yz(AT>yd0hc;cP`3rL*%~sf$^}}JQdkPUhy}@hDDlDDog41hSW-_=`hhBOE8dgJ?0Zo^fMj)j+kPb+)fEFm=?X0LihsuB} z6iC4bN>nJWhfbtH=BL58;#hÓs3fyyh03Ro#z0y%LDq8?uAAx8$Na0j&)3qbcH z6JN+7Vh>je11ChJxWT0kT-1SM1z80&{eknP1}u_v6il#`WuTzNsTCSGItsW-!PMds za8w|hfSjMqKxGyrP~#!N2o82!%E2XJMq+V%X#wa|7Gz~8rJMyuDW{+S(w356o`+W> z?&1-ek0FQ8;lCi&*~b%>z9Hj>py8!d&~+mydjX)ctva9Df zNIMUdx1kvMVl!-4vO&eb!yll60ya1W9dm_R51wuU&q#q2FjNxOZnK3s4p|Z!Um$0K zR`6rL#0_jCbh`k`by3hN4(=~RB*KM|EkpJtXq__B)osYiK*bq&k`8`?v4So5EK*cK z3@xQap!1!;x=V|4kfos|Lp-=FvsHjD?1hF;dQN^)Vop3PV@1RFdtkeD2&xC&<9cwl z;JG8v2&rRAiaXfkfW)H2+|-iPBGjX?!Od>Sp^=bvdZ05nAcstX`OpS6e2xd%nXq-c zP}jh6J!r`pq#OX70N?Ua1cgrt=ER}@LUqNR%!LPy`Uv?8W01(JN5No z>=aO`qo4qq^~uajhm|;>nPT{~1bAH{*klx+K{P_04J+B8A&L>d7&mM{^+60lFTBB_ z2MRFcW0k-O5<2aJqyaXFizEb^33N`($pLpwREuHvsDTFOVcyP5%SQwO*ko{&$LA&% zfF}u(OEfgnz>Cjx6w)-obz~Z7h#Tr`uy>({fkM0hts9_OBe5tQ9AKceQBWaJlLBG~ zs5k(bh3uC6q7+d7Bp$TtIv1XFQBQG#o$~~8WHiDk$mR%mn+{bE*b89)DS&6bU@0G3 zTOr1XA^O1^iIGz&QmERZnT#ZbtOL;^gk8}HPK9~UVpIdw6i_W&SPH8e;p?E_L;di8 zggG1(c#sfBDsVvg2Nb@jW`n{Qy4VU!wCN~hr&ikLB<3ciBq~%`DO7-i3Yzy2r^lii z35jx~v$vo@3pzUidZ%SPIA*|xfgAw$04$w?@@ig5W?m_*lZ=>G1(!XZ#X+f%E|Uh> zB)E$(=P96;gM6=`pbpapISdSRya%XnotdYAnzq1c8Csx&R;41xodV*9A+X(GU+AC| zE|9QSfVNSgcNsx*J!UzI#yN^$v|h^=cOYbJ_<=KCHV@8IXU^|PM+HbIjJB4b;wyNh?~n{hyK~B zLl>ojw1QSizY&>KL7G9s2cTuPp!5zgS5F-temaot42e1L{x`@ZkR#y-s6+IC zt81{sVG){}nVSkv*r2|qEy{T?>b{=7u81ytAPX6B`08UY?eNiI>) z$S+bz%&XK?fIA8@dkeA)><`e$2*QiR#0^{S0KJ*4x4bI2l)%&3N zTTq4ujW^~gz$^lFKf!bTpwt4_g|HrPYJsbVCllzPpgN+Pr4C-Rg)=9^jX=(Aq~u_P z0icYHl!>9GI#@)(Rsq`d0}r>rizHB9vxO-HaS-J{qNfe3xk2(sYkM`2^}*|E)N%?r z7r-qAxwF_7Od_lUwU}%{1VRY8nzlve=z-7N133qw7~1uMwxU3*|7?+4u?U5r-Y%rH zu?0mIN;4QD4y{^{yadt%OE8$aF!CZK$wEkIWe5u^P@f6bc!7z5nzryxMSMJ@aSR=Z zfOaULEodD`Z4PY&Kn20APedypCJLLu(}S1?JGwEy2+_(zYS%+`L92L3bq_9vK%oq6 ze}Ri2@Hws!D^XS|!0$tX$iWZVhZ+p;!D2cG-7;`GDM|$&hy>FOatdg21k@D-pN|Ah z&ESpkSQ>jo6t0a-3(e@R-B4Z4^|To4Fp7%f#eWa zjsv$$Kv&d5)xk0y)I@|!;Rc~)1aOq0UrmU4KCn7RRh#q#8J196oDL2AoY+?0gp05#;?E?3^?zDr9tk1URMc;V$hWY ze)%P-3MJsvcEHnq8TpyXsi4_8&@mP{sU@j7pj`?{nI%PuMU@J=x(c8T>*c8mppC!a z2ml#UP+FV;?rDJ|TcISQD8Dp4LjhDiWu$_}2Z|s=5ow^aWim^kSMp>+rawRifcpEO zZSonVMJ1rld`SkVV+J)Jl%(KM3`-Fh;fb7HVG9@#V&DJ(yAHI1A8as|@)zq+8^RD! zCj`2D0lu{YA)5wTr;t_#HX9*|7y$+4YseW6sAHgD4WJSNo}9qN3TD{@iFH_M3kwlM zNe$l41S(n}*Q`Mn@I!kL;H?6wuzN!hQ(T}@PcI(2;}4RFppsx`rIqF-gAUvRSF@m& zUp&O@RMldTMUY)~Pz|87Z%PXaK*vHUXe+=3L7OYUMnPQwE%LDQ)In8_YO#i<6=)O- z*=~s6K}SR@K(*>Yw+ulmA?SvR)FKUaq+3xT^E{C8N$>z{oL1!a)oMQK1Dp>q|`UNCBqgT`SY zX#liJ5OfGaW*O+%kJOwT1*p+_>YDJV!0zeX(wve^O5;8yB|alHF(tJ~T}L507Hj;d zBVBw7+R~x`(NL_Qkys3BK-hxIN9eAsVof~-=fpgPq*MiUxciZl2y}r;YKj8b{RKIx z3MHAKQ+bj63#v~+DI8p|fbt(Sh#}b$Jjw>2-h$7~!izBtXmII(^Do#8Na}%#<1b3U z85PuU$Vp6xW?N{+f@ue381Na8kgSlLlc@(S-9TqWg3C7zO>{k=(h0M0g10Ndj(1Nj zL7J_G%~V6Jg_c)Po(8zcL(>3ml!BZMDhMH7##R(SO+rKpsN@TRjGQDY1eYY{r6d-m z1eT^2RU)R%!E^uMdq_bM173y!^&*lvkgM{*!zR!^I}X#p;R7z-p_A{hT8Us}7iEAN zD(c`^f^}zMn!&AJsHu?tHe3c#ox)mopbQBy4CEnjDgkXVh2~lXRfVDqSUUsBhnZ;# zV5_55i(>T(ic*VH^Gd*#4%{ZtXbvRoK{ntGe~2xoze)B(8% zW-gK@aPJGE7aAWhNr++u4(1UNQbS%hpg*Fc~AR`~3WB|d?WQ}YP ztRO)+TnRKU32Iy==EQ@FV`%veN;fH>bH*#6JKfc7)pZmMA+_)_~Zi1PQ`K zcol%9!U1i(g9jaf#0*Pn(9=aggOuQ86A#+Gty-*sXhJ~J05s1*)F9>!An6w5L~Qm! zcD91{cq4XsgVZ9z2UI-A!&)9X3Xo_9FAuOqV(Y+t1=0<7Iz$B|{-Met931L##Dy&m zam?6MhweT^9lUZbN(Jpk&rHlI25;O0t=CDlf;$;oz`#p&L`W!sy``WIF#>$v6lgXp z8DY2;Xi!XDK~+^D6-#BUUR+rWI>M?%!w`$ZAx%^0(J@e`fh`2JZVEkT z0~%s_knlvBvQz-yjf~_H#C{5p$;dej$z+IwqCqF0LgE^v7HWrvx+#o1u(;I2)vEokNdDMjn0l|lT1VhX5j0vhIlnFS93kV(aQzKKQIpnT$pSevC< z1Uf()Bn?UYko1aLSO!!EXQbxjpyn=Ukq5T{Tze|OlqMDwK#RFzJ?GpMO<2VNQx0)E zw)_P(7b$c>nLMa84{Cip+$;r1_Z4a+#-Iw437{+CO7kE`-;|`n`nm;`B^mj73dJD5 zL5u<|?}0Q7Knqgg)*-lT*16>mV8tzDi^ojB4#}SS>`N@en@nwlcpfbN2DWtI_K1h*{ zuYs%tY9WKY0`8iqgOBb*@R7%W5Rt6}ZPmjM3&gJs)aHjP1#xgGhlQ596>Nh>F|ty4 z(AFY(8zu-EZv{;zgHC1FE3PaqNzK)O+yLv7nO9n&ju>twB2T~!RZq^(gCtIPjDv#- zG}f&LS{s_5lNz2`k`a`em;$Q95W}vZbVyo^Ac9vN5yv{<5kSzAGh0Xp4LTqQ9-YLR z4?%4OQ+Zp{xgP@+T&jXg~@Rtd>B# zNWtJ3cCE-PLF*2YVlTAUlBx&tFf>iUf&)?^8tAV$qEO zk0ik)6`+PfOB(P(Gj!u2w`k(ffvyX1{Q+qfw&ZA4Jr>76{^J`gHS3w z#6S&r9~Y=7g6{%DGXxycpxPaLRUoL+#V`$2aH6bZh1c=8OowjS50SkS1eg7R*D5%@yW#A1*?K*uBIWG11x9K{xRMni6_K-utv zD$xQA%_3-`EGF3+m`23erVxvujU9Ne95OQjozsPS7orl%!8#j)Yzo>W18jB$VhEZ- z=#VnhXiO!r9tutcprIYmnR2MMK$Sx#J#rH>^E4105xkWhOrsJ~76Y|)opVz()C(%X zy$^LASjP!5{s}5+pckIA(qmaNv*o4kUZPbA>onCNiNoj#ketJ4+L=IX;Vjqt10@YEVh0hAXsUjR za?oH^QC?zBP9>xcLHG#NoXbco2HlgXfn>6df;xty)HM-pJG==1$#f(mK_P?cOGIjb zB?4H|frK-5o_bMfUOXs0q7V0jgC8ad4SDQFfIJPo;~*Z?Q&Nuy8yFuCZ@uF6AIwm7 zWZ!}M>!5}dwEY4dql3mL_Gv}Zrt8wB+1lcGum$Y>SRQg~yA#M}bf z%?8RnpbOwYYb?s9{Gf=KZ3eILmiwy6jX~fP%1ER@I%gG051ta zF$b&#B}?HAVMGc7H^U&yt`$I|%Ahj{6jWhtFf=(}H?stO3@peX&@xD*AtX{7?!~2`B|@O) zFQ_2a)PtK2H&>lt*$MJ8D6xXtNEqo0B#5-C0HhXVlO8ntfQIg1wGw0~5YyY3BH(HI zpwc{0j}bIx4_faGnlA<&@d^p|XmyY+>ahyi*n*fq(u6mq!1X;eha=`ypxFd9To8^# zO4^WC3uL4WQi)?9BSX>xU5o)4AA{&b8bT^2s2?<33Yp4391U*YDacC zwt&Gs^Z{9AkXQsdG#I`;16=Na=2H;WI4IH;Kr`*20=+05TqC39Ey!`gNJ|CKbtCIX zEIWW48jQ_AXe4QXbTx+dl_n& z!7T#yGE?(XG@#pqZ6S)GO%9NWAPn~qL>7ALE-po|aMgpT29G;|3R&bKCXlJ5g(bu= zP`*N07ZD0Rct#IgT4d&=qK>A4Y=OHS#DNTmL5zaCO9#5_11b#P3yYdXu!Ssaa~`3v zg{iZJF>nPPOrF>fhN*}9g|aY)8HW+dF3@v&kir?QSObMF2t#u=%p|zObPzjfp~I5! z9i*t?43Dp()Z)?{&`KRh-GZ%fhsYowR76P;54RaITL;Q^kSZ3QS&-T;NEI1aAC|#h z$OZ%d(vpJG5)FuN5MBjq0`;q);fUbVj>OU@}x0c|6Q7VMR4=2tUx;<6d-4H zz>+s?$hNdN6_Vb;J_2WTR3Xq+)@VKh)%ux5pxaHs?JCe&e9(b-kll#z0gWDlCI(dc3FtmZNYLZr0-uKr4p)$oi8&w>Qx%Fp6ZgfS z*^A_&%mQ%4L1Pu{%7XNw#1zo2lO?4E;1JRSB?j10&&8F+T(C>b!(4-cJ^lSaeOm)P zvt&arYhT)8Qx@t{Li6kvC{sUvjgf-N`DEvPioGuBH@Pgl26P)|-ZOf^bOGEFf~ zG%zwXGch$wu}n)gG`27>NHjOFG=PrH;xNVpW{jzMidl+fidm{^(TUeT!nj58>rdpULa;lk$p^2rbsgZG_L6UizDK29SQNzY4DKRxM*(}x2IK{-! zB+)X($RgD|DbX~^G||-3EF~4cG4QZSN-;CGOtVZfNKH&ONj5Puu{1X@Hc3fMHcCx4 zNH&6vvSS1>#5M4+F*8ZFG)Xf~N;6M1H%c@#PEEEjPBlm}GD$NsH?d4I!tWY**d(PT zS{fM|7$l{bn;WH=CK?$Ur>3T)7@L}$6D?A4#Sz3U@R&|c zG&WB%OiME~OG`9LO-VINHMBIgFitg3GPN`^F$KA$mJ6x8(ox7S<^s)WL#l&}#Nv#c z%p_QCkeg!41)Yob1kZPY7uH%q)aIs`BC7-6r2|^mlbMoOk{X|zVycmv4mrlvFybwK-l+1MKC`pQ`23VtBMrs8_6gqbUDm_6f<{&B+Z53dqM8m9v z-*^O43T^s=E&!@5)+;VaNi8bUK{=-hqDl$mCGZX1;OQe!?FHHwo|u`Fnxd$npb-E) zryEf@6%^$s<)r4?YoaeTS5Qz@fVl-+i$ZKrP(W_)D=5I%8-q_{g>Nng`5JOL5n9y+ zHVRs~+M-o8I&f!!mv6wOLxX&D6iV{rQ!D45YD<0CjW%^r~mjcBw?r+OEtLWYeIk6!J=QlVG>xg03Qm84UJ+B51A!6uRIM zTezc2^HRWCQWX+lV&H&CPyki98s!<8$r&))Q}a^54JU|s3hMgmntBQf36M}q0PX4p zH7Jl8S>V&s5!I?fd1g)y$PX}^Qd4vkGSk4uWtM>SgHDl4%mLlNQ;7%{Jq6IA=LqM4 zgB@%gwB-agrUdFruxUx47GNqYK=RWRic^#GL3dDrw<5!wQ2@F_I7J~TH7&m=RRMHJ zY-S?pjx^9kw9uX~B*r1fyes7ArGi`!ZMmgD+?AN8ke>qzLs%?;Z7YGcbU>bhHic4C zAclcwyNdElOF+ktft>>G6e_?E>W4JHKq&!|=@Jsa=>@W*8)|h4S^-!}=0thULrGy8IgET=`51|AINL)cd0jvha1{9<9povRE36X8=Y?ZX3iA&R3 z0b(}T7^o9a4S-&+?VO*NmXn!WqN9+Cs3bDeP+}e-0dgE9724T>LKl+o&`W!zkc?CX zr~^}AClrIqdxTpRH0(gDph3wVoR?umkPavb$K;_F_)5u%pk4V%pu7j}1E!`ZlxJju zwt9jN3xFhCkWodYd7x7h^%MdilgH4A!!Qq&hd^1mC^Z?h0UH`<3Tc@|#U-F?szKR7 zJvFho5*GCex}iu(LtPKeeKC14c^dGW$)VGlszqq^mZpNL0wOJ^f@dJ34Pp`L5fr+h zP))7KEYX10SyM*$_)b-;BZG`~ix zqsXbpg6vlY&AjI)rYPh?ZZj)_Y~(~qqL9iHRyZT)RB&>G1}D6Vgm%`D@+oMg4qO^v z4GO9_6>Jqys!rJ8BT|GwnoNoC(;ZS2KXL#6pCREh`07n? z0D?!t6QT7cp=z@vBNbF4VW~4AMR@|aPy|?sOE2GeFDJz#f3L%|QKgM4s1GE!HilR4oSeDM4Lr zcwPXdd1#^m^|>J#0&?;{lnkm}z^$t*4bg$8`FUQT{G*vTLjAPj0>gAz9)*MdYrI43_{4|M6D1}L0$ zsRtlKGSIRTS~x?8#UO1>XbgZ3=`G0z57lU-mZj!_S{W&s#RWNum0TIAIR&7moF^o; zB_f&=uy_JBR6({EOA?Dpz$@VqHs&d)f||MD)U3zFr2#ri25~n~Zenr< zs09T&+XZ}w# z5p*O?38>Wrb|9hw3vC$|r9vwp$S5Ji1A3ZV)Qd!2B_%AYzCg(a8hdDkEa(8xoWx4- z&66nU52QuY8d7e8&P#yxHH&jnQwyvWz?25K9=s zI9F0q@`Q}NgIf5d7^Ae{I@UF@xDu)gRB(YjfKoIjruF11t{G4(SiT z%W9Aiiiscrh#FX2Bez5l`2keS+8UrG&B|guP@o}t2C3=ks7gSE7g7@gbj>d4cuhT& zW+Bv9prC}i8{{r%9D&3&Knte8;jF0*8bFEFQ~*^h<*DjL;AEYu0P6k1b>m7ZAfJLt zAy~9S>nf15QIi$O`ut+O(!89^yll`Q5yV@d6XvO{-F)?R8Zd-(pJZ4qZg&-=9hs7KEN?i z37%vDZ`n{t%`3?)0-Xqkx7J?O?b|{rVAySkI%@s(cte&Piq_b6803Y`SHB3NV7}$M5N=izh zU|9uF76I+PDbdZ$0}a~37uteKWN;WmQ*lu$xMc$#;R6{3s&e7RA`&3D&5dYXgSa5o z8Xy7@h-sh(IH)cJ3G1jAB_XY5fsY)6^u~ft@<}n(NP|_XSVx}I^k9`8*u~H)I$8-7 zJE}!i3aUjqF?pcwLuLus{h$Fsuy4V$bb6U3sky}(nvg|MAkBK9>o%ZOYf)+e=sF=K zCCEY!khCTwB%lG4nFkswECJhGT$Bt769rq)%yU6%a*1AgY6)cM3nZ=qO4hlMk*{dj zl51$HhX)DN3>^hrI_Qb(}^Cyd^I$F-IX4G&!t@C?i3>0cAzd zxC6}lprZndlB2D_21Cw4)-corEiDBHq&CPq(GW?{nlVF7D@d?ocwfC7nkLeq&7)$N z(~$yzD-(1D7bp?M$J^Q>EnEjXCo@eU2h`rGEC!96lxf--ftwv5XM^%|v|+5Rt-7vm zZi*=+s~}Zh5CQO<30%366*Q_)ivVyL0apY*G?0etA4a;-dir`LMTupZ#d?*wIW!6k z(7GQp6BD#`Kk#)x28Jev3 Date: Thu, 6 Nov 2014 14:56:19 -0800 Subject: [PATCH 041/295] Bump version to 0.3. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a6e0cf..a7ee86e 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ if py_version < '2.7': REQUIRED_PACKAGES.append('unittest2==0.5.1') REQUIRED_PACKAGES.append('argparse==1.2.1') -_APITOOLS_VERSION = '0.2.1' +_APITOOLS_VERSION = '0.3' setuptools.setup( name='apitools', -- GitLab From d088a81304325bb53ec121544b91958855920229 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 6 Nov 2014 15:20:29 -0800 Subject: [PATCH 042/295] Test tweak. --- apitools/base/py/encoding_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 0309438..8a6ecc2 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -237,7 +237,8 @@ class EncodingTest(googletest.TestCase): msg = SimpleMessage(field='field',repfield=['field','field',],) self.assertEqual( encoding.MessageToRepr(msg), - r"__main__.SimpleMessage(field='field',repfield=['field','field',],)") + r"%s.SimpleMessage(field='field',repfield=['field','field',],)" % ( + __name__,)) self.assertEqual( encoding.MessageToRepr(msg, no_modules=True), r"SimpleMessage(field='field',repfield=['field','field',],)") @@ -251,9 +252,9 @@ class EncodingTest(googletest.TestCase): # pylint:disable=line-too-long, Too much effort to make MessageToRepr # wrap lines properly. """\ -__main__.TimeMessage( +%s.TimeMessage( timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=protorpc.util.TimeZoneOffset(datetime.timedelta(0))), -)""") +)""" % __name__) self.assertEqual( encoding.MessageToRepr(msg, multiline=True, no_modules=True), # pylint:disable=line-too-long, Too much effort to make MessageToRepr -- GitLab From 5660168ddc48f1fe85fc381ab45e99ddb39dc217 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 6 Nov 2014 15:22:15 -0800 Subject: [PATCH 043/295] Delete spurious file. --- apitools_public.tar | Bin 307200 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 apitools_public.tar diff --git a/apitools_public.tar b/apitools_public.tar deleted file mode 100755 index f53b90bca7c593c342d5d6359e3802bf2ced015d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 307200 zcmdPXXP`MSFfcGMH#KDd(FO(v=rl-}!NADK%-GD(*a$3dZf;=ApkPXKmoYFfloppH z7AYt&6y@ia;IogAJU+dHSWZvBB)>v0GcS{dZUjZJnTZK{yu5y@o#8oWMsgg zU_e7R!Hpi_@gH51UlGfdnpc*SSzMxEt58sBWX{DEU6NW{lA2c*%axK^P^@69ke6Sa z%9Wg-o12)IQmkOhrJ$goP>@-mkeOFpl9-dDkXVpelAoVb4Az*LmmUjN0n!CBshA5M z+@p-a6#{ztL9UK2zOH(?DKv8!f&6c1VuX?Z4b99(^FPfZ1j%4!c-H0OidKwvD$UGE zQ3x(cEGaFH)yOC*DJZtm*DonbEXyp`P0rNIFG|->E=tTyPtGhU%GZZwd%fbabo->D z#JuDT+uX$BlGGwi5_D^FaV5Y^N>IowR!CGx&dy99JyB^jv-&_Yr-Co?Zq0qhQ(`lFL`GGnnC0xC2?&PYiuNX<)0%}dTqEyisO zR5jEn=hC90)VvafVsIpKafM{07NsgAg0MnbYPmvYZf<5CNI_0wQF^LEazx5hzfLQVUB{i%W{Rv=ow3OG;9U6pB(yiYgU~OA89}i%Ph(6oM2WF{J=Ovbu7NsiW=fTX(O)bgD zPbt>nQcy_A2ZbO=A=v#2$@zI@sYS)0B&Csb zHQ)*+2VzyRJ{L%1h-;8RJ4Ig7EnL?VY78K>Dl!EIfP%#5C45T(Cv$&)vGpQ7038;ih0a=-v zqL5Un5DclwiWLkMN-~P_OVcwHEJ4*;Nk(R|LP~yeX>Mv>i5}Rk3JMBPcjXshI4n88 zpt2}4J)=Y+zdSFsNFl#SAvLcgv!qfXv9u&3zbLZ`Y#%7l6coT-LDvDU|1yge(u)%F zN?^q!G#ro|keZ&Dqu>fQ8p9E#d7u(BKMibtVlvnim@D#96cTfCp!VmNWTZkJ2(Gz7 zRc~^BUP)1Yjt;mwf^u@ej@AMB22@EwN@7qgnx6+X393+`JhKE;IVF}ry{)IIQ#3CIzdc@VY^ zs5(ndECn|kVCfzr1rC8Ch1|rv#Pn29Diov^C4;JI4O0VEO_0NK^FdV|IN)Ih zm6jBjB<7`nVzM|Ru_(0|9E=*7AS07f^HS3?LCqgj(~;bXi1Nt%QY8hA{2~RV$o$eG zB~9c!k(j3d3X`(Tl+wf;kVAZs(g)NfsTHY3$(h9RD(y1IModII&|S zwcz~HqU2O1Q1;42wi8lqr>3PA6{V&?WI+vq+{A2<6}kB-nQ58H;L00Z*Ml-)W?phm zDL5#SN=p>-@=FwQGIKLaQd2MjqY!K8Z5CuB0{GXPY zUJ7ZPD5PcPq#_j`{z+M>$t9RhPt2=?IJhXaxHJcxKftY|+*D9gD>FF}mO;SnskHo} zT(I?^fCY=>K)GoOi3*V50UM{IfZ{=@Ytg)!oS$2e3Cd9UV8=nd32k7drl5KlmNP)5 z7ou?lZexQ}s6uXPN@k)$No4`Dx5M*`vM~yU^8BJ~P|AgrL!fkw$O4&pFmL1+L0t_A zI;gjD6H`(Z63Y@Za}twsQjuC>(8QtxDosHNFgXzvnMsujiBMw@rCVNpNoF#%d7oJf z4XM-=h0HuquqKw2fT}QXLI5Q!kZYkvXe8z-q*f&67UZOYIvxc@`DK|Y5RD*}js>9h zcV>k`Qff|qxh5jNxTF?kmL-;CmZd6y0;CuvL_jG3WC!*zP=NRz>Htvu>p=P?nZ*i8 ziN&cYpyn(nE2e<#0Hyw-{9H&`1+og9FhO~tJR>t1R2;&Jrqq#2f{X4^iVBvpj~H2CgkKi(zdD9R>8@gJykDYKElpRIu^Ikbo&l%}vaNRR+bW z1&KxAlmg0C;PNgvwJ0^GQXwZZFB@EjC1vJ;>)pJ>+*D0a?gvF>W?o5ZQCea$xRQV+ zeUJn^P%#_@D#26p(-5)l4644Mbv)J>M-308EDH??kbUp~g!M+ip#w=r;K&6fACwXf zoQ6}NEfJVW`H-*yyFR}NGen_|N2D!K0OWyMC!qc?)Rf}Vq}ElEvLf|>=X??Iso%4{G_si3iiqWrwfWF3XF z)S{%s9B?`+F9L-{9=Ho%ng^|46hOHEYElU(&|yvCd~gFSv!oc@Bm`#`aDh;a6zvL_ zRT0!EPy+{HM}D3H!c7XfiJ3W|Y7R6cTdbo1sgb~CIixX>pQBJ*SzMBu3$Ob?MR;a$ zacL^3lb#H&BcUoFsRUH>L0T)Ii~;s+ex3q6&_Rg=R*XPOU1%x*MFX^L1q!#6%;MzI zVsI@FwlNo6Ugo8P8&aSIq66|U)Q71Ru=*I)udp^~a(-TML1uDkera(|r9y6EQ8uXP zD^dU#N}$#e)SQ%5NJk%1Dudz(6gJpXBB-iR^2;w#NK`=1t9nZC#x-0Rt$hvmHppY3 zv;wMe@HTmn0}xa`=Av1ukdaudkd&I5r%;rdoSIn%X$B$N1NC5WX;Lw0)FiJYrxKL! z@{0;UrF3cvxWStV>6Cy%7g7)z=_$B_n(v?pafbH@Am*m(DFl~7s(xr%#oC!c%4W!= za%y671|r<@ixfa3M@f~C-U7H?5Sd@9kO*p-7NnMxCg#A3(enJFoRsoRP?IVzKTj7N z^TnBEsX7XI`MSv&pm8Q0g`)h*#GH~!-L#_AR2_xPqN3EY{A5r;4h;rGc8B)=KqWV* z9s>2YQgsvxia;fQDrQNSp9XVBL1|J>W^zuYLP}5 zb(R8B3hB3jnh&r{019nzSqyG0V^j>tO&Vx*4k|QrArWMZ6hQ%rC7?F^(2pIB)QaTP zf)Y>$0}U)fG9$R~Er#@NH5CdVeuaiEDE2@@wK@tJiDjt@pjI9%n!x?&{IoRCh+Dow zacWME4wTHy1q~8HA_iU#fXcMIVzBBANHqyFs2r3)6~J)>vo*1xAP3al%+CW2`hv=9 z(6}$uX~{W>nYqOZ5Vc52F9|e0iI(=k#T2OA%u7v9EiO(h$^_@mw4%(sbVyb%$w%^q zA-WaiiN&DA45`2M6g<-uBJ)e3=76T*1LSlL%NLm3@$U!^q8jwa8 zs9c4&t&8%Di*>;P2J%L7ekrK^1mS1qDI_Z7B$gMKW|n}wnvlh~Jy3__ z7onGmnR&@L3ph~qR$Kyhcrmm`jxYz@e+M473iAvduS+-LxoY~XSRt_Iw=0w>@!P^}7eI@mpUT6v&G z8Yn(NJ+{(Z9dN|~iZ)Ps$}cT}nu5}(gk&;YB^ju=g?5F&(TYe~xv8loAeSkm<>%z& zmxCf6F-&F!t7kP5H6cDNEiO?=2X&J`&I4y@P<5DD0Gg|Tha}W{P-lWV^B6@gXiBJ{ z5|V!qb(S?K@ZfePA=wETxB*!T@inL)1?sP@0y6_;^gIt8}IV69>Ekq;;9nhGX0=#zA zfu?;OP_dYj3TiijqY}xG#1cr21{$V<`VBHs1InCe?g0sbf*)3-f_x9IzriCZkXbXB z8DPhy;)YKBEl8P z80;YoPE+Wf11Gk`5`~=9#NrZ#{5)7$1@aWMcZtZY1v#KjuobwB0GH%YHzL9s5kkn` zN-S1@#Q-QY(A{7SGY}++lE`3nB*-!b- zg6bS--U63cke)WQWQ>^VzB}wU?2rEtQ(#Rsc97yK*0j8Sy6%rUT%Wt!y$w9 zCHWw=MG)hl-F%QgVJQPtJ%f9;pn*w*lkEt(WV?aeLI0P{w6TDCXIywh+GcHts=7u^N02t@S0L8Bm$rZgxWK@9>aL)7+=AO(-IA~&NT5n>ANib7`sKt(8W`?Xk4 zAv7;1wYV4*5elgl1v#0?nV{}3sNzjUI0Ba8LHVc$6 zu#M2pF(h~87eSkPNywwm*di8D!y)?@(%i}~EdjSZk zYT!UdtD*C*3Sh;c6b14+EDB*M1<_3ftwhO$&K`j(^y2(HaFq^9{>i1qCHc9a>1mK_ zGV>HtQj3#|GLt~zUXlT;J|QEuFq=T-8>qPnZ@EJigJKs{8^F{-3rA2D0LcmFdI~O? z#o(?JXwo)KAw01NG=NhH&jfI%f+jaW9a!+V2&jvt0G~4f$0B$NBQs9{Itr$Ph%jj0 z2aT9Rdyn8nE}$VDXc)ueFuw>^=@x_9QYc-~VolI!Y+{~*l4G!fXRwljlVh-FFf4Gx zJwrVFLqin89fN`#{X#rlgBARPkf-JS-4q=CA{D$n{akbuQZqqg(4gF11j^}fpJe8O z%9RvQqXjxbh{!?UDLZgg32M|L~d6`{{c6xdnOp zxx4zh`i1By___u;dpP=qI68Uyc!orRN>l|m&k#S?V94?TN2nPAjzJ-w&Y?bzK?(t( zK>_~3u8`Um)b}sW&jHOW6r~myDwZWSp& z=UG!hi4A$86|*x7NqZK03O=A&P;iifH2GvECV>{yRq82t<|*XmE2NfzsxwfgfS8e& zuaE1Os;L9c44~O=aBzeBd!V6Q zm?@=@LLX0h*8rC{iNy*jsX3WRpmkxOkV#QU2aQMMR6;`tW@U*&VsZ&6%@#pQIgI=P zS=xb8eSoGZU^zS|6I>)f8U*0VBPB66F I4z5MOB_K#QWZoZ?WMM4m(n6TeGV_u% zQ&RIvpi>E;!EMNxE@(myYA7iDL56`wL=%%iYvU4&AamxRS{&*%$gD9)DRLd70Ge$m z1$D8&jbl)$m0zTgnFozbP{Rx)hA6whr4>>nVNOISz+44N(K-3W;NEz8ett@MW)AYm zZni>kNq#{=VtOhlc5@3#K@Lw#%*+9=yaTO*$Vn^BgRF-EITPeVP;x?F2?Cl004)+o z%z@jSU!(xCIJH<)2b>N;{Q+?NBb8OqV1W;&C#IBTf+tDA<0A?M`NhSVkQ5CHGq7%` z#~?YtQcuA#88mbW%B--G7vxSyM9U5-Q-o(^=A7*K4|2zBwqnM(g{l^ppXH@e>$j}nUy z+`P;Zczy(>2XOHUO2Xg_t&pFT10CZ7H%RqC#SUn-P%$V|7H5JM^q0W;SDCQBiw|V2 zm7an}etBvcXeA7&PY8AnD8RtsiDWRsf8Z4@dGH*Wp9k_eq+0@=R|0tzCJZV_72w4v z*u9|10Z?ZZIt>L4L^ezzBe5tK-24QkeNcjhha%J&nR&^eWwh|wGH~q;E|XJ>K+~nr zK~&Im0@}E75@={Iu^7I1DL+l2GQYG48pWV61=$Apc`JkMeRH=|ul$e~I zT2ic_6denjrY%YZk4!*oJFr9Z%kvb#rD|qcB`D;OdeV9d8ZP;H>LsvYBd9|_j)NJl zsHp(%t%JL%#TogfIVlQBsR(y~sK}~Nu z3P=k=6*NGOhOTW)RnoKuITF?%0rmPJ)faU595hr0nr+GiPsSJIWP*!Ea2Ew+4Ag3H zX9_f=2kNlrCxOQ!k;jSQLj($mC159mhBXK*Ifbr<)-}>I0BeDEQ}MJ|K%*p}9u{WX z1G?r6ntI&g((uNNbw1E{}7&@y_m)?N<=O?vvpsU@Wa zdIgoVa}ECW{|2Up28IkM`+p4$O$?1j_kYnYMi3?|E9#dP7wIQu=IN*Al_?ZdmSp7T zaVc{tDj#IFjlFWl&oQ z+B#1L%~K6su4h*Q~+;Y!m|Ydxjq1CphkV544SimOwo~D9VjbMtu|0rphjh& z3|Vds9%#T?7jP*lDFuVd8c=rxv8-B8NlA%|3$pM-0n~y7O_p$Rl@wK4fhtZ=r3D&q z0GCpb?R3x~K2Pvk5!a%k{33`-(0WjX)T(%}QkcHd;#7zLL@%fXfGVV+iNhLj7RUoT z6jD{f+W-ZL$=RSm0&tTKG;UZ>0$v>jT2_$=TFe63wg+mlg7T6=060^DRvLmwr9u57 z|6o0?AlJZ9&mdQq_y9*|Z%236U~RK~*= z2iwB)gdW7jFc}R^E-ujYDMG1@g1V8OxjHB_U?eR)(DY?qiiUb=9%zkgafuNqI85~n z)j`Q0xAMfI^n%2q;#5%Z7=cxD#XAOghWPvY1jmQD1_gWi`zhEes2k{kWVyJ&c@Lb` z^uSCFaC!#yBW%@STYRDZg!<4H!#qfuN9rosszbvNvSSo97!2ySXO!k9=7G{BI8ztp z*n+c-mA-y@W=TeAl3sFtF1EeOQ0qZCD!<59-5Io-S-}~!of$14Q&NjX<-Wv5J=dhW#*;C!z45`AwEvc zD=DgsFUZf#D=D_E1_vC-XX>D_#QdDpc&MrBR-h4gN@W z5Otu9!^JRFn2Hp%6;R3xZ3Xn~4>2aSqNFGhVGfeh3o`TaQd88e6woq1L?6h-pr{4; z2&r5`k_0(0GcR4;3Wv=g6QF*EO@P=!icpXYJjE$!s1|FgE2t`HU?w(b;Dgl1LTpx6 z2&fG3gvFZz@}L7aQj$S^`pmS<)S_Z!%b;x^1uH8B|6orAN6>5lXwk2gl>)R46b$VU z!a7~hR0OhGJtV&%GZ~~8uE-^|EHx*;0JO9XB<=&=jGmbaOYkJ>0vQ3VhkWx>Kq(c; zcBsdpbK17*Q1>Frz3kM=^8BKdVvJ0zIdXPNBihWviKVCS>*(j{<{BKLmzhT+j}hqq z85kNGVD$ft4M+QbG>Q%sW1-`Y3R-#vm0So;Nks_~FE@p&C^fmX2(+eO7p4eSokO&u z7&^)sG9jP`Kf{G~2f=~zKS}4m85)`zn~t9UMvp{<6#wz@nR%Hd@$pn23k1bKzVqLV z4AIB`jLl3&&wryw^fO=|{{s!xfX)+0Q9w!*;29=ZsSg>p0S~zpWT%6M4PYZKsTC!u z;NDG123LH1L1IZpe7u6K0+O%>Ojt)DJ|5IJjE^5}N75y>F)cyL|Det1G^qqY`Jc4% z-^j$oe6;+>j4kSmpvON_ey5scp!g@P{5LT*9i9K7TKMBLXteyt7dpdV4n6-DR8qeR z0Ofzu%70@6^U?C3E{P9I{HNw6=Ytl$#)C#*snHn##Xs)y-^kd&5dHi=Gh>6%_CH@*4Ryo{*LcuAiPR#{fxhaR(0D+s#ttaT2W@@L#2YSL z3JQ?;!_ZfWJs1@fvQsPZMxp}fMA*_)TxyIoLF)mrdIDJsG%mrOhD?4cC}6r#N5K$h zl5oz?Eyyp;OM!+%Wk_W~Dt14C7Cu)XTke@xlA4}c1hE{PGC{p-WEIFs7vv*o=tEr{ zo>`LNnpc{O%{q99hh$_HqbdUNz&S-hAkaCQXnHB&W0JUqX6~*M0H7dKC&;csW#G4NJ&jgEX^sgg&Cm-w;Pm6Gt=|( zi&9hK!6p(ijk-x18obaL9SkWO>LN&xV@q*53ZUen0NVNsK3FR;2PxMghaQSa;22j> zP{&O1>iCkpM`E!b=)!>%sA1UBpMpXjSP4$MG3-QyGc+wh42TCEtA%U|N}j-(EnPvA z+?2Z#-K$`y!73wHXv-@kwYWqBQKjjD_|A#NshZ&PHBwU36hJi{Xn|e;sM>{vT5)Pl z8Z;l~7N;xNqE&9-gm0^!n3N1&^Z^Rc@_2|sc-f6%W^QphDDx zEKSVO$SqFS0~@KMke3QJ2*QOZhs%Oihoq#Yse_XS$c0ct{PGE!13JVT>Q``u1`8ey zkTFn?CYhPU7nc-)Y;kl<)`nOE5`=h6y;=$EMkOl+CDmf3TF@c@kZf@Y@Pt8k)`9>3mfk;Y_ z3`GqesHKoFf@O72@Y%Mh;NrR%Ir&49C2AuUhvn+kwdy!+#S^jM&;+kGLFA>p(wv-H zqy;4K@&nZ`xy9+At!toNzNwIUx7apX9h}J3V{ru%vVE}F0wsu8vMq#}0=5&WP=S=@ z1i}`*z=k#IK*eWqNotBMo_YtN78HJ1+m5!;LQb^&peW=arhtP1J;#?Rd=5!;q^)&G6Z&mA2~Hsgdy8DnT`7_FHD8c_nbBA}HS zR6E#VxN}``YKevh*lGEBsp?3s)Px2P$Ymf!(G_~2kpj>H;|kERda$-F)}(@^?Dj17 z%P#@B58f<>ws!Cuq*st%08N(|elXNg$Vtu9z^fluWoiV`i!-f5LLJu3K%_!wJ&x$H zBIhtr1VYn~dbLt!UO{OIxXb{LM<`h-80aV{fd(Exjqd!?5|DIlEu={eJ>9?-k_bDV!j+1T^pvenf;b)N|0ZfKZOzA;Y$)0wn=Rg^$txf`o}JgaU;mIQYT6 z2`UwA5Tn(t6i6-OhI}Ch@j3Ri1}f{o*%MX};ZsaPd4|szT=^5JSR?K`{^(|vl;`6x3lu2Osv7EVh%X=;B|w=J1tTd3*X*G3 z9qKh$?;jG{nyB@TCuC?ABW^(E#e;VJgQguoBfzlX9FX^sV#n2w{MbRY14lUc<>#fM zhgyDKDh|~e>d;i<>ZcCZRje15Sd^KVSEA?Y7wU`D=s*cUXejHYrivQUM+Y1C8~jf_QqcxphPr(#XKjL`T6~N5M!(!N^!g!PrJ(L=6K?P~i(IbTB#^;MyG|qh759 z(g1cLD0)CH(KRsFH82V>GPW`{wlXr+gSbN5z`)ADK#7PE0nqRaxTOY5Q_!#o$qz~` zC_)Y{Wrc#uoXorutCY-Q(D7`xNr@@CXKnKBq&&L7XyO);^ zx@WYsC=Y!6FW4i>3Me)p9Y~Z5-T((al?QZMdj;t90+6>M@r07RG{9kC3n_Bdbzmcj z@KH=1m`tn=jf=jR%$Wmc+Y~=qMS;|24qm; zQ&Lij;esbEz23HC2*t1X-{NR#;S! ztOq(K2)_I@IX?$@~dVK=mmXSA2Y8PEI^%BOT;?3w2OO13bzM zorVVsfT{;*`v4{k>XbMZ6(v?83qYchi%+D1=Ha03KKzx-4TJNL) zI=>Zk7HMv3Ng`y~Dd=#1(0QtfW%-#Y3OT7I;Cp_-VHOnN%mvb3l$s7Ya~E_$btdTK zSn%}V3=``^Yp>TL6>DFD!^RJw*b)a>yVU9tbj(*_NqSA`;t5Wko9rkEwA%z$%1YKtD=MG+?6 z97K2pwBQ{alyEg6`H<6fu!g6Sl9F?NURi2U3AhkQ&n!#LLl_S(N5GSr3XpODTm&g7 zfC_#+(CY8xjMU_8q&U(+=mmB4fDS1^QX43E zax&8(CAS_ln}RDUP&x#q14yNc60Y$Ganu9?N#caf1lQb{MuLU0nh4F}xa>uw1F#nf zBr%AOKxwQ9l1@@GlS?$BH9-3}K&vEl6u=$?r+HM@fK@;|3CYMH0ciONE+0TdW*#`z zL&_0QQxUrFBNiolVeuEg~Q%{z1|o zq?7=~D_A$QjDVg12r>qg3_26Zon1MXe|AcQ_qu1j@MfUnO9^wNbb*AP2`DLo*BaPD z+bM_+f{TBslaDLNAt(hIvPmer2u<;r1+MWxMuE;jg=|^`_d7s$%Rr(WQeGe%4oyQ4 zhpK}T86-QY>nNCkj6y1A@M{C7Zgm|6b5QmG_ePO2C+;+hk|4oZ5_}U1)LF%#ETo_S z%i2~7jtbC%0DSduB9awgZD4OeH9-|-rWS*=fj61ugReJ*wrW5ZX`mXQpa4Gm8n%@c zW-E5bgB4>q2bA$3K@E0~f&#LOzCkMCUUw4Z`8!HeHI$1U?*w7Er9JTggOP%-+-r66g{vI!MAr1 zWG*-@U=KD>IR%MxaLPg{rJzM4&U6Mc8XP9r?T-f)d+`B5{vrPHA^!0Jks%)be(}DJ z0SdMXkbO!Z1Hf*xf;5!DY?OvFWO@~(4w4O^3LzXED#1>IDg?99)YWpKxGXXNe6n~+ zX+ch^28s-_n?cR_VhwQl5AWCpfHoJR7!K)#g2(s3wE$ZE3T`xmWkEfIV1GYY=z!b+ z%H7b~38yQf!Hc^Q#%RVO+Xx%Kz-aS;9g1jufHEp#W^^Yg3|+%Ql#!6IFKOy z52QfVQ7FwTEe4OlfOaSr*@C(d;JAg_0g4R0l+xUSVhwP=32F?IBauu)O+OIJz#1TG zAY1OB7DBc}Vp@n99|#kW7x91$!)6~O2tdw`2PuJj5Q}}F72XQA3MASH?m-~MX;CW3 z9OOY6l=KCWhIRFE=z}CVaJa)0Ca6?^cm`6!Ln>2Sh`r27O-{E&+%b5cuE zp~D%_Vh$2Ws1AgdHl(@}Qd&UW2~F+LMhbY)uNXc%1qla3O;AHQ2XwXzJOh*C54av! z+5sC68U+RihDJP=NYGJ;$KrEMNLt3@Xi&BUSN0GyJwcrguyv>r11b}gl$1OY^HOpk zH=%$p2}{gTNKPyUUswt0u7EBvMIFITPAq_2Pzq_}L1r#NlSReFp!-wcE26<22}n~2 zi)<)d8aj{*j!V5V$Z~2(4V$NcY!}GI#KZzP;UOg^a7PimND)5JTa;P=KDP&EB&etd z?Yb#0DblEbj2D8piGhs*xi2*bUW$Sfu7WKnOd!dZdTAUfu5cxAWd+CuiFv7zdAO8( z&=tdwyTzdqr~{ggs02;dL6UolLRM*Ui9%Xt9_WHx!nqObX)NJKWFkg*0-k~qLa52N zSP!{Zik5Ie86J1Kg(U7G@N^Pv@c?+WtpcQ4h0OSZD|JZ1!WHEydYQ%fpt+&M5)DYq zr+~(6z=41x=D~8v+7Lrd@Gt`BYqAPCh@WxAKgh2NXfuGIb$)uqB}D}w1|&`DD5$GO z>Z<1Is-~!Gg2M&8_l$7lgUm;Yd(fDPCKqOv1a>6W@(U~jk9l|`YHA>uqqj@YdJPbk z^{noTE{)ee4OjQLh=iAb3j+nK-WeUg9}jD zVkYp~3GliuNb!kic*C0a<(WAt$)E)vkd;B$S4`=FdcpCD1({Gw6tr-vD@o2Eq$s&4 z6?6rAW@1ipd`@N(A?2_Vn4miNLM_57ONtU9%|k*;LD!nagO+R;q!tlU3mQv|2VMS4 zP-RI`VqS3?VG}{)HV9isOIc7(7?~JL`JY%&U_`aD9~7zt*8iFqo1?G)H8D0GJ^vLH zbTr2}%YVm$g5><%+{C<;V%_BY+=9fC%%q%D1*jPG^i9y-T+q!9MTrV&nH8CN(8>T( zdF<<7Y&uqCiTsRcQZ`z*og z^;}%tJpEkbp)-Y$F1LD0YEo%BXde~mq!DnZ64JR<56;LhSAhGhSRp?Tb~z~IQd97n zG<8ibJZ5L4CZ^=178k4QC?K2cSX^A1o2rnPuTWYHD*uZW5+RGtlJoOQit=-EGV{_E zpf?HXscUk%`8c|RE~5}g$>p`2JcvdEhPm7BV<4o5?-)Xq7d74z^5l-EC@x`1Ri1n-2@8CQt^pJ>BS1R z3hAjOi6teF&5=3^>OrM>pzU&wMWEYQ!E5$F0}9Zi8RJ36G8C6o=73JHMIIC`&4cWF z2AQb?nV;43O)bgDPXYDHASPs{fnBJFFa$E*p`egfnwP9#tAOG%z0BNrkT_@+D`*`9 z$mD`lkTPgC2Q3qY3Tl89LM#Dyv7y3xAR07Hq5xtPq$Yy}q77ofyOcnyZoxa0K&C6e zlz^^KElLNKBYL2N_Hq)FQ#C-V0?-!#gC?L-lS@)lbQ6=4Q;R{f(GYV$B*;2j5CON- z&R}UQc5br?Mf#U{bQ;5HdzXsw$P)#cZmweEchN9HOoSaI9^2EFn(887C z(t-lej=jX390kzAXSn{n{Jdh&I$zKd2!wsbC5c5~OF=_LsX1wI7l4MiKu!RyVE`>& z1WnCA&4RcTtTVej1(a4GA|T};pTRC{#Hlqmu>xi%=x`xa-5^sG@{1He^MN3D!jf2S zW*$rvL<8IuUGUOdh#*Ke$PLi7?!~Ak!IK);@gTLZqz1PI;k4q)V!hnNip;zcSXzTv zfhb@VY!$$TmmWe0If#l&i_#L4Q{yx9;)^p){NCY%` z1@?11ic28s@VXV`Cy)uCVJ`&*#E>Uk6F8|t3(TO>JZNww7NwU#gG)(CDX26LoIPM| z3g`+3Pe@q=O1X)klVx=wg&Nd}p#233#i=C*jGy^n>4XO)~DqIE6G|(tA$Y_WGpflks6+i|;;u7Qou-l|K(`8jH+(>*cZ;E#g1}2?;VHoYRQVT|f=ejSzSX>{)S`TlSXyFcPH7R^ z>WfUU3n8w-ETr)y6}Zzui5Pns7M`jA8zU};Y$8cfC`v6Z$p>xY0A2m9keZmBp#T~( zR4C2|hmf*DNk(D`sOU>h%u^^xEdn(WL2X5FYAXgMEYSA+AlCpN$SM+0K8KXwWzmLK zkYoe$sDdrH9s?T+TfbkFUaYBL1NAGc$bkhnRFk@2eu+YAUVdqMhC)GpG4zOPP>LwF zR!9S_n^I6s(NRdPC;-<4pvF*6YGQGTf@%tAH6f&(1}WM=#UIFdnvhyrFCJzkq}D=l zl^x7g&>{-mRp33_iFuX8x+yoGNGCz2U<4OM;5<+Ynq`5h zQc!@XQx64;Lbo)4Go~I`A@l$PcosHRj+pj{fgioM5b_(%s`9-;j zC5T|tQ2-rh4sr%)ZV?u##idE$W_v1lEo5$fN@iN60;toJT2z!@qyU}FELKP?N`<87 z)D*aPpbm@&B`ie$1+sP&TrWaqFtIrSYAz^UfD$u`W1*)!L358QIE8@r4T9Gs!5nn2PT)Vbhn26mojF*paMgH~a~tpS@uOjs3XXBH@c z^JqFK%JcISl2eO7che(pcLJ?91O++NZcw8VsjHBx1M2tafeQ$5D1j3t$Zhc44NAb^ z99gWR0BU&RuFXMp0N6PSx%nxub-@Y>NI4PW2uQC97V=2dwuTlctLi9dX@M;Qr$I=i zt>BbcoC*&i9R+v>g-v-T6{RL-D`e)Cls%+Y}7EwI_()zEMu@K_3H zxGe>oz(Jd06T!o5Ir;eopq5BMPAVu1!fXJ=8rt9yv{O^816nPWTTlYE1?p(T24YYd z1F1DYEInAY0_yiyt7oL<6sTK)ECjW{i=!a|v5;#JAnHMbCYgyjFn!=lcR`w=YIPLU zK?7%?W076`-PEDY^ZX*{K53{cK_xnrS&Y0v1(Mvr#V5EU3hGON)Z()`F)sz0O+eWO zsSbi#paYE^O;FbkNhMTT54~Rx4LFct5Nn|diy_@=1zT9~z|AT~9a~GukB4mX1$#k9 z6TGqk6rOrPrFqcyC~^!y2RA^M;^Y>TK$g4Z7L>*~1T=02IZOf6afJ0pQy|^Vg8Y)yypl}NAe%0@A*fqW1lj`x>0m1q zBo={Bgv-n;&P+*F$j?g!Wgw6@6cmuV>d1bug06dpx(}^$?w4AYS_Iwx3-7;!7RAE5 z??}au9y}Bvg@%G|@YWWnS^$L|IOw5)4Jn}2-ShKP zk}6a6)Iq)j#WHAW2DA_o8iimjpwW5on(mU+iV_|0nP70Cq}24xlnOXEH7_MIr2=XP zw8jFRLk7#!(3BPpDH4#YW@sdXixLG}uqAp0iA5#guDFJ}g1RQ?lZ--ivH)yZK~a8g0W=)pM@Aw72HH;ppJWOOI&c&~Llhcoj%A6NIiO}etQ0D?g3c#F z6)33dW#wn)X@F+>!TJ$JJZOnKqzhG01WMOnX-K3(Em2ZZ3Mfh~D9Q&7c7rn_xc937 zKA0U``GAT=&^ShBUMhHe9aPA}f(7akkjufr09B_4ZdVn9DoPFY)QZd!bxqJXE=U=Q zRADJd3e7NekOg4V%JYlBqtj4*phH77p}lTMX$rC~HBSSqP7~xfLr{sY0Oo_H13(S( z%sd4|MU?}Z`BE=SEdou|sK=t)45|!aV;jhpLPs}{eG82^NO6H9SRrAW2s)4@9Wna} zD>O>;ib0+3%seDH9kBbsqi2v_IMhmTS)Z8)4i#{+ibpTF^%NYzagkPC+i_K;E`e&?qh`(u9>a;Oq%1Vxg%7stc{ua|CbxRzNgl z$}>_+GE$2`5f6$7P{|B+0=R64ZKN!LHJTt}eXvd@QdbYVhe5S6Wb{T4GZMg_gfvrO zz5uoLFbZ3+I{2U!d~hGC6Q!q!GMEDze1Vl)kYONbnFTcvnlr)uV9@jzxMou=wgL?w zg7l&&11*;YWip)pgxrM)O$y+7TUY@OI_Vi?9?EIOU;)tL%}mJTA0)3p!yeRd1GiGZ z%}$U~L~9GC1nN4Fn;;lewu7c8K=BAKWwAC3A$Ea`E!In|C`rvr(NITk9ii9^vlCfk zQEF~}St?i~s4Rd;LA?()8s`0!{CH57g=RmfQQ*=XxpV^!aDq}FNGaHEWQ7VEpi>Zv zQj1gbO2CCwdTL&3QDTk)NHw_p1BZsP0_OCcV+wr8P!}8+X`sEhpfPVqQU=wqkbx`z zyj17h6mYP^n}%=(*uAh}IRypKr4V@~@x_@{pt)G&sbu%m65qrMSR)`cFTEr~LsLOp z!5Hj91<<}_&~P-e?}{}Q>|n>&fT9v&IH(d;P*>1GwhL;80<4k&2dsiEWJq2wB( zyeP2%dFUIiR1fMT_tX-P)SLnhb#)!k>J5m$Y*C#9S;_!%3J61-V+)~BrP4Bsic3I+ zQ9MM}Rvo+`0HjMn0W1l2VLV8QjzT>6$_-FU8fG0h`jDCwF?ksJL0UmUp{lC|tFj?> zfdd~hIj^9tpl+qEgKi@vNrD^$O_u60dC*!|1L8Q4;SfhbhjUS9y1JoKRwpqL>(>mg|Z4R(WE49}gIr6Y3h6-g(lWhAr> z2{r+|%mvxsn6{z@A&TFSHNvK$pkioET+~@u za&8K=@GUOMFFBz-2mU-l8P2s6-(#4>b6cT9lYvl3A9jP@Gx}YSV#BYz2jYAb;P05KyMFQ-Jq& z(1uSy*61jt!DfdbQ{^D*AYq8P;08L)1FojghY&z32|#B}fTI&M!Ceen+y&`oA_`7* zNRI$CTmo&zgW^Cv0NhUiwX#4{Q=nuBI${CtJJ9k2JtXxYD@XjCd2sp@{ z;2;BcBZCPC8Yo4AIx8T*fEIYcI{NuVkih~-pn`n}8t_rDMQ&Q5tdVk0ErHevps8&Q zP1p#PEyVAT^aRbk;i);v`MIfz3K}7m1*za_7hE#rf@Y9GXVZgbelqj(^fVy_M0rML zPAYgE1}sY;H5EYxXF-l$a&8K^3k&I`H4j-puIswMWqEL zF!Mm+09rHx(g#h&Aph#2H@QbwK4V$`3z|s^2Au1EK|LqvDqm1h2HK-hoLW?tnVbrl9Dps$1+6blttiRKgsVFlY3g5N~Y!e=b2l^qK*A&Wab^V0I6wT-%C zL1t(H=*C54K~Mxc7Gyf-WTxhoz*Is7gCQ{u6ZHel-RERhr8*X5y5^-6{JDT`+kt6UHPDMY|B#>3QIFnA?uNg3rdSJ^Gk~rpy@y%GmlGIAwRLSB*Q2f>=p%$ z{33;vqWl6BQB6HAs1DHTQlI?v^wc5^b!2_un+Mc2^@>wVd_WtQG+^pogM$2nG$E(= z#fJv@#QV7Vxrcaw&hRrZv4FQzP(lgkX)?a4DVd21rQqOHK&srpH6HkI-i-Vd@c0UN zereS#Ju!W=mx9g+FzyVUg5B25L~#EQ(^(p&|QIA|gaHNkZ#q(T!N+@;VZw#AU^H9%&eT2>5N z!k3D}fe=Zo8i;io)CdyY1sNHGqy^+thfoev0(Chdd(NSU8z?Bi;ul#DsLhj}S_DpH z8sHCyat;hsUvKv8DfP#N|A*#-#O>! zrDdj<7J*mUfHNK>$K@$F26%$bp9bd!E(Hb0oE+G3YM@m@kj8s4Y#22!za%w9As;lX zS)N&(ijh%^QgaePy%DfitrUWwTm?{42U`eQ&<(c^ocX}3wc;~VtQ0(5uqy$V>X21k zRtg>=Apr_ysYOZ1c?QE&kbUt5iA9OI#VDl)NE|fbm6)4al3D~QGcYt4mZla}V$%(l zAgDdRC?&NhH3gdiP$>maj~-O2fL#c82|RmZxF{vHIJqdZ05tn&rGUf%nSr6WD7COO zwYUVlkrLdSwE|B>L)4}!K&8Rc@9+|-0JO9ow~k`4Jc<^G9pEkhR-l15(1-`o9sQ{|up&c%T#H9RENVN;m1h*cd8J3%%r78ICEJ(c#Nvsh2 zpt8mBDf#7jpjHHOvF(`#?#O`x4PGgMPDlrBfIw7QFrA>;VUTBFQ3yKD9>h;b%qg)2 z*LMjD;4}(%C_I6I+yW{;!6zsn+-y6DfP*QU@+IL3t0F zDR8MX!BTpIJw!mCDHeUuyhcE$85W($84SmnedeIj8X66VY=g^N77*2tTtry8B}6$= z4#Q=zp#kVjr8LkmDs*@ll2DaV; z&@r8O@TfOaV|-9*VQFSjDmdgEOG=80HS&|PbigM~LS{|U@{7Pc(4Z-3#~ZBGTvAd5 zt^hz=2TDqc^hyf~;L8a>;hbNhkdasn(g8719b!310IUx@VhJkPq5EYZsu0qUfKgC@ z+&HK1m#?5&tN@(|N`)-zf*cY9D(%4$rJ!1@2buyzj0P%zT@JPmvb_#u325jM6a?|{ zpb!N|AyHuox(yJT*syiFG!hFk<6(_Xa4Ld^0ImquA`0Eg2$Fhi*i7P7E&P&K6OwBQuIU1JFpPgi{M$Rg8V$txo-A~p&j zrw4VU#krs`ftv|km;hR{1Mv>rI!G!3jrZ!IRTps2fd@b03yU?7w1d?`eFOG%W*SN$ zf*KE?X=pdlL#uwl^9M{-Ymd~tF{YA$Fh6UkEhC%#@@SL8B8U3bqt> zb{-^+A`%c<$V0*%+;oP-5Nh6q1sg&?BsM|G3P}u}9l_}mwylSVEUpf{Ee3MoG0TM-J5M{saK;|SY|22aqQ%2de7eBeEL z#re75Gx|WyY_JKC?jLvy4cINppo7tM9fO@cJwZDl!9AGb%$(Fb&`Il{U1gwN5O|am zboxV4Q7Y&Nc0JIYUyyUup@x7v=cx*y{ziF5KB(X+N(HwU!N*2`D){_z@ERJBL!gza zt{rH{64U_5pi?nuo(L?0NP#(-NuVqZaRWF2An^(ulVl~bVghR`MU z(7|9xTMw-f0Fr|?_&{+B9l6cPOe#tQT_FT+0w^f>x&{Y3y1NF)`}(_t`nbY&dvYl# z#5)Ff#(TR)g4)@jrbxWAkEg3&NW7;Dx=66AbC4@|A{eYLz|q;;(H&%zI+!2q><>BM z8oWj|J~Y@hDBjWC)eouzTH2sBL%=N%q+xz=V?zhjzK%y01+6C2Q2=!hAqwDu42e_l z=9nCa6lgd*9@<~91$QZQphDnGkO~vaE6oM1{sx7SttrN2K9c3&jrLHpVQV$gbMliC zbD+r{B8Tpz#FP~93f#n;c#yl}K^sp}i;BUDq3It>a~zatbQCny;|mg#vlG)n$Cjwa z7boWzq=HvlLi-!At^!ia39HLMnIFB?0h+GR15FP@F0;a_51JIgUH)QlUkQ@#L9PSs z#ZLwI6XQ`r0klXCn>aXJz#4P%(_s+E|2)_IfJc0=t&VU|xfNCrf4{QP056Gz!R9zqr zIs>c8OhZq);6H3UMUFzWB_%Z+%*!m!P)JDyEiF$3@1iToNG(oyvSgwPXXh3@58#*B&4?cqjbhaN-XhWA!Lo!t^Vn`iQ z$$*FK6_64`IjFglr(U7}o}{RRE}uqDA3C7*+sUaQA-FGd!HEzwT$`C!mY)M!f0UV$ zs+*dYmYNI>36NuhHOW*QgDO^xdQaC70^Cz*m)G7?a7J7#h|H%JW%@yw44T! zd!WXmEx!iEww^+8YDqCp^;QWYZ3662wX9=;zm*itzckohScOp z@}QHf;uF(B)6+0Di0U1p4pK&e!UNWe1@C|a$1S!L?^&V_TC$&%S)z~vnshA!%|wH) zA_k4}gC-)NI*Us(b8>Vu5_8gY!S}?1i?I}N;DXkyf%X-{hr4wYK$onRrz+%Tre~CZ z&(eUJQj!lossuFN1U{(+eBf7R3FwN#{N&6;&`cC)g9W&1KuZ6R$Oc`i1K!Y7q>z{g zp0H9V2Td-3Zv}#k(re@J)U6cMVLNGxLAz>GG{DUcKiKkxU~r2D4W`#CaP+|qUW{~Sapnd>%ydn276jTO)E(L6Y5V;*$HK$dKq-3M-XIaB-Izxu0pl~yn7u|xI;CAM*2Xz_7#c} z%OMw8W#$!^q$Z|-rX)Z|-Q`33+6oH!IVtd#7}$~6oCrxEpi}1Ju}FhD-=I4?KwWDE z1&olyt^;l_*#Btx8L2;m>`~OPL{bDc1Bs71YKSBOHVrk0fJ=KsClfp@rvqLS1?r6F zr9;PE;TK5TC>X#pImS3YbgU0?Gc-7DfkF>4m;=7rJTEmJykZ1$wHUayisVY@mKAVx zp!Xob?gRT58rL|}4CvU$pj7bX%usJ5SpiL*pjbc>g_RRXLSP?&U4kSCnsr2sNr080 zMLViuG;2W42bEy~knN|SnZDHG5~w2~@^i+t=)K~XAbOE+i(D`<&8 zej02+1!S!$DAZwvHd1#U7I(09exTKOh(UR9G(pZig}4JWw+C)+sTP9=Zb9eqfR4e0 zx=#(zFCw3c8I1SqXH22h!;Q&F>*R1r9lAs)i~<3mcD;k^_ z$r*{DZ62vb&@MAbAxffDcgjzxw2H}71r4WRI1myK=)23X7@(j5y2DfxTBbk~8?td| zDGpZ6;L47WAuedjLrn(|DOk{w|s0IQp*j9jLD0nD=0sx93F%QdQs1*n-2SF@@ z1OwDpFohr=Ve81bq$Y#TdMM9GEdno^Do#}Z-KPpVZ5Fi1tQfX8IVZCWe3T-nj0#RI z$^>`hkee@{QUH7%5i}T}_QTseP_>Y|zF?~p(HkkY;A{pOWJVsr0XKm_7Q3VtW49XS zOK^dr1D-sGIzvfG2}u`t_jh_`S!y0wjY1-9{V&#Vg*OP0w$(s~a20Ifr^4xh8X_V2 zQ2%LQa|$>7hUi%wAC2@YA%?Xx+VDd2HWuqsWE z2jHi6;j4Plj7KTm!A2>7PmTtaOE3>8fbWk69f1ehj08=Z3aSdodO>LubTJ*M_=Y4t za14MZN)mHZf7W`O&fsPjdjmTi@LJ}yt7sTl$$b3;79F3#tfig3kqw!_;`pIXwU+@aRBT_h!jex0~$(& zj%Gt-QP!A2WwG?@q2_~2PpHMv5P%p1l?NpX^ez>|WyI(NB`46faPTT(XYiU%ur^S+ z1ZsLOPL1ezY&C;@7b0?N4xpppd?Hcyi}yY0azayC4a)#GD2qtp-Bdw&t$ppb!%(7^H(@_+`k zaRu5lkOSJE4~{Er3ZWw(C>DWEu?LTBK-~tKk|1ONsGNjMn`GvJP9iSNDbY*J$$?zP zSgZl{9^%4j4HO$e2{=Bn7<7tBY6@h7E#y8!Jy3H4v?35(PCye9h=NT1e}k>F)BuH~jzV;-4(Ql)SaAy)J%XI!4+%cdF$AeODPX6-`mSK-KnoU#Dp*bi zrGUKRlA_Y&62yoaED3;T?4h9ws>i^6D;))tuz&`Yl9G~hKGYzEL%cu&&`>z6qXDmnP>hCUa*!J_Qz_U_ps+>JgPzDxBq0F{c1=8}8vvTw z%}g%QK#Dx@3_WNx9W}<00~MNBK~|$;lthIy1)`)gbQi(;RB+E}fKv>@RnT%s2NIgF zwO-%=1*-&gDj*&NbBoa}fO|PxN1+UI#UO}<&;|+@9ETSm`xZ370UE`GO>#o)0fhj5 z<6x;4^|p^P&}m`dyEmZ831k(-8zA>YXM#%N=wo#pgj~&mj=Tu(78k~ixi-% z6_L#WH|SC|)CrjbTC)h+W(vNu5X8XiI;5)`!O;iJZwPN-$#byHKcEe6D3u~dW9}Xi!MZqZhl#6QBh_}syZmQse_ybbrwhw!V;Kf z@Q7kwi7u#?hiFMGD98crG0M!()34AiFE7^x4XEgr7Uh8Zx~VBxT>~E9057_R1wEwO z0JR*sD*(w$-~9I>Xi$>_${uVdW;a5fQh{ zycFp2%Sr{<1vv^S`KiU=wGpYQxuC%<@bR~xK@-RfFg)!++yz~U17CpycRe(^K@kcn zt8GEy3ZFj!kJ^Eb+sOkR(`l<-U5hd-2+b_p!#eel<}rAV6`VD} za}==9fEHb-Zh~e2u)~qhtwb>q)SX65UxL+wyolMqfCLZ36`*D$dP@ea&jRrc=qMBL z)D5bikj8ajgD*%nqqq-}@(=+H^%PRM8Jt=IT8arBF@@$9Xh=X?gph6(G{J(_+hpeC zfNBI#E=fu(&P-N-jv0dnA&|Ag^9#1*1}=9&MnGH73W*9inI$DTsR~J{B_*jv3ZO&= zUdaRYoHA@$4?L6wJN^K)4;HcF7cu<~@6zibiGlAUD99|y&(A5=g?LKeP|pBTY@xUt z>Yr%vguX6h1qrCwL>LBk6Lvk2?b(=RCOF9v)TawBOf&P+AqJ#ZWfp+DXE})_sfcw^ zpw)DsAz`fXg;v%p#zdDY1%ulJr8GX<_LMZ0K`^?~u7HUocHHAQF7o2{;X&y&D z!EPd`$^rMniOM9IY0%x&(83+nVW7;8wlfdrR#43Z>E40vCxdJm21kzqcu*eZZfNj< zxnODJ!T`k#*jNNq2W$WatGi$s9BcqM2oa6|TcrmUj8CcrhXiy$1+P)ymKUfi4p||K zFd5_!=r}60yZ}#^qc{N+<_fkDkArR21BpN)GBXXDbUG`0c)RKJA)xL;A0+hHw3AhMWX+TO`aByOCHrQw#h-*OZfH)OT;?fqV6gZL5Fk7NUE7Vqaz<{lXL@QJZ zBRZgsIan}b4`6V_z#Adp0td4V0v3iepdhB$LMUk1BDdQiPJ-6AXmulU`a>(4Ftp{T zR6;C-`VN!_kwzTh;Q%j{Az2NnISh&$#7I42@&W2LXqp9YL_p~iq4^C(4&qECOCW7= zXh#ka1MtBf=yQ2Wg<=-^xLYiQ(-Hup5+7`@lL1#h~Sskn2h-6+jn?fzE16OiqQbQ$&pAB`&Id__B{+yUG>yO;QsA_N+LZ)5Dl<(1?s_biz~nHkK}}UiaRUuB zBoRH7$ixy|C_QelvoPWdDT+``v&Cr-B+)>V9eOvLkn14@J!Fj=e8?A?vylr?WOKkC z2Ter82bJbQ4x@s$b5K$ql21?)8@O!5QpiH$6Cz7W;pz;#h7;P(fbF{irv!Kt7i$tj zP1IPD1-6DU_^JcY;DZ8mVR^AaQEFOZa!Gy>xNglz1+8|-Ooqm)A@n39;60&1V;FAaHu2+KYgDg%}NXjqCfO!_=6IppV?ZP5$Q2E`ZD4+RQ6lI}3O!H^!fGCH=z<&%n*9NTJM72A2DSk8Y%+KCdMlbIA>l3H9s(OIzg&wn*AHa0;&|JB^g*lhItS8VY`Z87ZU zzjDFOV+EB1NN2DXrNSg2M{Rlf!}yT1T8-cubMuq4p$E05=jW&Ar0OLW6o5LF#jsOW zLAz!^Nei?V459?Y!++>0TGGU?7-SptEFEX?uKUoU%&6;%7d-Km$GSdyFpauC!BpoR%}U z(ChQE%R##mLAx{|dlbPd^T6jPqu2myWPqBm$PEpM)@XHXE&>lLW0L>}H8u&5BO$&; zwHP#<4jsFP455S9#)I-LD1wtS^dOB)(1z`t%p@Zn1@+RRocw~+Jarv~#L|*{&@ow% zGdDmv7?S_vLE2%79Z5(Jk_I4yaSFByu+l>h#?VlQ-n{`1!(#X<4qR?3$;{DFPJj@ENePIW9UPAvkheFi0J$aQ(e8j#?G?70S|M_Ah%+A;-gQ7$U+ zEC!`M4WztthoPLjxWbpmRYW^R%F1+C4cHy3ErNl3gIV2wE^f+Gyz63}QES zIc%Y2tDajKk0t@?!Et4#Da3=bXM8+pen34wJ~uHlFFsx!6wydoA0(KWWdt? zgUw)3)aFCB6o32A*xb|rz5QouWMV$r{zDE!TCuUW|G*0db8|tLi07xIf)}tQL(T*T zHw-{qKtWwT$gXtIGLl5lFc9oWaj#&1Kk&dBtZ@ix>Y16qc**%WIiNEZ^7D#eLa?=5 zFg|G72gZi(J)y?2|KL;q!Fwi)3X);%P%s~CA^zqhV*dk{WA~l&QFq2d7eJ_^?6H6e zLTB+HTS{PpE}6+C@ZA_NLAXkgEKD$<65N2qtr)5RrXFrcP-;Pux(-(?7Z+E&bAC!H zV7^ z7$j9-v%oWhAd^6qCg|Q$kWpM**v3&oBPyW5$l_G+0cDUeVm+?-P`@BoXMcA;&nQ=y zcsEa1AD4Jv$AAD&KlflrYb)M4G&scHHy*ksBi`BH#nl;24x$f51|k#^8Q_YdgNqB) z9q>&o2+b=>P0mlx%dARG0Z+gdLsq7OYa(#P2fA@PH9fN!wE6)&2@942wT=?ePs&6x z3wqcJXao=1BnRgmNL?EQH4k)V0oV=&1$+Szot#q)TIhhV9o$%fw7wLQb3i9DLpd;$ zp*A)$Q;~95hf5E#*O@ zt|j>(A2=3dhUDkx6oby_($K`^G(_OXqaO4C+2#Q6+@re^c5pPvCU`Xm^Ab2(;aYL~ z1>^ycM?e>!KuVwdH1K#d>|}kgLC`b`jSQR~M2jD2IDt(-IE4#)MdkvU-p(vmNXdkp zoeA$@NzO0LNdaA!k(>`X>?##>bPLq$pdwTce9~H8I$Ft`13D(H zv{(VW6JG(-QK04&s6WD`ppXbYHw$cAW{Cpa!h*zN(DqZvygR6u176_J z(0T9N(wve^P;^*I$gL!h zo*k$W4s`+8$KH7$W8sE@tuM~VFDijvwS(>xQ0o991<8Bxz4@Re3Pq)PpnFeJ6cS6o z27&xxXkdVJqyp^lzH-ncHfR73l6F9c5rc=ki$R5YCTQhEr9w(RXsvrLc;Fgt9Hiq0 z_Lrv%bk5w<1$+RqLTPa+_|Qn`73+z43aFMt0xmBfbgoWmPHHi_-$1dHlb@FkItsQl zCl!2w18fBbXjnHX6_P?zQ^0KrgquLI2W}&Q=D#6X4tnW9aY>Pmf+IA*LG3B<5S#)u zOK2n(7nkP3f}jX|;v8f_Rxu=9B&DY2DdZ-mq(a)j;ACF}I<_yd1hgC@1CbcPYdlhr zj-SEh(wg1?NQ2 zO#nstxeBla-umgOdEi6?xt3YMRsk}&4ccU>p$-vL*HJKl&ro~jrKIMSc!LMeH6Uui zwTl8&BWzm~R0(ZTMdW3un1Vtv__BqVJay15)C%fa8W1mo+yRLns1j(Y z8Ilh!sNvcb6hJm17us+;Kx4Yt0tF|3?J3Mf#r2Guo)J1{_=*MJxcU5kz!KZx7~nSKKm0iXcSEl$^qhi1I^c#s)- zpkM^`VHLm`4WtNq&L5=D03Lb(XLN9H1R6sqdpe=J-9V8D+Q$J>0-~XL4|FeiY7Y3e zpyG72CEOrousFjOu+W$U-J%V0EB43+HxNL3qEjKeg24$5>{zhdVdjHl0wD}C&lco& zoXG@iE^X3@wz`747Dg(8Ugb+5m4F8RAVohkePpIV^Bg2=B3B#gkh3d6d+b2Bpnz_@ zH-I0{1-YOlw>Ul8%Fw_7eAEG!919ypMBWk#PTAmN85M#HQj;?ib3&rP2UNlpmsEka zG-+CaQaUK@fyR5#4xh+RODj$-f%*ZYrns~qwMYZSK**9}P@)H|w2F@hck)1Lp>r(| zGZbtUAPiL7Ky9t~cCdJwa~c^(=gARCFy zWFV!WJOnwlx2PZ)e1j%ZCIcxzj$znEIq;x^F_2d66sPM|rY08YfbP`M%gxU#$$&6Z z5-TB$jQmoVY-V0*Nh(CNI5jyx59#DBQ0C9gOfJfYN}xp^NE#AvdZi`FkWkPt&;&KT zLH$+mL@_v&KuHNY^Z}ap1rK|`+TqZBUp4z6rchHbS4dGRuF6gB$q==P~^fO72HpSIt!!< z8u(D9FnQE)+^@mZ8O#(ShNlD2u1-uv) zwn`Or9~0#e>OogWns9u;`zylS~K@Oze zD4HrzjKG5wGPauuy22jW@eo^(4qgHW2xv2aJ1o|qu?dSbP`#oDx!FWr4|cS^f&%C& zLYQ8V^TFj9I2C3VgZd(gdC94u+1b)m9q?_8@HB|3H98io5!wcZrA&w`v20#NI@t-5 zgu%K&Nn1eybUhY0{emms6wn!PnR&3#2Q^ed6+d*f2%e91kAkgpSuk?i~XM7euv!twJ=^ zo3Jfl*hXc+H5S-anh+zP=R3ewPs8;>hH{YxG{HJSfdiQ^(8SaaiX^CKqAS1+mnOdZQnYLgp9JB}lTLv-!Ie>Ja2@Ne^@VW_fv?$p5kU)gH z2jmja(V)Y+()uBod9Y1)?L1iFCCM@*8 zRv-?3f+*Gldk89(2|79!RFi|3Wr3!2V3`}9y%EYmZ3RSS3Jybf$pY`$f&&^{go6DA zYJ$Mmv4P4WXh{HTNK} z5>Q@2ouWa@wO~6SjzkMpXd40Q@?t$m3I#U<5Mc=IS|B+Z>>8b{Ry zZhXM3h=sbN2yQ$$exTBzrmAZmX2A!Qh84}=5)#r2#+~HQ0vT=uw5)_mgJTD8SqX6u zQdtR3bD)J4phAP<~b|KUnjM5RMR7EKy5xE{xbt9LJU|Uo1mW@y) zSjtAI4Axu>akVYz&>PUXd0;L$e!+bQXo(41eFIB9*vm!@q+AXT0;I$btw|xJ6KG2x z)D8tm_ZU~H3T`|V>$xPBq=LpZAljk%4N>D6=G0p%i+5 zGT3#+Ak(2bA!GAUj$TS?aY<$#_!M1eUl&OPlx^{>i~=)Gm9WyYS4vHkajiN5Gj&-pvHr`EJ&#cVFs?G1hpJIjsmHW2%I>K z=xM4Wuh4=_jTI~8mKK+QrWbQT+w@W(^U$EwpI@4nq5v6^2Gv07P=A8vl2ReJZiC$l zDmWEvk^GKQu_Go(5Hlm1;H#k&azUHlQ;Ul7ixj};d4OhYAX|9B#TdwNux_vgNvR;| z)QUvV)tUhaK^iH|QuTD%9z5TH}tt7}2-fEGtk zC&R|UAT6m-NH&L>Qw*zpF^UJ|&;(@%q=U^cOA55801d++)_;H!7qqAVmo4BJf+c6L zB)BMroE4*h6r`Xb5zv?f{M1$O&IT zDJZQWhlNIVYNaj6W~g#ou*nGZU~VyJ8(V5#iUuh8flL6&Ln9p|1W|+@@5Cf{kmpch z8leeRmLW#lpplBO5f)*f)(SL{K$0dX7@?^V97R|XFYG{FY^&lxCy8W&&K(C&Lc=f5 zQz%O;$^?y8WTt^50doETQl~H}H7CCuyz~P!AzzZ14Z5SR2z(K7W{IAHX9?)$J*d{i zoMP~@h`dx#CqJGPtZ($N-&%51QmG$Vr8a zIb?#DW`Ig{aBS!(fNwB_WMoi1h$RDq51iFN2{=&1LR%!5*#?rfV23-YgYALt)j=(n z!7^a|_={xFYF|)$0(xMvf~^8*eH9`xgUvxI7$C(wN@WCbFrw_mG!kqGIM6`}2b6j- z(tmMD5n3?7as#A44s{>IE|4i;GfB%8u6TME&|DCdnwwaFr(FcPWjY^pa2R;PM6Wo% zv?w_hnkgV0a@#{t-6#W>(EJU}M#ysoI2V9I+e^jJOlS)mCP!;9L2LtsCT71H6hBb? z_|pz#Qz>R+DYQ7X$TP2?v;^FRN7U%>Ys|qmKnfS=usv7=;%1PeAY4#I0Zk}S_n@RZ zE?6#sl?jmM667*8H2=bKiXK!gESRuOf+4~NRH%WR0xAnYG<32abjopJPHAd9)G*Lq zFlZcsZkzxYP_V<~LFo!qBEV`NaI#gfRe)B;A^D)}rU5b>lCwa*2c2GA44pPofD|pT z1I3}i0Z|LiQP7T6acT*c;ss)Gc4{S}t_M2+nzvvc09gua^gzr5os9%a+MsjAL9)=4 z2$6);Ja@?v-M^s z&};-BY06B4BoV!$)B@1mfyON?0wBu3 z5daM)@WmfyCeYDU@G?YDAi*YJ;NcH(2Iz3V#Nt#l6VR>ipx6S1GNeq_02SI`FGBmo zFn11k_&^Wo2!%9!L8Fsxuo*fXh4PHdIvi=pcWHYl^p*%Gw2R!VB zn2LZFuvQA-K!#OopxK17ROsXfXxyZNCs{>9LpaKr63EbEQ3&Jm~0G&6EG!_DuM5@_A*#j~j22FvW`U*N^4(W4&7M*~n zq>wT|9%#8VL?z7T^wbi3_g_GpHsJgQ(g@Rrqv`@X6Kp?h0~p+P$bt~CTr6xB9AqLM z47ChVKfC&c`hrSzv`B#Lx&w_YgZc!}0t8xF6D~u7LCIbLlvs*VL7Tuq85Ndn!2T%) zB^XEwL@P3qON)w9^Gb3m6*ANF@{3Xt`2$?S7K8h>ko=dI59xsCk{ z%)HbT1r1PU&dE;)t&dgm%ZK-u(m>-*O0aCBsi)wUUj(ic3QCI#@{3bJiyQO7Q&`Xo z!!q;IK}*Opi$SAvg{7e3!i>bC+_cgh1&!j&Jn$ij3MHV@s2Duh0V;L!AnUd>^Yb+I z6g*OkQWZd>NtwBzbKF1$Pf138aVjVT(82_|KL#}XQ35`zP6Oj)#?-{(%$!QlfLama zPO}uvL03p1$2&%`pirDx203?*4uu0W#X}ke#E+mTXoFYvz*_B4Q^2JPa$ycNRs&q~ z=R(esfr^0|W=LyEAZ-PB-UW}lWT#dZLlcEAfDOP=vVi4?D_NjcA9z27rP}%1<7^fBu7^v9X~k zgMpEenX#Fnv5|oRgMp!$p}8@Gf&t}L6E>l=xFoSiL4lztKfeT@YC`h(^bTSSOoPxxn;#By- zWKe^Qpof#eeFf@x>J?Oi+w4Up&>kgd?g{^?5TNsgu$%|so>~Gr=?Z=XLt-}UTw<6I z?4&Trc?`-5#s(IkcAtWYktK8+6L^~!Xh0e~B?xu59#@cSaHy|qyr-YDzi)t#YY1qC zp0R-iSBSrVysx8QB<$=h(1`~oMwVRhL9Q;IL9Wgr@xdXEA)&$G(+xn9kh%qCuCITX zt4n-zx4p0??uN(on7qTHPF|P!^cQ91}w78N>8PpF?N>u=N zmrL^DN9TeJgO0O+#~DD;3oe#m2@86|Dg2aDq!J3;|A8Kx0uDL!Gwwhe)6()mOM##v z1!6)C23?Ay9;*qCOB@LvOJau{x(Xeri;pkP$uB7eorR^K0pCX#AD@$&hq|K#w3Ze+ zBI67>EifoCFFjQQ>Z(1xp`ZX7!Gg$xwS#6}5Jn(t(T8gR)xRZR)v0-)omU7Wkk#wL)kAxVnRz7|sd*`y z3c8?G5*lDrK>J_}LCGIHi>R&(^%>$$7EntSoa#V3e!#^oJjsEbsg9xm(uxJG=7psP zg`!mOk`2(}5AfzwXrc#c1EnGGELDD*0_eP#+=8Oi;$rZ2W>9!4E5NS&wFY&^k`qCL zHzh@&W8phvl@&ndgD0**Mrb7`=4mNFjDYzR>!DiVMH19DaBe{_U?Hbag6{$Y7njf+0W|`}ThXv1D`T;m z1UlNeGCnb_B((_HB#?K(Qo3L%yq-m~4p|#^J2Q*pi&7ycO(5F|bv;-J<+iiTJOvyL zghZsFB~S`b&&f{)FFrzbFsKd(l`Np+nqaj!D7!$bT4*^3IyAkcC^NO#))d5qI1tpj zvNZ*Vuac5dFev+jbiyvPQvjWf0m>VY_0b?H@Hhjw^$BWq7NizIkDvz?>lul8DLJW- zTn1_@9vkb@fva#AZ&K_f92IjQN8UEPp91E5#}9qEqR4A+6Ur6LhDi=hK< zFZn{}%PUf$L$n}gz=GXM!BGL``I3Bv;?z9oQ3W86WP-N9Yh;3IR@jaPQ%wcG(%huf zBGB3ss2WgwfHvXh7LIYNdK5$Yj{ zOHV-ooT@U5L4BqasOuC!#~Y*;L6$2e=9NI|pQ8L61(0dr0Ys$E7m}HvftuXJyhPA8 z5wIaiiOJcZK}Kaz%F0MA&QM6n1a0Kc%+CY)F*6OcYzlnEE+|dtD1eH5sE0tSoItHW zaE4V-09`mzoSc!G3z~<6b+bTiWyoGOkak#-0*PwsK@@|VNZ@mgbK)T);4Fq@8Z?{g zA-n;NMbOM2dgy|zK~@=!FbFnK1&YE-P*qg{uF*6={LIv11#JaGs8^H~K&?v!*zyw4 z*$Wz);NXI`DM4p|z#Em|`~bn`}PPMV%VaB8ZKLTWms3#F_8 zxhly@Uq7idy;u*j^jI&yC|y6ZxVSXc(9put0_+xW`v#su6%^pJi3$oJDbS=Qxcvg^ zos^Umz%E^bbO}M_2x@=uK?YHWDA;e1cg03iou**aK{Sem|IZC0(R{q zxWLr|i6DAiV13Y9LJ!jHLAh5Jxu~#3=79YGO`71mW{XHLU_NN@4?Z;vvIcaQrc+`{ zF!*$7pUgb)7&v@_A3CI<2QIZW)SVMc(=$pGP}L)M4nXx-NfBtY!wR%|LPr65;s)4C z1zQEsi~yLOT2P_@8K8ie4H|ie+lsh!0uneNcfvZBina>iE-I{Lg}R#pyL%xPDS$me zk_$l=g96Ai4>o-ql$r>hSfiO+(X1lD&7ehAd1?8OBD(}!5W@lqUY*-2z^ZKxlq8SZ zFaf7(aFzl089>|0z{NjGBgsa=)W86gU!e*?83bkzib6$O1)P@wOFAdiVDvF>j zL}*b2GCHX!H8C4%6)cuPUeHhvN`<62*q&%mJqx;1*G9G24q9PBny=u+=?bdFdeGq} zkg-S@$pPSG2kM4`QVXPNL-rCh6Xk$H9I>cBwIUf3jF>3~?hle(o(WHR3Pq`jDL9;t zQSgB7V1~I8v0R5_+YyC_0yz9hw;tR2Nl>d8bXrug0%%YYv}`RIdQTZ>2m(~&f|DIA zK46I$q*$*wCpEPIksy)M3aIV|4Ls^OC+2|efhtK2Ps}WF%P#__4{!xn3GZDY!UV}w zqI5&{(t)!BD5rtyeux8ctvGW@EJ;LbUm+We2H zk)h$}{0}+tNva;4^FN?lr4@>ki!uvJzzyE?)V$Q9M9_(O&OV;tp-1?XN@7WBMJ9aq z1+;bsHv0lCg<;%$_<`S`0zM}*4}P6YQ4XxtQ3O+5Tv-gAqk$ah4_bK-Dz@Mk$RrjN zBNd&FAD2AM62i1>rjt4Z^3%U&r+^oZ?sv>BUplhN^8U#9`8+1B1xK#r`Z%ExrAty06DJ4;% z!b+h6(#KH;?K4uhf~>a%4Vd^sR_;U2wbkLOh1Lh?!3(Jvl$4a965!>PkYoT(zo5`@ z_VI)*uf!IfplE_$5L67>1_U{DTmhS3K?`ltu*yK|$W+j>Qqbx!P~_{mxVm}zxyC2u z=jWs*=D})3^_={4SPx2F2eKp;5)29o>OT4Duy$TCWJ?-2X7L*ZX~E$z3nEFpVUV^V ztRlu?C|D6>+zh4?n+>4ldT?*&gHwMzWStT;#VaTv(giFGu%;uuvdq+S$kG*EJaCPE4<$b1giDO?IzBMP=YmP=W|&p*V~3O@1zn(xR=%LL!XRFn$8pB8j_ zD0r93cRSe%&*xt|xh+6X3-o>~Ga^)xg=la)T8b4@g0lAeBU{;<(yUAh_3cKOQt53F^Vb7vvY0q-9osCl`9&$^2{9670otO?nA37SdD0j-4uZ3@*$%_{>XQe-K;(gM)Yf*RG}&;o_KI$8=< zw^Be8fz<^H3hJ=vRJTI-4q*~fsDl(BTAmOS!25VXL(M6vSVB_|?PN&M9yo}15P<;k z4a^asS?uD}_{6-F_@vCd6b zXdDq5+MvEVC?A1R8!U%_q7hV0feN)^(EV;1dZ`r!iFqlY;!p!73o#0uZS&!(Kvz5? zRKN@brRl^JP`c3rxji0aM1E0aJjfD+YG|NCItY5Ox=#br=tD$~9(Yn6hZ&$k3Y3>& z4Pr=MR8mrME6qy=B~(z!4sEJ}hE|I~ZRrxw`c;G%=???DnKsb1&2zdLRw-rs49RQ!7n+mrA>WW;5R5@t=GbcYE zG-9a?)tH(KGFw@ppfV>juf$3rC9@b*TiAj&-lQg%q^9UXF1+J{WO`7^0dJsyj*9Y4 z%*^uz8xL+U6jv7OL2N6I24y&qdeC8$$l{1w;9&DUAfJHRLy(FW)VMEps)U#d-tDIe zs){t?!K;kpMp< zLICfzf0)*gIubH6)ngg|qvp^bI8gDg|28l%GciLO|1&W%8omD+9CoxOaE||l33;WV?P=7fUGysoKN4r#jCI2UZu9>FV;6DQc1OEJPY+`JPe*dS5>FE9M zw2O0OlX2#MAL#ZDP_~CHUQ+-k4e$~&_y`lY#Rt#!;N~PG2Phz|k%o=wgLY%)=s{b` z`9+{C3g@7z$<547)yoAH5THB@QV0@>hl^lQotK{nmxgOV7DUwp-WCFO0@zjY5I#%| zbZQaC%`8bR0{1Ck00hKeS*A z$oh83I3b7)PCluiTSH-E5uo+wiJ5tz+hX#I6u>LoVO0wB)+NvwL|(c=BH9KI(7+CO zeH>^7FX&DtbQLE6-O* zEJ`m0SCNRt9InpdJ*Tv||&1D>uZDJejTC-9&=cpw0>7!Blk1%=|$q}^!N5L_`6Xtzr9HSod4JsADkp}iHTIho%pn(r@Kgc@p3L1p-K(Pk8xC%6Jl$lqO znx0w&DSJSp!=Q1f#1c@07P5mMv?3q6xEo{?C=N4|Q$a>RnF^VC8L35?;PL}JNe*5c zl30+b2b$J)EXV|{q|QuEMK}kt0u&l1Rtmo0AOfv>P0LINm%gB6kdj)QS(KWB8XO=O zfKNt;rXTR;5zsAyD8Uap^**z>1TtF&(NdgR0$K8e2n7Xgg=kRdftII*`ZhAr@$c<^8@*cIsE4;G}{X`qRL%sd5fY=VLxC3C?X2r3RC z%|Te154Ih$&xMM+8D|nj&Xh8sIQaT@eS0N}g;Egg^WdJQU`5MJ-(8c=j2moheco6`11ZeF!Y|Ro> zNoE=(#!=H5xZMLv5YW^BHvuvX4sr-o29}jUBebAx3GwmJVi9zwHRyyMNJvA15IL*B zU58Rmg6dGrETjp_zM!OsRuh9;OrRCY;AupR3K=X4O@L^1FSNgkRO`a4PjJ8>_F*YN zl!9!rQUKL$3ZX$h;Q9(|JGiC-HLjuimq7J2O3ne*ndDTvum}f-b7DazEE9kxN1<0Y zfu+!Dc?D4Yf#Lwr>I7sBU}uB813oyeBsl}j&M#6>2PLUuD}DX)@^U@M%DTjY%wo`V zqCQv?eDd5e1yV?YVhg3k1BwE0<)xzlSs(@KJ16EAqs0X1Bsb74!eFJaOb@mKn#GZW zADW&(;RxxVBDFjSG&&$j3#1b?SD%-z0BUeSE|rBfG$2Y)dw8H`3ut={I3;gB_GC^l-!j5!8o>&2=ACMu4OmX?GYusa ztrYyh%MBBAK&QlkcUULpfJQD8b0O>Y5c}3k5c~cV6re^16r~m<7NtTmBsl1yp^dU- zP66DX0(%{n1eHM-zZ7Mrg6|vx->#dMnGV^c2J!=1P=e}2m}kJoK&*#0e8A-dIL&}A zh=()+p$mW@>OfOh3JRqKphNz#>Inr)q3Hozq^tn-BDmiH?o+|ac(6PueZ#U5k~^T9 zkirE~PC?uRbqCJWjS@?cwF01IgW?HW6dt%l1MMGoE&`pL33f0v<$%jx&|zGeC7`U4 zUs|M)3%ZcH2z=}lBw-+x2Vjrnr0K!>vtXZsY)AGjD5K`2>7kVsAW?8s!1iuI{RxQz zaOr|%5&S4$Br&izKn1gFMQUOe$MtSh9tB{K(!5dA%eo}z?NdfF9m|;l%vQqF(tblJ2RdCF!MEC`~ zl@L6q0j^(5lao_ZK~0#XN_fnHiW^WX8twpSa4A5~ZY)YI$S;D-$>$^%mmr!?@HP%8 z{^4~t=y;vPlGHNLogyXR3ENcAL3H37$FY|HpsQ;@-9%`bf>v#?;s6?m;ED*!28T0@ z4QoUqrITV%Pd+anagjI3fglXkqleo30N>&YIVc{sS2#0I0lG;ODV(9H2fQo=su0mY z0gbytcglcc4K*c#Q!dB_$_mgdm7Ar)y9xYzxkFa4sYtLc#$WP@u{O%7l(Pg8Tr{ zfou(^QL6z}sD~^8FT?auoAEHauowl+ncyv-$ibGMlLHDHP~rrSg22-_EJr}?r%FNr zwX48ki#@4;lYmBIUM1Yq*m`#8$rR>5nCC%Z397y_5{uy*onaebK^vZnQd3g%K;t*X z>WJlskg+$IGQDJkBE6#2w4&7F4AhW=FDwLm2RRnOoBP0NsR*(x6Pz57vk`oiA4VF2 zBvf#QP*PF?OQ(YFILRr=1RZ}5P6E)A5zt1YLB(rHK4`^KW?p(uDl8W9tm}iMk zzJswrHD(g%MuD6h(4csJkpgs(7SaLH0hMZ?y$_(624T=5U$`gWJL-`8ub>lsK|?D= z#U+TQmO>_Ir##32P+1K+rM4J)Drag6VuTzLClCW5bqHke8yv+T&3Fba;iWgMCW8!z zgBvyAp%}O!un|nq*}pldpc9Ni-7d(SIy^Sv!<&$T80(NKtbA8iaLr4BtWhh;2br3j z0jhqB;f6s*w-Lw3fRkx_JUsr8I~Jgx8>pS*o|B)Hm;(=L9PtEdpF;;k;T=n8)eSZs z67^8~;BjsXHx_0atOA2&a8RiSNdchIhZM+}8>vOmavEwLR58309S@y80hM3XwJ@VY zGEx=5wQgc*NxlMfjxsexArm@W3N;L*Bp#$BJ`;3w5qy?VFFC(7uLN=$6DV9l^D+xd zQx!a2Ksz5Hr^kXS=EP#qGQU&>&~zAR9N*JL0lEVoB?!R%AFu_brI{&uAi@M1<)xDpsAAlV#Hul9_Wbh;>4UBSi`Io9959; zhvk(bNI|V`qgt%3T5Jct+7FrnpmC!EIt~Lg-d9+fUy=%|p5il8AVX>npmDv-WJo^) zwu~AYZ;&_($@fgrfJ8pHcn6P!Lscl?j3iJ714kt^kOM%QUUI?9S+S%l@GyLSQ7))> z1?6>6p$*zC18S^-3mPm*NfXq-EiKA}9FvSXbOv!Qs9=V4j=_-+IwL3#GUtgA1t=*C zEkS^CTxvxLXze;E)>BZ`BAw-;0I6jlvxu3H+jSu>0i_L4a~oQgK-R%RDu~=f=pk6J zD23L~&>#nm-WHVZRtTK*jBlngbYnGr)B**i9Pn`!yg55;TX1 z2npC84Ty^&O%l+7P}=I?bxokNPr!K?*21N#MW6%SQo)%LQeHtE5p8ItYZ$9n1Ua`w zU0Yp8!4TR4geH8*f(E_PJdD&0Dlfr>P%x}!g!aWz%OGq?8c$;dbfO!$tpYxV8k)9| z8xhcw2$b}&8Ut=s6hWropxUt{H>fG#NlVa~FQD2qGY>R^lamQb#`$^rAk(4Q1hq&| z29;mM;MR`EMxd*5|z&sTPY9->-38(;gM~njakV&v^kb=@YNQ{8Z z1T`cyK(j57GZ8^o0TmUerhu37!#RbesYR6#1<;8uh|^%ELoI`J6(EHXXuphtx`H;W z=><`$t)LE`HPkoMGmOcDT$7iZn3Z3oqmY}KmtO=Pe*`Ba*pV&ZwbXEzf=&%80o7X)AfIkGCTs6+$V z9nfT|tbncxG;~%BUJ8<5o(GC5uxmhvJ;U-f*saP6-l?et$fFV91qVqi-Y!>fEJ2CFEmq7$SqEf z&Q7fat#Ak11IlC&QFV{};u7!)o8VNEqL-Iil9Qhd_DgPYI%p4Fd}&^01!%4ooLZm( z1U|_D5?`RxcVGo2C_TYk0n=WPSP2@GgH>VR(|Vw$C@a8oIk@@&t;m9ndqTpexTGjP zGhe|L+C1>|*MqbZJpIAzzClf)+{6knZmIE3u$kWIzPS;U@g^Mlt;unY_ zaA4Si@_u|~J}3!;!yD4J0y!4s`T_;m9quuCF`xma#1gRYK}imD#IZsN=twPvd{8?D z)C>U~V+S4Df`%m|Y_P-ua`;1?0-~a=bd8{cBhUgEDI4oS?y%5MFD*&awNM8gc?r@3 zN+aNc7-{+nZSoH48qBg7UVE41Bf2%99x`Z=75Ml_Z~{XvfT3ntDL`5;;Gq+cGeAKJ zDn>!AThtB{_F@?0=r*uTuu2-VcfBYR5dn~302j27Fog(!%PdeK2u``+yP3c&SP>JG z2dUCPWoWz(w77;`3x{L?Otpf#j)D<9#Rq^6j|RtgQEG7sv||Y$i-WiVoEJf5BJx5) zJ+RIqXu5}#ZV-i#4Pl^jY@neCx6C~ilC&Th1ln%_tw&O@RRC|5%PmfaBwUan;`B+- z=rH(DD^Mx`rCaD=7c2oFCO*I}NI}hC&{zhQzQyUFOb`!s7c3t@6CtQBL|R+~>1m;# ztpsltL#Ld;egh{=ztnP1$b=G73WSV{CxY+70rjoG(eDWAb7kfw7o~#Ms)3Rdbd(m< zP*q4Q%g;;!AApyZlbKusJz)(bl$ZnBidG3eI1hVrQ*Z>^1Zm@gdO(n*4~a%ts)KqL z6hTnmAb0ch@=_~GG$74-kiQVKWYF3b=Wq#RJ}fV_9A0Taq6;}+LCwI|umMdPKo@Qy z^_CUjyVU=>=7>TxL;)n>Yl5nCXyX``FOgQ=f(-_R9n>^PD-MyC5cvo; zPzy2~vLXYKpAhvkIBCEV8fXd#v||u-nrBg2DroaVQ9fw=2&o_gI{-N;g0ed(ErDo^ z85c;p0^N27>VkqdErRMksN=yKi(xqsrR#=bm``Ss5xC0)8m>wN9ej{i44(4<4}F1e z_5=@mfzC<*1&spc>Iy`+3uFew36O#b)W!kv%Zu{!z$?!%x`5D4@St)J)arz<9S1i# z5$#R&+{}v96zD!(W!Q~@u#;Cn{Y~)hRnXx9@CGMJ0bZPnahEVyM?6%Ox(@6V6i}2} zfoEpH(G4oe!G%Un4k&wpyVs~=E6^$e+AIZ48-R?^11;rA%AiRFaX2@)V+ATw5PnyOr(k_>lC@Sy0^K&1SX61Nq*|<`uBo63 zZ>@vw2!qKYs(EP6u?5ev!mE1F8FsLM2E{&Tm=&D5py2>+*g~{|#!*U2i@_^p73^#k zj13H62@Yc*12lmP&i#6zBjaF2B7AfnYB*%JAJnD>4bFfMRtGHyE71i70JI$g+P(^D zZGo4zLI!jdGNB_rkf2kr1s`P#?goOig2rKDiemCW7&M*-tEKcHEh}i72yBZwv@H&9 zI}vE+feI3sQQ(GXQE_S!qQM7s3#hjZ4KS!hP~SpFr9d4kc*@nwEJkd5foy&U6^0n$ zi6{y|=R`t`hm9$rr3~1JhC0+*Q27DgW|NtxfND1=oxm~$w3>m>(Lu(LP%e?dF6 zAy+c!#TTU(G356ePVtJ&inJ3(5hY zK^oYq1|n)uScro)ytj>XCO0Y{1?RBE{L&~)xNT&gIvM^|u3iO<3 zsJX-y$Ef<@^N>h8bnq6-3ZTFTbt^zqv#{I*Y9PakdxZbsB|7qaE5faiC;=ZRgxf4M zMc9o*-qtwECNc!D?EeMN!hlcEE{-n&-Ks?C9dP*f{~DSYnVX~S|2H=hMK3uAoUs&`d#bNNRD32Et)_ zAii^AaVp{*w8Y}#)S{9gP%}HED6;_E<_DjGRti!Cs+&>I%?1xIgUZ2@qC`jz(~I{C z_VTsEh}z)d64R91l*d&|t>!2SP216L7d1lpmnx zK^m7B<{?y|#XAm9!R$x0-$0oHdVvWzh9JIzx*yz_Lb4RnSkgeM(4ksEXLEzQTTqQ? zNd|iEX)4H{;Dn=r?0tkNG%3)|TBLM<-36J&@p+{=Ikq81rJzG=!BfPL=(SZ(R38NH zgY3G5I>{I`-vGKy8Onz^5t?8@G3!^FlLH$t0Vh{z{2;|CN{R+GvQSeJB65&}1~vVF z)Ie_O!ERSsYF-KGrX*Vh9D$DFDMNFN#EvKf@k$n_LT$n@0L7Dr=Aa=1q~qb0MJ!8GV8(hd7O(9_XRQP)(5ORY=_wixGH;2mt#Jk32Z|u`5D} z0k9Hi_=DM)QQ%k%^;~LB8Z>)>RmOwt0F4Yq8|x@2f#N|)MZlaX9rY5LIG?jYVqI?UI33JnDUddz%B&^QneE3(xYM}E6|;TskM+0%1_Dyo$IX* zmQuG;0QsO6oF>86kQ_KL3uzKG`AJzM1`Oz+Kx`Qpw7RESJyG3C!AwU%JxSe4K|K|8 zS{AqzRfhxsC~?Ju6r$%eNN8&m6y+DB7M0-7#SpciZI&Rlm8fmF?9@t7od<1k+M?S7 ziEh*qz)TZQb|!3Dk~)6NP>Teo|zAtf=~dh%>i!#R{&oe3(6cItH5{n z<)uQ_gr?;eLAUrTgU@J9EXphf-PHwJ-c<}bm8>`ubesq1<}}d6G`J~QtdLm@y3I2q zHK!o0GzT(l2)^wTd`cRqeF%y?(4Awb^=mwiHeiaT9wKpoGZSchssJ><14&d!(y##+ zq;kMBPoqEwGM$F*07wYKO~dL)oF-z}jZ)Hr%05T}3VrLenYO%U}^u38<8ql&l0P z_rMYmH>p=Ef%cJrO;NH^04u5m@1li{|3dmDh?);<9wa7^VhtgmpOl417Hl&iBfyHl z=75tAk~NS@0xSXwIG?Wci30lyPoLmq}jX*P4 z1)v?7Dd1iN*qz`>Km}U`kTQ^4z|I4;tH3u4gH5yrIRG>oq6cDV7@C_J=_pw0C|GLh zAZ)cow8%iZK`N27SejevD46Ofm?4$I*pyiq=qMN(=qMPO5K?HUqX0sdXzGyDFJ$o% zD9ZIxN^=W}k&o3z_zqNufxM(^V5w_h37YCn%*g@!yI9>y0o7OPhL+}*x(24Y24*0I zhL#ovx`qb2MkY|6p{{|UuAwFTW(SZL6%=BjM{8qGjo?Lku;Cq83V?L?!IQ3#_{GqR zstuI(5DhSBTtXX%pv6<*sD?GqJ@ZP;Opr?^Wd+a>52#!R*E67M2E5{{vLIDKJtsde z9h$45sWv}76_iq7gIzE;BT{W?at3rd7cs?K3|$h5mTJrM%aPSUx2HlB8G-8DqRhm+ z5?i=kVToV?Jx{+7GZRSAC@7Sr7FEWV<`rkAgSLKx4x<52T^EDTF;7n|f*5Cv$GA`w z^KwBR0ByDa-}{5hJa|q~P%yz`G}vV9IS&*ez9soy;CO~rKhR}Zcv1vpCIqC}t0do7 z0~{;h`UD!`#U(}XxIi}zHKAbDiZI?KA6z(t(gUU~xy9+=eQ}`e)!^WU@<6%1*wzH@ zg3L5XOD4BC9p*)7PY;qm!2y8XBaoB}$wZlHut|J`DbQ4*V5@+kA8aaA9@=97&)tAs z2U?R1pP&cL>lT;hlqlH3-3glH0TmnZAQys0|6o?bqZ}Hnplv5PuzkN!G3d?LTD?5KTs>AtK^^RB z*gQVy2nR^LrT`j#0ACjm4hT^E*(yM-1fPxz4F_AWL7+Q3%5A|$VT%D2-`iRk80aXV zc-%JH(8yRv!Nk-ITI*o=0bEEUCkUuh5P1d?09;&n5BWwKi_cBW1f3;L(cH(VW(Ec< z^M9aoY~l+N(?JVYDyb1h_~-u&j13Ji?*B72A3gt@8bMFc4D9oNN=izO3ZU*RXkRDn za4qoa>r}`QAjP1a59(azCgxRw&P;;bwg#IpOwKQ;#Bxk+ zB&b&aS_1|$26{*>s8UA}hYWl`Pn-d@BET!qbQE$jb2CezH7 zgY0s$Rfm`jIXur+9lS$FT}PpyC^ZGN)eGr{8YLwq(1JHi|3O!G6zeEtr=}Kw8i}CI zirJtBA}F&cfN$l3Y?DN+ih(*9d@dMxrZOHh>4|oFqK<+i>~KNw%2|a(1wD{^H8f#| z`hW{@r~|CP%bq~9t$NTp5#C?YQ2_0tgsOwp)e6b^MUb^B;48@>OUJ?^EZiv2k{-R>#EPI)P%{Fu1zQgkG$Hxfsd)+rBOn7L z;Fz-lZS{jVJ2$Z+Gq*HX0lL;7G^&%DoL`gz-mO{*N+WvUrLmw!3F!D3_!01M2NXfo z7iZ*`=A?j*xCCp1EgwcC4?GXf-qFs0;8hNCl}yh~-Y8z=9+uE6^qvh@0~g zb5lVH0OWOuBzW@y$dt6ylH?4~`s)1BqU2QALS2wIko2d4_@H4OkO7cL0&fXVEP)Jn zC8a7Pf)4}&Sps$-B<$en7IuLmC<&#ZWK>8itQb623tI078vTOo{7p(#fQJj0f&w_3 zK@X?^4`&q>B*Rh`tV;v&ZBc45#4}(grKW&xzymoPnufsZxFBoeVM)+d0hH(UQc_b3 zKn&QXGEjd7ssuT~*g{T$0Ow#XP;(TV$B~i{bfJ?1=(vKMR7lysaf5F+CNWS;5Moi%~Ij!OVu5Ot6X>RWOJ{ z089H1>1>;L(2)j|F8?BM|AV2qv5_I#`Y$sx1B21^U-U>eINN{D`MJ6Ic?w900dk~z zDx_iu-SP+C{hnWh+Q7(7{edkdsug ziRs~yN>7GgE1i#I4CFQm&~1jP;9Kgj8Vhct!Ci%Zoiy}f3+M%<(6w*yyDz{kyMjvC z8aTuf9Q-TVKzCYWX%#yc!H=N=wT;~}b5fzhsp{^DsgB@S$@ryHOR#8y z9^~zq3_iJr6a`4Gi{;|t;({J!om`ZdnVy_kQk1W0rQixa7$3Ze2|OeZx`?0{a_1fB zoNLI5pPE9UKEztqB*oxv4(^KM-FHbt|ZEK+ys9IZPH3D+m=3J}hR`!A=29RVb+IW#wn)X%y!dm87O30Hz1tP}YFQoDsyv9>Q-5J~k2BpVj$R2y}RRYM~2Mrp7i%*h8M z6PRh>ducT^;f{gdc@EA+nAuQ+paK?7~P=mmr4 z>AcEIu+`=;ZxZh+9CdGaszQEQYEe;UN-Ahv0(5);sLoXYxf^tAALwEb@Etx1CHY*+ z3Z>wim%&waeol%4^l(d1qb3J@90Pa(jh=#QW(laWE+{Pq9UGVlI*%bG6*QOwG6UJO zprSRUD8B$w55YnJdV!BS_WZAbY=B;Ra;lzVK>@Zvf>(sF<`8lM2akHVJG&}m<`tKK zsy}E$6>`}H+9C6h=mgiUm?bT!c(+wgNli;E%>mtRrKOc!o>C0999-gpvKc5i^;}Xx zI|tp9Q}OK641$iLgyz9VupoIBoXbGPzk-4~$k|Bc2W%KBH3hWI9CS^hLZSjl4a{rc zA_=U~1z)NIT|@4kTB4DhQ>>$)l~|Mx zqL9;ONl_)tg|ONsd#G>N7e)U*LP1KQ|@g*P;IL6Hv46UqwU zT{N)$*`Oh$Ovq_2;2qjYsS0VJg?!*10K^Vu@Sbzfy#y(#WvMx!m665J>j}VD6M*i< zDlJLX2c5*PpPO2e2)Z;5>~u)b$Cnm?mWCr@RUOo&w9<#Gg&77~4q23!n4=Fh(HCl# zewm>@EFtS7CV|zV;zKEzJW>sAT5n zfz>18LqP$lO;Q9tuNM}Lu$~cUPd+GXrh=;O;`}^_xqJPGU)E5%m5n$k@38WF`p|v!Hbg z;8X-EpvXwjNvS1?#HVVo?}$p&;66N==^2vzbU-(8_1ft{w#~&>|HrE@g!Tl;&GvK|w0`3g!gxlwy8qi9%8;D11T7 zs$p^{tGP4s%ej<62k=97ycdHSpZP^4#d->EnH37)Y3QU%1@PgR;G5~dBDwh~r8)2x zsJo-9F7(!wWbom1Ac4%hl+=RMyp+_u5{0Bp&|+@TA$RGeiACUKO0yRS5K>%uv43b6>7H5F=D1%piL0V{_v2f6W+C*qA18Tyc z78M8;;9?(I_2C$f(E#a>R);p5)nnmnThVH_c&K$GR%-E93L2T|dHF@DDVhq7@T=Yx z$gR~tfuR6d!3DYn9a1KMMpmHN8TL!(Z0G$Vz3T^shD^0;pN-L?z5h!$FnKnlh9%oO!l z*tjiBC@(EP8hH>Fy8>_l0!q-}-Zku=4)8R9o|})Od$5jzI%sAmJ|{mpF(RJA--3)p3S9KG zT9TiiQ><51nx_G>N(U)OS0T~C{^PZeqP@;gf5(hcNwIVqqF)uw8 zTtMh31XmW9q~^L-WR`#jD&ajJP}F#&7NsI70F8eu<>e!}Lm{yQv@JvlI=GeryEzu5 z8N)8Hc4+!hQqa{^C@qGZOa;5opfbNyp(qt};vzIYK*l6O`U?AG3KB`0iyo8a!^jRS>p#E^Hi{R0AcYbB^}mKjM#kvtKTJ%FOh?y$AO#`K zm^jyeI2J?3L!t8opq0JF8Hu1<3YRF;z4`(!R3uQyo&@|E&&xt z1}(3M<>HDDat(G33UhUd4-N8+clK}$3I^SFpk$?QuN)ogps1pzuA!-=tz&Jg1f5ZW z8UX8bDJdy|L_ilhrz(_Zq=KvE{37sxba|?JQK~{wXQL;O^)O9#4YZbqrcV zlA@5G2WbI-0u1+%D7<4085D)=ch*-9$w-ZN%`3|+%FhEGpAG6Ep&YFT8eLS#OiNS1 z?ildJGhCpl1kg#r@x}RRCFO}lsi5k&ST7Y~F1T5w4i2WE_+WpxkZ{K!SJ2i!@H7Op zPzKMZr53?W0G$mAzhlJ(GRO;Zpg#Dz6>#K$Dgq3fQS?DZO_93H`k)hExWKJtP^c%D zz$afwiw9?7;vqRFzcd9lker;K3mZd*L;!3!QyCh+pmj)G3ZPq6LHFlCVib4aqt54m zYIIO;Qc!@msz6IApy?lUI|z76RUMQ<@CX(EsusFReaP@D=rgC8{Fo0kq5DgpNqkj~5icOJmT=_r(diW2CC%McJ7 zYy*6bxj3~1GL8;bj<^5-+{;VNK|Q+**<7T;P0zC=wJ0$u2X^2WGz3wN2DP^^X0t#J z0LNNu}tA?1_Pb^A@Rs^mU1&MhnL8+j_$I4Oz5=$~Pa=}w| z&>MF_S5y=w<`#ptuRzqqgZj|mNoR0~q7+%6#coAVvmnC+pq(|)2_a~Q9;6*qJ3*C! zjRPeR6l?Whrog4u!TXp&#uX$MCFX*rOHg%y%!OX+4GCDVx_HoPGth|=ptBIFRf}su z5d)S1mk7!VZuvzDx%ow)i$6*dGjoa+5|i>vOB9rfK=%QZrKTv9=NEytjT9v26=&w> zDS^iJpgjy_1;`DXR{Hvoo?T{YNt#}MQM!IcNp6mQQChN@skwo2acXi&W`3Tov7V8h z5!hCk+40F4iAA76M|DVV8ZM6FXl)EvLnaeai^@_{5dO=|1BW=o@1WuWJid?uw*yNf z0qhtBur~0)&m|dpu#<-|oB~Y0?H5SzL~|KgM1-Q14kF=9&zY2IG7KyA70addX^#nF8&%QYdBrsIr zdT9apsDAKLlH&Y=jLhN^(8OPA3N#>;6><{`z}uuUQj1dUp`zd;C&1wY$_UY5+aYIO zL2D`R)Jm`q;6YEif1xv^3b{~!gR)E}sNo8>4ZGhn(;!DGf=VDzAxLNIk%Atx7VM!xJ-0-13WpQcH?JR}GgGRmOuZ zlFBV80pB$hU!Is*Vrymqjt9`CnlOFE3W>R(MFODe474#FhZtc& zU7sBcD1gk22brm0t6-#{r3LXG%%NPMatze9ECP+_ftOHa7J;@eDCFhm zRizf?gO;nM7L_ID=zwkzC{Kk9e3s-ZfY+;KmMCPVfz1Hd#9%WrL7Nft6^e5~8xTSD zCnU5%eh1ehNGSzUf`G0uho}L~0Q$b$=Qja10qHPO}RXw=lHFOaML6RWEP*A}Q zO~<*Jc^YstAZkEdO=#M7Oa={ng4(IRnYpQ;{HXz&Hz_CqH3~{ni}H#=SNr9rf*XUd zgbj%$@Q^gP$CwMc-T>4P0~M1H?I07u@e6IOK}6w(f+O26zXWv8E%@Fv&;$r3}-lMX3s4hZQL#<(Fh|DS&fkMrtv5;u>U_l9rMl=n$}!d~j_Gz8^(1&vI- zR6PYHYlT8vB~1nJ@nQv_JP5lJ6_HKQoNNW&CJC|;A^{F6@U3wm_kr`T0_3^{5Z?-% z%|Rgy?zMpHTJRd)oXl+S!cFkJ8#p{6(E{B<7>+u^05Sj^hmd)?)M5pV#1e&^)WqTv zO-N9HZ%i#tg$Fh$@j|+qrFl^Ar>4Lo5HxU70FGer@naC5AY?&Tms+cX;;R5ujc2AQ zU^4`g7ciRD(5aEs6lZX$11{?kCAT^_s9-@2DG9;lIJgE!vc^gQ)}#jwK|po|Kx;ur z`H!R)?s9N_1zN`j%MQMv#X70Q8c++-a|5`{fdo8p`2lJUw4n|vih}cVQx%HAGYZ9^ znHJC%bMRRu3hG**AS^CPO-unL7j-RtEpBhVQ# zdQ_MQz(4+PU}kP;hQ9yP$ZT}{A0?h>#lb%Q58nX^JH8G!0AE~*J~9v4+z2~30JfF` zdNKvLCMw3Oga&s6z!EUTUij<;NEqpk0MM8w?2aq=;s3$lwgvR~LG0%dLe6YJIt&eT zNo-y^J`8h%OA_-^ z5{ps-L6|>pu%o+caJ;X- zOQ?@4sE=D&td|R3w$)V;)9+216)CqH_?z? zgNEv0)y_Viu6`l$o-Uxe6SnFg1*`)q73}I9QsinxWBvm<^g{(M!!sDF*Ff(@=*@H9>X{fgIFTqy;f}H1> zmz-0YlIogUP*MplM2ew-gyd4lSUGB;2eA+C2asw#^eBd?0ZT#L1UXe!y}A|@z2Ic5 z2fw5dS`dN{cZ6sJg(q|eupZQz5Ep7_sDn!^bsbG;SfcnGW{VP7k&=}H=pqQhRw5-R z(6Nv3s|G>thgyR&{H*{wh^9CdTrfEnWcq@~@ICX=^5OSRfy-!c5oD{55_;eXH;|jb zJFTEe0Gi{#3&|j(-0&;HFfWgUtdB=@gFx%aZL8HG7gd9HhZq?csMo^R&)ce3!$Sjl z@-Sjx4?1EJUs{x7is8 zGElh*H&YL~8lOn#DL_x)g3sWf7zXM1fO?Uj4Ce^n>I1n35#^p1%&j@#eL&E$YF*D_ zUB~iN&|o?!iVdN=e+Xqla40G(_~pZP0D%S^!6*8HWuR^apQ41B(7?hP$jJh!R)MDD zywq~o&KP+8gc_d^Q;=+07DJekOvw|#>nwXMWR6Ll%3|tGT zV@VzOf((`r!7W2v!>&g-{C=Y+=_b!^}mLe{l0qPu~U!YZw_Ann2FUHZa#wFf`IpFg4auFf@i19LT8> z)P952F{g$wPEL&ehnhs?x*7H(0a^UuIQ;QS{86Ejl-wEn-T znX&ok`afFr`H{`V-v3upQgW>TO&EYu0eGkmaqJGHWL3z?Oe#tQ9RLa*;pBpLdLd)D zaJ}Hxm6DPYs4oFIJ_>9YX!Zmtm4QZ|VFrL^CXg1zfX&lU0EvQ`FasU)6d+wx(0FNL zod1f)_kWBE1CyMj@@=M%68;`+eX@E(%i3*@qMww}ush~p-AO{_S z=C*PmTghQtmQjoYO}c{(K{X1^x8U3Cixt4DFd!3hpi^iRGC{NTD8@mjL*X~I;WZU> z!YZ`8tpM5x1X+rWVjjwpYAo(_%&P?5ae~;g3o%2XBrylPJEbIFAu$hhj4OD9ZfX&V z;h^FZVkV|FurLM7E9Anqw!q3th2Z@BJWx|5DOCY(48(9q?+UHU3~O4#8@)OTrA0Yl zOB56!9b33DpmrTfw-%{60y77^Vg}OA23K|90vg&b1T}LYn>-OxrA0ZA?b4vNl%PsB z9&~zkd^}2T6ru@S&E%xPTA6w&so+I)>WRh4nVF#J3Fw*-X!{B}JfaSFKJ+4M&~&Gb zYO$Ra=m1ghbrgDEo)v;ZU@)S3V}G$|E4;|$(;k_ZkpWd+zW4$xt(parn0 zpc%uwd+0n#-Ub6=b16Zek9o#gdwWVmsoZZPaoIY@Y(6K7ednf}aNnN)d^WgIvK=?w}ca zjAoXl=0PsGN`oY0sC!ZT?3Z5> znOfor8Y~5^HBU_eo1=j((I5;*TBDGd13DrJdWu?QY6)n519A=$T5bsjZ`*XO$Ve8_P?2tvAHpN``_4b^!!(9Ml(*caJK(L3qZ#?fR_6!q~w?9fjQ9S zA&@qFK_+COF1T=qoSmSMnwyxJqX#W)^NYZ%|3E_Va3R=4B19!4=xQdA7n&4vLH+ks7Ziy5N&2YOaipY9>xagE&*)}fC(TrB|)b3 zJpExZB^jWc3%Yt7|GXliV}oBE$bInnM{uo-Yo-#kArH$5u`bYPfo)`h#2DlP1<(TJ zP+vzUAJ_QM03Uxxm-rBmAlG0Ie;?4am#Kn{je?N@SA4LiZvb2gXu=D!wn`nm3kpR) zND_3*PcCQ`6JjPT9#%dhW(>hAWxxe&W-|Dg4R9X=(%^?U5pses%=yqt1#8P6vKbz< z-95iJ71ZcXP0WSN+vq4HXO!k;gD-mkO_pFZ+!9Mm^5bDe8TgtvNRtnF<_+9rj*kbI z!||Z_P=KE>4;r`1Nz;puPtHMD4eA^qalvg}s2Nb*APQL+;sH>v09-#HOF?=SAh(0M z6;S(7T&-XW+Wd|11wuF6?;sb!!T~gKXlP(!VQOyX12PiSzK@SDN-Zct?W#cA#h~*H zK#s@SFF^J+xOEP$%HR`vAbn_I1F{jlcL53w+`2)s*mNPH1Gip;8j!ES&Vd_;kkbP# z98N8Q&i^39!RA7j14BIqHUcdvfL9e`b0H*F!KOg9!@?BQ^8)t{QxkKs=`1bE!L6;d zCLF6gLFY0&97&?VDge}RGqT#|!%rA0a5 zWrPZ#b&rts{28gB1!+alJ~=3hgP8DRBf;9hh9p%gWF(e>+W5t(ko&$stzB?Hfkvi5 z9C%M4zsMRQmRJHhfFKooY60X#5%5|^SSJXatU;zjU4>Xs37IUjQgF-zpAH8jbbcZz=0TnSHS0l6f-TeoYmJAME?^b;MPTp13V+aP zx1a$4s9{hIARmBFeFG&4^o)h<1dw_xsSuDAU8oh35!c=y09o!7hxQJ*Tu)Dq~@TO*Dx2M z)rxbc*9q3#Xs9~Ul4~h;@YJ)ir-e%GZ0k@tsN)n6GQ%h_S4geim zX$zlMhs2tuCS=bq$RE(YA*`{W0dH`CT@2bVmspYrX^<*_b~)vMu8xK@SHammpfo8b zGa0nwq9`+|v?LXp?co)ul>+owS44XPRP>b=gN~>yg7+`MtFF9$?I>Jrh=`40kT$vlVO^;6co^lPMBtJRsbg=^i~Z_ z1*lNNG583&jRA6|7wB{)&}ugw@FC0PMVTe3um&nhTMpa;g<1#NG6Xr4I}tp&4X?MM zQyP%Y0Qlf0kkR1&Kd8t9ZB>Ho!w0eSQo-9zK_g8XFjqhZyED^}jx*0JE&(rU&(H){ z0iYZTs@K6Tg+vvWvs$1Nbxb%WG^Fec^I;ag`Ta+LLmvT5F2r?W4 zi6TgA4c#b^@i^iGmTF;BOwjm%m;;XuB%eTJ&}|^vv7iB3RHuTvsNj?0AniVIp9S4` zP&rGE2f%SvoLU5FU8aC1kKmpH4UOYHk`B755_H}>XtQl{PO%=|B1adrWnTkitX_6% zCHM?hPznOwVhh=x2{wsPLIi~nctj0W!h+0Dho(}{>=f8D;2~wOe?Yr8Ae&x6mg3qK z2rAeh+eMIlr3o%a6%eBywhAC$f=A%Nm8=FNx*_Q~6MR4yq8PGpea0XNn4A}vk2YDo`m`X?AQ2sgc^_upf)eKON!B|1l0v# zEzmkQvnn+Oo)c5SD|gjlsYgKrrHQQxS}&~tK0OYn4)7{Z)f9czVg+#58{U`#?ZpO{ zsh~az=GhfsC&7af5(5Y?L7Ebvbu-}dALKjOG)*dWy%acPASEYk^%}V69G_I0nUj(V zyA&L>+6F43hrEa>T0JqRL_HQX;+314l9>qcDyRhqGYz&-Em}PYv@H~(7VJk`16|b= zP>lz6gC4w@si3Q1r~`L!e0*?fNeHNc1y%;h8ql&Fw4)9>7p)84xC+`D2F|arfR5Ku zh}Tg7hlqkLSSDH>RY$B|5oq)jeDFAU-7+{mfp?QZv?`k))@f5t0Gg<(ODp0-j+8UsR!xkyxydl$x5SP?lN*Zizq*NX;$CEXo9RK`M0= zl1fWpMRsydYGP4Nr9w)6aw%l8NFgz=5^^{qBn)6ZG0ZT41tmD`BIiJG8x=YLg(wkW zxe6Nn$Y~3bZlN_kWU>;>fhG?E!#)^mEkGpz>S5)`mO%zfK!Xp^tOM)&lz=uz!$zP` zN0(q@Eg*9s`avu#V>9sa7wD`xOe4foRtlb=%QPYL;5zUlD`8Wm1x5LwMZ@5MqO{Ds z#2kgtARp*HdT=~Kis0hZJn-4gpw%#l6`imIR17}U5@H`HmSH1Kpe7KKAt3jIr1SDi zKts)-9q@W!v2y5+W6-IK;4!Dv#1x1xAYDOd#(~r>s3QvS1Oqk*no~gb!aN8Sf+ZY` z(GO7D8`9)a_bn|hfvsnPX@`y>=|LBgfNik_4=aJj#2{VoAh>}7KC>znHWCwBlmo8IkyS%H3R}B@zRVbOl`=GGfz!GkDAN{0 zM#G?9fFyXBkr3m+o9Q8oIY2`JC8fpSB`OM_8Y?poQYB#Phr_&t5t^VR4h~H{P&*F3 zb_C{Zh^^4f3oVO~d!R6N;J}0`gBRg2Nj*@b3Str@6j3A~XQsi*Y0$y7po47G5koMr zdjTMeOF+pRZ>&MAfJQX7hypE~d976H1T~n#SC5c5P zu6dAwSRGL9SX82;keZi*T&ZNHfkd#C4WL*9nV?{!U;qm!EaL^R0s`721Lrj4t{P&n z7JMa7YF>&0=rH~Y$li{;)bzxX%(7I7>mV}Fat+kPguB7c7LxN3)i3p&1l~0Pv05P= zJUCMXYJGr~DV8MWD1hb}5l+fW0d3dJ%+pBCOM$jW5pDuC^&lAw+~5NF)du1-%3KR> zYQk!^)D%e2rRJqXs}{#9Ky3iA!L3h-!$2_y?ad+vxj=&86o8R<;Q@^j*^rR21s(08 zVE|1*3fkx~0rPPYq~8G6gb_5L#t0;R2B(&Q{R7!41FjlCB_Vh4jMv%MG7${>4L%**{EXZ>1CkM1-l)uL*QKl zcocw(0Z52|E-?iae9*7}+kyxJ9nfh$;P!lC4r~((a@c^+e=jKlUt6CD8OnwnI}MEw z(8OF?X-*DkWf*wCA3ov(E_xv?7(^8To7hUtO99W0E7&T4&X)nD<@_Q@VumFFcozpF zwjoIZ8m{OK16ZQ~v^5Rv1hnc3)I&l}Ay8vb(gw_R5Q{(~G4LH&urR__ra_H@MU`GD z_{KX8MDr1*8DcIp1i`JO@*>cot3`Tvnl8_)IL&Qe4p7BXm#-9f0}wJb`Z?Mhon(;4Iz#0(9rWKdS{1no70EOCM!x({}Z0%*TpNd_odO7g*N9*A4P z(@~%tmz-D(>G^_&yYkZ%5*47mR#+Ps^HnGZ^{2tb7RUtX zO`af=K?DAfb`{j`5SyW)qz)LJg@_vc87Tq zG`x|VQ(Bx^mI|JdQ&0eBY=vA{~8$vQA6EK*7cqQtW_> zMCkO6m4aI;=voc1mEZys6rP~q0T)zY^B@BR;2=ac24r4pUJA?vkRFg9psO+AW~Jsq z`+Hgt+d)+iHWz^O=YU2ZbRk+HWTpL zhk2Hz>Ope20;s_V(X6A8UjXXLB<7$PlV3oLDVT+>m4Zf6etr(Jsi1u-2!(LpmgIxO z1C+Bt=OGmrLz+aOxPTS$pcIdG0yAvE2c)+F8=3^qY=eqr&}DMq`&U55V(H~$){%&j zN>HkUHc{iTwZM_8eURyph4V1&3O3*%g?b5GtHwhN1l7E7RnUePSRT|U1*b+>oIp}i zG3c;vuwTI2_QBmL4VVF7^|oN`Amb53&>$CK#fXrH)r&aq83Z+hAaxpO#T%$rijRlZ zY4Exa=0=#CVGhGyWP_W~pk6;r8>~1hPvkGcgBwqy`j# zP-V#da%`rfw7B(P>5zyVp@6M1jv1M-6b?(e#ODc!_t4kwz@i$dJUA3D+LWi)dil8 zQGgf-?(T!@S5RdFu?}Jh@(Kkj1r6xhAuGsrQsDXz)Pw`&BT%7Lo~lrs0Y1bLtPhlt zp)sZa+wW1Hk(vk1hOj<8$WDkmAa1UGRYRUO``d{EU^q5&J$L7RWpQBa3R0BBwv5y~JQXnGU0 zpB+++LNYFNIuhn?(3Cy&_)+NO7I=~gQhtJa3J@kd072eS0G*zMSf34Bc>}KAK(!-8 z3)&(QP-sC@H)NF(B*}qVOei}?;Wj}M0BC9e$trM+Lf5JwFGh#kjVMS^A`o>A3uN%y z7Sy?f6c>;j77x0d7&N2@K0zi2+KYh7!NVEcD~E;|*s)Lr@LY|yJ_@qqFCOf|`1GR0 zywaS+BGCRO%vJBjB}IuPsp*x_mNruPkBC?WTgW6NIG#W%=(f}f9s-2o7k#A_B&N~U z^?>?PO7z!IR`52`9*W?^wD zbSxJ%4~2aV7-&!zT)2S(0cU%ZQuoUT#}6d_5r-XsVhUn0>Zk!I79irdie^Mn zfEEMb6Mv``1)#NGcvr`vhdL-G2!whXtonrv9Y7ZlrKTvr7v3q9f>slQ>m5XV#_t5;*cQGEjqj_%4ovv`?UZ1Wl76ai9?&&_$o{oPxLr6q+5; zC`3xK1x=zt=1LKQprywk9jG_Uz!GO>zMiK)Xxlhqg)2l#uQ)X|8?>1PaX1(m*~S^N z%b+L~GBg6;1P5y?=I4RVlh=jW1x;DdyaXDQM&5`9%EqAL5|l<^WiKQwNn9L=LlY?U zurCdSYCx*kG4=?8>j|hjDlOlGnHH3qR+L(t5tLe73f@5osfdG15=&Awh+hl{H4Rr6 z6mZ9Kuy8F^byu3MhnO_9j;0A3U!unyLauMtrSgRk> zkI_SfHK;eLZf|d|4wV7Vv|-s90ZX5_0|=%96h00x4(cXLm?+o~)Ge0qJ9a6TZkk$Y|vIHBjN%aEKqNu zrd@;@un7n*YWO0=!A7BmFxW|uVhG!6G_W+<;zV#rfoz1N_u$kLm(;Yx(wq`ViVTMK z1EH2d0uuc&Oz_M%)Kj1xS)lW%b2IZ&!Sj@$8~Z?`4$z?zSgi?5C*b%2PZvSV1`qsz zccwy)49Ll;R46D)1s(ALi5F0`zz+b>Q7DBhU4q2G=B?Ps|p%+0WBKLPgBq^)C7&pz+48o;{|L&4rsTUj)H~}L;=(Rsd*^{`JlVK zQ}R=bLF-*%$0ER;1zV*78Fv94T3Mb7S+@$iAW;Y84Gm+kxuC-=ixt2NWkE}SVD5lf z0}T=6U9pg$f<}Xtf-}k)IiU7DD8M0#K$8k!*FwgUAUkSdi|ru8X^F+f`N^4yC8;Uk z<}S?fPEj0wxpu%v;uG#WmHtdN2yC!)2<7!}mY833_9O{swP@yD0Um+$p$6%rN~>OjK?mWE)yg09m-3NcV~1U#4m_AIpF0Xhpr56fjv zsJ@2WrUI?+LFRxv{gi|OVn|)JSOHYLs0kTTOT#9iU>O5+jSgzL1Iug3+d;r*HD;Ejg6^Zq z0G<5@?pDB}44Umh85JBg;4%<2gbT|<;4%YdF8EXeSccL=-Dm=Fen@3ODr64{yd*)i zNWeKfJ|3QtePL(gz*bNpTMY3tXwf8MWg1ELBlf$%V+z@BkTh<4ktf`U4^4O%1svGW z07cf0Kgi(PVL^r-VDLUCc85dvX5jKA$TYNc1&LUcG7&irKnr;i6(DH@UQpsHAdxk~ z7Nj9o8H0v%!3hK83O#t10f!NaPH-IsnxDx@gDqeH9aaQcLk(%!;Z8sx!^!a?Jh2k= zr81=Pm07Hir~ucc56)z%DbVr|(mu&8PLD6j&o6;ZZNOFQfyxzl+SgErYKC@al@;I` zp%ZSP>a`rS8ykG>VJ7IJ`tm&3(h+a~7NnP+T<|d_pZk^6a zvjXp#0FM=e)?y`=lq4o+fZL7Wz)vg&->wGQb_F&aa`blEjba?jt3q3kdRmmn$*k8OHa^I zNJ>pkEQLH(o4>N>Cj+qN2BUVzt* zf-cE`EaL)Qepn01AqdOiYtX<60^6c(xC=nR3*ILOTD1ZmqlP&iB#@F>1i5g$*wz4Q z324JFw1xnmBMs97u@k%p43yary#VMQVUTgS)T)Eq%Sf{v=w3nD1P2;A19=RzTG`(l z5^Ny0bC9cJh^q@UvB2BTuwhdLTkyq|V4p)*LZi6{lycBkM8nUh0(WCw<30VH{Xs_t zhPcAA0%)x_a+sj)YlJ%kHarR55DSWX^j0c3CgS0T6d+bhxu=%+Bo>!ARhFb`fCCB| z{F(~d3XmnSP)ndy0%#@%(w&6a35h!7<|#%m5a$L#)G&hQSwz~06fR&3A#s67K%iKH zE>VDN)B|&%5rkzYFQhx504mbKSsAjp8e|I0OnA>4R!+hDHE7|E;XzRU7BXTAP63b@ z0L=+QErlCYoLXY7kdm3AP?=u}J}C-5z62XYu!pv~pd%%q0UFSJ7PusW_!A}sOC)$) zL(J}049^l-R>Ab47Um$g;&d3{X>{m(ASAItOI7g7yB6p1&uavu*Jv2Mz6qYs6oS3(8drXGBE9mR)=rwg&fF&H+~_eK$8U7TX@0gXb|k- zEdsX@AbWN&=2+pqY~;cm6v#S|Z6mNE9g;{OCtN^Qzay9UdZ4}tco&?8x-K-PupLpBF*t0% zPJo*TT08+teIV08MFYq}#DO7TwV>1kSE33DTHKaG!Vb%1nKN|k7CcLgbc{XN4roOH zxm*C;lg}(lErx6%1Us!HA9PPTtmB>oZd$6QKo1>;=mt9-x5L2UjCHd!Vu3n*{W7Rm z1K!CEsh6OYC3MSkaB5x&bW{QP@~uNor1xh9+Xo1;~-uG3G7?Jez$$O@F8h zP^x`mJ1Fq_9Of_NE(*wfh>ZturLZPFj_v`>IgoWw;JO2L51j(&I3jQ`!OhTCFjP=P z9X5po6)b>3!($=-F8&(HMTwc|$(bcZ`I=S=kZTE{*WQ#TR)RX@MXAZ4^A8n3`{*D; znII2fw0kjc$v|`zK^oNE5;JqaQ`qp+vXB<^!*V@r&P1UAG(44}kXQm<@vi_GE(4hZ z!s^hi@1PDl%*&882MbMNcbI@fjg)O4AnT!ZEi8k8N@hr18=Q)qMnUWU6G3NEU`cwQ zmbwo3KtJT_7ql@7QZ}X{-Pr}sR`Ajg>|$KGDl-jiCVD3Ye2W%pY5z2-aNK}Lg1~1Afd*N?VGnl>%n+0U3RIvHD5QufETQ9ou<<2~ArffY1xt$z zN0$fYSp2Rh;Bthm;6^8C2@Lp{0dSaNYb}F{v!Ybc;Sr!Y4A4RpSZM)iG=nxa!%l$! zhc|dzDPl+lPnnyUrhw25Ep0*DuEB0m03CiC4^CGKwrJ{fAx(FvI#{?M3VlQwf?__j zumGj}Xwb#7V8gATM^mB~NI36A8A%dI_y44)=IPVK>jnk}2Ii)w;Cny~3=Gg|kTlx; zKSss|W(*3ZG;s}@p{1ZSqM*P4nsr2zLg5mUM^QXh?^GJ{jjta8VL93EL zSEJ<@l_;cElz4=X9&3Yn)HDJOz3a|?#G`Vm)aa4weA%K?uA(5Aw z5}%R^I)kL3B)^FAq5yyXH!(0aK+peX=7uJt`JaZ-4mTQS{&xlE*VGiy!J{SlMFq(U zh$K<0$HfI+PF<3ioeI4J0CZXm_-GoCQWtpYf!z9>3c2ztF(n10x}+$P3%Wu+GZnP* zxkLdn(TcKtB{L6VwL)eIsMVQbA5)W}XsKT!5}NNlea2O@W+c zTmm`>xi~d7w^$)3GaGa4#hFPtkfS{d zN{b5ei&Mepp@7!Hzf#1aKV z0|PxSP#pm|HcKHfDH(Q=UVcGp5ooy!j9-#kQ347zE@*X9ln+U7P@RZmhg}{TyxMF#Cl{Uo_Qsy>8V8!HAbLaDMguyc_p^+APNH=d6ie9=jj(>Yy=)BL2??hNsyIO z5Q7X23=r2WBgX{HC;3JABOy5_vACGxNC3wtDEdG%yr4}Zg{6r(nQ58H;0YB-jDQzc z6)ZorBmDhS3qTrDK(z+5DUAcqo49Kj6EFVX{h8WA`;pkf%5VUaD9xEO$0j5n%5nH_qG7c4PBU5{`l&P)tdi4@x4 zl@>_nFA-Oa!ptF%iNQf$Y^4AWMnonC)p*c^42eo82U6u&Dfp#=u6$O2Spv?2NM%5N z9xNL`bc2dqG_8m-0*^kV;Kp8XK%E7#1)jSgrG^5ILIXulN@{XGxN0s2SpZ{$rZ+P4 zl5e%{YFzcXhKv<}*qhO3tB!f?b=ydTC(B*^RUF7+Bpu0+P^7GOaO7cM|6D^@w!AqBv zR9Myn9Uhujk_c{RWP%P6%Yk0Xh+M#cVid)e?QUPy}_6APz&cVdEi9^kORo z*NT#&MCb@mG3YK?XwJ_pR)G2!q!*IipaQt!3{+vD`5sv*G}VLi8f@+qmVBXdh{(m3 zf{Dry(EJB87!-=grkYZgFEB$8*@vhJ*bvhyF7P~RIB0hZD3OA1|3hjh=)i(p2XymI zab|v=4y4B$4=$#m_a7-KDS=G@_dX%TGpw+I976?55&7W0GpOu@YKKm;f;;;#BlW=L zeE{eHx}=;`4QLc2IRKP=klY9=RpY^WKtWp!I-ay7A7&Wn5Y3!aB-MI|6phdbjckH? zp*0+&1O$}^pwI(*9cBXYftZx9E;0gGO62!4W3JT?!DJ7ttqR{Zub56`jF3m|S zNeu@{XlN?vf>#MZ4Tr}fR5^56DabyU2)I$At)OlNT~-B>12se|Ky9!bs5d}@pz0I3 zLRMET25nIWT~(k_p$9TXN1;LwmYXy4^FUb$qESHsWFlzq639^4IE)@>Fh&DOgAT-Z zpi`Z|S3<=@LrPsi0c9B=%ur0axHPFawXifbuLMc2f&wI35XxbG2e}BAQ{!E83rZ@X zSrt}UgEKQYilCFapc(zdyh;re1xUpPw&{H2fnDxPqMrUgTerT9lTU3_j8vQV_#ZUNJ1c#DmU-0GkA!D@jaB)^l`n_DwBG1a+w$ zK>Wlc&EMiy#9j#90DM z3J~8S#~W^|psFx!gZ50(%2DLl1RDwqtJETJjsaJFM##-7a702Q1vX^{4GCzX2Acz0 zcm=EJ6l`Hh0Tj=miYXpZD?$q!AJEo9s9ErO5NaAyCIwBgIH>1 zXp~tGR5HK~0v8bA;)uwi02J(cplv$f!WLUOXAE^5x@R@i_4PrEcXhyN3Q`JzvkavC zMJvrw?LkUVdXOP29R+ppfK)NTA_!9E+JbCH9OaD^&R}D31vc0+sE5E8$CT#AgT;`; z7?Pn7l>n;Sv4#;i$nllG;26TN30f3G9Rw;lQSCwxV`!2D2Q!}B2{i<+3?($cE&-(+ zNR&a8d6XJ@E|_K7^ubYz^#SW{%GRR0d&eL05mf}`2)HY zIW+|~OaYtc0vCI##jp;Af@-lI_I!t4`#`*fR8GZ%DtA!8pf)!^6%cZZA4!Lfg1ITg z7|_B^=*(hfejdamSl+|*4K(N=sR3jwC>jZP(hAKEh@;R1)r%|hN)js+Y!#Fs z)r68Y)FJ2!p+ThpNg?O~0F{NuG_=5i)_y1%6lAatEXEK`BeY5v$>;FlX=uF&uDw7h z7+DQ`BnE6Ea&ZT%QlPpaqiLX81=b%&?Y@IkJS41Otz?jk2~;BB0+Pr&32ZF9FhMmO zy+j0g6P&M!s&$}VgaiRN&w~o2VjTr_P-#kP-9v%h$PU5UJ_VT#!|KpfQml^X0_p23 zK+_?azK67VbrjUWLocLy9a<4VQ!&^Gq|O_(d4rVvps`700zi#&tQ9WQKtkCZi#v;< zu7d;s)Etn*VB;n6U>Q*90%pJ~7%&giE`u(mfT+_03nP_v;FPWanaBVqG~~e`a7gLp zgD8@rxw90(9ASs-+IfgVg1 zxT=QLHQ=ozV8bD{W#%bF!+i+ar(O*93z8R+#`IzlrhvNc;KLRnZ6DB$weKD-?FV=I)&(BFs%mdqx&mu(dp!g6yCP9`VA0>t9Vq6A+Iy9hI zMCbq=F8~i{g=na~3bqQW#jzkSLj@pV0rf0u#zxdH3JRb;jVkDfF^~Z%;5rnl2V^Ta zS%M0`{E~cdAc4jdK`98-E`_tf4KE!~KMr)_B_bujD_w9tfflCFQUDsyU_+sa70r>5 z%mNieHUXv5MllEM^2{^^0)rhOpFz!qw-BM`fLq|T>hL57b{|sB3)+L7aD}e?vRz1iD6vMPZeE<$!a9S@$9r*!gAE;`m zT6l2#2iZ!9&?EE`}R;>i5 z7J$Y@@{7Rf4yyss@)mkBAgVX@AY*i}A^=qqatILGaDz*JGGz2;;0GMkm6VhqJCKlC z1lR`UKp6stQN4)K27rbjM$HK77(h}LI582`DugFvc+V6z76CB{)hQYv?;u7Vz}A4O zL2$%EdxD^f4eCO4&ER1{1yy(ght#2(RYjs2d{AWp^ElW+V4p(+5h?&K)nQJDW)8TQpfLu^7Z}Mwy&$m|c1SvMFrnp1 zcntw<&^p3*y?N$>j$enBWROABVrbODh7pn1uRv8OfYyXU4{ZlGe2NN^!Q~1ld8g;+ zr{|=?W?L0(6{0J^9dhto0L%c;Rp$_e>Y%1*1>^t<@Rgz@=mZUrw{LX(9ED=UB&-&!enrh(SWrh?A7Do-s+ zRY=bV4NsTkgF+BK-3(b!0$S}0?KFcHrK*FjiwEuB1qCc5OBUEb5tKKG(klVtp;__tb#-w zC}n_(R8X=YWrW899%^XDf((NgjV1x|8nj1?CWE3J9GIBd6MeV^b4nh?6j<8DDyRfa zcA)mVYO%T!sC|h#e+^ZDRW}YnXj*`lFVF@OT0sr+H)@Ly8jA3O29&B(U==&K%m6P- zgtn>R1q@O(i|Bkpnrz5Y`9*O6>Jp1btwLkBs1vA%GfGKXp0XIgR4WUb z%pl1SUV#%F{8E68&B4M2w~bIIfr1W{l@RF&HB+HY4l5vr*9cpI*IY<11u>)sGY(Y~ zto;abCPvu88zNv|LW>D-u0Ztzv|s>TDF#`A0tr_53OZQoMznEh-!fz+$c0( zfE|X>e1doeDZzr0D|~4usP;gKez**La1Gs|4yvHhPPjUVBhVFLwxyvV25wHH*a#XZ zfEa?U;D97x!~zhgMPR=|jYKP}kqw8K3+w!X`U0@_BC;x^C&i#P8K`=K zECxW)4Xu|Dc^)+6h&Cx$tfPQ?IxsOe6*3tJN>y-UK&cAOfhHtWd%^JsiVwu(BYYhS zSPmZaDEbgNAt$rA1f11i@cG$h-iI0mv559&~qQ4x?= z;(c9%gB{&n6(H9@!OR8OQ>^EbSdto&nVSkV2s9K48rcBNyMeNX zhNhJQb`9WUro&asg`7mdg2j+A<6_Xb9Jqo6hZH!9P)5i>o`e*Q5buH6M2@3F8fq!f zp>ohMKyd$oiX3&2+tsZg!_$zyBfJt&&&^LM%}E8FEerJ-@+5~kC|JN_&maXzlb;ZY zwEUvn#1eEV$Ot3CK5*9*F_57S9;O5>V8)^sI-rk37B*OlVQ(#D;1%ivNH{3iDqzMt z_>@|3&?1J_AsHX3k^sv?VgWS14vID8u#JX=8TbGhn4O@IDu#`ugG~TeHb^Z?upl_# z!6(szxd>N)?q~q5d@o7_>w~04a0*f<)+wo=^~ez2SRI10*$GmV-Pg6(rJ~&$STwL88Lw!Qx!yJP=9sNQ;=YfGE9X6qc z&qlN?3iUHo6m#z*q|J@!_dw_TU^eN1+j5AxB~XzIo|gcRYr)(Bi#)LFK_w+wGf%MM z18g+-9P0wuj2?;=ETs;;h=m7kfXp{}Z`p<1k24C)JlwSYVK5Ef{{ z6|4tbshv6{tts;U}bmq8>!egSy}c`m@)&66f2618 z#X}Axjt8CEOr2AJ@Sp!-Xl!n1gm(UqshQE}`9I`ED2cjop8pY?3BHW5BtJg~T(m$B z&j1Z>B`QFYh(b%> zoc#23(7_szi3HG{s0E2d#i`KqBVbKCJ~CugMQCThR~FFqwd8Qg~gABkB4x_(a~B{La(Oi3}+2nF!62-)<25QjA>GxPI6(qNw;xdL%vTv8?Y zgj2}A2Bc-B>WCv^)U6ay{0#28sDtLmA>pE~qflK7KHs*M3p6|%o(i5HO3Z`oB?hNo z&`?83QDSl`>>L&sSP}$ZXP2iA9(ctXD$&SJiH%kV8w0-81=4X=P=JR5WCk5qD8Q1S zUPzFmvunJEt4{!|o<$rSZ>69PjuNmTApbxH$v_7pg3|;_guxOF=y>bWlGK32l8j<3 zyYEm!51eAa+rvR;bb)TSE6M?9AidI}9B|_TTmvhhhIzC)=mfmbq8#;D9dsFJA_v(H zIc+zwAQN)(7wBZ){5;U9-L?wq`p~uu76+CeKpG*< zQfO4bU7`b7F$URTpra74qflI$mX--x{R>y4SA;mV9@7etxlmIfrYLADV5)+d3qO_y zwrnrI2s*o$Uxc>p$T>eHRRLOP=R>xwBq~70+Tf`HwBx4~blMTPOn}cgfyQ~@D_G!) zL4}PDwB(Nm4Xi*m*%j+3fD%SLbc#y{bl+M^W)Uo1!-E){_Th@44od|U$dKEsbU^bc zpxa+c@{J%O;B*Qf@J4Zmf-Pzi1SeSNcqOE60=p1&{&!9$WY<|*KC+_x(vpji5Hu_h1|T_H4=Ral zX<2GfF=XZp)1!J&No4iNL9Ad4OA6o+M^=U$d7zG9etrooE|HaiqYpG`S`3p_uvGvH zVsR>19HO`cbcHhLIzL3WS6yE}IVZm~ML#Jsy*N1&R`G*c8pZk`t@;I(RRw# zG%u*f0P0PEX$&`l?vRJwlmIdeTn>Q-z)*@vur^TIgoY*56Yzm>D#WKkL9 zd=_Y#3cXYg>P+-@2VzhP-oStzMim5Y#2~^QY6gOo8tb&MyMp)(34wL5xDN0UnRxso>=}pc#t9loasAj=7+?f*KB<_=Yzs z(sS~Y5_6E6HjoKitSt#pI>gbUf;tNv0!RrHCB&ei3Mp@(jzL5`E}tTdgfn`p6{^ zw4DO4*kNkmF$798&=>&~Kta&{tp-#fs%4PW2R^VXH#0q>L?JP!I9~yL{9{Refo=|{ zJFNh97^I{DH5Z`*;IxpFriZmP0#&1~qYw=}5gppxhDt!1)uej^9x)0E>WKxJppsl2 zYNWaqxG@XmLBkoAj1Vm|gtL(nre1teYC%q7a;gTXOIJ{tlbKgym6BPUn3R)h3!3pv zO)g1I(FNTXUR(@!LveC`L29ur)DCC}7bBtqA`dRt9H1Rc&{BZU#zmjoTCyW!TIC=Lg*S!P40DLE5#rHA3XE=|nVpaqC8?0hs^_vv~K^ z67a-$DrB8HxcLtaIOq^As1;q3nWG08PzoqYEh(vlX~{1wfuwh6AVamlOb4G|0=g9k z>?CBf;YX7s!fx7z?Yakz_<(XUsJ{XsN@iQ*bejrDO%Iii6a|FwcX!Z?Iwm<|K8f!FniW z!!sx-G~i_nN&ydRwIgLQz2uzC_@dIBRB#!I=2^n_Ktr7vTR@#e+FJx|7!>336PgcU z=cI#nia-uo*MscQ19xh$9ixueW&qAR(8U|Y;M0FVp@E!-KzzOUc&M8YMI=}!T!$WV z76)YxaM=mU`(Sl28^HTw;F^fG22~pYyP$PwF{&mkc3~F#V7H>`BVd^`xO(z|jBsO= zKhUtlVG%qW2-|_nWb{Us9@q!?3?^~lAF3SZ{14^FzC&$b0N?X~`~C+*b2CE&wD}(s zbMw*j-{Ij$O9syQA5fK-n3tjpTD+?OTK5Av!>A-5G?D}=_fzvA$9=(OWuVKKb25`) zB6#L(@{3`Lpmi0DTU=QToyURB79|!GfM!PE^;BX(0W@#GjYv<+NlY&Wo#72;K=s2$ zkkIFW5Vs!Uo(V#o2F0xorf$EiEZc1fBGpT9KSnTAW#y3ORievVcZiR~N1+zO*PuPhAs_6Ht^Q zyCgKoM?oXMNI?nY9DQg%TuBphHbg37d=>64&~Qj;ajG8lnmKh`uGH0qcpKy{JPrf5 zGN8i&&=Gl1_W?W%2#YhAxKmw6dI}mYso=faU|T`0P0$W@L4w6t(;;#TC_W{#2-%~dIVG97i6x*zeTz#n^B_AGKrLukBQHfE z88nxm0lJkH91c(k&{Qxqw3G9Tic*tHpyQsPN#N9CO+5wl5Q41#$xl;o^Ko`NA;%iRYy=G|fZbLO8gK=LU2&=cWUWcDLU9IoW+*8Ysv5M-t|&hj6l7?b zH@7sW1T^)F$ibiqy@Ghi)C{=i42`USBGAc#pezAa0ll*wbo4MN2^52d3_xiDp8Mdr zNKZk-H?b0yhmw%5Fa+CPkdvxVl9`)YjG2r;Hsz&5QxT}wpPgE%uA>0HB@t5IfRbQ# zD)>T5&;<^lSOf_{`p6*v;!YyaRyU~3Mm00E7&0IYnL7fNX9_9#<#{>zi7C*41O>N3 zN@|%NXbVemeh%cMJE(z~DKNi+4wZun7N;f`rIrxopv)9h=Rn$J3Z5>Yu!ngP6x7gE znF{ahq31%B;6rf;RvRF01Scw@j0eXaiXGsw2INE&3?7^SIUIT)BP^Lcgche3DS$o}khTJhTN<3@*Y@>kCv0-hTkO^a5OX zC#5Qs78IlwC4(*lK~FUx)1g@o#Q~sf9-o|_SC*OwItN`{M*-9r&;woE0WO2Vi%R_* zeO=?7{r$pR{X#tb{UBurk~%c`V2A>BB-evdcp_+Wp(GP@o@kLmaY-dKgTVt1TtgKT zh!YgYgND!Yjp8BI3tBaVWH~r~Ac_L{p}t7~e4+PnFsC6M8l;-chu(5;t9BL(TH zC14>)kb{O@ur#m0u?V&vT!DZ_cwi=>Uty1aWjJ)Vft21-?sX+9+-CFdg0bt4Li;3f%Z=@w+! z5v2NsRBoVCL=`}HJ%dxC9!dm50t;z%GD0>bKN&Vs>6TiOoB>}RT9;kBG04Ex#R*-8!fe)G$L6kqRqyh0B zsP2L_bszrkP1FLD$&bm@W@dntn~-2fkEztgb-4+fb#*^{Sc=isRgHd zBtAqhD0!zT*eZZb1N**M14)e@_#S8I@??BGY%~?QaCqU9rt(hqb zwhHB`Nf2dlLFlk5l0s-x0aGzV0;&O;Ry_StQw4gy^2;xA%P-AK0jq+=5Xds5yTL(0 z0~>S(c^RCIVW&Q01{q3ngZo3l78<>95pWp*9wmqAK*J2&wnHq3N`W&W7NO<}P*#RC#NcbDAdv)Z#Dj7t zq%(xC6o9AzmjbX*134Dj`2}_BQj<&a!B?k%&h3B~CYi-0#Tqcpphh^T28GxU?`=Y2 z4$F1*d1;yHrI7QP!H1xtrAl>wNS_$dLrtv!*$29Vptx8;R~K#rr2Pe+(1W&M!D$-Q zv<7Pd^(H}81h^f8RDL7(|Ddi!TA~C`pUBEkk~{PO7>Iw7Re=HwVj!%TLze|b26&wq zIA35l0i*^R(uqmMpyC@^W}_Phiem+orE&_k3a})m0BJmerlBC$hu|^WG3bmL-%Ununbb~fEy=Z2SJ;mU>^s7&aEfY%ZN%E8d{*) zI?y~CDAmBuOelt|rHltr1gr@cvXP4<{*z`h^rNkh?vwgfbWCwD6J)NWOQ*ov!fv7&>=X zOp0G&!3**Ss9FFODNx6NnicTmja!OEfq*yB z(vfpcCc=LT3i-u)$r-u%DY!g{Y&ht;dQkZS?vH>o3V0O)G@XO9JUC(ql?q@*;80aZ zTAu(KjVV_LH)P=*I5f9|tpkS|?4&|i`hx_l2GTT$o^x&r*hxxCN-jm2ptG_-1AmY> z17|JJlnwM2QScH7=t}RP(md$&V?0ayzf>tuRr=;MmcDq2Cvc#jV!=cKynr= zRM2uIk_vE0LQMrlDdY$cm_BgM#1)t@MGD}nKEoXq0NVm*j$#nCYJp#8SU;_C6>z=)4m z2OXXVat6pJpg~$liv>Iu;ZzAR)g?2zL_-tYH;f0{9v=^0SqCZ2AR6Lf!%iTbFpU~8 z>vh0^04aY#mSv|_#-m#3omvUfSq#&P-%eON7H%t4yIXz{#K}lD!|Vdt4iDngJnW0> zz_H?&U#?J`TA~08BZb5Q&;q)m%)}DVmC@iE91}}GqOkc{NVF!Ug0*KP78F#17A)cj zaL`&Tq$q*~DJUY4%Q{e*1#&Y{rsx&q7r^}t%KV`69Gu)C1s<}&jwvb5xhWdzkYZC^ zM*)`mAq6hTPHf6SsWBf(H8?ARtOA|p3r!ZF6Alnb3zR;PO@jD{mUI00%tl-P0c}Dd z76VaZ2QdEqzb3|J2I%WQ%#2Kp859igIhk-W z5-Y&AxPpsw(4}4DriS?L1s>>Mv=OPeWqq~qHdLeu4TNIZmhOGG6%G{4tik}G;|=v zdr>9Y-aD8uILx3~3cP6q8gSqPC=_5VLTDQdQt*I>q~M3Z!1JLF#^z*Lj~Ucd0Ap~F zfj2XNS)ic^*k}{DIRsV&AwX#vwv7uk45910JhE*Ss|za0vd@1)5sFgumW9&mXV(Vwje$?u>fQuB9ZBVw^^m8Kq^CI z?a=NJJk+pkk3v=st*4NcqYEJ`2M23u5$OH{7!yMp6zYhLaU~hZ3eZCxl=hH@vOp1m ztQ>3yN|eA1MiE5Tl9-bd4<48V1#WR_2{>ix#e>FN;QexlJa{M`R2xEP=2F2*QR5sv zH1*(W!Mf3xzC#Q~HV3+>8+7n)YKjKvxH7P;UVJ>lp`bhTVA;VvwFH{4z{kyjrc%Kg zKns(>dpf}NDb$fL!yyZj!6``(p$!}=pv11OS`0N4bi4>Op`xpXB|DHfvV*|O)Zyp& zLEQpf=M2iDC9p&a=YULr5%?CP_3DgJzjktm?`N67P53y+( zY7)q1^_V<(e+)XXim*U0H7}(Y)Z^7q*F(P95upNn9!xR#0y=~s)Ob`+sjC))ASli> z5K6Im0n{)8l1SGaR+u2$Scqz zXNthndf0CMgr)}YE&<3qCz3}XZi6X7<*7Rsr5D34_k<}?2X%7`OH+$WtQ0^;;wWV1 zfmWPBb0YW%9 z1C8>6Drfj)0eEBuk`N)G3-c2sT_U7GT|G##jjhB76`>FvF#YOat)MFqz;z;M+#EDg z?^uutbuReyR{KwsjYn|v z;z3J#Kv&X$RwRKBB?J2h9NM6CsSLTZ6})6FIUh8;1{wSTB@swDRIC6l@IYNL9a_>lFoX zTtNa2Y$Zz41dZB2Uf{~zYo+TwkAQc+wkfIe@OoK+Ek?Jed(1mFStw+ktD+LYXz=R+) zEY#Fhi)~elbkub~Ywe136w*^mKrVsRtDxYANTE3%u3u9JDG4AYOVCX{5UUV^uzY|J zfu<`IkAVVq5Tq*6fS!jPNI}w1#3K*3^b%2EOzLjS0{NB;^X=6>18piOCtz zRyrsTf$C>i=?ogSh2~UHnE)>!KxrPN7Agngg6c}_B@iqhASG?kkzb(TjRq%fP&S3O zZ$L~)dkosFvjwFJ@OU*OV$d2sDB*%!=s_BVdWi)Esd*`|VgX_p!X0Ra;cP{LtVS~s z);dBr66RTO{iRwAYSe(PGJ*yaBoINKgdB4TF%+rXz*f#eyILSez;~9Sb={!0qQnoJ z1F;lDB7BZWElQA@3f5?GPc4B|3C`f!TLW~~t|so90VVv2ECGnm|4?sfD4|;eu4>&= zOCV(caZXJvN(XloA?Xg{YfxVi+#k_F?M&FBq(P+Q2VS`YjZ9Fg*F$j`aspHUbtoVf zqNhTL9#}U3S_;}iM4>#C3WJnnip^ShT7@xCOos)sdJ(v34&A04RGJ4#2O21fv87LZ z?Ep{#0_v253JZ7yL|f?^qFfURO4@K|s1}2&T978>Z-+>pt=|qozc36vEcS6 zq{RrH&IV6YKwHl)smVE@tL8HEVbj4-s~}BYcs2n=C#VpEW;c{A_@KT6tW&L!f-n#? zjhKiuWeRFagG*Sb72v5kB&Wm1L{y6vsu3H~ads0R0e~&cp^j7lH>jZL5tPGGd;y;NJ79h8AkcVn8MrT1Wh=!p-fN_zEg06xw-1#6qP=(lTGDeXu zDo6%h0|_cp6(DJ~7(9w#jiv1l^)@(2z!3s635KDPpfJH4Wx(_rs&WE-Pf(i z52= zh!3I33BBe4k78gOV}j2AgBq#GkpX3cwSh+;Ag1H3i(poRF5tqODxf(C6cW&l*$S%Q zxkHSS9O?{E9wQ=sAZl8OYY}{P=)Ew(@xJ~pp+2CAQCM)o+cz-h!}KEA9iLf}nhR<; zB8ln6Cl-TFBi7JF&iSBqLTE?afXZyN6+NJW8dTJPx|uqlt$Lv11<7oPeXz6#Zghf_ zf>&BVCZg08U`yT05_1p=6~KnsDnzTp4yZ0KFNdB?4LW@qG#9L&SXz>y4_dSj4%XD% z#LOI|CN3-()Z-zSV-<+qH)yp)_&Pz?Zz2E;@+EO)`;0Vz-*bHX4p zzeqt9w7DJ>nR@ZyvrECd7r_M`C|2MR0UL3}7;Z(g6`JPJiy0KNKpgPIKT;UmBC(N2 z*WeBSX+lp8XnY+7P#D{S2pwp`$jwhl&B19Na_T^0qoomu45XQe( z17(3*$eI{c07w(aMo3c9iwA9>gf^oP6XGBR@Dd5;MwHGFsEJ0TqrrteWXUMVa`>PN zxCw@JA_<}l>NOlKIn)L+mV^)OHpIi%H$r_04i}K`!Sgua${53E(EbFt9fMFt*uSuv z1F|&=X$zFPV@U~OU%4)1nN(_#0&F-PR1|=3XMx2&I0nE40XRBwHj|@~HGo#AB6Y7o zaRh7c#>YdJY6qo)2WUbn3m}U);9iHhBr`uxMlS^R3GDxw66hy`9vC*hzK+i)3w<95UIl#ORb`Q*P3JRFTfL}f=%arD& zfcyqGSpl*h1Z)D#k=RmWG*|`L&!FrM=7IA_W*Q>fVw(d3=V`EVux3OCbWbgTW;pP@ zTv$Q_sX%}^4Q3mXfth*m9FClQ@CAokVrEWi3TW6b4|F3U=ztCdSkVHTPY0h@0dL)b z@)}Y+!801zcO{htsR--fLec8TIzXWYTd@F2lnM&!j-aL-+`f{0g(T4VQN_^R44}az zn4>^uLY4u6!yKX?LPEkA|oK@>T{fdM5I9l5Q!f95$VWgLt+m+*Hc`an4StM z5q^JMT&1?MQ~ zY(SEBF)7-b8ukG;rFoepc6v^l_9~#oZZNyRE(EpdLFYh0&B1zZ4RXUBaZeEV)+3Pn zK%RkCNFZBv6hPL25-Y-7J+Sc_>L7X0{4_`fOiM;)8t94>1DJC_a-h_SzGwp^9uFE4 z1GS*ly`9vp6bub?6x4k|EF%!h9mFyQu|hyB69Y&)TS1{1Y6sj1a9RZMV1rJO>;Z1V zKop||1h&RYXkK<+et90myO8Dz$ViA$(3l5Z)Cuymrh=A&hM|Iu4a6--Gm6lgRZ!-d zVYN$2W@0?b#4Wta$%JfL1#dca%go6E&)|S+W{7goS#oKakhRj-x+0MIY1oo@=wdC1 z6TnMeAbtQfE^;A>RUMQOb5k)K1S>uu2I|4mF(~#S1vyG$2S+g^17R^MS{-H?v_%QB zA9VU&K?%HFgGC$63D9jb;47_=8#L;E`Oy2)z->*?{1d3L32IxT1Qzt54vsrZ7#l9JRSP*DJlWj%-lEF7Q}3;JdqXz2h^3o`^--hh0H z5`5~Rd7!f+KpRCBK%0KRAqPGpM+X{~a1+2@1PwyxC+FvY3L>0dgDXNhFbEV2FmX`0 zfQksv)rp`0C`v6Z%?0hLL=47)%?2IQ1)8V?*#{0Zm;vC6?7;4U1{i$Z9{6}c(4u^h zx!~#m$>&Hb{J`eJ0umG|#n9Dwkm%J>P**LEhh%jPgsq^_YfVtrOvy})263S27g`2D z^dicCXpV<0fB{$ z6+V}xhq5>moG!7I6Gf>xi6xn3si4@-KxAr2wdn`zqXa=!fv&Ja_H)g?o&_)vm`@3RukHdhvzQDsvo3qfiz;lF%46gmzIxKoFRoTSS2W&Q2b+y!b1cs z%*Yg5I13>O8qbA}MZ%;}90rny*sLCl(1oM~i!N{?M(LD+?>j?i2VZFj85$|JjYcvT z6r9+^QHBVLaasv(H-TERkZ}+24s3+2u;c`4zJeP-u!}(;(T^A&g(`#|?gt5HNNNQ) zYYFIus6^2W9+Qg)Cu3Wv;Se6gagY)h+?GX8N06X|Ew%FnRalT}4R%f=G+QXZhN8fY zMo2vXT^xl-=Fp4^F#x&@44#zWCLqNv#CC8Y0@0)nE*8L!gjE!zD}N()jFOH#3nWq=J=htB%xC}_YkAh=)# zEu;kP;wVlAmCaC>WB3=Iia@LOt7}mvEMa9SSPdd%VJQfu-3^IW&?y(mkaLu6!Q;Wv zh~_+`p8;;A<>bQ@*}|^;gxLj;8F1Vp=N;Ja8rU^3>r!(v(-csQ1LZ1p?52UFpcxK_ zVTj>Jd~HUs3qUy!VY4czWl{t>trT+CEOZ16QJ{h{7R;T{J`!4K2})DQE(EPFhECN( zBM&o3kkmlyDG-4fLhxihn;Gmfl z5@QOGF;XNW(8i6yApnkYaKQpff(qb95Nvo5G)M$?7{UyQ^OO}lOVo?OhkK@`fVM|f zDwO8sWM+eoyC}&>EKw*=RZju!c2EGF3j-<;6hPAonZ=;9wsaKAGcuDQ8-SG+a#IuY ziWM?J#wCFccLeo?K__CQ<|!nWfKNtIfE=X)c4K))W=<-Gzvvh=py0q6EYPG0QUj^d zz)=p%XO1Z;&}li)$R*NUXE|x02^_@S8!SD-#w3)KlpIr1pyN)^IWX`@F?`(}*dD}= zKgfaPkgN_~90LwJ2p@UW2GXJj)nLVtgalCnO-0C>Kt&5^ijClYbMOQxD3L?*$jL6C z^;$WVpe|fVVrHH~qCx_wAWDGkjQ}SQP-e&7Bv6NzF3^D#aJ7ighB<%%nd$^L$rNnC zn^nQlhr0h4H2Mf>enBk00tfetqn8gFn{ zfR_`324R9+14CVdL*hMyM*Ahn=|T55_0njzSW z%Hq-_s7|Oqic6CqVF_OU4V8oMb_AWN1r-CU2Q6Ji40IsNfbLR9F&ZVbaa3RE#)A!k zItLmw(8z+E{gPS)tpu@N@(G?7097=aTmyYH5Zd`q&{a3kQ+6`*^WsZVi%TfH3k?7H zPX@*&#-?cZf0>&Ynlg-@|3rtxf%E*QkksN5(7BP&!($W@(-SlEib2=W=72Y^I|g_b zgYGp4T~JY!nqHcdSd>!?kphZ`pb?Y$2p!o_=8s*}OcMR~1_wfu4QLt5j3|p(GfR4k|D>H(w znNv^7Ob2(Sk%W>Vg|}XrAxu$mNq$jcdMZRDmWzwa0pSxpC^I)PFEKr}2z+*MNNR3@ zOJ)&dJzRcKW_o5`Vh-puAIRCz>8T~j)ZyR1Jt>u){LSp(@(_O)e~sj# z#LV>M%#xyfO)CZ8#OzeWMPaZbHKBu63I&xV8Tok%MtWwjHmfq|;+>>aI2T%xD=WYP z2h>D2(lb*?$xkf?ohl8!n7KGRv%oVArl|a;Cg3w6aJ|!88ee`a8N5#Gy!{uyHZa*jaiy=-^jS8wI&l zqoAM;N|>Nii$Hf;sTMJZes^FT+p<&|XS zm4ZA4@92O`0hJ{v2P=S^5Q)XbsYNBOg{6r(8U~;S7}&kgMg~Y5cn1Ke+yE7W;0uRR zK*=b-D6yy#y#ERPWMPn|G%g3mt&IUapm%_{JF>>VLHMA9PU+Vp$Fy z&i^zpG%!SK|Ct#X8;rL9=n(x3INN{FW8x8eY@u~o2{Gq?f=V{jrXT2*8t9py;8|zT zUS(*D4%Ri!FX9648AO+drb*bnu<$cTLDxS)jey*Wotgq4PslIg;sULT1NFwyM)N=& z;gpinf}B(hb(j)$9R&^Wx?gbL5md;PB^G5S=9Q>}M%AHbenLY8(!s-j<|ow2;57%u z;A5;qKo?JduJZy79_MC)PCo~&v4biEO&#QC?u7ZC?uw&6oUxpjwdb!@cBBK zNu?#J#R?k5rO6qf9nVlBz}wxxqX*zAr=rXP@X6wuU~$mV+X|@_B}Iv#yKnMSAU8CE zPXmX{Q-Su%C08crfNlXTfgI)vYR-TTeFvZ33<@X(P>o!a3()|MsgjIT(B+Wj`3kv- z1;wBvN>hs=S8;)E;RIEf$_n5;M8#J6pnI!QbMgyPi;AJ!?Z8{@;B5{4GDCgPc%5z< zXimO_OFE$L(WrFw^9JP4;0rLAS1w$ zXbY1>ucAPe2^2#k*%lgv#d^UdMVWc&;28tZJ*wb4{K6cAJRSW);Iq5xpv#ccQ;Tre z3X{cStpbL8V8oEQVf(h{Ix-EJ+qSL2jlcU~yh)ZW2xpLnZN8i{V|joczQR z0(K_mRpRg{I30t>fQ$9Kiu3cpYx_{z9MI?mQy7+qL2|Ypbi*q+UTZ<)l!)}^<{#wi z7=oVIGV@A|jc~Y$aIE8Who@hNu@Tr6Aor@5Qf3Wi+JgBv6wRW{yb?1L9NvNEE8KC4 z&o^c!sJugUJ6Q+P0zKW1S|&X#UdI*h(Q3TuVIV0pj?!g zfyp8H6fy>-tN@x_2F<)=7Nw>@kCZ4z4h1wf>LG3Rg)}ok(FGY;1SK%Y85^*q3N;XR zJ8m&FaYA~(pi!j)NW%!cOeqCCuU)L5k*Sxer{I>KujiClqzRg#g(hu;=ZitjFzEPC zd_1(F0`H!I5(wUd(Ulb(!IyUAr$O4H3b1txMX3tOrA0-lc_lfO;5KGnx`GC%iI3I_ z)I_mAFR?fobdorzl?4h7&>7?K;6=3vWEOY@1Zb5+S*k*DDrl{PLQ!g3YEf!law@om zo0kuc0A`6G92}?H)^-4If1HjP(%7NIO4H98?m zL^%%KnhqXj1Jws$PlK%k*+Q~^!4n~HH=>0)PHzT)$D_fv(#4Z-cY*u}jmq50;*!*& z%v4DB1}9Y1T%TVAOD`x{4(wpif>xM*Q1=fM0+?H7eG_v)oxIc(1!!>x>r#WqZxvJ( z;BJ6NM|?bVC97L$P7Zj{3&_cksu}8VXj)d+gO2;=CKiCEG(aP`h$MiNo1?8_6|@zi zLG>?a=QT93$HzOSq_6CJ3ze^P$G?whn(GqG;#oQbYd=O`UVvKP>WC^5#%ngNYiyx^C$jR$C&LO2(Z$?<7(%`44C=+T6F2ipQz=pFQlDJk%g zmi!_GXbp>H$t$R`MDq)L>u_3WP7Zju0#PlXCv;G}BSJwB96hkihR_b$stuZDi^pgq zz!Ds0J|y-83%D(isy;JK0WN};{817aZi`(qi&B%38Xlk!Du&ecB?{2VIrt6HX#Pi7 zZHsUQI6R;U9a{as+ep!H9iSyUsHTDSWu_@)7UN2EU=^U`iNN?$5k}&Jrz5EAkXiz0 zy&LFByr>R@ggAJ8H*PoLZA+u2YgnXUaZYMpX)dJsgk0nxk{-k@P>m>MzaD55FR>^^ z10K{UYEe>!UQudpepxEAQfSeOz5^Im%pi&~Xb}%}f_rKSBEvdWf@=jx?!aN7f00Wj z?2Ht!LP%i15)8tpu*`_zBCsxSk^t3oklPoK%0N&OMp-cqzS|uo*@GzmQNnqgeFCx+y zv|zy2;0Fz6CugK4XWN6lQBagxmYH8#jM{rZDoDVppuqzZ26b>?*%;Guq*??`AEbwZ zsRL{WcpVCug{BK^8|bn{NSL9?Ld-<=1}G#!4H(Ci6x8ww8eE{@N9ooh>TpP;B1mwl zRhF1jnp&)*0Or8L7_8p#-L&?0F_s|92l=(xbrROIF5kh2Hy z4`85L0quuE0~H+huo4Dx%_V66eqwPkQV9m~A51-xe?Z6Eq{44ELXHW{h9FEkC@dk- z0_p*SjtZ>MQP2SS4czd+3^YwpZUf1Kt_)5^jJ{N57C^M&4sB>~D?lOwtd77?4=AL- z1tcQZK!Xd}5UgPa-@FMj0~~fBM}s{D@h_?g;5iHhTLo|kfP)T29yOG~CLs&u7b&RQ zL&>C)Vn$QFLw-}b|z(#du?cNWIOb#~ z7DICeBt)~zKuB{a^OCaMwP^S`zRH(a=ya)*bsBWyW>6`=00T3@jHprstgL^^27UD5b z0Dz+s8o1DKL$VmzN{C@lrHF1GWFSwq7&e55H0ll70|QkLUf~WJUP0WDh-@=a(dw9z z;s`&$GN1@F845a(3>vjir|Li>5H#ihQ-O$N@L*6Lc*G7gznhqYFb29st0W&b1O<<1 zq>)U-2}R(+ZBRLsoRONF7_E*_gfJ3%>S<~osOl^RHx9x13#1Lw+rZ|fN>BxcC^w+~ z0>`=zlFQ@K+yaRnS+C50Yx=(yGif{FaxT589zbL}r6IJ|#6dzbLUJzXSIG|)9527|Rg9SLi5gTo1=4eT9ACIZz|hyyC1@dYhPL3Jy#aj?<}l3DSY z3Qnq;;1ij_bNOJK;WHUfM6K(os-HDuIz^_=&bs(N+O=q!%e21V|SER(QfQ zV15x~S2M^zpwS&z2tdLPBnnz21WJ0))Hy(vBWlqGNdaItLIYkw0i}e|fgX|vIV%%- zkQt;MgJ?0LrUgXq(13asT&6?f1-vR1t{&1x%`XD=fS_vO2@z53gQ^j5oP*t|j*vh) zUIlUv3b@gxqo4tD8mxFhZKgu^79q?)*;WL%9JGEOw3r^gNf=x#Abf^a}N{0%vp9?g<3SFrNturcNc?n*^fd;ogIR;!Zp;o{! zJ&;BL?WF zgVlhJ<;f|@%u5G{8EAJlIN=v7Bo?JAL*wKFOqZF0~tl}I@dl-$5>f&@0wQa1$!^~@A?D^MMSXog{F7(fhz zwhTap5{fHvH7lUr109Zw(TK@T1ueuvZP|bg21O*OErN5g5SrEMpgtP7ag~vZ)Ubnx zh#r!wK^7pjjX+#=a9>K@3a%cSA`yWA)c|P^fzmK&qaR^3AyDGQp%&~;h!x<70=1}@tW-=vq4zbx<*| zD$tI1=%oYT4ewyw#|mF4lm07uxSbvKwZ$E$F&=NL5SNY~*x_In)QU3mOg(KR^ZJ z!8HXqCx8P3tOh=72Ubd;jgFRiq4@^Y2v`OJn+N7#u?P81IMARK=+0kAV;7d8GxPI6 z`whScR)f+AczHj#IgZxfMx;X6xzlLw0Xqa~mPo-CQg(oy1+CUlwSY=1hvJgNlFVet zUUulj8GKOyayEt}0O(1B(34bP%jw`}cR>`vx`jx6DD&>kdsyn}Sp16&Gk`_|PLIVq4CWB%a z>|tAw9bgt{nICxT4PsXYaxnxStwFO0WVsG_Y7cA*H00psBdP?DV~{HwxDtp9!86!! zF@!c~y#`-?1Xlvl20FqSO)Kc|U)-xQ^vW_*%ORJ)fC^!7`UeFf++4Jk70@ynzN!I9 zKh&@A&Lis7d}2xpVp9xK=pk2)kiY{+FQicdsTU#H4ZNZUe42M6e4C8|r2PkOia@o2 zM^8XAsrir-#q&~AQ=n&sgFOfKSW#*kB1q9=0&EZ@VS@)wq4fw@BS1@{2H10=9)bs8=_DX2+m=ok68EIH{GAUCzJ?VSNfB{))})u9GKPn89G3KCr)Ll8Ba zUU5lcQAu%mW=VzyXgH`Ozo;M?ZJms|Ch9rpaJS$v29|R`>WcN$5i7e;ViL0A4>A>m zV@)@>-(3t%b6_ukj6`c8qWA!L91~H^V6hh(N1y^gK>^|ja_6H^9fm&tfh`+A%W6<& z&P^-72cpF&a-oZs`awp3g9o0Vz#^cO2};Z0q=-m3D4YCX zNgS*Jlty611`hSnV1*!)Aq%wO=^ZQ%G60JkY;9o^DKP86^EjyG3|I&`(!u3A=(aIZnI$=? z;G6+E+R!JnxCA^*fl?5lT7|FQ1~wR`JTnb4ln@Wq0XfnXv=N420|^mANJ9^}y#}@x zTxzJ7Ap8Rl6PSZQ)Apbd7I4&pWDzEUTRpJh8KjVcX#uGQhp!GeI-r3Ewik5HEs`q{ zmZ7&

hAU!4?vMa6U1`J193{9$g2nYQcE}e3Kiz3IV$kHo64&2*jP>mNVRLTyrj< z8Vk9cBfp4&h6&h}xN@XAXx0ak8?iVK(tv;;QHyQrD#on5dAH3OX=ep**n? zwB;f>KQ~pOv;dm35ion85#6Ahhd=XKs)~fbPN}T zr@dkN1poPOMurAv#%Sk%7@HZJkDmX58Isf&!Fm3NV{tL~a9agXlE?%tQ&K3-NCa)2 zOiC>&2W{p9T}Dw1E}siBDLEAcx{)S3wX!_FCk#%o$|SwQEIkD)=xAa7S55Ny#a(I8_0BbQ7q}3p(Kl?0nE^cc3MYnfZC( zF?EHs1s?JV&MzwQNzF?y z$#BaoDlXAT)KN$Rn*yy*b(3=oG(Zb#5;Y+UYmziUogD>`Fjy_zXP~LHVhuGbgj8QUQEdFxY3wIjM}rkUH~7X37TH0NKH{FE~(4`UwX`? zpy1-_20r-0&(YU4-r3(T%+(JxbOq{U`uK;t2E{u&`nvjn=4sJX1cMHiQb$&!qX6ZG zdxm(#hx)m=27v}U{rvr0)nh>|a&WN3gSLLg$3vIFfrh3Fpm)8e7GsQ0ft(Jy=AbM! z53~Rde4QzVqGZq^F^Qnd2jfA#+hWMQ)Zl=E*Qu!XfZCAY^sa#_t^*2u+jz8?gVem> zDjm{7M{*IY?oS3?e+71<0&L41j*tacyimi?T@Kpei{=?nxM3f)1ub51PRvcsfv%G% zP6eHr4!$on9@hT^g&$~tcdA}-X%gtBt>{GEDg#~1SWS&+N8PAcO?4fGBJ~(U)#4Z< zQ1=0D24rLi;&W(v1(~InlV6@%qyZi-!S1S%d_r!**7|eK&nwPMNi9++2c5KDlCO}G zuaKAoYMSMwCW4lWfkL&UC^0h!F^^xGmy%jkoSa{Td_GPBXzT_&jg$b&q@b-TpkwJO zp*x$QUV&ZpQXFkyWf%*)Q%gZxp*Y&mDi)lZz!?VY<>G>zOwhe&(9nhKmc!+FA5i!Z z3IUL@pcAdZjoJ8kTtNnp42)=qHi(7Agn~A>@d2s#&Y*%2?0b+Cp(ZOSDFvh! zfkvhvC4XizINTIqr6%0NAU%jIqaGa>@2DFE%{8FnEFNXo4%9=S2+1r?$xP2Ift(Bq z@(9!`r5 zX!3zeg0c|EdZhLE;JGPKdIRmFhL(A@>hKyxNl6KMUMpz10{BodXd48i2XgEMhy_nR z*ou4i)DnfF%oI>_61+V)8PrAtuipZ-T{80&Ag8N?!U3`(3!2T)2RNbL0oe!k9yYtc zXR*QeIDkeJpfm2Etv_Ik^V1ZH3sRFa(=wBxrohL9A+0Nr`Nh_dqCx@e;L-vVcOjQ& zh>>i_2pvc>iah!dHCn`h)PtP?Vt^Vdkmv>%+Moad*Y=PkhN^FE*!oCPxpgVAbAd2 zm|#oX;E8rzp23#j!c!HBGxAGwQa~pLgC@p6X%1W+LedT-Fq5FePGHX}LvjM>+!XMY zDPZf06(BjHC>4AuO>uE%8R-0&%nIWju38W6C?Sj_gfuv0E7$7)_VQDeLO-AY`;4mJwjSDdwn(09? zh+Kk$9HvS5{BGwA&}rK#nMEM=I*?OfQf)yO-h$HrsG!L&QOGaW165FZsTG;UCB+(0 zHc)b-RsB`~Pq%+CYe1(T6jl$cxsKKvoEL?JgbJ)=Y+IT3V7 zd}*;lNfG#%Hc(Q8Mj6y#%oB#;JzdBdd&{c*oe}h8S7HKyYJULh)oDFhC zv^ps4k*~Rgrx7a!j9h@?foQnOSopzq3JU7TNdm(%SXx24{!$(6fns$l1;}ul2B^ue z0TI+hHW=nL&?0a+8<7Cip<78v#0c2NbZN!Wg=!Q%6AqsueUD3%V;cIXf{u6|}4fMI0VPuv;8aWT6~{ z?dqU2JupPU4uO_1$vMRuT3S#gAiKd0Z1i)pU@HgVMIpE)0L>Cmlfac1v|ItTi_pzT z&dH1~D$Pj+2MZScNEh!S8wF}b<)pz*q{8YtkjbE}r+7>ToyM-93OP{$q#W)L94-Z| zNr$V!=~(FHAE53LSPkriLE_vDvV$~dgLJ@Mh0W#AmU1x>0S!9{r${v_n za_oWohlFr~>me?rz&#hpVRN87364k9gsrYxtOr&F@&Pz$5@!X>s8}Hpbd7mRYF-KW z9=z0w66n=1;K~qMfukKR3cA+-bm}!+5VS1-yhnBX!By5PORU^5{mLdrI%19LKy&@a;gFD+N_ z%malc)EuYaM2I86BOtmlRy=eYB6MGp9Y$49q?%oc_pB9 zqo4q5A!9}Xnl@0-5IhJ5G;NfcSK^$XSCU!*8ZFLAgN77n;1QJG;EN3s^C}e*i_*c3 zD9})IZej^&cN#befm%-aIiO)IusI0-qD45e$I+w|Y!yJg8L$crKVqsyvW*MmQ&1v^ zj|W|m0-vMN0^gMaNyCT^Bf3$bwk=2@Xe1h<7PeFhwM&$e4<35XOIHAA!h(FzjqUlM zd=UZwNP0Qr3c=Jo~ogqSe%@h3F<8*B_?MV z=Oh+qK(AAQdIr>?18s+~vx7Q1zqCXLG|UT4vQSY-^40;n9Fkm-`-$M=D;3a;Q0L+T zm#FZDtP8C35L%R@foRQ^7Uh6brjnA9Yehj~9_ZEqg|yUig~X&J(Ah)aBUeGQA0_$V zBi9gZXob?E97xXv+y_Q65@m!7bf-Kl2SW>J6vbSia0YFe(Nl*ld`6Ln#2lnY4W7$^ zrXb`E{*bLP;A6BvnF!UD;J^m8Y(aIPjzT=7+XHQFg4gpDCxfSjKzAPLfu^>>ds#Kq z^}srzZdM2R19X`|d3ibX7K6ls%wo)&3_zm#s>L8wlv-Q@s#sy$IS`Hjv7jyh)k>gz z<_5Yy2p&x?`N^o!1is1!bU3H2DJ;goI>6&U@BmXt$%owR3c6bu)HR13XH;6011(YD zDnXtF4c)*-%i%U-CJ<2F0ZTrht`k@rcw{aovA85Y6*kF^5C;wL$Abp;D~b~H(o;1c z2XMiz`-R_krhtejsJTd6il9^cdZk66t0VGILqk)qC^a!f1G-LJK_RIqH8C4p%0XPC zpaIkF5fTyrPBA*56bo83pcfkC0~P^amW1jBNMb|WL{*+x1e!(HP=_9v2%13$Wd%@N zAsi11b;vDw3W;f;b^_!)H?Vs^g*YlkwGot|K?j$h1Tf0=w~(X{s#D-A3&76TP1acEJ*(1_RN`5jI^%tRxnmAw~ zfVTb%RADCOr69JM(Ek1>QzH`-^!q=}j7&!Ff1*p8z`6blRMdbPaIm8r!Br|~B}#ri zw91FAyFy#31#0b<7bOKq+Xn@p#!xq#%0p0wh!&M8PpK!}bOb5?Sz=H>RrV*@x29ku`ngvfVKcZQ-X=`15-f#F7T`*be#;S)oKNv1BA>AW)_3Z0L`Q*KsS|?R2HN{BvX;( zkhXfF9R7h#Wg2MbI7k(k0c}o$T5;eBZjhTH_N9Uj3I^}7F1CW-)&P%02h1U zphd_+RD%}?B<5HtXh5?G=)h8N$XP)~XEYT&)4&~XP*^}C40dlU+!@KBUH`=e`FSbe z1ds>b#}D2E4muzV)OZJZ9CQaa7kF-lO97G!kap68)&W2-Ez>Xrwc5bW#i76mqyQ;V zDBw_M3{eO64=yz(AT>yd0hc;cP`3rL*%~sf$^}}JQdkPUhy}@hDDlDDog41hSW-_=`hhBOE8dgJ?0Zo^fMj)j+kPb+)fEFm=?X0LihsuB} z6iC4bN>nJWhfbtH=BL58;#hÓs3fyyh03Ro#z0y%LDq8?uAAx8$Na0j&)3qbcH z6JN+7Vh>je11ChJxWT0kT-1SM1z80&{eknP1}u_v6il#`WuTzNsTCSGItsW-!PMds za8w|hfSjMqKxGyrP~#!N2o82!%E2XJMq+V%X#wa|7Gz~8rJMyuDW{+S(w356o`+W> z?&1-ek0FQ8;lCi&*~b%>z9Hj>py8!d&~+mydjX)ctva9Df zNIMUdx1kvMVl!-4vO&eb!yll60ya1W9dm_R51wuU&q#q2FjNxOZnK3s4p|Z!Um$0K zR`6rL#0_jCbh`k`by3hN4(=~RB*KM|EkpJtXq__B)osYiK*bq&k`8`?v4So5EK*cK z3@xQap!1!;x=V|4kfos|Lp-=FvsHjD?1hF;dQN^)Vop3PV@1RFdtkeD2&xC&<9cwl z;JG8v2&rRAiaXfkfW)H2+|-iPBGjX?!Od>Sp^=bvdZ05nAcstX`OpS6e2xd%nXq-c zP}jh6J!r`pq#OX70N?Ua1cgrt=ER}@LUqNR%!LPy`Uv?8W01(JN5No z>=aO`qo4qq^~uajhm|;>nPT{~1bAH{*klx+K{P_04J+B8A&L>d7&mM{^+60lFTBB_ z2MRFcW0k-O5<2aJqyaXFizEb^33N`($pLpwREuHvsDTFOVcyP5%SQwO*ko{&$LA&% zfF}u(OEfgnz>Cjx6w)-obz~Z7h#Tr`uy>({fkM0hts9_OBe5tQ9AKceQBWaJlLBG~ zs5k(bh3uC6q7+d7Bp$TtIv1XFQBQG#o$~~8WHiDk$mR%mn+{bE*b89)DS&6bU@0G3 zTOr1XA^O1^iIGz&QmERZnT#ZbtOL;^gk8}HPK9~UVpIdw6i_W&SPH8e;p?E_L;di8 zggG1(c#sfBDsVvg2Nb@jW`n{Qy4VU!wCN~hr&ikLB<3ciBq~%`DO7-i3Yzy2r^lii z35jx~v$vo@3pzUidZ%SPIA*|xfgAw$04$w?@@ig5W?m_*lZ=>G1(!XZ#X+f%E|Uh> zB)E$(=P96;gM6=`pbpapISdSRya%XnotdYAnzq1c8Csx&R;41xodV*9A+X(GU+AC| zE|9QSfVNSgcNsx*J!UzI#yN^$v|h^=cOYbJ_<=KCHV@8IXU^|PM+HbIjJB4b;wyNh?~n{hyK~B zLl>ojw1QSizY&>KL7G9s2cTuPp!5zgS5F-temaot42e1L{x`@ZkR#y-s6+IC zt81{sVG){}nVSkv*r2|qEy{T?>b{=7u81ytAPX6B`08UY?eNiI>) z$S+bz%&XK?fIA8@dkeA)><`e$2*QiR#0^{S0KJ*4x4bI2l)%&3N zTTq4ujW^~gz$^lFKf!bTpwt4_g|HrPYJsbVCllzPpgN+Pr4C-Rg)=9^jX=(Aq~u_P z0icYHl!>9GI#@)(Rsq`d0}r>rizHB9vxO-HaS-J{qNfe3xk2(sYkM`2^}*|E)N%?r z7r-qAxwF_7Od_lUwU}%{1VRY8nzlve=z-7N133qw7~1uMwxU3*|7?+4u?U5r-Y%rH zu?0mIN;4QD4y{^{yadt%OE8$aF!CZK$wEkIWe5u^P@f6bc!7z5nzryxMSMJ@aSR=Z zfOaULEodD`Z4PY&Kn20APedypCJLLu(}S1?JGwEy2+_(zYS%+`L92L3bq_9vK%oq6 ze}Ri2@Hws!D^XS|!0$tX$iWZVhZ+p;!D2cG-7;`GDM|$&hy>FOatdg21k@D-pN|Ah z&ESpkSQ>jo6t0a-3(e@R-B4Z4^|To4Fp7%f#eWa zjsv$$Kv&d5)xk0y)I@|!;Rc~)1aOq0UrmU4KCn7RRh#q#8J196oDL2AoY+?0gp05#;?E?3^?zDr9tk1URMc;V$hWY ze)%P-3MJsvcEHnq8TpyXsi4_8&@mP{sU@j7pj`?{nI%PuMU@J=x(c8T>*c8mppC!a z2ml#UP+FV;?rDJ|TcISQD8Dp4LjhDiWu$_}2Z|s=5ow^aWim^kSMp>+rawRifcpEO zZSonVMJ1rld`SkVV+J)Jl%(KM3`-Fh;fb7HVG9@#V&DJ(yAHI1A8as|@)zq+8^RD! zCj`2D0lu{YA)5wTr;t_#HX9*|7y$+4YseW6sAHgD4WJSNo}9qN3TD{@iFH_M3kwlM zNe$l41S(n}*Q`Mn@I!kL;H?6wuzN!hQ(T}@PcI(2;}4RFppsx`rIqF-gAUvRSF@m& zUp&O@RMldTMUY)~Pz|87Z%PXaK*vHUXe+=3L7OYUMnPQwE%LDQ)In8_YO#i<6=)O- z*=~s6K}SR@K(*>Yw+ulmA?SvR)FKUaq+3xT^E{C8N$>z{oL1!a)oMQK1Dp>q|`UNCBqgT`SY zX#liJ5OfGaW*O+%kJOwT1*p+_>YDJV!0zeX(wve^O5;8yB|alHF(tJ~T}L507Hj;d zBVBw7+R~x`(NL_Qkys3BK-hxIN9eAsVof~-=fpgPq*MiUxciZl2y}r;YKj8b{RKIx z3MHAKQ+bj63#v~+DI8p|fbt(Sh#}b$Jjw>2-h$7~!izBtXmII(^Do#8Na}%#<1b3U z85PuU$Vp6xW?N{+f@ue381Na8kgSlLlc@(S-9TqWg3C7zO>{k=(h0M0g10Ndj(1Nj zL7J_G%~V6Jg_c)Po(8zcL(>3ml!BZMDhMH7##R(SO+rKpsN@TRjGQDY1eYY{r6d-m z1eT^2RU)R%!E^uMdq_bM173y!^&*lvkgM{*!zR!^I}X#p;R7z-p_A{hT8Us}7iEAN zD(c`^f^}zMn!&AJsHu?tHe3c#ox)mopbQBy4CEnjDgkXVh2~lXRfVDqSUUsBhnZ;# zV5_55i(>T(ic*VH^Gd*#4%{ZtXbvRoK{ntGe~2xoze)B(8% zW-gK@aPJGE7aAWhNr++u4(1UNQbS%hpg*Fc~AR`~3WB|d?WQ}YP ztRO)+TnRKU32Iy==EQ@FV`%veN;fH>bH*#6JKfc7)pZmMA+_)_~Zi1PQ`K zcol%9!U1i(g9jaf#0*Pn(9=aggOuQ86A#+Gty-*sXhJ~J05s1*)F9>!An6w5L~Qm! zcD91{cq4XsgVZ9z2UI-A!&)9X3Xo_9FAuOqV(Y+t1=0<7Iz$B|{-Met931L##Dy&m zam?6MhweT^9lUZbN(Jpk&rHlI25;O0t=CDlf;$;oz`#p&L`W!sy``WIF#>$v6lgXp z8DY2;Xi!XDK~+^D6-#BUUR+rWI>M?%!w`$ZAx%^0(J@e`fh`2JZVEkT z0~%s_knlvBvQz-yjf~_H#C{5p$;dej$z+IwqCqF0LgE^v7HWrvx+#o1u(;I2)vEokNdDMjn0l|lT1VhX5j0vhIlnFS93kV(aQzKKQIpnT$pSevC< z1Uf()Bn?UYko1aLSO!!EXQbxjpyn=Ukq5T{Tze|OlqMDwK#RFzJ?GpMO<2VNQx0)E zw)_P(7b$c>nLMa84{Cip+$;r1_Z4a+#-Iw437{+CO7kE`-;|`n`nm;`B^mj73dJD5 zL5u<|?}0Q7Knqgg)*-lT*16>mV8tzDi^ojB4#}SS>`N@en@nwlcpfbN2DWtI_K1h*{ zuYs%tY9WKY0`8iqgOBb*@R7%W5Rt6}ZPmjM3&gJs)aHjP1#xgGhlQ596>Nh>F|ty4 z(AFY(8zu-EZv{;zgHC1FE3PaqNzK)O+yLv7nO9n&ju>twB2T~!RZq^(gCtIPjDv#- zG}f&LS{s_5lNz2`k`a`em;$Q95W}vZbVyo^Ac9vN5yv{<5kSzAGh0Xp4LTqQ9-YLR z4?%4OQ+Zp{xgP@+T&jXg~@Rtd>B# zNWtJ3cCE-PLF*2YVlTAUlBx&tFf>iUf&)?^8tAV$qEO zk0ik)6`+PfOB(P(Gj!u2w`k(ffvyX1{Q+qfw&ZA4Jr>76{^J`gHS3w z#6S&r9~Y=7g6{%DGXxycpxPaLRUoL+#V`$2aH6bZh1c=8OowjS50SkS1eg7R*D5%@yW#A1*?K*uBIWG11x9K{xRMni6_K-utv zD$xQA%_3-`EGF3+m`23erVxvujU9Ne95OQjozsPS7orl%!8#j)Yzo>W18jB$VhEZ- z=#VnhXiO!r9tutcprIYmnR2MMK$Sx#J#rH>^E4105xkWhOrsJ~76Y|)opVz()C(%X zy$^LASjP!5{s}5+pckIA(qmaNv*o4kUZPbA>onCNiNoj#ketJ4+L=IX;Vjqt10@YEVh0hAXsUjR za?oH^QC?zBP9>xcLHG#NoXbco2HlgXfn>6df;xty)HM-pJG==1$#f(mK_P?cOGIjb zB?4H|frK-5o_bMfUOXs0q7V0jgC8ad4SDQFfIJPo;~*Z?Q&Nuy8yFuCZ@uF6AIwm7 zWZ!}M>!5}dwEY4dql3mL_Gv}Zrt8wB+1lcGum$Y>SRQg~yA#M}bf z%?8RnpbOwYYb?s9{Gf=KZ3eILmiwy6jX~fP%1ER@I%gG051ta zF$b&#B}?HAVMGc7H^U&yt`$I|%Ahj{6jWhtFf=(}H?stO3@peX&@xD*AtX{7?!~2`B|@O) zFQ_2a)PtK2H&>lt*$MJ8D6xXtNEqo0B#5-C0HhXVlO8ntfQIg1wGw0~5YyY3BH(HI zpwc{0j}bIx4_faGnlA<&@d^p|XmyY+>ahyi*n*fq(u6mq!1X;eha=`ypxFd9To8^# zO4^WC3uL4WQi)?9BSX>xU5o)4AA{&b8bT^2s2?<33Yp4391U*YDacC zwt&Gs^Z{9AkXQsdG#I`;16=Na=2H;WI4IH;Kr`*20=+05TqC39Ey!`gNJ|CKbtCIX zEIWW48jQ_AXe4QXbTx+dl_n& z!7T#yGE?(XG@#pqZ6S)GO%9NWAPn~qL>7ALE-po|aMgpT29G;|3R&bKCXlJ5g(bu= zP`*N07ZD0Rct#IgT4d&=qK>A4Y=OHS#DNTmL5zaCO9#5_11b#P3yYdXu!Ssaa~`3v zg{iZJF>nPPOrF>fhN*}9g|aY)8HW+dF3@v&kir?QSObMF2t#u=%p|zObPzjfp~I5! z9i*t?43Dp()Z)?{&`KRh-GZ%fhsYowR76P;54RaITL;Q^kSZ3QS&-T;NEI1aAC|#h z$OZ%d(vpJG5)FuN5MBjq0`;q);fUbVj>OU@}x0c|6Q7VMR4=2tUx;<6d-4H zz>+s?$hNdN6_Vb;J_2WTR3Xq+)@VKh)%ux5pxaHs?JCe&e9(b-kll#z0gWDlCI(dc3FtmZNYLZr0-uKr4p)$oi8&w>Qx%Fp6ZgfS z*^A_&%mQ%4L1Pu{%7XNw#1zo2lO?4E;1JRSB?j10&&8F+T(C>b!(4-cJ^lSaeOm)P zvt&arYhT)8Qx@t{Li6kvC{sUvjgf-N`DEvPioGuBH@Pgl26P)|-ZOf^bOGEFf~ zG%zwXGch$wu}n)gG`27>NHjOFG=PrH;xNVpW{jzMidl+fidm{^(TUeT!nj58>rdpULa;lk$p^2rbsgZG_L6UizDK29SQNzY4DKRxM*(}x2IK{-! zB+)X($RgD|DbX~^G||-3EF~4cG4QZSN-;CGOtVZfNKH&ONj5Puu{1X@Hc3fMHcCx4 zNH&6vvSS1>#5M4+F*8ZFG)Xf~N;6M1H%c@#PEEEjPBlm}GD$NsH?d4I!tWY**d(PT zS{fM|7$l{bn;WH=CK?$Ur>3T)7@L}$6D?A4#Sz3U@R&|c zG&WB%OiME~OG`9LO-VINHMBIgFitg3GPN`^F$KA$mJ6x8(ox7S<^s)WL#l&}#Nv#c z%p_QCkeg!41)Yob1kZPY7uH%q)aIs`BC7-6r2|^mlbMoOk{X|zVycmv4mrlvFybwK-l+1MKC`pQ`23VtBMrs8_6gqbUDm_6f<{&B+Z53dqM8m9v z-*^O43T^s=E&!@5)+;VaNi8bUK{=-hqDl$mCGZX1;OQe!?FHHwo|u`Fnxd$npb-E) zryEf@6%^$s<)r4?YoaeTS5Qz@fVl-+i$ZKrP(W_)D=5I%8-q_{g>Nng`5JOL5n9y+ zHVRs~+M-o8I&f!!mv6wOLxX&D6iV{rQ!D45YD<0CjW%^r~mjcBw?r+OEtLWYeIk6!J=QlVG>xg03Qm84UJ+B51A!6uRIM zTezc2^HRWCQWX+lV&H&CPyki98s!<8$r&))Q}a^54JU|s3hMgmntBQf36M}q0PX4p zH7Jl8S>V&s5!I?fd1g)y$PX}^Qd4vkGSk4uWtM>SgHDl4%mLlNQ;7%{Jq6IA=LqM4 zgB@%gwB-agrUdFruxUx47GNqYK=RWRic^#GL3dDrw<5!wQ2@F_I7J~TH7&m=RRMHJ zY-S?pjx^9kw9uX~B*r1fyes7ArGi`!ZMmgD+?AN8ke>qzLs%?;Z7YGcbU>bhHic4C zAclcwyNdElOF+ktft>>G6e_?E>W4JHKq&!|=@Jsa=>@W*8)|h4S^-!}=0thULrGy8IgET=`51|AINL)cd0jvha1{9<9povRE36X8=Y?ZX3iA&R3 z0b(}T7^o9a4S-&+?VO*NmXn!WqN9+Cs3bDeP+}e-0dgE9724T>LKl+o&`W!zkc?CX zr~^}AClrIqdxTpRH0(gDph3wVoR?umkPavb$K;_F_)5u%pk4V%pu7j}1E!`ZlxJju zwt9jN3xFhCkWodYd7x7h^%MdilgH4A!!Qq&hd^1mC^Z?h0UH`<3Tc@|#U-F?szKR7 zJvFho5*GCex}iu(LtPKeeKC14c^dGW$)VGlszqq^mZpNL0wOJ^f@dJ34Pp`L5fr+h zP))7KEYX10SyM*$_)b-;BZG`~ix zqsXbpg6vlY&AjI)rYPh?ZZj)_Y~(~qqL9iHRyZT)RB&>G1}D6Vgm%`D@+oMg4qO^v z4GO9_6>Jqys!rJ8BT|GwnoNoC(;ZS2KXL#6pCREh`07n? z0D?!t6QT7cp=z@vBNbF4VW~4AMR@|aPy|?sOE2GeFDJz#f3L%|QKgM4s1GE!HilR4oSeDM4Lr zcwPXdd1#^m^|>J#0&?;{lnkm}z^$t*4bg$8`FUQT{G*vTLjAPj0>gAz9)*MdYrI43_{4|M6D1}L0$ zsRtlKGSIRTS~x?8#UO1>XbgZ3=`G0z57lU-mZj!_S{W&s#RWNum0TIAIR&7moF^o; zB_f&=uy_JBR6({EOA?Dpz$@VqHs&d)f||MD)U3zFr2#ri25~n~Zenr< zs09T&+XZ}w# z5p*O?38>Wrb|9hw3vC$|r9vwp$S5Ji1A3ZV)Qd!2B_%AYzCg(a8hdDkEa(8xoWx4- z&66nU52QuY8d7e8&P#yxHH&jnQwyvWz?25K9=s zI9F0q@`Q}NgIf5d7^Ae{I@UF@xDu)gRB(YjfKoIjruF11t{G4(SiT z%W9Aiiiscrh#FX2Bez5l`2keS+8UrG&B|guP@o}t2C3=ks7gSE7g7@gbj>d4cuhT& zW+Bv9prC}i8{{r%9D&3&Knte8;jF0*8bFEFQ~*^h<*DjL;AEYu0P6k1b>m7ZAfJLt zAy~9S>nf15QIi$O`ut+O(!89^yll`Q5yV@d6XvO{-F)?R8Zd-(pJZ4qZg&-=9hs7KEN?i z37%vDZ`n{t%`3?)0-Xqkx7J?O?b|{rVAySkI%@s(cte&Piq_b6803Y`SHB3NV7}$M5N=izh zU|9uF76I+PDbdZ$0}a~37uteKWN;WmQ*lu$xMc$#;R6{3s&e7RA`&3D&5dYXgSa5o z8Xy7@h-sh(IH)cJ3G1jAB_XY5fsY)6^u~ft@<}n(NP|_XSVx}I^k9`8*u~H)I$8-7 zJE}!i3aUjqF?pcwLuLus{h$Fsuy4V$bb6U3sky}(nvg|MAkBK9>o%ZOYf)+e=sF=K zCCEY!khCTwB%lG4nFkswECJhGT$Bt769rq)%yU6%a*1AgY6)cM3nZ=qO4hlMk*{dj zl51$HhX)DN3>^hrI_Qb(}^Cyd^I$F-IX4G&!t@C?i3>0cAzd zxC6}lprZndlB2D_21Cw4)-corEiDBHq&CPq(GW?{nlVF7D@d?ocwfC7nkLeq&7)$N z(~$yzD-(1D7bp?M$J^Q>EnEjXCo@eU2h`rGEC!96lxf--ftwv5XM^%|v|+5Rt-7vm zZi*=+s~}Zh5CQO<30%366*Q_)ivVyL0apY*G?0etA4a;-dir`LMTupZ#d?*wIW!6k z(7GQp6BD#`Kk#)x28Jev3 Date: Thu, 6 Nov 2014 15:24:31 -0800 Subject: [PATCH 044/295] Simplify package requirements. This drops the pinned versions (which aren't really needed anymore) and pulls out several transitive deps (to avoid overspecifying dependencies). --- setup.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index a7ee86e..090b480 100644 --- a/setup.py +++ b/setup.py @@ -28,34 +28,21 @@ except ImportError: # Configure the required packages and scripts to install, depending on # Python version and OS. REQUIRED_PACKAGES = [ - 'httplib2', - 'oauth2client', - 'protorpc', - 'python-dateutil', - 'pytz', + 'httplib2>=0.8', + 'oauth2client>=1.2', + 'protorpc>=0.9.1', ] CLI_PACKAGES = [ - 'google-apputils', - 'python-gflags', + 'google-apputils>=0.4.0', + 'python-gflags>=2.0', ] TESTING_PACKAGES = [ - 'google-apputils', - 'mock', + 'google-apputils>=0.4.0', + 'mock>=1.0.1', ] -PINNED_PACKAGES = [ - 'google-apputils==0.4.0', - 'httplib2==0.8', - 'mock==1.0.1', - 'oauth2client==1.2', - 'protorpc==0.9.1', - 'python-dateutil==1.5', - 'python-gflags==2.0', - 'pytz==2013.7', - ] - CONSOLE_SCRIPTS = [ 'gen_client = apitools.gen.gen_client:run_main', ] @@ -63,8 +50,8 @@ CONSOLE_SCRIPTS = [ py_version = platform.python_version() if py_version < '2.7': - REQUIRED_PACKAGES.append('unittest2==0.5.1') - REQUIRED_PACKAGES.append('argparse==1.2.1') + REQUIRED_PACKAGES.append('unittest2==0.5.1') + REQUIRED_PACKAGES.append('argparse==1.2.1') _APITOOLS_VERSION = '0.3' @@ -83,7 +70,6 @@ setuptools.setup( install_requires=REQUIRED_PACKAGES, tests_require=REQUIRED_PACKAGES + CLI_PACKAGES + TESTING_PACKAGES, extras_require={ - 'pinned': PINNED_PACKAGES, 'cli': CLI_PACKAGES, 'testing': TESTING_PACKAGES, }, -- GitLab From 916952a6c79498320ae93532ac101b0eca78e7ef Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 12 Nov 2014 16:59:57 -0800 Subject: [PATCH 045/295] Make two optional dependencies looser. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 090b480..b96ea6e 100644 --- a/setup.py +++ b/setup.py @@ -50,8 +50,8 @@ CONSOLE_SCRIPTS = [ py_version = platform.python_version() if py_version < '2.7': - REQUIRED_PACKAGES.append('unittest2==0.5.1') - REQUIRED_PACKAGES.append('argparse==1.2.1') + REQUIRED_PACKAGES.append('unittest2>=0.5.1') + REQUIRED_PACKAGES.append('argparse>=1.2.1') _APITOOLS_VERSION = '0.3' -- GitLab From 2316c6b3b1a59c9a683f26b38e33dc927fdb6096 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 17 Nov 2014 12:47:20 -0500 Subject: [PATCH 046/295] Replace 'google.apputils.basetest' w/ 'unittest2.TestCase'. --- apitools/base/py/base_api_test.py | 7 ++++--- apitools/base/py/credentials_lib_test.py | 6 +++--- apitools/base/py/encoding_test.py | 6 +++--- apitools/base/py/extra_types_test.py | 6 +++--- setup.py | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 4ea978f..6d43dea 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -8,7 +8,8 @@ import urllib from protorpc import message_types from protorpc import messages -from google.apputils import basetest as googletest +import unittest2 + from apitools.base.py import base_api from apitools.base.py import http_wrapper @@ -47,7 +48,7 @@ class FakeService(base_api.BaseApiService): super(FakeService, self).__init__(client) -class BaseApiTest(googletest.TestCase): +class BaseApiTest(unittest2.TestCase): def __GetFakeClient(self): return FakeClient('', credentials=FakeCredentials()) @@ -109,4 +110,4 @@ class BaseApiTest(googletest.TestCase): if __name__ == '__main__': - googletest.main() + unittest2.main() diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index ecebb25..e4e461c 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -7,8 +7,8 @@ import StringIO import urllib2 import mock +import unittest2 -from google.apputils import basetest as googletest from apitools.base.py import credentials_lib from apitools.base.py import util @@ -27,7 +27,7 @@ def CreateUriValidator(uri_regexp, content=''): return CheckUri -class CredentialsLibTest(googletest.TestCase): +class CredentialsLibTest(unittest2.TestCase): def _GetServiceCreds(self, service_account_name=None, scopes=None): scopes = scopes or ['scope1'] @@ -51,4 +51,4 @@ class CredentialsLibTest(googletest.TestCase): if __name__ == '__main__': - googletest.main() + unittest2.main() diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 8a6ecc2..77224d6 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -8,8 +8,8 @@ import json from protorpc import message_types from protorpc import messages from protorpc import util +import unittest2 -from google.apputils import basetest as googletest from apitools.base.py import encoding @@ -74,7 +74,7 @@ class ExtraNestedMessage(messages.Message): nested = messages.MessageField(HasNestedMessage, 1) -class EncodingTest(googletest.TestCase): +class EncodingTest(unittest2.TestCase): def testCopyProtoMessage(self): msg = SimpleMessage(field='abc') @@ -266,4 +266,4 @@ TimeMessage( if __name__ == '__main__': - googletest.main() + unittest2.main() diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 8dd816f..457c606 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -6,14 +6,14 @@ import json import math from protorpc import messages +import unittest2 -from google.apputils import basetest as googletest from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import extra_types -class ExtraTypesTest(googletest.TestCase): +class ExtraTypesTest(unittest2.TestCase): def assertRoundTrip(self, value): if isinstance(value, extra_types._JSON_PROTO_TYPES): @@ -172,4 +172,4 @@ class ExtraTypesTest(googletest.TestCase): if __name__ == '__main__': - googletest.main() + unittest2.main() diff --git a/setup.py b/setup.py index b96ea6e..9ca1a5f 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ CLI_PACKAGES = [ TESTING_PACKAGES = [ 'google-apputils>=0.4.0', + 'unittest2', 'mock>=1.0.1', ] @@ -50,7 +51,6 @@ CONSOLE_SCRIPTS = [ py_version = platform.python_version() if py_version < '2.7': - REQUIRED_PACKAGES.append('unittest2>=0.5.1') REQUIRED_PACKAGES.append('argparse>=1.2.1') _APITOOLS_VERSION = '0.3' -- GitLab From fe749c3876cfa6d38eebf31c9644be1b8ca766fa Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 18 Nov 2014 15:42:59 -0500 Subject: [PATCH 047/295] Set minimum bound for unittest2 dependency. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ca1a5f..15a4ace 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ CLI_PACKAGES = [ TESTING_PACKAGES = [ 'google-apputils>=0.4.0', - 'unittest2', + 'unittest2>=0.5.1', 'mock>=1.0.1', ] -- GitLab From 00da26ada7a3bccd27aabb28fa436866b07fd2d1 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 18 Nov 2014 16:00:00 -0500 Subject: [PATCH 048/295] Really make 'gflags' optional. If it can't be imported, then we don't need to fix it up inside 'CredentialsFromFile'. --- apitools/base/py/credentials_lib.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index e467a39..7b7e16f 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -14,7 +14,11 @@ import oauth2client.gce import oauth2client.multistore_file import oauth2client.tools -import gflags as flags +try: + from gflags import FLAGS +except ImportError: + FLAGS = None + import logging from apitools.base.py import exceptions @@ -192,8 +196,8 @@ def CredentialsFromFile(path, client_info): client_info['client_id'], client_info['user_agent'], client_info['scope']) - if hasattr(flags.FLAGS, 'auth_local_webserver'): - flags.FLAGS.auth_local_webserver = False + if getattr(FLAGS, 'auth_local_webserver', FLAGS) is not FLAGS: + FLAGS.auth_local_webserver = False credentials = credential_store.get() if credentials is None or credentials.invalid: print 'Generating new OAuth credentials ...' -- GitLab From 9be51563fd4de3b6aaac0e536c693c8bae69957c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 19 Nov 2014 08:07:46 -0500 Subject: [PATCH 049/295] Use 'hasattr()' rather than 3-arg 'getattr()'. Incorporates feedback from @craigcitro. See: https://github.com/craigcitro/apitools/pull/11/files#r20564006. --- apitools/base/py/credentials_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 7b7e16f..b4d660d 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -196,7 +196,7 @@ def CredentialsFromFile(path, client_info): client_info['client_id'], client_info['user_agent'], client_info['scope']) - if getattr(FLAGS, 'auth_local_webserver', FLAGS) is not FLAGS: + if hasattr(FLAGS, 'auth_local_webserver'): FLAGS.auth_local_webserver = False credentials = credential_store.get() if credentials is None or credentials.invalid: -- GitLab From 08a927cac44354611d4a30cf9bdab916a78793ef Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 20 Nov 2014 10:43:05 -0500 Subject: [PATCH 050/295] Make project name unique. 'apitools' is already taken on PyPI. --- .gitignore | 2 +- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 536e655..c044de5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *~ *.py[cod] -apitools.egg-info/* +*.egg-info/ build/ dist/ distribute-* diff --git a/setup.py b/setup.py index 15a4ace..9c02940 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if py_version < '2.7': _APITOOLS_VERSION = '0.3' setuptools.setup( - name='apitools', + name='google-apitools', version=_APITOOLS_VERSION, description='client libraries for humans', url='http://github.com/craigcitro/apitools', diff --git a/tox.ini b/tox.ini index f25e289..c1426d3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,5 +4,5 @@ envlist = py27 [testenv] deps = nose commands = - pip install apitools[testing] + pip install google_apitools[testing] nosetests -- GitLab From be5fd5b84bb3c0a05ea72237982fcc118022bd3a Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 20 Nov 2014 10:56:32 -0500 Subject: [PATCH 051/295] Remove redundant (now wrong) 'provides' metdata. --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 9c02940..40ca39e 100644 --- a/setup.py +++ b/setup.py @@ -73,9 +73,6 @@ setuptools.setup( 'cli': CLI_PACKAGES, 'testing': TESTING_PACKAGES, }, - provides=[ - 'apitools (%s)' % (_APITOOLS_VERSION,), - ], # PyPI package information. classifiers=[ 'License :: OSI Approved :: Apache Software License', -- GitLab From 623d5d2268258dc547a9d372915603f73eb5d575 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 20 Nov 2014 11:23:03 -0500 Subject: [PATCH 052/295] Include the README as part of the PyPI metadata ('long_description'). Note that this requires changing it from Markdown to RestructuredText. --- README.md | 29 ----------------------------- README.rst | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 4 ++++ 3 files changed, 53 insertions(+), 29 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index 606a3a5..0000000 --- a/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# apitools - -[![Build Status](https://travis-ci.org/craigcitro/apitools.svg?branch=master)](https://travis-ci.org/craigcitro/apitools) - -`apitools` is a collection of utilities to make it easier to build client-side -tools, especially those that talk to Google APIs. - -## Installing as a library - -* `pip install apitools` - -## Installing the command-line tools - -* `pip install apitools[cli]` - -## Installing the testing dependencies - -* `pip install apitools[testing]` - -## Current status - -There are a few imminent large changes: - -* finish the protorpc -> proto2 transition -* switch from httplib2 to requests -* better retry support -* R client library generation -* optional support for `dict -> dict` as the signature on client methods, - doing the proto conversion (and validation!) under the hood. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..4b84393 --- /dev/null +++ b/README.rst @@ -0,0 +1,49 @@ +google-apitools +=============== + +.. image:: https://travis-ci.org/craigcitro/apitools.png?branch=master + :target: https://travis-ci.org/craigcitro/apitools + +``google-apitools`` is a collection of utilities to make it easier to build +client-side tools, especially those that talk to Google APIs. + +Installing as a library +----------------------- + +To install the library into the current virtual environment:: + + $ pip install google-apitools + +Installing the command-line tools +--------------------------------- + +To install the command-line scripts into the current virtual environment:: + + $ pip install google-apitools[cli] + +Running the tests +----------------- + +First, install the testing dependencies:: + + $ pip install google-apitools[testing] + +and the ``nose`` testrunner:: + + $ pip install nose + +Then run the tests:: + + $ nosetests + +Current status +-------------- + +There are a few imminent large changes: + +- finish the protorpc -> proto2 transition +- switch from httplib2 to requests +- better retry support +- R client library generation +- optional support for `dict -> dict` as the signature on client methods, + doing the proto conversion (and validation!) under the hood. diff --git a/setup.py b/setup.py index 40ca39e..3a4f0bc 100644 --- a/setup.py +++ b/setup.py @@ -55,10 +55,14 @@ if py_version < '2.7': _APITOOLS_VERSION = '0.3' +with open('README.rst') as fileobj: + README = fileobj.read() + setuptools.setup( name='google-apitools', version=_APITOOLS_VERSION, description='client libraries for humans', + long_description=README, url='http://github.com/craigcitro/apitools', author='Craig Citro', author_email='craigcitro@google.com', -- GitLab From aa40aecf4d1ac236c7eab3ac36d99a1c5584518e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 20 Nov 2014 12:22:33 -0500 Subject: [PATCH 053/295] Apply results of running 'python-modernize'. As well as additional fixups on the way toward "stradding" Python2 / Py3k. --- apitools/base/py/app2.py | 74 ++++++++++++------------ apitools/base/py/base_api.py | 20 +++---- apitools/base/py/base_api_test.py | 2 +- apitools/base/py/base_cli.py | 2 +- apitools/base/py/batch.py | 14 +++-- apitools/base/py/credentials_lib.py | 13 +++-- apitools/base/py/credentials_lib_test.py | 7 +-- apitools/base/py/encoding.py | 10 ++-- apitools/base/py/extra_types.py | 11 ++-- apitools/base/py/http_wrapper.py | 19 +++--- apitools/base/py/list_pager.py | 2 +- apitools/base/py/transfer.py | 40 +++++++------ apitools/base/py/util.py | 11 ++-- apitools/gen/extended_descriptor.py | 4 +- apitools/gen/gen_client.py | 2 +- apitools/gen/gen_client_lib.py | 5 +- apitools/gen/message_registry.py | 9 +-- apitools/gen/service_registry.py | 25 ++++---- apitools/gen/util.py | 16 ++--- 19 files changed, 148 insertions(+), 138 deletions(-) diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 2a90d55..5301874 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Appcommands-compatible command class with extra fixins.""" +from __future__ import print_function import cmd import inspect @@ -12,6 +13,7 @@ import types from google.apputils import app from google.apputils import appcommands import gflags as flags +import six __all__ = [ 'NewCmd', @@ -28,12 +30,12 @@ FLAGS = flags.FLAGS def _SafeMakeAscii(s): - if isinstance(s, unicode): + if isinstance(s, six.text_type): return s.encode('ascii') elif isinstance(s, str): return s.decode('ascii') else: - return unicode(s).encode('ascii', 'backslashreplace') + return six.text_type(s).encode('ascii', 'backslashreplace') class NewCmd(appcommands.Cmd): @@ -44,7 +46,7 @@ class NewCmd(appcommands.Cmd): run_with_args = getattr(self, 'RunWithArgs', None) self._new_style = isinstance(run_with_args, types.MethodType) if self._new_style: - func = run_with_args.im_func + func = run_with_args.__func__ argspec = inspect.getargspec(func) if argspec.args and argspec.args[0] == 'self': @@ -102,9 +104,9 @@ class NewCmd(appcommands.Cmd): fail = 'Too many positional args; found %d, expected at most %d' % ( len(args), self._max_args) if fail: - print fail + print(fail) if self.usage: - print 'Usage: %s' % (self.usage,) + print('Usage: %s' % (self.usage,)) return 1 if self._debug_mode: @@ -124,7 +126,7 @@ class NewCmd(appcommands.Cmd): def EncodeForPrinting(s): """Safely encode a string as the encoding for sys.stdout.""" encoding = sys.stdout.encoding or 'ascii' - return unicode(s).encode(encoding, 'backslashreplace') + return six.text_type(s).encode(encoding, 'backslashreplace') def _FormatError(self, e): """Hook for subclasses to modify how error messages are printed.""" @@ -132,7 +134,7 @@ class NewCmd(appcommands.Cmd): def _HandleError(self, e): message = self._FormatError(e) - print 'Exception raised in %s operation: %s' % (self._command_name, message) + print('Exception raised in %s operation: %s' % (self._command_name, message)) return 1 def _IsDebuggableException(self, e): @@ -143,22 +145,22 @@ class NewCmd(appcommands.Cmd): """Run this command in debug mode.""" try: return_value = self.RunWithArgs(*args, **kwds) - except BaseException, e: + except BaseException as e: # Don't break into the debugger for expected exceptions. if not self._IsDebuggableException(e): return self._HandleError(e) - print - print '****************************************************' - print '** Unexpected Exception raised in execution! **' + print() + print('****************************************************') + print('** Unexpected Exception raised in execution! **') if FLAGS.headless: - print '** --headless mode enabled, exiting. **' - print '** See STDERR for traceback. **' + print('** --headless mode enabled, exiting. **') + print('** See STDERR for traceback. **') else: - print '** --debug_mode enabled, starting pdb. **' - print '****************************************************' - print + print('** --debug_mode enabled, starting pdb. **') + print('****************************************************') + print() traceback.print_exc() - print + print() if not FLAGS.headless: pdb.post_mortem() return 1 @@ -168,7 +170,7 @@ class NewCmd(appcommands.Cmd): """Run this command, turning exceptions into print statements.""" try: return_value = self.RunWithArgs(*args, **kwds) - except BaseException, e: + except BaseException as e: return self._HandleError(e) return return_value @@ -183,7 +185,7 @@ class CommandLoop(cmd.Cmd): cmd.Cmd.__init__(self) self._commands = {'help': commands['help']} self._special_command_names = ['help', 'repl', 'EOF'] - for name, command in commands.iteritems(): + for name, command in commands.items(): if (name not in self._special_command_names and isinstance(command, NewCmd) and command.surface_in_shell): @@ -218,7 +220,7 @@ class CommandLoop(cmd.Cmd): raise CommandLoop.TerminateSignal() def postloop(self): - print 'Goodbye.' + print('Goodbye.') def completedefault(self, unused_text, line, unused_begidx, unused_endidx): if not line: @@ -229,14 +231,14 @@ class CommandLoop(cmd.Cmd): if command_name in self._commands: usage = self._commands[command_name].usage if usage: - print - print usage - print '%s%s' % (self.prompt, line), + print() + print(usage) + print('%s%s' % (self.prompt, line), end=' ') return [] def emptyline(self): - print 'Available commands:', - print ' '.join(list(self._commands)) + print('Available commands:', end=' ') + print(' '.join(list(self._commands))) def precmd(self, line): """Preprocess the shell input.""" @@ -268,8 +270,8 @@ class CommandLoop(cmd.Cmd): return True except BaseException as e: name = line.split(' ')[0] - print 'Error running %s:' % name - print e + print('Error running %s:' % name) + print(e) self._last_return_code = 1 return False @@ -304,16 +306,16 @@ class CommandLoop(cmd.Cmd): firstline_indent=default_indent) + '\n' if not command_name: - print '\nHelp for commands:\n' + print('\nHelp for commands:\n') command_names = list(self._commands) - print '\n\n'.join( + print('\n\n'.join( FormatOneCmd(name, command, command_names) - for name, command in self._commands.iteritems() - if name not in self._special_command_names) - print + for name, command in self._commands.items() + if name not in self._special_command_names)) + print() elif command_name in self._commands: - print FormatOneCmd(command_name, self._commands[command_name], - command_names=[command_name]) + print(FormatOneCmd(command_name, self._commands[command_name], + command_names=[command_name])) return 0 def postcmd(self, stop, line): @@ -337,11 +339,11 @@ class Repl(NewCmd): """Start an interactive session.""" prompt = FLAGS.prompt or self.PROMPT repl = CommandLoop(appcommands.GetCommandList(), prompt=prompt) - print 'Welcome! (Type help for more information.)' + print('Welcome! (Type help for more information.)') while True: try: repl.cmdloop() break except KeyboardInterrupt: - print + print() return repl.last_return_code diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 2ee5fcf..77acf95 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -3,16 +3,15 @@ import contextlib import datetime -import httplib import logging import pprint -import types import urllib import urlparse - from protorpc import message_types from protorpc import messages +import six +from six.moves import http_client from apitools.base.py import credentials_lib from apitools.base.py import encoding @@ -321,7 +320,7 @@ class BaseApiClient(object): @num_retries.setter def num_retries(self, value): - util.Typecheck(value, (int, long)) + util.Typecheck(value, six.integer_types) if value < 0: raise exceptions.InvalidDataError( 'Cannot have negative value for num_retries') @@ -419,7 +418,7 @@ class BaseApiService(object): method_config.response_type_name) def __CombineGlobalParams(self, global_params, default_params): - util.Typecheck(global_params, (types.NoneType, self.__client.params_type)) + util.Typecheck(global_params, (type(None), self.__client.params_type)) result = self.__client.params_type() global_params = global_params or self.__client.params_type() for field in result.all_fields(): @@ -437,10 +436,10 @@ class BaseApiService(object): for field in self.__client.params_type.all_fields()) query_info.update( (param, getattr(request, param, None)) for param in query_params) - query_info = dict((k, v) for k, v in query_info.iteritems() + query_info = dict((k, v) for k, v in query_info.items() if v is not None) - for k, v in query_info.iteritems(): - if isinstance(v, unicode): + for k, v in query_info.items(): + if isinstance(v, six.text_type): query_info[k] = v.encode('utf8') elif isinstance(v, str): query_info[k] = v.decode('utf8') @@ -468,9 +467,10 @@ class BaseApiService(object): def __ProcessHttpResponse(self, method_config, http_response): """Process the given http response.""" - if http_response.status_code not in (httplib.OK, httplib.NO_CONTENT): + if http_response.status_code not in (http_client.OK, + http_client.NO_CONTENT): raise exceptions.HttpError.FromResponse(http_response) - if http_response.status_code == httplib.NO_CONTENT: + if http_response.status_code == http_client.NO_CONTENT: # TODO(craigcitro): Find out why _replace doesn't seem to work here. http_response = http_wrapper.Response( info=http_response.info, content='{}', diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 6d43dea..f6508c6 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -102,7 +102,7 @@ class BaseApiTest(unittest2.TestCase): request_type_name='MessageWithTime', query_params=['timestamp']) service = FakeService() request = MessageWithTime( - timestamp=datetime.datetime(2014, 10, 07, 12, 53, 13)) + timestamp=datetime.datetime(2014, 10, 7, 12, 53, 13)) http_request = service.PrepareHttpRequest(method_config, request) url_timestamp = urllib.quote(request.timestamp.isoformat()) diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index f9d7d1a..d55394b 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -54,7 +54,7 @@ def DeclareBaseFlags(): flags.DEFINE_enum( 'output_format', 'protorpc', - _OUTPUT_FORMATTER_MAP.viewkeys(), + _OUTPUT_FORMATTER_MAP.keys(), 'Display format for results.') _BASE_FLAGS_DECLARED = True diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index eaf5eba..bd07cd0 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -6,7 +6,6 @@ import email.generator as generator import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart import email.parser as email_parser -import httplib import itertools import StringIO import time @@ -14,6 +13,9 @@ import urllib import urlparse import uuid +from six.moves import range +from six.moves import http_client + from apitools.base.py import exceptions from apitools.base.py import http_wrapper @@ -66,7 +68,7 @@ class BatchApiRequest(object): method_config: Method config for the desired API request. """ self.__retryable_codes = list( - set(retryable_codes + [httplib.UNAUTHORIZED])) + set(retryable_codes + [http_client.UNAUTHORIZED])) self.__http_response = None self.__service = service self.__method_config = method_config @@ -91,7 +93,7 @@ class BatchApiRequest(object): @property def authorization_failed(self): return (self.__http_response and ( - self.__http_response.status_code == httplib.UNAUTHORIZED)) + self.__http_response.status_code == http_client.UNAUTHORIZED)) @property def terminal_state(self): @@ -169,7 +171,7 @@ class BatchApiRequest(object): requests = [request for request in self.api_requests if not request.terminal_state] - for attempt in xrange(max_retries): + for attempt in range(max_retries): if attempt: time.sleep(sleep_between_polls) @@ -280,7 +282,7 @@ class BatchHttpRequest(object): # MIMENonMultipart adds its own Content-Type header. # Keep all of the other headers in headers. - for key, value in request.headers.iteritems(): + for key, value in request.headers.items(): if key == 'content-type': continue msg[key] = value @@ -338,7 +340,7 @@ class BatchHttpRequest(object): Returns: A new unique id string. """ - return str(self.__last_auto_id.next()) + return str(next(self.__last_auto_id)) def Add(self, request, callback=None): """Add a new request. diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index b4d660d..d08fcd9 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -1,8 +1,9 @@ #!/usr/bin/env python """Common credentials classes and constructors.""" +from __future__ import print_function -import httplib import json +import logging import os import urllib2 @@ -13,13 +14,13 @@ import oauth2client.client import oauth2client.gce import oauth2client.multistore_file import oauth2client.tools +from six.moves import http_client try: from gflags import FLAGS except ImportError: FLAGS = None -import logging from apitools.base.py import exceptions from apitools.base.py import util @@ -137,7 +138,7 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'service-accounts/%s/token') % self.__service_account_name extra_headers = {'X-Google-Metadata-Request': 'True'} response, content = do_request(token_uri, headers=extra_headers) - if response.status != httplib.OK: + if response.status != http_client.OK: raise exceptions.CredentialsError( 'Error refreshing credentials: %s' % content) try: @@ -200,7 +201,7 @@ def CredentialsFromFile(path, client_info): FLAGS.auth_local_webserver = False credentials = credential_store.get() if credentials is None or credentials.invalid: - print 'Generating new OAuth credentials ...' + print('Generating new OAuth credentials ...') while True: # If authorization fails, we want to retry, rather than let this # cascade up and get caught elsewhere. If users want out of the @@ -213,9 +214,9 @@ def CredentialsFromFile(path, client_info): # Here SystemExit is "no credential at all", and the # FlowExchangeError is "invalid" -- usually because you reused # a token. - print 'Invalid authorization: %s' % (e,) + print('Invalid authorization: %s' % (e,)) except httplib2.HttpLib2Error as e: - print 'Communication error: %s' % (e,) + print('Communication error: %s' % (e,)) raise exceptions.CredentialsError( 'Communication error creating credentials: %s' % e) return credentials diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index e4e461c..2ad5fc8 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,12 +1,11 @@ #!/usr/bin/env python - -import httplib import re import StringIO import urllib2 import mock +from six.moves import http_client import unittest2 from apitools.base.py import credentials_lib @@ -19,10 +18,10 @@ def CreateUriValidator(uri_regexp, content=''): raise ValueError('Missing required header') if uri_regexp.match(uri): message = content - status = httplib.OK + status = http_client.OK else: message = 'Expected uri matching pattern %s' % uri_regexp.pattern - status = httplib.BAD_REQUEST + status = http_client.BAD_REQUEST return type('HttpResponse', (object,), {'status': status})(), message return CheckUri diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index c44897f..bb15392 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -7,10 +7,10 @@ import datetime import json import logging - from protorpc import message_types from protorpc import messages from protorpc import protojson +import six from apitools.base.py import exceptions @@ -171,7 +171,7 @@ def MessageToRepr(msg, multiline=False, **kwargs): s += ')' return s - if isinstance(msg, basestring): + if isinstance(msg, six.string_types): if kwargs.get('shortstrings') and len(msg) > 100: msg = msg[:100] @@ -283,7 +283,7 @@ class _ProtoJsonApiTools(protojson.ProtoJson): try: field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) except messages.DecodeError: - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): raise field_value = None else: @@ -350,7 +350,7 @@ def _DecodeUnknownMessages(message, encoded_message, pair_type): field_type = pair_type.value.type new_values = [] all_field_names = [x.name for x in message.all_fields()] - for name, value_dict in encoded_message.iteritems(): + for name, value_dict in encoded_message.items(): if name in all_field_names: continue value = PyValueToMessage(field_type, value_dict) @@ -475,7 +475,7 @@ def _ProcessUnknownMessages(message, encoded_message): decoded_message = json.loads(encoded_message) message_fields = [x.name for x in message.all_fields()] + list( message.all_unrecognized_fields()) - missing_fields = [x for x in decoded_message.iterkeys() + missing_fields = [x for x in decoded_message.keys() if x not in message_fields] for field_name in missing_fields: message.set_unrecognized_field(field_name, decoded_message[field_name], diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 4b15683..b848ade 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -13,6 +13,7 @@ import numbers from protorpc import message_types from protorpc import messages from protorpc import protojson +import six from apitools.base.py import encoding from apitools.base.py import exceptions @@ -101,10 +102,10 @@ def _PythonValueToJsonValue(py_value): return JsonValue(is_null=True) if isinstance(py_value, bool): return JsonValue(boolean_value=py_value) - if isinstance(py_value, basestring): + if isinstance(py_value, six.string_types): return JsonValue(string_value=py_value) if isinstance(py_value, numbers.Number): - if isinstance(py_value, (int, long)): + if isinstance(py_value, six.integer_types): if _MININT64 < py_value < _MAXINT64: return JsonValue(integer_value=py_value) return JsonValue(double_value=float(py_value)) @@ -121,11 +122,11 @@ def _PythonValueToJsonObject(py_value): return JsonObject( properties=[ JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) - for key, value in py_value.iteritems()]) + for key, value in py_value.items()]) def _PythonValueToJsonArray(py_value): - return JsonArray(entries=map(_PythonValueToJsonValue, py_value)) + return JsonArray(entries=[_PythonValueToJsonValue(val) for val in py_value]) class JsonValue(messages.Message): @@ -190,7 +191,7 @@ def _PythonValueToJsonProto(py_value): if isinstance(py_value, dict): return _PythonValueToJsonObject(py_value) if (isinstance(py_value, collections.Iterable) and - not isinstance(py_value, basestring)): + not isinstance(py_value, six.string_types)): return _PythonValueToJsonArray(py_value) return _PythonValueToJsonValue(py_value) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 8c3ee28..9ef0faa 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -6,13 +6,14 @@ currently httplib2. """ import collections -import httplib import logging import socket import time import urlparse import httplib2 +from six.moves import http_client +from six.moves import range from apitools.base.py import exceptions from apitools.base.py import util @@ -28,10 +29,10 @@ __all__ = [ RESUME_INCOMPLETE = 308 TOO_MANY_REQUESTS = 429 _REDIRECT_STATUS_CODES = ( - httplib.MOVED_PERMANENTLY, - httplib.FOUND, - httplib.SEE_OTHER, - httplib.TEMPORARY_REDIRECT, + http_client.MOVED_PERMANENTLY, + http_client.FOUND, + http_client.SEE_OTHER, + http_client.TEMPORARY_REDIRECT, RESUME_INCOMPLETE, ) @@ -129,7 +130,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): url_scheme = urlparse.urlsplit(http_request.url).scheme if url_scheme and url_scheme in http.connections: connection_type = http.connections[url_scheme] - for retry in xrange(retries + 1): + for retry in range(retries + 1): # Note that the str() calls here are important for working around # some funny business with message construction and unicode in # httplib itself. See, eg, @@ -140,7 +141,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): str(http_request.url), method=str(http_request.http_method), body=http_request.body, headers=http_request.headers, redirections=redirections, connection_type=connection_type) - except httplib.BadStatusLine as e: + except http_client.BadStatusLine as e: logging.error('Caught BadStatusLine from httplib, retrying: %s', e) exc = e except socket.error as e: @@ -148,7 +149,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): raise logging.error('Caught socket error, retrying: %s', e) exc = e - except httplib.IncompleteRead as e: + except http_client.IncompleteRead as e: if http_request.http_method != 'GET': raise logging.error('Caught IncompleteRead error, retrying: %s', e) @@ -161,7 +162,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): break logging.info('Retrying request to url <%s> after status code %s.', response.request_url, response.status_code) - elif isinstance(exc, httplib.IncompleteRead): + elif isinstance(exc, http_client.IncompleteRead): logging.info('Retrying request to url <%s> after incomplete read.', str(http_request.url)) else: diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index d8f5971..0509b83 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -36,7 +36,7 @@ def YieldFromList( response = getattr(service, method)(request) items = getattr(response, field) if predicate: - items = filter(predicate, items) + items = [item for item in items if predicate(item)] for item in items: yield item if limit is None: diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 610ef2d..8407365 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """Upload and download support for apitools.""" +from __future__ import print_function import email.generator as email_generator import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart -import httplib import io import json import mimetypes @@ -12,6 +12,8 @@ import os import StringIO import threading +from six.moves import http_client + from apitools.base.py import exceptions from apitools.base.py import http_wrapper from apitools.base.py import util @@ -38,7 +40,7 @@ class _Transfer(object): self.__url = None self.auto_transfer = auto_transfer - self.chunksize = chunksize or 1048576L + self.chunksize = chunksize or 1048576 def __repr__(self): return str(self) @@ -121,10 +123,10 @@ class Download(_Transfer): chunksize: default chunksize to use for transfers. """ _ACCEPTABLE_STATUSES = set(( - httplib.OK, - httplib.NO_CONTENT, - httplib.PARTIAL_CONTENT, - httplib.REQUESTED_RANGE_NOT_SATISFIABLE, + http_client.OK, + http_client.NO_CONTENT, + http_client.PARTIAL_CONTENT, + http_client.REQUESTED_RANGE_NOT_SATISFIABLE, )) _REQUIRED_SERIALIZATION_KEYS = set(( 'auto_transfer', 'progress', 'total_size', 'url')) @@ -242,13 +244,13 @@ class Download(_Transfer): @staticmethod def _ArgPrinter(response, unused_download): if 'content-range' in response.info: - print 'Received %s' % response.info['content-range'] + print('Received %s' % response.info['content-range']) else: - print 'Received %d bytes' % len(response) + print('Received %d bytes' % len(response)) @staticmethod def _CompletePrinter(*unused_args): - print 'Download complete' + print('Download complete') def __NormalizeStartEnd(self, start, end=None): if end is not None: @@ -290,10 +292,10 @@ class Download(_Transfer): """Process this response (by updating self and writing to self.stream).""" if response.status_code not in self._ACCEPTABLE_STATUSES: raise exceptions.TransferInvalidError(response.content) - if response.status_code in (httplib.OK, httplib.PARTIAL_CONTENT): + if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): self.stream.write(response.content) self.__progress += len(response) - elif response.status_code == httplib.NO_CONTENT: + elif response.status_code == http_client.NO_CONTENT: # It's important to write something to the stream for the case # of a 0-byte download to a file, as otherwise python won't # create the file. @@ -348,7 +350,7 @@ class Download(_Transfer): additional_headers=additional_headers) response = self.__ProcessResponse(response) self._ExecuteCallback(callback, response) - if (response.status_code == httplib.OK or + if (response.status_code == http_client.OK or self.progress >= self.total_size): break self._ExecuteCallback(finish_callback, response) @@ -591,7 +593,7 @@ class Upload(_Transfer): self.http, refresh_request, redirections=0) range_header = refresh_response.info.get( 'Range', refresh_response.info.get('range')) - if refresh_response.status_code in (httplib.OK, httplib.CREATED): + if refresh_response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: if range_header is None: @@ -619,7 +621,7 @@ class Upload(_Transfer): http_request.url = client.FinalizeTransferUrl(http_request.url) self.EnsureUninitialized() http_response = http_wrapper.MakeRequest(http, http_request) - if http_response.status_code != httplib.OK: + if http_response.status_code != http_client.OK: raise exceptions.HttpError.FromResponse(http_response) self.__server_chunk_granularity = http_response.info.get( @@ -651,11 +653,11 @@ class Upload(_Transfer): @staticmethod def _ArgPrinter(response, unused_upload): - print 'Sent %s' % response.info['range'] + print('Sent %s' % response.info['range']) @staticmethod def _CompletePrinter(*unused_args): - print 'Upload complete' + print('Upload complete') def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=None): @@ -674,7 +676,7 @@ class Upload(_Transfer): while not self.complete: response = self.__SendChunk(self.stream.tell(), additional_headers=additional_headers) - if response.status_code in (httplib.OK, httplib.CREATED): + if response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True break self.__progress = self.__GetLastByte(response.info['range']) @@ -703,10 +705,10 @@ class Upload(_Transfer): request.headers.update(additional_headers) response = http_wrapper.MakeRequest(self.bytes_http, request) - if response.status_code not in (httplib.OK, httplib.CREATED, + if response.status_code not in (http_client.OK, http_client.CREATED, http_wrapper.RESUME_INCOMPLETE): raise exceptions.HttpError.FromResponse(response) - if response.status_code in (httplib.OK, httplib.CREATED): + if response.status_code in (http_client.OK, http_client.CREATED): return response # TODO(craigcitro): Add retries on no progress? last_byte = self.__GetLastByte(response.info['range']) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index cd882a7..18a9bea 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -2,13 +2,14 @@ """Assorted utilities shared between parts of apitools.""" import collections -import httplib import os import random -import types import urllib import urllib2 +import six +from six.moves import http_client + from apitools.base.py import exceptions __all__ = [ @@ -46,13 +47,13 @@ def DetectGce(): o = urllib2.urlopen('http://metadata.google.internal') except urllib2.URLError: return False - return (o.getcode() == httplib.OK and + return (o.getcode() == http_client.OK and o.headers.get('metadata-flavor') == 'Google') def NormalizeScopes(scope_spec): """Normalize scope_spec to a set of strings.""" - if isinstance(scope_spec, types.StringTypes): + if isinstance(scope_spec, str): return set(scope_spec.split(' ')) elif isinstance(scope_spec, collections.Iterable): return set(scope_spec) @@ -99,7 +100,7 @@ def ExpandRelativePath(method_config, params, relative_path=None): raise exceptions.InvalidUserInputError( 'Request missing required parameter %s' % param) try: - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): value = str(value) path = path.replace(param_template, urllib.quote(value.encode('utf_8'), reserved_chars)) diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 9ddf932..66afd73 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -19,6 +19,7 @@ import textwrap from protorpc import descriptor from protorpc import message_types from protorpc import messages +import six import apitools.base.py as apitools_base @@ -154,9 +155,8 @@ def _EmptyMessage(message_type): message_type.fields)) -class ProtoPrinter(object): +class ProtoPrinter(six.with_metaclass(abc.ABCMeta, object)): """Interface for proto printers.""" - __metaclass__ = abc.ABCMeta @abc.abstractmethod def PrintPreamble(self, package, version, file_descriptor): diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 3d654ef..e30a8e8 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -193,7 +193,7 @@ def _WriteGeneratedFiles(codegen): if FLAGS.generate_cli: with open(codegen.client_info.cli_file_name, 'w') as out: codegen.WriteCli(out) - os.chmod(codegen.client_info.cli_file_name, 0755) + os.chmod(codegen.client_info.cli_file_name, 0o755) def _WriteInit(codegen): diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 1c5df9b..3e5b656 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -5,7 +5,6 @@ Relevant links: https://developers.google.com/discovery/v1/reference/apis#resource """ -import json import logging import urlparse @@ -72,7 +71,7 @@ class DescriptorGenerator(object): self.__client_info, self.__names, self.__description, self.__root_package, self.__base_files_package) schemas = self.__discovery_doc.get('schemas', {}) - for schema_name, schema in schemas.iteritems(): + for schema_name, schema in schemas.items(): self.__message_registry.AddDescriptorFromSchema(schema_name, schema) # We need to add one more message type for the global parameters. @@ -99,7 +98,7 @@ class DescriptorGenerator(object): self.__root_package, self.__base_files_package) services = self.__discovery_doc.get('resources', {}) - for service_name, methods in sorted(services.iteritems()): + for service_name, methods in services.items(): self.__services_registry.AddServiceFromResource(service_name, methods) # We might also have top-level methods. api_methods = self.__discovery_doc.get('methods', []) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 1ddcb81..c93d995 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -7,6 +7,7 @@ import json from protorpc import descriptor from protorpc import messages +import six from apitools.gen import extended_descriptor @@ -109,7 +110,7 @@ class MessageRegistry(object): raise ValueError('Malformed MessageRegistry: %s' % mysteries) def __ComputeFullName(self, name): - return '.'.join(map(unicode, self.__current_path[:] + [name])) + return '.'.join([six.text_type(x) for x in self.__current_path + [name]]) def __AddImport(self, new_import): if new_import not in self.__file_descriptor.additional_imports: @@ -239,7 +240,7 @@ class MessageRegistry(object): self.__DeclareDescriptor(message.name) with self.__DescriptorEnv(message): properties = schema.get('properties', {}) - for index, (name, attrs) in enumerate(sorted(properties.iteritems())): + for index, (name, attrs) in enumerate(sorted(properties.items())): field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) message.fields.append(field) if 'additionalProperties' in schema: @@ -333,8 +334,8 @@ class MessageRegistry(object): def __AddIfUnknown(self, type_name): type_name = self.__names.ClassName(type_name) full_type_name = self.__ComputeFullName(type_name) - if (full_type_name not in self.__message_registry.viewkeys() and - type_name not in self.__message_registry.viewkeys()): + if (full_type_name not in self.__message_registry.keys() and + type_name not in self.__message_registry.keys()): self.__unknown_types.add(type_name) def __GetTypeInfo(self, attrs, name_hint): diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 2b3a832..b20a9fc 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -6,7 +6,6 @@ import logging import re import textwrap - from apitools.base.py import base_api # We're a code generator. I don't care. @@ -90,7 +89,7 @@ class ServiceRegistry(object): client_class_name, class_name) printer('self._method_configs = {') with printer.Indent(indent=' '): - for method_name, method_info in method_info_map.iteritems(): + for method_name, method_info in method_info_map.items(): printer("'%s': base_api.ApiMethodInfo(", method_name) with printer.Indent(indent=' '): attrs = sorted(x.name for x in method_info.all_fields()) @@ -103,7 +102,7 @@ class ServiceRegistry(object): printer() printer('self._upload_configs = {') with printer.Indent(indent=' '): - for method_name, method_info in method_info_map.iteritems(): + for method_name, method_info in method_info_map.items(): upload_config = method_info.upload_config if upload_config is not None: printer("'%s': base_api.ApiUploadInfo(", method_name) @@ -115,7 +114,7 @@ class ServiceRegistry(object): printer('}') # Now write each method in turn. - for method_name, method_info in method_info_map.iteritems(): + for method_name, method_info in method_info_map.items(): printer() params = ['self', 'request', 'global_params=None'] if method_info.upload_config: @@ -145,7 +144,7 @@ class ServiceRegistry(object): printer() printer('service %s {', self.__GetServiceClassName(name)) with printer.Indent(): - for method_name, method_info in method_info_map.iteritems(): + for method_name, method_info in method_info_map.items(): for line in textwrap.wrap(method_info.description, printer.CalculateWidth() - 3): printer('// %s', line) @@ -166,7 +165,7 @@ class ServiceRegistry(object): printer('package %s;', self.__package) printer('import "%s";', client_info.messages_proto_file_name) printer() - for name, method_info_map in self.__service_method_info_map.iteritems(): + for name, method_info_map in self.__service_method_info_map.items(): self.__WriteProtoServiceDeclaration(printer, name, method_info_map) def WriteFile(self, printer): @@ -190,7 +189,7 @@ class ServiceRegistry(object): printer() printer('MESSAGES_MODULE = messages') printer() - client_info_items = client_info._asdict().iteritems() # pylint:disable=protected-access + client_info_items = client_info._asdict().items() # pylint:disable=protected-access for attr, val in client_info_items: if attr == 'scopes' and not val: val = ['https://www.googleapis.com/auth/userinfo.email'] @@ -212,10 +211,10 @@ class ServiceRegistry(object): printer(' credentials_args=credentials_args,') printer(' default_global_params=default_global_params,') printer(' additional_http_headers=additional_http_headers)') - for name in self.__service_method_info_map.iterkeys(): + for name in self.__service_method_info_map: printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) - for name, method_info_map in self.__service_method_info_map.iteritems(): + for name, method_info_map in self.__service_method_info_map.items(): self.__WriteSingleService( printer, name, method_info_map, client_info.client_class_name) @@ -276,7 +275,7 @@ class ServiceRegistry(object): return True field_names = [x.name for x in message.fields] parameters = method_description.get('parameters', {}) - for param_name, param_info in parameters.iteritems(): + for param_name, param_info in parameters.items(): if (param_info.get('location') != 'path' or self.__names.CleanName(param_name) not in field_names): break @@ -346,7 +345,7 @@ class ServiceRegistry(object): method_info.supports_download = method_description.get( 'supportsMediaDownload', False) self.__all_scopes.update(method_description.get('scopes', ())) - for param, desc in method_description.get('parameters', {}).iteritems(): + for param, desc in method_description.get('parameters', {}).items(): param = self.__names.CleanName(param) location = desc['location'] if location == 'query': @@ -385,7 +384,7 @@ class ServiceRegistry(object): """Add a new service named service_name with the given methods.""" method_descriptions = methods.get('methods', {}) method_info_map = collections.OrderedDict() - items = sorted(method_descriptions.iteritems()) + items = sorted(method_descriptions.items()) for method_name, method_description in items: method_name = self.__names.MethodName(method_name) @@ -417,7 +416,7 @@ class ServiceRegistry(object): request, response) nested_services = methods.get('resources', {}) - services = sorted(nested_services.iteritems()) + services = sorted(nested_services.items()) for subservice_name, submethods in services: new_service_name = '%s_%s' % (service_name, subservice_name) self.AddServiceFromResource(new_service_name, submethods) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index e07cd16..c3b6dc3 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Assorted utilities shared between parts of apitools.""" +from __future__ import print_function import collections import contextlib @@ -9,7 +10,8 @@ import logging import os import re import urllib2 -import urlparse + +from six.moves import range @@ -163,9 +165,9 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( 'user_agent': user_agent, 'api_key': api_key, } - client_class_name = ''.join( - map(names.ClassName, (client_info['package'], client_info['version']))) - client_info['client_class_name'] = client_class_name + client_info['client_class_name'] = '%s%s' % ( + names.ClassName(client_info['package']), + names.ClassName(client_info['version'])) return cls(**client_info) @property @@ -255,9 +257,9 @@ class SimplePrettyPrinter(object): else: line = args[0].rstrip() line = line.encode('ascii', 'backslashreplace') - print >>self.__out, '%s%s' % (self.__indent, line) + print('%s%s' % (self.__indent, line), file=self.__out) else: - print >>self.__out, '' + print('', file=self.__out) def NormalizeDiscoveryUrl(discovery_url): @@ -276,7 +278,7 @@ def FetchDiscoveryDoc(discovery_url, retries=5): discovery_url = NormalizeDiscoveryUrl(discovery_url) discovery_doc = None last_exception = None - for _ in xrange(retries): + for _ in range(retries): try: discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) break -- GitLab From dea6f9bde42f4a63aa903e7090da8f75e73e7bb8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 26 Nov 2014 22:41:48 -0500 Subject: [PATCH 054/295] Make dependency on 'six' explicit. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 15a4ace..79ab9dc 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ REQUIRED_PACKAGES = [ 'httplib2>=0.8', 'oauth2client>=1.2', 'protorpc>=0.9.1', + 'six>=1.8.0', ] CLI_PACKAGES = [ -- GitLab From d14323195960b781a197331a5013b18eb30fe87d Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 26 Nov 2014 22:45:52 -0500 Subject: [PATCH 055/295] Use a more precise spelling for installing test deps. The 'google_apitools' worked, because distutils normalizes hyphens to underscores under the covers. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c1426d3..e9d2654 100644 --- a/tox.ini +++ b/tox.ini @@ -4,5 +4,5 @@ envlist = py27 [testenv] deps = nose commands = - pip install google_apitools[testing] + pip install google-apitools[testing] nosetests -- GitLab From ad0ef8f6e39b5ebc1e650b3e879d3f7dfea807ac Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 3 Dec 2014 09:22:31 -0800 Subject: [PATCH 056/295] Make one tweak from python-modernize. --- apitools/base/py/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 18a9bea..1097890 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -53,7 +53,7 @@ def DetectGce(): def NormalizeScopes(scope_spec): """Normalize scope_spec to a set of strings.""" - if isinstance(scope_spec, str): + if isinstance(scope_spec, six.string_types): return set(scope_spec.split(' ')) elif isinstance(scope_spec, collections.Iterable): return set(scope_spec) -- GitLab From debb501f06c48ed78547221581088470f93e5fe2 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 3 Dec 2014 10:26:26 -0800 Subject: [PATCH 057/295] Simplify import generation for generated clients. There's a complex web of trickery in place for how I write import lines in generated clients, mostly to make things work both internally and externally. At this point, it's just confusing -- simplifying for now and I'll deal with merging it internally later. --- apitools/gen/command_registry.py | 2 -- apitools/gen/gen_client.py | 17 ++--------------- apitools/gen/service_registry.py | 2 -- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index eac21e1..675d5fa 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -446,8 +446,6 @@ class CommandRegistry(object): printer('from %s import cli as apitools_base_cli', self.__base_files_package) import_prefix = '' - if self.__root_package: - import_prefix = 'from %s ' % self.__root_package printer('%simport %s as client_lib', import_prefix, self.__client_info.client_rule_name) printer('%simport %s as messages', diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index e30a8e8..c4da20e 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -36,11 +36,6 @@ flags.DEFINE_string( flags.DEFINE_boolean( 'overwrite', False, 'Only overwrite the output directory if this flag is specified.') -flags.DEFINE_string( - 'root_package_dir', '', - 'Ultimate destination for generated code (used for generating ' - 'correct import lines). Defaults to the value of FLAGS.outdir.' -) flags.DEFINE_string( 'root_package', '', 'Python import path for where these modules should be imported from.') @@ -148,18 +143,10 @@ def _GetCodegenFromFlags(): raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') - if FLAGS.root_package: - root_package = FLAGS.root_package - else: - if not FLAGS.root_package_dir: - FLAGS.root_package_dir = outdir - FLAGS.root_package_dir = os.path.abspath(FLAGS.root_package_dir) - root_package = ( - util.GetPackage(FLAGS.root_package_dir)) - base_package = FLAGS.base_package + root_package = FLAGS.root_package or util.GetPackage(outdir) return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, - base_package=base_package, + base_package=FLAGS.base_package, generate_cli=FLAGS.generate_cli, use_proto2=FLAGS.experimental_proto2_output) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index b20a9fc..0942ccb 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -176,8 +176,6 @@ class ServiceRegistry(object): client_info.package, client_info.version) printer('from %s import base_api', self.__base_files_package) import_prefix = '' - if self.__root_package_dir: - import_prefix = 'from %s ' % self.__root_package_dir printer('%simport %s as messages', import_prefix, client_info.messages_rule_name) printer() -- GitLab From 5a1bab1df7a474cff57f7f5cc066b19a1c286a21 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 3 Dec 2014 10:37:36 -0800 Subject: [PATCH 058/295] Add defaults for client ID/client secret. --- apitools/gen/gen_client.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index c4da20e..96326a3 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -53,10 +53,10 @@ flags.DEFINE_string( 'Use the given file downloaded from the dev. console for client_id ' 'and client_secret.') flags.DEFINE_string( - 'client_id', None, + 'client_id', '1042881264118.apps.googleusercontent.com', 'Client ID to use for the generated client.') flags.DEFINE_string( - 'client_secret', None, + 'client_secret', 'x_Tw5K8nnjoRAqULM9PFAC2b', 'Client secret for the generated client.') flags.DEFINE_multistring( 'scope', [], @@ -127,13 +127,11 @@ def _GetCodegenFromFlags(): client_id = FLAGS.client_id client_secret = FLAGS.client_secret - if client_id is None: + if not client_id: logging.warning('No client ID supplied') - client_id = '' - if client_secret is None: + if not client_secret: logging.warning('No client secret supplied') - client_secret = '' client_info = util.ClientInfo.Create( discovery_doc, FLAGS.scope, client_id, client_secret, -- GitLab From 384627080ffa663592bc0c3399075f63e25e8348 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 21 Jan 2015 19:25:49 -0800 Subject: [PATCH 059/295] Add custom query encoding for prettyPrint, and fix a bug. It turns out the prettyPrint flag is particular about how it's encoded (it wants either lower-case true/false or 1/0), and apitools had been sending values that got ignored. Digging in, though, it turns out there was a subtler bug: we would never include a query param whose value was False-y if it was in StandardQueryParameters. This was just prettyPrint, but nice to fix anyway. --- apitools/base/py/base_api.py | 10 ++++++++-- apitools/base/py/base_api_test.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 77acf95..e88bb80 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -418,12 +418,14 @@ class BaseApiService(object): method_config.response_type_name) def __CombineGlobalParams(self, global_params, default_params): + """Combine the given params with the defaults.""" util.Typecheck(global_params, (type(None), self.__client.params_type)) result = self.__client.params_type() global_params = global_params or self.__client.params_type() for field in result.all_fields(): - value = (global_params.get_assigned_value(field.name) or - default_params.get_assigned_value(field.name)) + value = global_params.get_assigned_value(field.name) + if value is None: + value = default_params.get_assigned_value(field.name) if value not in (None, [], ()): setattr(result, field.name, value) return result @@ -438,6 +440,10 @@ class BaseApiService(object): (param, getattr(request, param, None)) for param in query_params) query_info = dict((k, v) for k, v in query_info.items() if v is not None) + # The prettyPrint flag needs custom encoding: it should be encoded + # as 0 if False, and ignored otherwise (True is the default). + if not query_info.pop('prettyPrint', True): + query_info['prettyPrint'] = 0 for k, v in query_info.items(): if isinstance(v, six.text_type): query_info[k] = v.encode('utf8') diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index f6508c6..f6993ff 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -24,6 +24,7 @@ class MessageWithTime(messages.Message): class StandardQueryParameters(messages.Message): field = messages.StringField(1) + prettyPrint = messages.BooleanField(5, default=True) # pylint: disable=invalid-name class FakeCredentials(object): @@ -108,6 +109,23 @@ class BaseApiTest(unittest2.TestCase): url_timestamp = urllib.quote(request.timestamp.isoformat()) self.assertTrue(http_request.url.endswith(url_timestamp)) + def testPrettyPrintEncoding(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithTime', query_params=['timestamp']) + service = FakeService() + request = MessageWithTime( + timestamp=datetime.datetime(2014, 10, 07, 12, 53, 13)) + + global_params = StandardQueryParameters() + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + self.assertFalse('prettyPrint' in http_request.url) + + global_params.prettyPrint = False # pylint: disable=invalid-name + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + self.assertTrue('prettyPrint=0' in http_request.url) + if __name__ == '__main__': unittest2.main() -- GitLab From 901342e3fca6edceb1dfacd3c63e60d92cf5e72b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 30 Jan 2015 13:19:47 -0500 Subject: [PATCH 060/295] More Python3 compatibility. Fixes needed to get stuff running in gcloud-python. --- apitools/base/py/http_wrapper.py | 4 ++-- apitools/base/py/transfer.py | 3 +-- apitools/base/py/util.py | 11 ++++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 9ef0faa..41bdcd9 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -9,11 +9,11 @@ import collections import logging import socket import time -import urlparse import httplib2 from six.moves import http_client from six.moves import range +from six.moves.urllib.parse import urlsplit from apitools.base.py import exceptions from apitools.base.py import util @@ -127,7 +127,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): # wants control over the underlying connection for managing callbacks # or hash digestion. if getattr(http, 'connections', None): - url_scheme = urlparse.urlsplit(http_request.url).scheme + url_scheme = urlsplit(http_request.url).scheme if url_scheme and url_scheme in http.connections: connection_type = http.connections[url_scheme] for retry in range(retries + 1): diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 8407365..4a1412a 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -9,7 +9,6 @@ import io import json import mimetypes import os -import StringIO import threading from six.moves import http_client @@ -568,7 +567,7 @@ class Upload(_Transfer): # encode the body: note that we can't use `as_string`, because # it plays games with `From ` lines. - fp = StringIO.StringIO() + fp = io.StringIO() g = email_generator.Generator(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 1097890..1ef0b46 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -4,11 +4,12 @@ import collections import os import random -import urllib -import urllib2 import six from six.moves import http_client +from six.moves.urllib.error import URLError +from six.moves.urllib.parse import quote +from six.moves.urllib.request import urlopen from apitools.base.py import exceptions @@ -44,8 +45,8 @@ def DetectGce(): True iff we're running on a GCE instance. """ try: - o = urllib2.urlopen('http://metadata.google.internal') - except urllib2.URLError: + o = urlopen('http://metadata.google.internal') + except URLError: return False return (o.getcode() == http_client.OK and o.headers.get('metadata-flavor') == 'Google') @@ -103,7 +104,7 @@ def ExpandRelativePath(method_config, params, relative_path=None): if not isinstance(value, six.string_types): value = str(value) path = path.replace(param_template, - urllib.quote(value.encode('utf_8'), reserved_chars)) + quote(value.encode('utf_8'), reserved_chars)) except TypeError as e: raise exceptions.InvalidUserInputError( 'Error setting required parameter %s to value %s: %s' % ( -- GitLab From 4bd261083749d5b0e23fc25458457d09761b0f0c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 17 Feb 2015 16:57:03 -0800 Subject: [PATCH 061/295] Update the storage client in samples/. Hopefully the last time I'll do that manually. --- samples/storage_sample/storage/__init__.py | 1 + samples/storage_sample/storage/storage_v1.py | 3149 +++++++++-------- .../storage/storage_v1_client.py | 794 ++--- .../storage/storage_v1_messages.py | 104 +- 4 files changed, 2083 insertions(+), 1965 deletions(-) diff --git a/samples/storage_sample/storage/__init__.py b/samples/storage_sample/storage/__init__.py index 5ff5bb5..6426fac 100644 --- a/samples/storage_sample/storage/__init__.py +++ b/samples/storage_sample/storage/__init__.py @@ -1,4 +1,5 @@ """Common imports for generated storage client library.""" +# pylint:disable=wildcard-import import pkgutil diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage/storage_v1.py index a882e17..97627e2 100755 --- a/samples/storage_sample/storage/storage_v1.py +++ b/samples/storage_sample/storage/storage_v1.py @@ -13,6 +13,7 @@ from google.apputils import appcommands import gflags as flags import apitools.base.py as apitools_base +from apitools.base.py import cli as apitools_base_cli import storage_v1_client as client_lib import storage_v1_messages as messages @@ -30,6 +31,10 @@ def _DeclareStorageFlags(): 'history_file', u'~/.storage.v1.history', 'File with interactive shell history.') + flags.DEFINE_multistring( + 'add_header', [], + 'Additional http headers (as key=value strings). Can be ' + 'specified multiple times.') flags.DEFINE_enum( 'alt', u'json', @@ -72,7 +77,7 @@ def _DeclareStorageFlags(): FLAGS = flags.FLAGS -apitools_base.DeclareBaseFlags() +apitools_base_cli.DeclareBaseFlags() _DeclareStorageFlags() @@ -103,10 +108,12 @@ def GetClientFromFlags(): log_request = FLAGS.log_request or FLAGS.log_request_response log_response = FLAGS.log_response or FLAGS.log_request_response api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) + additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header) try: client = client_lib.StorageV1( api_endpoint, log_request=log_request, - log_response=log_response) + log_response=log_response, + additional_http_headers=additional_http_headers) except apitools_base.CredentialsError as e: print 'Error creating credentials: %s' % e sys.exit(1) @@ -114,6 +121,7 @@ def GetClientFromFlags(): class PyShell(appcommands.Cmd): + def Run(self, _): """Run an interactive python shell with the client.""" client = GetClientFromFlags() @@ -135,7 +143,7 @@ class PyShell(appcommands.Cmd): 'messages': messages, } if platform.system() == 'Linux': - console = apitools_base.ConsoleWithReadline( + console = apitools_base_cli.ConsoleWithReadline( local_vars, histfile=FLAGS.history_file) else: console = code.InteractiveConsole(local_vars) @@ -145,17 +153,17 @@ class PyShell(appcommands.Cmd): return e.code -class BucketAccessControlsDelete(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.Delete.""" +class DefaultObjectAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Delete.""" - usage = """bucketAccessControls_delete """ + usage = """defaultObjectAccessControls_delete """ def __init__(self, name, fv): - super(BucketAccessControlsDelete, self).__init__(name, fv) + super(DefaultObjectAccessControlsDelete, self).__init__(name, fv) def RunWithArgs(self, bucket, entity): - """Permanently deletes the ACL entry for the specified entity on the - specified bucket. + """Permanently deletes the default object ACL entry for the specified + entity on the specified bucket. Args: bucket: Name of a bucket. @@ -165,25 +173,26 @@ class BucketAccessControlsDelete(apitools_base.NewCmd): """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsDeleteRequest( + request = messages.StorageDefaultObjectAccessControlsDeleteRequest( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) - result = client.bucketAccessControls.Delete( + result = client.defaultObjectAccessControls.Delete( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsGet(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.Get.""" +class DefaultObjectAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Get.""" - usage = """bucketAccessControls_get """ + usage = """defaultObjectAccessControls_get """ def __init__(self, name, fv): - super(BucketAccessControlsGet, self).__init__(name, fv) + super(DefaultObjectAccessControlsGet, self).__init__(name, fv) def RunWithArgs(self, bucket, entity): - """Returns the ACL entry for the specified entity on the specified bucket. + """Returns the default object ACL entry for the specified entity on the + specified bucket. Args: bucket: Name of a bucket. @@ -193,22 +202,22 @@ class BucketAccessControlsGet(apitools_base.NewCmd): """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsGetRequest( + request = messages.StorageDefaultObjectAccessControlsGetRequest( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) - result = client.bucketAccessControls.Get( + result = client.defaultObjectAccessControls.Get( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsInsert(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.Insert.""" +class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Insert.""" - usage = """bucketAccessControls_insert """ + usage = """defaultObjectAccessControls_insert """ def __init__(self, name, fv): - super(BucketAccessControlsInsert, self).__init__(name, fv) + super(DefaultObjectAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -241,6 +250,11 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) flags.DEFINE_string( 'id', None, @@ -248,9 +262,14 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -260,8 +279,7 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', + u'The access permission for the entity. Can be READER or OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -270,7 +288,7 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): flag_values=fv) def RunWithArgs(self, bucket): - """Creates a new ACL entry on the specified bucket. + """Creates a new default object ACL entry on the specified bucket. Args: bucket: The name of the bucket. @@ -288,17 +306,18 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. + role: The access permission for the entity. Can be READER or OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( + request = messages.ObjectAccessControl( bucket=bucket.decode('utf8'), ) if FLAGS['domain'].present: @@ -311,52 +330,78 @@ class BucketAccessControlsInsert(apitools_base.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Insert( + result = client.defaultObjectAccessControls.Insert( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsList(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.List.""" +class DefaultObjectAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.List.""" - usage = """bucketAccessControls_list """ + usage = """defaultObjectAccessControls_list """ def __init__(self, name, fv): - super(BucketAccessControlsList, self).__init__(name, fv) + super(DefaultObjectAccessControlsList, self).__init__(name, fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"If present, only return default ACL listing if the bucket's current" + u' metageneration matches this value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"If present, only return default ACL listing if the bucket's current" + u' metageneration does not match the given value.', + flag_values=fv) def RunWithArgs(self, bucket): - """Retrieves ACL entries on the specified bucket. + """Retrieves default object ACL entries on the specified bucket. Args: bucket: Name of a bucket. + + Flags: + ifMetagenerationMatch: If present, only return default ACL listing if + the bucket's current metageneration matches this value. + ifMetagenerationNotMatch: If present, only return default ACL listing if + the bucket's current metageneration does not match the given value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsListRequest( + request = messages.StorageDefaultObjectAccessControlsListRequest( bucket=bucket.decode('utf8'), ) - result = client.bucketAccessControls.List( + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.defaultObjectAccessControls.List( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsPatch(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.Patch.""" +class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Patch.""" - usage = """bucketAccessControls_patch """ + usage = """defaultObjectAccessControls_patch """ def __init__(self, name, fv): - super(BucketAccessControlsPatch, self).__init__(name, fv) + super(DefaultObjectAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -377,6 +422,11 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) flags.DEFINE_string( 'id', None, @@ -384,9 +434,14 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -396,8 +451,7 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', + u'The access permission for the entity. Can be READER or OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -406,8 +460,8 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): flag_values=fv) def RunWithArgs(self, bucket, entity): - """Updates an ACL entry on the specified bucket. This method supports - patch semantics. + """Updates a default object ACL entry on the specified bucket. This method + supports patch semantics. Args: bucket: The name of the bucket. @@ -425,17 +479,18 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): email: The email address associated with the entity, if any. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. + role: The access permission for the entity. Can be READER or OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( + request = messages.ObjectAccessControl( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) @@ -447,28 +502,32 @@ class BucketAccessControlsPatch(apitools_base.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Patch( + result = client.defaultObjectAccessControls.Patch( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsUpdate(apitools_base.NewCmd): - """Command wrapping bucketAccessControls.Update.""" +class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Update.""" - usage = """bucketAccessControls_update """ + usage = """defaultObjectAccessControls_update """ def __init__(self, name, fv): - super(BucketAccessControlsUpdate, self).__init__(name, fv) + super(DefaultObjectAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -489,6 +548,11 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) + flags.DEFINE_string( + 'generation', + None, + u'The content generation of the object.', + flag_values=fv) flags.DEFINE_string( 'id', None, @@ -496,9 +560,14 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -508,8 +577,7 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', + u'The access permission for the entity. Can be READER or OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -518,7 +586,7 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): flag_values=fv) def RunWithArgs(self, bucket, entity): - """Updates an ACL entry on the specified bucket. + """Updates a default object ACL entry on the specified bucket. Args: bucket: The name of the bucket. @@ -536,17 +604,18 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): email: The email address associated with the entity, if any. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. + role: The access permission for the entity. Can be READER or OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( + request = messages.ObjectAccessControl( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) @@ -558,382 +627,454 @@ class BucketAccessControlsUpdate(apitools_base.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Update( + result = client.defaultObjectAccessControls.Update( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsDelete(apitools_base.NewCmd): - """Command wrapping buckets.Delete.""" +class BucketAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Delete.""" - usage = """buckets_delete """ + usage = """bucketAccessControls_delete """ def __init__(self, name, fv): - super(BucketsDelete, self).__init__(name, fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u'If set, only deletes the bucket if its metageneration matches this ' - u'value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u'If set, only deletes the bucket if its metageneration does not ' - u'match this value.', - flag_values=fv) + super(BucketAccessControlsDelete, self).__init__(name, fv) - def RunWithArgs(self, bucket): - """Permanently deletes an empty bucket. + def RunWithArgs(self, bucket, entity): + """Permanently deletes the ACL entry for the specified entity on the + specified bucket. Args: bucket: Name of a bucket. - - Flags: - ifMetagenerationMatch: If set, only deletes the bucket if its - metageneration matches this value. - ifMetagenerationNotMatch: If set, only deletes the bucket if its - metageneration does not match this value. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsDeleteRequest( + request = messages.StorageBucketAccessControlsDeleteRequest( bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.buckets.Delete( + result = client.bucketAccessControls.Delete( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsGet(apitools_base.NewCmd): - """Command wrapping buckets.Get.""" +class BucketAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Get.""" - usage = """buckets_get """ + usage = """bucketAccessControls_get """ def __init__(self, name, fv): - super(BucketsGet, self).__init__(name, fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) + super(BucketAccessControlsGet, self).__init__(name, fv) - def RunWithArgs(self, bucket): - """Returns metadata for the specified bucket. + def RunWithArgs(self, bucket, entity): + """Returns the ACL entry for the specified entity on the specified bucket. Args: bucket: Name of a bucket. - - Flags: - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. - projection: Set of properties to return. Defaults to noAcl. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsGetRequest( + request = messages.StorageBucketAccessControlsGetRequest( bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['projection'].present: - request.projection = messages.StorageBucketsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Get( + result = client.bucketAccessControls.Get( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsInsert(apitools_base.NewCmd): - """Command wrapping buckets.Insert.""" +class BucketAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Insert.""" - usage = """buckets_insert """ + usage = """bucketAccessControls_insert """ def __init__(self, name, fv): - super(BucketsInsert, self).__init__(name, fv) + super(BucketAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( - 'bucket', + 'domain', None, - u'A Bucket resource to be passed as the request body.', + u'The domain associated with the entity, if any.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + flags.DEFINE_string( + 'email', + None, + u'The email address associated with the entity, if any.', flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the bucket ' - u'resource specifies acl or defaultObjectAcl properties, when it ' - u'defaults to full.', + flags.DEFINE_string( + 'entity', + None, + u'The entity holding the permission, in one of the following forms: ' + u'- user-userId - user-email - group-groupId - group-email - ' + u'domain-domain - project-team-projectId - allUsers - ' + u'allAuthenticatedUsers Examples: - The user liz@example.com would ' + u'be user-liz@example.com. - The group example@googlegroups.com ' + u'would be group-example@googlegroups.com. - To refer to all members' + u' of the Google Apps for Business domain example.com, the entity ' + u'would be domain-example.com.', + flag_values=fv) + flags.DEFINE_string( + 'entityId', + None, + u'The ID for the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, project): - """Creates a new bucket. + def RunWithArgs(self, bucket): + """Creates a new ACL entry on the specified bucket. Args: - project: A valid API project identifier. + bucket: The name of the bucket. Flags: - bucket: A Bucket resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this bucket. - projection: Set of properties to return. Defaults to noAcl, unless the - bucket resource specifies acl or defaultObjectAcl properties, when it - defaults to full. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsInsertRequest( - project=project.decode('utf8'), + request = messages.BucketAccessControl( + bucket=bucket.decode('utf8'), ) - if FLAGS['bucket'].present: - request.bucket = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucket) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageBucketsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Insert( + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entity'].present: + request.entity = FLAGS.entity.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Insert( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsList(apitools_base.NewCmd): - """Command wrapping buckets.List.""" +class BucketAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.List.""" - usage = """buckets_list """ + usage = """bucketAccessControls_list """ def __init__(self, name, fv): - super(BucketsList, self).__init__(name, fv) - flags.DEFINE_integer( - 'maxResults', - None, - u'Maximum number of buckets to return.', - flag_values=fv) - flags.DEFINE_string( - 'pageToken', - None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) + super(BucketAccessControlsList, self).__init__(name, fv) - def RunWithArgs(self, project): - """Retrieves a list of buckets for a given project. + def RunWithArgs(self, bucket): + """Retrieves ACL entries on the specified bucket. Args: - project: A valid API project identifier. - - Flags: - maxResults: Maximum number of buckets to return. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - projection: Set of properties to return. Defaults to noAcl. + bucket: Name of a bucket. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsListRequest( - project=project.decode('utf8'), + request = messages.StorageBucketAccessControlsListRequest( + bucket=bucket.decode('utf8'), ) - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['projection'].present: - request.projection = messages.StorageBucketsListRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.List( + result = client.bucketAccessControls.List( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsPatch(apitools_base.NewCmd): - """Command wrapping buckets.Patch.""" +class BucketAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Patch.""" - usage = """buckets_patch """ + usage = """bucketAccessControls_patch """ def __init__(self, name, fv): - super(BucketsPatch, self).__init__(name, fv) + super(BucketAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( - 'bucketResource', + 'domain', None, - u'A Bucket resource to be passed as the request body.', + u'The domain associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'email', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", + u'The email address associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'entityId', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + u'The ID for the entity, if any.', flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, bucket): - """Updates a bucket. This method supports patch semantics. + def RunWithArgs(self, bucket, entity): + """Updates an ACL entry on the specified bucket. This method supports + patch semantics. Args: - bucket: Name of a bucket. + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. Flags: - bucketResource: A Bucket resource to be passed as the request body. - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. - predefinedAcl: Apply a predefined set of access controls to this bucket. - projection: Set of properties to return. Defaults to full. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsPatchRequest( + request = messages.BucketAccessControl( bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['bucketResource'].present: - request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageBucketsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Patch( + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Patch( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class BucketsUpdate(apitools_base.NewCmd): - """Command wrapping buckets.Update.""" +class BucketAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Update.""" - usage = """buckets_update """ + usage = """bucketAccessControls_update """ def __init__(self, name, fv): - super(BucketsUpdate, self).__init__(name, fv) + super(BucketAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'bucketResource', + 'domain', None, - u'A Bucket resource to be passed as the request body.', + u'The domain associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'email', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", + u'The email address associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'entityId', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", + u'The ID for the entity, if any.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, bucket): - """Updates a bucket. + def RunWithArgs(self, bucket, entity): + """Updates an ACL entry on the specified bucket. Args: - bucket: Name of a bucket. + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. Flags: - bucketResource: A Bucket resource to be passed as the request body. - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. - predefinedAcl: Apply a predefined set of access controls to this bucket. - projection: Set of properties to return. Defaults to full. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + id: The ID of the access-control entry. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. + selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsUpdateRequest( + request = messages.BucketAccessControl( bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['bucketResource'].present: - request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageBucketsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Update( + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.bucketAccessControls.Update( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ChannelsStop(apitools_base.NewCmd): +class ChannelsStop(apitools_base_cli.NewCmd): """Command wrapping channels.Stop.""" usage = """channels_stop""" @@ -1043,594 +1184,703 @@ class ChannelsStop(apitools_base.NewCmd): request.type = FLAGS.type.decode('utf8') result = client.channels.Stop( request, global_params=global_params) - print apitools_base.FormatOutput(result) - - -class DefaultObjectAccessControlsDelete(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.Delete.""" - - usage = """defaultObjectAccessControls_delete """ - - def __init__(self, name, fv): - super(DefaultObjectAccessControlsDelete, self).__init__(name, fv) - - def RunWithArgs(self, bucket, entity): - """Permanently deletes the default object ACL entry for the specified - entity on the specified bucket. - - Args: - bucket: Name of a bucket. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsDeleteRequest( - bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), - ) - result = client.defaultObjectAccessControls.Delete( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsGet(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.Get.""" +class ObjectsCompose(apitools_base_cli.NewCmd): + """Command wrapping objects.Compose.""" - usage = """defaultObjectAccessControls_get """ + usage = """objects_compose """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsGet, self).__init__(name, fv) - - def RunWithArgs(self, bucket, entity): - """Returns the default object ACL entry for the specified entity on the - specified bucket. + super(ObjectsCompose, self).__init__(name, fv) + flags.DEFINE_string( + 'composeRequest', + None, + u'A ComposeRequest resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, destinationBucket, destinationObject): + """Concatenates a list of existing objects into a new object in the same + bucket. Args: - bucket: Name of a bucket. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. + destinationBucket: Name of the bucket in which to store the new object. + destinationObject: Name of the new object. + + Flags: + composeRequest: A ComposeRequest resource to be passed as the request + body. + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsGetRequest( - bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), + request = messages.StorageObjectsComposeRequest( + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), ) - result = client.defaultObjectAccessControls.Get( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + if FLAGS['composeRequest'].present: + request.composeRequest = apitools_base.JsonToMessage(messages.ComposeRequest, FLAGS.composeRequest) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Compose( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsInsert(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.Insert.""" +class ObjectsCopy(apitools_base_cli.NewCmd): + """Command wrapping objects.Copy.""" - usage = """defaultObjectAccessControls_insert """ + usage = """objects_copy """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsInsert, self).__init__(name, fv) + super(ObjectsCopy, self).__init__(name, fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) flags.DEFINE_string( - 'domain', + 'ifGenerationMatch', None, - u'The domain associated with the entity, if any.', + u"Makes the operation conditional on whether the destination object's" + u' current generation matches the given value.', flag_values=fv) flags.DEFINE_string( - 'email', + 'ifGenerationNotMatch', None, - u'The email address associated with the entity, if any.', + u"Makes the operation conditional on whether the destination object's" + u' current generation does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'entity', + 'ifMetagenerationMatch', None, - u'The entity holding the permission, in one of the following forms: ' - u'- user-userId - user-email - group-groupId - group-email - ' - u'domain-domain - project-team-projectId - allUsers - ' - u'allAuthenticatedUsers Examples: - The user liz@example.com would ' - u'be user-liz@example.com. - The group example@googlegroups.com ' - u'would be group-example@googlegroups.com. - To refer to all members' - u' of the Google Apps for Business domain example.com, the entity ' - u'would be domain-example.com.', + u"Makes the operation conditional on whether the destination object's" + u' current metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( - 'entityId', + 'ifMetagenerationNotMatch', None, - u'The ID for the entity, if any.', + u"Makes the operation conditional on whether the destination object's" + u' current metageneration does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'etag', + 'ifSourceGenerationMatch', None, - u'HTTP 1.1 Entity tag for the access-control entry.', + u"Makes the operation conditional on whether the source object's " + u'generation matches the given value.', flag_values=fv) flags.DEFINE_string( - 'generation', + 'ifSourceGenerationNotMatch', None, - u'The content generation of the object.', + u"Makes the operation conditional on whether the source object's " + u'generation does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'id', + 'ifSourceMetagenerationMatch', None, - u'The ID of the access-control entry.', + u"Makes the operation conditional on whether the source object's " + u'current metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( - 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', + 'ifSourceMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration does not match the given value.', flag_values=fv) flags.DEFINE_string( 'object', None, - u'The name of the object.', + u'A Object resource to be passed as the request body.', flag_values=fv) - flags.DEFINE_string( - 'projectTeam', - None, - u'The project team associated with the entity, if any.', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', flag_values=fv) flags.DEFINE_string( - 'role', + 'sourceGeneration', None, - u'The access permission for the entity. Can be READER or OWNER.', + u'If present, selects a specific revision of the source object (as ' + u'opposed to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', flag_values=fv) - def RunWithArgs(self, bucket): - """Creates a new default object ACL entry on the specified bucket. + def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): + """Copies an object to a specified location. Optionally overrides + metadata. Args: - bucket: The name of the bucket. + sourceBucket: Name of the bucket in which to find the source object. + sourceObject: Name of the source object. + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's + name value, if any. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. - id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. - selfLink: The link to this access-control entry. + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given + value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether + the source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on + whether the source object's current metageneration does not match the + given value. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( - bucket=bucket.decode('utf8'), + request = messages.StorageObjectsCopyRequest( + sourceBucket=sourceBucket.decode('utf8'), + sourceObject=sourceObject.decode('utf8'), + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entity'].present: - request.entity = FLAGS.entity.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') - if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Insert( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['ifSourceGenerationMatch'].present: + request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) + if FLAGS['ifSourceGenerationNotMatch'].present: + request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) + if FLAGS['ifSourceMetagenerationMatch'].present: + request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) + if FLAGS['ifSourceMetagenerationNotMatch'].present: + request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsCopyRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['sourceGeneration'].present: + request.sourceGeneration = int(FLAGS.sourceGeneration) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Copy( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsList(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.List.""" +class ObjectsDelete(apitools_base_cli.NewCmd): + """Command wrapping objects.Delete.""" - usage = """defaultObjectAccessControls_list """ + usage = """objects_delete """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsList, self).__init__(name, fv) + super(ObjectsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, permanently deletes a specific revision of this object ' + u'(as opposed to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u"If present, only return default ACL listing if the bucket's current" - u' metageneration matches this value.', + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u"If present, only return default ACL listing if the bucket's current" - u' metageneration does not match the given value.', + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', flag_values=fv) - def RunWithArgs(self, bucket): - """Retrieves default object ACL entries on the specified bucket. + def RunWithArgs(self, bucket, object): + """Deletes an object and its metadata. Deletions are permanent if + versioning is not enabled for the bucket, or if the generation parameter + is used. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. + object: Name of the object. Flags: - ifMetagenerationMatch: If present, only return default ACL listing if - the bucket's current metageneration matches this value. - ifMetagenerationNotMatch: If present, only return default ACL listing if - the bucket's current metageneration does not match the given value. + generation: If present, permanently deletes a specific revision of this + object (as opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsListRequest( + request = messages.StorageObjectsDeleteRequest( bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.defaultObjectAccessControls.List( + result = client.objects.Delete( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsPatch(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.Patch.""" +class ObjectsGet(apitools_base_cli.NewCmd): + """Command wrapping objects.Get.""" - usage = """defaultObjectAccessControls_patch """ + usage = """objects_get """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsPatch, self).__init__(name, fv) - flags.DEFINE_string( - 'domain', - None, - u'The domain associated with the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'email', - None, - u'The email address associated with the entity, if any.', - flag_values=fv) + super(ObjectsGet, self).__init__(name, fv) flags.DEFINE_string( - 'entityId', + 'generation', None, - u'The ID for the entity, if any.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'etag', + 'ifGenerationMatch', None, - u'HTTP 1.1 Entity tag for the access-control entry.', + u"Makes the operation conditional on whether the object's generation " + u'matches the given value.', flag_values=fv) flags.DEFINE_string( - 'generation', + 'ifGenerationNotMatch', None, - u'The content generation of the object.', + u"Makes the operation conditional on whether the object's generation " + u'does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'id', + 'ifMetagenerationMatch', None, - u'The ID of the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( - 'object', + 'ifMetagenerationNotMatch', None, - u'The name of the object.', + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', flag_values=fv) - flags.DEFINE_string( - 'projectTeam', - None, - u'The project team associated with the entity, if any.', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', flag_values=fv) flags.DEFINE_string( - 'role', - None, - u'The access permission for the entity. Can be READER or OWNER.', + 'download_filename', + '', + 'Filename to use for download.', flag_values=fv) - flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', flag_values=fv) - def RunWithArgs(self, bucket, entity): - """Updates a default object ACL entry on the specified bucket. This method - supports patch semantics. + def RunWithArgs(self, bucket, object): + """Retrieves an object or its metadata. Args: - bucket: The name of the bucket. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. + bucket: Name of the bucket in which the object resides. + object: Name of the object. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. - id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. - selfLink: The link to this access-control entry. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + projection: Set of properties to return. Defaults to noAcl. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( + request = messages.StorageObjectsGetRequest( bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') - if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Patch( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Get( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsUpdate(apitools_base.NewCmd): - """Command wrapping defaultObjectAccessControls.Update.""" +class ObjectsInsert(apitools_base_cli.NewCmd): + """Command wrapping objects.Insert.""" - usage = """defaultObjectAccessControls_update """ + usage = """objects_insert """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsUpdate, self).__init__(name, fv) + super(ObjectsInsert, self).__init__(name, fv) flags.DEFINE_string( - 'domain', + 'contentEncoding', None, - u'The domain associated with the entity, if any.', + u'If set, sets the contentEncoding property of the final object to ' + u'this value. Setting this parameter is equivalent to setting the ' + u'contentEncoding metadata property. This can be useful when ' + u'uploading an object with uploadType=media to indicate the encoding ' + u'of the content being uploaded.', flag_values=fv) flags.DEFINE_string( - 'email', + 'ifGenerationMatch', None, - u'The email address associated with the entity, if any.', + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', flag_values=fv) flags.DEFINE_string( - 'entityId', + 'ifGenerationNotMatch', None, - u'The ID for the entity, if any.', + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'etag', + 'ifMetagenerationMatch', None, - u'HTTP 1.1 Entity tag for the access-control entry.', + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( - 'generation', + 'ifMetagenerationNotMatch', None, - u'The content generation of the object.', + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'id', + 'name', None, - u'The ID of the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', + u'Name of the object. Required when the object metadata is not ' + u"otherwise provided. Overrides the object metadata's name value, if " + u'any.', flag_values=fv) flags.DEFINE_string( 'object', None, - u'The name of the object.', + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', flag_values=fv) flags.DEFINE_string( - 'projectTeam', - None, - u'The project team associated with the entity, if any.', + 'upload_filename', + '', + 'Filename to use for upload.', flag_values=fv) flags.DEFINE_string( - 'role', - None, - u'The access permission for the entity. Can be READER or OWNER.', + 'upload_mime_type', + '', + 'MIME type to use for the upload. Only needed if the extension on ' + '--upload_filename does not determine the correct (or any) MIME ' + 'type.', flag_values=fv) flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', flag_values=fv) - def RunWithArgs(self, bucket, entity): - """Updates a default object ACL entry on the specified bucket. + def RunWithArgs(self, bucket): + """Stores a new object and metadata. Args: - bucket: The name of the bucket. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. + bucket: Name of the bucket in which to store the new object. Overrides + the provided object metadata's bucket value, if any. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. - id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. - selfLink: The link to this access-control entry. + contentEncoding: If set, sets the contentEncoding property of the final + object to this value. Setting this parameter is equivalent to setting + the contentEncoding metadata property. This can be useful when + uploading an object with uploadType=media to indicate the encoding of + the content being uploaded. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + name: Name of the object. Required when the object metadata is not + otherwise provided. Overrides the object metadata's name value, if + any. + object: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + upload_filename: Filename to use for upload. + upload_mime_type: MIME type to use for the upload. Only needed if the + extension on --upload_filename does not determine the correct (or any) + MIME type. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( + request = messages.StorageObjectsInsertRequest( bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') + if FLAGS['contentEncoding'].present: + request.contentEncoding = FLAGS.contentEncoding.decode('utf8') + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Update( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + upload = None + if FLAGS.upload_filename: + upload = apitools_base.Upload.FromFile( + FLAGS.upload_filename, FLAGS.upload_mime_type) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Insert( + request, global_params=global_params, upload=upload, download=download) + print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsDelete(apitools_base.NewCmd): - """Command wrapping objectAccessControls.Delete.""" +class ObjectsList(apitools_base_cli.NewCmd): + """Command wrapping objects.List.""" - usage = """objectAccessControls_delete """ + usage = """objects_list """ def __init__(self, name, fv): - super(ObjectAccessControlsDelete, self).__init__(name, fv) + super(ObjectsList, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'delimiter', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of a file as distinct results.', flag_values=fv) - def RunWithArgs(self, bucket, object, entity): - """Permanently deletes the ACL entry for the specified entity on the - specified object. + def RunWithArgs(self, bucket): + """Retrieves a list of objects matching the criteria. Args: - bucket: Name of a bucket. - object: Name of the object. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. + bucket: Name of the bucket in which to look for objects. Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsDeleteRequest( - bucket=bucket.decode('utf8'), - object=object.decode('utf8'), - entity=entity.decode('utf8'), - ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - result = client.objectAccessControls.Delete( - request, global_params=global_params) - print apitools_base.FormatOutput(result) - - -class ObjectAccessControlsGet(apitools_base.NewCmd): - """Command wrapping objectAccessControls.Get.""" - - usage = """objectAccessControls_get """ - - def __init__(self, name, fv): - super(ObjectAccessControlsGet, self).__init__(name, fv) - flags.DEFINE_string( - 'generation', - None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', - flag_values=fv) - - def RunWithArgs(self, bucket, object, entity): - """Returns the ACL entry for the specified entity on the specified object. - - Args: - bucket: Name of a bucket. - object: Name of the object. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. - - Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsGetRequest( + request = messages.StorageObjectsListRequest( bucket=bucket.decode('utf8'), - object=object.decode('utf8'), - entity=entity.decode('utf8'), ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - result = client.objectAccessControls.Get( + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.List( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsInsert(apitools_base.NewCmd): - """Command wrapping objectAccessControls.Insert.""" +class ObjectsPatch(apitools_base_cli.NewCmd): + """Command wrapping objects.Patch.""" - usage = """objectAccessControls_insert """ + usage = """objects_patch """ def __init__(self, name, fv): - super(ObjectAccessControlsInsert, self).__init__(name, fv) + super(ObjectsPatch, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, @@ -1638,99 +1888,329 @@ class ObjectAccessControlsInsert(apitools_base.NewCmd): u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'objectAccessControl', + 'ifGenerationMatch', None, - u'A ObjectAccessControl resource to be passed as the request body.', + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', flag_values=fv) def RunWithArgs(self, bucket, object): - """Creates a new ACL entry on the specified object. + """Updates an object's metadata. This method supports patch semantics. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. object: Name of the object. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsInsertRequest( + request = messages.StorageObjectsPatchRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Insert( + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.objects.Patch( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsList(apitools_base.NewCmd): - """Command wrapping objectAccessControls.List.""" +class ObjectsUpdate(apitools_base_cli.NewCmd): + """Command wrapping objects.Update.""" - usage = """objectAccessControls_list """ + usage = """objects_update """ def __init__(self, name, fv): - super(ObjectAccessControlsList, self).__init__(name, fv) + super(ObjectsUpdate, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, u'If present, selects a specific revision of this object (as opposed ' u'to the latest version, the default).', flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) def RunWithArgs(self, bucket, object): - """Retrieves ACL entries on the specified object. + """Updates an object's metadata. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. object: Name of the object. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsListRequest( + request = messages.StorageObjectsUpdateRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - result = client.objectAccessControls.List( - request, global_params=global_params) - print apitools_base.FormatOutput(result) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) + result = client.objects.Update( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsPatch(apitools_base.NewCmd): - """Command wrapping objectAccessControls.Patch.""" +class ObjectsWatchAll(apitools_base_cli.NewCmd): + """Command wrapping objects.WatchAll.""" - usage = """objectAccessControls_patch """ + usage = """objects_watchAll """ def __init__(self, name, fv): - super(ObjectAccessControlsPatch, self).__init__(name, fv) + super(ObjectsWatchAll, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'channel', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u'A Channel resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( - 'objectAccessControl', + 'delimiter', None, - u'A ObjectAccessControl resource to be passed as the request body.', + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of a file as distinct results.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Watch for changes on all objects in a bucket. + + Args: + bucket: Name of the bucket in which to look for objects. + + Flags: + channel: A Channel resource to be passed as the request body. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of a file as distinct results. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsWatchAllRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['channel'].present: + request.channel = apitools_base.JsonToMessage(messages.Channel, FLAGS.channel) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsWatchAllRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.WatchAll( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Delete.""" + + usage = """objectAccessControls_delete """ + + def __init__(self, name, fv): + super(ObjectAccessControlsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) def RunWithArgs(self, bucket, object, entity): - """Updates an ACL entry on the specified object. This method supports - patch semantics. + """Permanently deletes the ACL entry for the specified entity on the + specified object. Args: bucket: Name of a bucket. @@ -1742,46 +2222,37 @@ class ObjectAccessControlsPatch(apitools_base.NewCmd): Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsPatchRequest( + request = messages.StorageObjectAccessControlsDeleteRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Patch( + result = client.objectAccessControls.Delete( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsUpdate(apitools_base.NewCmd): - """Command wrapping objectAccessControls.Update.""" +class ObjectAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Get.""" - usage = """objectAccessControls_update """ + usage = """objectAccessControls_get """ def __init__(self, name, fv): - super(ObjectAccessControlsUpdate, self).__init__(name, fv) + super(ObjectAccessControlsGet, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, u'If present, selects a specific revision of this object (as opposed ' u'to the latest version, the default).', flag_values=fv) - flags.DEFINE_string( - 'objectAccessControl', - None, - u'A ObjectAccessControl resource to be passed as the request body.', - flag_values=fv) def RunWithArgs(self, bucket, object, entity): - """Updates an ACL entry on the specified object. + """Returns the ACL entry for the specified entity on the specified object. Args: bucket: Name of a bucket. @@ -1793,370 +2264,165 @@ class ObjectAccessControlsUpdate(apitools_base.NewCmd): Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsUpdateRequest( + request = messages.StorageObjectAccessControlsGetRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Update( + result = client.objectAccessControls.Get( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectsCompose(apitools_base.NewCmd): - """Command wrapping objects.Compose.""" +class ObjectAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Insert.""" - usage = """objects_compose """ + usage = """objectAccessControls_insert """ def __init__(self, name, fv): - super(ObjectsCompose, self).__init__(name, fv) - flags.DEFINE_string( - 'composeRequest', - None, - u'A ComposeRequest resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'destinationPredefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to the destination ' - u'object.', - flag_values=fv) + super(ObjectAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( - 'ifGenerationMatch', + 'generation', None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'objectAccessControl', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + u'A ObjectAccessControl resource to be passed as the request body.', flag_values=fv) - def RunWithArgs(self, destinationBucket, destinationObject): - """Concatenates a list of existing objects into a new object in the same - bucket. + def RunWithArgs(self, bucket, object): + """Creates a new ACL entry on the specified object. Args: - destinationBucket: Name of the bucket in which to store the new object. - destinationObject: Name of the new object. + bucket: Name of a bucket. + object: Name of the object. Flags: - composeRequest: A ComposeRequest resource to be passed as the request - body. - destinationPredefinedAcl: Apply a predefined set of access controls to - the destination object. - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsComposeRequest( - destinationBucket=destinationBucket.decode('utf8'), - destinationObject=destinationObject.decode('utf8'), + request = messages.StorageObjectAccessControlsInsertRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['composeRequest'].present: - request.composeRequest = apitools_base.JsonToMessage(messages.ComposeRequest, FLAGS.composeRequest) - if FLAGS['destinationPredefinedAcl'].present: - request.destinationPredefinedAcl = messages.StorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Compose( - request, global_params=global_params, download=download) - print apitools_base.FormatOutput(result) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) -class ObjectsCopy(apitools_base.NewCmd): - """Command wrapping objects.Copy.""" +class ObjectAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.List.""" - usage = """objects_copy """ + usage = """objectAccessControls_list """ def __init__(self, name, fv): - super(ObjectsCopy, self).__init__(name, fv) - flags.DEFINE_enum( - 'destinationPredefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to the destination ' - u'object.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the destination object's" - u' current generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the destination object's" - u' current generation does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"Makes the operation conditional on whether the destination object's" - u' current metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u"Makes the operation conditional on whether the destination object's" - u' current metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifSourceGenerationMatch', - None, - u"Makes the operation conditional on whether the source object's " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifSourceGenerationNotMatch', - None, - u"Makes the operation conditional on whether the source object's " - u'generation does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifSourceMetagenerationMatch', - None, - u"Makes the operation conditional on whether the source object's " - u'current metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifSourceMetagenerationNotMatch', - None, - u"Makes the operation conditional on whether the source object's " - u'current metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'object', - None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the object ' - u'resource specifies the acl property, when it defaults to full.', - flag_values=fv) + super(ObjectAccessControlsList, self).__init__(name, fv) flags.DEFINE_string( - 'sourceGeneration', + 'generation', None, - u'If present, selects a specific revision of the source object (as ' - u'opposed to the latest version, the default).', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) - def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): - """Copies an object to a specified location. Optionally overrides - metadata. + def RunWithArgs(self, bucket, object): + """Retrieves ACL entries on the specified object. Args: - sourceBucket: Name of the bucket in which to find the source object. - sourceObject: Name of the source object. - destinationBucket: Name of the bucket in which to store the new object. - Overrides the provided object metadata's bucket value, if any. - destinationObject: Name of the new object. Required when the object - metadata is not otherwise provided. Overrides the object metadata's - name value, if any. + bucket: Name of a bucket. + object: Name of the object. Flags: - destinationPredefinedAcl: Apply a predefined set of access controls to - the destination object. - ifGenerationMatch: Makes the operation conditional on whether the - destination object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - destination object's current generation does not match the given - value. - ifMetagenerationMatch: Makes the operation conditional on whether the - destination object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - destination object's current metageneration does not match the given - value. - ifSourceGenerationMatch: Makes the operation conditional on whether the - source object's generation matches the given value. - ifSourceGenerationNotMatch: Makes the operation conditional on whether - the source object's generation does not match the given value. - ifSourceMetagenerationMatch: Makes the operation conditional on whether - the source object's current metageneration matches the given value. - ifSourceMetagenerationNotMatch: Makes the operation conditional on - whether the source object's current metageneration does not match the - given value. - object: A Object resource to be passed as the request body. - projection: Set of properties to return. Defaults to noAcl, unless the - object resource specifies the acl property, when it defaults to full. - sourceGeneration: If present, selects a specific revision of the source - object (as opposed to the latest version, the default). - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsCopyRequest( - sourceBucket=sourceBucket.decode('utf8'), - sourceObject=sourceObject.decode('utf8'), - destinationBucket=destinationBucket.decode('utf8'), - destinationObject=destinationObject.decode('utf8'), + request = messages.StorageObjectAccessControlsListRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['destinationPredefinedAcl'].present: - request.destinationPredefinedAcl = messages.StorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['ifSourceGenerationMatch'].present: - request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) - if FLAGS['ifSourceGenerationNotMatch'].present: - request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) - if FLAGS['ifSourceMetagenerationMatch'].present: - request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) - if FLAGS['ifSourceMetagenerationNotMatch'].present: - request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) - if FLAGS['object'].present: - request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsCopyRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['sourceGeneration'].present: - request.sourceGeneration = int(FLAGS.sourceGeneration) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Copy( - request, global_params=global_params, download=download) - print apitools_base.FormatOutput(result) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) -class ObjectsDelete(apitools_base.NewCmd): - """Command wrapping objects.Delete.""" +class ObjectAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Patch.""" - usage = """objects_delete """ + usage = """objectAccessControls_patch """ def __init__(self, name, fv): - super(ObjectsDelete, self).__init__(name, fv) - flags.DEFINE_string( - 'generation', - None, - u'If present, permanently deletes a specific revision of this object ' - u'(as opposed to the latest version, the default).', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', - flag_values=fv) + super(ObjectAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'generation', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'objectAccessControl', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', + u'A ObjectAccessControl resource to be passed as the request body.', flag_values=fv) - def RunWithArgs(self, bucket, object): - """Deletes an object and its metadata. Deletions are permanent if - versioning is not enabled for the bucket, or if the generation parameter - is used. + def RunWithArgs(self, bucket, object, entity): + """Updates an ACL entry on the specified object. This method supports + patch semantics. Args: - bucket: Name of the bucket in which the object resides. + bucket: Name of a bucket. object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. Flags: - generation: If present, permanently deletes a specific revision of this - object (as opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsDeleteRequest( + request = messages.StorageObjectAccessControlsPatchRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), + entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.objects.Delete( + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Patch( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectsGet(apitools_base.NewCmd): - """Command wrapping objects.Get.""" +class ObjectAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Update.""" - usage = """objects_get """ + usage = """objectAccessControls_update """ def __init__(self, name, fv): - super(ObjectsGet, self).__init__(name, fv) + super(ObjectAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, @@ -2164,285 +2430,107 @@ class ObjectsGet(apitools_base.NewCmd): u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's generation " - u'matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's generation " - u'does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'objectAccessControl', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + u'A ObjectAccessControl resource to be passed as the request body.', flag_values=fv) - def RunWithArgs(self, bucket, object): - """Retrieves an object or its metadata. + def RunWithArgs(self, bucket, object, entity): + """Updates an ACL entry on the specified object. Args: - bucket: Name of the bucket in which the object resides. + bucket: Name of a bucket. object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - projection: Set of properties to return. Defaults to noAcl. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsGetRequest( + request = messages.StorageObjectAccessControlsUpdateRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), + entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Get( - request, global_params=global_params, download=download) - print apitools_base.FormatOutput(result) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) -class ObjectsInsert(apitools_base.NewCmd): - """Command wrapping objects.Insert.""" +class BucketsDelete(apitools_base_cli.NewCmd): + """Command wrapping buckets.Delete.""" - usage = """objects_insert """ + usage = """buckets_delete """ def __init__(self, name, fv): - super(ObjectsInsert, self).__init__(name, fv) - flags.DEFINE_string( - 'contentEncoding', - None, - u'If set, sets the contentEncoding property of the final object to ' - u'this value. Setting this parameter is equivalent to setting the ' - u'contentEncoding metadata property. This can be useful when ' - u'uploading an object with uploadType=media to indicate the encoding ' - u'of the content being uploaded.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', - flag_values=fv) + super(BucketsDelete, self).__init__(name, fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'If set, only deletes the bucket if its metageneration matches this ' + u'value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'name', - None, - u'Name of the object. Required when the object metadata is not ' - u"otherwise provided. Overrides the object metadata's name value, if " - u'any.', - flag_values=fv) - flags.DEFINE_string( - 'object', - None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the object ' - u'resource specifies the acl property, when it defaults to full.', - flag_values=fv) - flags.DEFINE_string( - 'upload_filename', - '', - 'Filename to use for upload.', - flag_values=fv) - flags.DEFINE_string( - 'upload_mime_type', - '', - 'MIME type to use for the upload. Only needed if the extension on ' - '--upload_filename does not determine the correct (or any) MIME ' - 'type.', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + u'If set, only deletes the bucket if its metageneration does not ' + u'match this value.', flag_values=fv) def RunWithArgs(self, bucket): - """Stores a new object and metadata. - - Args: - bucket: Name of the bucket in which to store the new object. Overrides - the provided object metadata's bucket value, if any. - - Flags: - contentEncoding: If set, sets the contentEncoding property of the final - object to this value. Setting this parameter is equivalent to setting - the contentEncoding metadata property. This can be useful when - uploading an object with uploadType=media to indicate the encoding of - the content being uploaded. - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - name: Name of the object. Required when the object metadata is not - otherwise provided. Overrides the object metadata's name value, if - any. - object: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. - projection: Set of properties to return. Defaults to noAcl, unless the - object resource specifies the acl property, when it defaults to full. - upload_filename: Filename to use for upload. - upload_mime_type: MIME type to use for the upload. Only needed if the - extension on --upload_filename does not determine the correct (or any) - MIME type. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + """Permanently deletes an empty bucket. + + Args: + bucket: Name of a bucket. + + Flags: + ifMetagenerationMatch: If set, only deletes the bucket if its + metageneration matches this value. + ifMetagenerationNotMatch: If set, only deletes the bucket if its + metageneration does not match this value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsInsertRequest( + request = messages.StorageBucketsDeleteRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['contentEncoding'].present: - request.contentEncoding = FLAGS.contentEncoding.decode('utf8') - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['name'].present: - request.name = FLAGS.name.decode('utf8') - if FLAGS['object'].present: - request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) - upload = None - if FLAGS.upload_filename: - upload = apitools_base.Upload.FromFile( - FLAGS.upload_filename, FLAGS.upload_mime_type) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Insert( - request, global_params=global_params, upload=upload, download=download) - print apitools_base.FormatOutput(result) + result = client.buckets.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) -class ObjectsList(apitools_base.NewCmd): - """Command wrapping objects.List.""" +class BucketsGet(apitools_base_cli.NewCmd): + """Command wrapping buckets.Get.""" - usage = """objects_list """ + usage = """buckets_get """ def __init__(self, name, fv): - super(ObjectsList, self).__init__(name, fv) - flags.DEFINE_string( - 'delimiter', - None, - u'Returns results in a directory-like mode. items will contain only ' - u'objects whose names, aside from the prefix, do not contain ' - u'delimiter. Objects whose names, aside from the prefix, contain ' - u'delimiter will have their name, truncated after the delimiter, ' - u'returned in prefixes. Duplicate prefixes are omitted.', - flag_values=fv) - flags.DEFINE_integer( - 'maxResults', - None, - u'Maximum number of items plus prefixes to return. As duplicate ' - u'prefixes are omitted, fewer total results may be returned than ' - u'requested.', - flag_values=fv) + super(BucketsGet, self).__init__(name, fv) flags.DEFINE_string( - 'pageToken', + 'ifMetagenerationMatch', None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( - 'prefix', + 'ifMetagenerationNotMatch', None, - u'Filter results to objects whose names begin with this prefix.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) flags.DEFINE_enum( 'projection', @@ -2450,206 +2538,202 @@ class ObjectsList(apitools_base.NewCmd): [u'full', u'noAcl'], u'Set of properties to return. Defaults to noAcl.', flag_values=fv) - flags.DEFINE_boolean( - 'versions', - None, - u'If true, lists all versions of a file as distinct results.', - flag_values=fv) def RunWithArgs(self, bucket): - """Retrieves a list of objects matching the criteria. + """Returns metadata for the specified bucket. Args: - bucket: Name of the bucket in which to look for objects. + bucket: Name of a bucket. Flags: - delimiter: Returns results in a directory-like mode. items will contain - only objects whose names, aside from the prefix, do not contain - delimiter. Objects whose names, aside from the prefix, contain - delimiter will have their name, truncated after the delimiter, - returned in prefixes. Duplicate prefixes are omitted. - maxResults: Maximum number of items plus prefixes to return. As - duplicate prefixes are omitted, fewer total results may be returned - than requested. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - prefix: Filter results to objects whose names begin with this prefix. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsListRequest( + request = messages.StorageBucketsGetRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['delimiter'].present: - request.delimiter = FLAGS.delimiter.decode('utf8') - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['prefix'].present: - request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) if FLAGS['projection'].present: - request.projection = messages.StorageObjectsListRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['versions'].present: - request.versions = FLAGS.versions - result = client.objects.List( + request.projection = messages.StorageBucketsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Get( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectsPatch(apitools_base.NewCmd): - """Command wrapping objects.Patch.""" +class BucketsInsert(apitools_base_cli.NewCmd): + """Command wrapping buckets.Insert.""" - usage = """objects_patch """ + usage = """buckets_insert """ def __init__(self, name, fv): - super(ObjectsPatch, self).__init__(name, fv) + super(BucketsInsert, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'bucket', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u'A Bucket resource to be passed as the request body.', flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of default object access controls to this ' + u'bucket.', flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the bucket ' + u'resource specifies acl or defaultObjectAcl properties, when it ' + u'defaults to full.', + flag_values=fv) + + def RunWithArgs(self, project): + """Creates a new bucket. + + Args: + project: A valid API project identifier. + + Flags: + bucket: A Bucket resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. + projection: Set of properties to return. Defaults to noAcl, unless the + bucket resource specifies acl or defaultObjectAcl properties, when it + defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsInsertRequest( + project=project.decode('utf8'), + ) + if FLAGS['bucket'].present: + request.bucket = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucket) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsInsertRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class BucketsList(apitools_base_cli.NewCmd): + """Command wrapping buckets.List.""" + + usage = """buckets_list """ + + def __init__(self, name, fv): + super(BucketsList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'Maximum number of buckets to return.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'pageToken', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', flag_values=fv) flags.DEFINE_string( - 'objectResource', + 'prefix', None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', + u'Filter results to buckets whose names begin with this prefix.', flag_values=fv) flags.DEFINE_enum( 'projection', u'full', [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', + u'Set of properties to return. Defaults to noAcl.', flag_values=fv) - def RunWithArgs(self, bucket, object): - """Updates an object's metadata. This method supports patch semantics. + def RunWithArgs(self, project): + """Retrieves a list of buckets for a given project. Args: - bucket: Name of the bucket in which the object resides. - object: Name of the object. + project: A valid API project identifier. Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - objectResource: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. - projection: Set of properties to return. Defaults to full. + maxResults: Maximum number of buckets to return. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to buckets whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsPatchRequest( - bucket=bucket.decode('utf8'), - object=object.decode('utf8'), + request = messages.StorageBucketsListRequest( + project=project.decode('utf8'), ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['objectResource'].present: - request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') if FLAGS['projection'].present: - request.projection = messages.StorageObjectsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.objects.Patch( + request.projection = messages.StorageBucketsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.List( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) -class ObjectsUpdate(apitools_base.NewCmd): - """Command wrapping objects.Update.""" +class BucketsPatch(apitools_base_cli.NewCmd): + """Command wrapping buckets.Patch.""" - usage = """objects_update """ + usage = """buckets_patch """ def __init__(self, name, fv): - super(ObjectsUpdate, self).__init__(name, fv) - flags.DEFINE_string( - 'generation', - None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) + super(BucketsPatch, self).__init__(name, fv) flags.DEFINE_string( - 'ifGenerationNotMatch', + 'bucketResource', None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', + u'A Bucket resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'objectResource', - None, - u'A Object resource to be passed as the request body.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) flags.DEFINE_enum( 'predefinedAcl', u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', + u'Apply a predefined set of default object access controls to this ' + u'bucket.', flag_values=fv) flags.DEFINE_enum( 'projection', @@ -2657,195 +2741,148 @@ class ObjectsUpdate(apitools_base.NewCmd): [u'full', u'noAcl'], u'Set of properties to return. Defaults to full.', flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', - flag_values=fv) - def RunWithArgs(self, bucket, object): - """Updates an object's metadata. + def RunWithArgs(self, bucket): + """Updates a bucket. This method supports patch semantics. Args: - bucket: Name of the bucket in which the object resides. - object: Name of the object. + bucket: Name of a bucket. Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - objectResource: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. projection: Set of properties to return. Defaults to full. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsUpdateRequest( + request = messages.StorageBucketsPatchRequest( bucket=bucket.decode('utf8'), - object=object.decode('utf8'), ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['objectResource'].present: - request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + request.predefinedAcl = messages.StorageBucketsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsPatchRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) if FLAGS['projection'].present: - request.projection = messages.StorageObjectsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Update( - request, global_params=global_params, download=download) - print apitools_base.FormatOutput(result) + request.projection = messages.StorageBucketsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Patch( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) -class ObjectsWatchAll(apitools_base.NewCmd): - """Command wrapping objects.WatchAll.""" +class BucketsUpdate(apitools_base_cli.NewCmd): + """Command wrapping buckets.Update.""" - usage = """objects_watchAll """ + usage = """buckets_update """ def __init__(self, name, fv): - super(ObjectsWatchAll, self).__init__(name, fv) + super(BucketsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'channel', + 'bucketResource', None, - u'A Channel resource to be passed as the request body.', + u'A Bucket resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( - 'delimiter', - None, - u'Returns results in a directory-like mode. items will contain only ' - u'objects whose names, aside from the prefix, do not contain ' - u'delimiter. Objects whose names, aside from the prefix, contain ' - u'delimiter will have their name, truncated after the delimiter, ' - u'returned in prefixes. Duplicate prefixes are omitted.', - flag_values=fv) - flags.DEFINE_integer( - 'maxResults', + 'ifMetagenerationMatch', None, - u'Maximum number of items plus prefixes to return. As duplicate ' - u'prefixes are omitted, fewer total results may be returned than ' - u'requested.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( - 'pageToken', + 'ifMetagenerationNotMatch', None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) - flags.DEFINE_string( - 'prefix', - None, - u'Filter results to objects whose names begin with this prefix.', + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of default object access controls to this ' + u'bucket.', flag_values=fv) flags.DEFINE_enum( 'projection', u'full', [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) - flags.DEFINE_boolean( - 'versions', - None, - u'If true, lists all versions of a file as distinct results.', + u'Set of properties to return. Defaults to full.', flag_values=fv) def RunWithArgs(self, bucket): - """Watch for changes on all objects in a bucket. + """Updates a bucket. Args: - bucket: Name of the bucket in which to look for objects. + bucket: Name of a bucket. Flags: - channel: A Channel resource to be passed as the request body. - delimiter: Returns results in a directory-like mode. items will contain - only objects whose names, aside from the prefix, do not contain - delimiter. Objects whose names, aside from the prefix, contain - delimiter will have their name, truncated after the delimiter, - returned in prefixes. Duplicate prefixes are omitted. - maxResults: Maximum number of items plus prefixes to return. As - duplicate prefixes are omitted, fewer total results may be returned - than requested. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - prefix: Filter results to objects whose names begin with this prefix. - projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. + projection: Set of properties to return. Defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsWatchAllRequest( + request = messages.StorageBucketsUpdateRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['channel'].present: - request.channel = apitools_base.JsonToMessage(messages.Channel, FLAGS.channel) - if FLAGS['delimiter'].present: - request.delimiter = FLAGS.delimiter.decode('utf8') - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['prefix'].present: - request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsUpdateRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) if FLAGS['projection'].present: - request.projection = messages.StorageObjectsWatchAllRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['versions'].present: - request.versions = FLAGS.versions - result = client.objects.WatchAll( + request.projection = messages.StorageBucketsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Update( request, global_params=global_params) - print apitools_base.FormatOutput(result) + print apitools_base_cli.FormatOutput(result) def main(_): appcommands.AddCmd('pyshell', PyShell) + appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) + appcommands.AddCmd('defaultObjectAccessControls_get', DefaultObjectAccessControlsGet) + appcommands.AddCmd('defaultObjectAccessControls_insert', DefaultObjectAccessControlsInsert) + appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) + appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) + appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) appcommands.AddCmd('bucketAccessControls_delete', BucketAccessControlsDelete) appcommands.AddCmd('bucketAccessControls_get', BucketAccessControlsGet) appcommands.AddCmd('bucketAccessControls_insert', BucketAccessControlsInsert) appcommands.AddCmd('bucketAccessControls_list', BucketAccessControlsList) appcommands.AddCmd('bucketAccessControls_patch', BucketAccessControlsPatch) appcommands.AddCmd('bucketAccessControls_update', BucketAccessControlsUpdate) - appcommands.AddCmd('buckets_delete', BucketsDelete) - appcommands.AddCmd('buckets_get', BucketsGet) - appcommands.AddCmd('buckets_insert', BucketsInsert) - appcommands.AddCmd('buckets_list', BucketsList) - appcommands.AddCmd('buckets_patch', BucketsPatch) - appcommands.AddCmd('buckets_update', BucketsUpdate) appcommands.AddCmd('channels_stop', ChannelsStop) - appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) - appcommands.AddCmd('defaultObjectAccessControls_get', DefaultObjectAccessControlsGet) - appcommands.AddCmd('defaultObjectAccessControls_insert', DefaultObjectAccessControlsInsert) - appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) - appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) - appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) - appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) - appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) - appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) - appcommands.AddCmd('objectAccessControls_list', ObjectAccessControlsList) - appcommands.AddCmd('objectAccessControls_patch', ObjectAccessControlsPatch) - appcommands.AddCmd('objectAccessControls_update', ObjectAccessControlsUpdate) appcommands.AddCmd('objects_compose', ObjectsCompose) appcommands.AddCmd('objects_copy', ObjectsCopy) appcommands.AddCmd('objects_delete', ObjectsDelete) @@ -2855,13 +2892,25 @@ def main(_): appcommands.AddCmd('objects_patch', ObjectsPatch) appcommands.AddCmd('objects_update', ObjectsUpdate) appcommands.AddCmd('objects_watchAll', ObjectsWatchAll) + appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) + appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) + appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) + appcommands.AddCmd('objectAccessControls_list', ObjectAccessControlsList) + appcommands.AddCmd('objectAccessControls_patch', ObjectAccessControlsPatch) + appcommands.AddCmd('objectAccessControls_update', ObjectAccessControlsUpdate) + appcommands.AddCmd('buckets_delete', BucketsDelete) + appcommands.AddCmd('buckets_get', BucketsGet) + appcommands.AddCmd('buckets_insert', BucketsInsert) + appcommands.AddCmd('buckets_list', BucketsList) + appcommands.AddCmd('buckets_patch', BucketsPatch) + appcommands.AddCmd('buckets_update', BucketsUpdate) - apitools_base.SetupLogger() + apitools_base_cli.SetupLogger() if hasattr(appcommands, 'SetDefaultCommand'): appcommands.SetDefaultCommand('pyshell') -run_main = apitools_base.run_main +run_main = apitools_base_cli.run_main if __name__ == '__main__': appcommands.Run() diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage/storage_v1_client.py index 630fa96..4d5024d 100644 --- a/samples/storage_sample/storage/storage_v1_client.py +++ b/samples/storage_sample/storage/storage_v1_client.py @@ -9,19 +9,20 @@ class StorageV1(base_api.BaseApiClient): MESSAGES_MODULE = messages _PACKAGE = u'storage' - _SCOPES = [u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] + _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] _VERSION = u'v1' _CLIENT_ID = '1042881264118.apps.googleusercontent.com' _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' _USER_AGENT = '' _CLIENT_CLASS_NAME = u'StorageV1' _URL_VERSION = u'v1' - _API_KEY = 'AIzaSyCBivqENU2RLBuz8XWfAjQiT5SAFCHHg6U' + _API_KEY = None def __init__(self, url='', credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, - credentials_args=None, default_global_params=None): + credentials_args=None, default_global_params=None, + additional_http_headers=None): """Create a new storage handle.""" url = url or u'https://www.googleapis.com/storage/v1/' super(StorageV1, self).__init__( @@ -29,343 +30,336 @@ class StorageV1(base_api.BaseApiClient): get_credentials=get_credentials, http=http, model=model, log_request=log_request, log_response=log_response, credentials_args=credentials_args, - default_global_params=default_global_params) + default_global_params=default_global_params, + additional_http_headers=additional_http_headers) + self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) self.bucketAccessControls = self.BucketAccessControlsService(self) - self.buckets = self.BucketsService(self) self.channels = self.ChannelsService(self) - self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) - self.objectAccessControls = self.ObjectAccessControlsService(self) self.objects = self.ObjectsService(self) + self.objectAccessControls = self.ObjectAccessControlsService(self) + self.buckets = self.BucketsService(self) - class BucketAccessControlsService(base_api.BaseApiService): - """Service class for the bucketAccessControls resource.""" + class DefaultObjectAccessControlsService(base_api.BaseApiService): + """Service class for the defaultObjectAccessControls resource.""" + + _NAME = u'defaultObjectAccessControls' def __init__(self, client): - super(StorageV1.BucketAccessControlsService, self).__init__(client) - self.__configs = { + super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) + self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.bucketAccessControls.delete', + method_id=u'storage.defaultObjectAccessControls.delete', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'StorageBucketAccessControlsDeleteRequest', - response_type_name=u'StorageBucketAccessControlsDeleteResponse', + request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', + response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.bucketAccessControls.get', + method_id=u'storage.defaultObjectAccessControls.get', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'StorageBucketAccessControlsGetRequest', - response_type_name=u'BucketAccessControl', + request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.bucketAccessControls.insert', + method_id=u'storage.defaultObjectAccessControls.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[], - relative_path=u'b/{bucket}/acl', + relative_path=u'b/{bucket}/defaultObjectAcl', request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.bucketAccessControls.list', + method_id=u'storage.defaultObjectAccessControls.list', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/acl', + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/defaultObjectAcl', request_field='', - request_type_name=u'StorageBucketAccessControlsListRequest', - response_type_name=u'BucketAccessControls', + request_type_name=u'StorageDefaultObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.bucketAccessControls.patch', + method_id=u'storage.defaultObjectAccessControls.patch', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.bucketAccessControls.update', + method_id=u'storage.defaultObjectAccessControls.update', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', supports_download=False, ), } - self.__upload_configs = { + self._upload_configs = { } - def GetMethodConfig(self, method): - return self.__configs.get(method) - - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) - def Delete(self, request, global_params=None): - """Permanently deletes the ACL entry for the specified entity on the specified bucket. + """Permanently deletes the default object ACL entry for the specified entity on the specified bucket. Args: - request: (StorageBucketAccessControlsDeleteRequest) input message + request: (StorageDefaultObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageBucketAccessControlsDeleteResponse) The response message. + (StorageDefaultObjectAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) def Get(self, request, global_params=None): - """Returns the ACL entry for the specified entity on the specified bucket. + """Returns the default object ACL entry for the specified entity on the specified bucket. Args: - request: (StorageBucketAccessControlsGetRequest) input message + request: (StorageDefaultObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) def Insert(self, request, global_params=None): - """Creates a new ACL entry on the specified bucket. + """Creates a new default object ACL entry on the specified bucket. Args: - request: (BucketAccessControl) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves ACL entries on the specified bucket. + """Retrieves default object ACL entries on the specified bucket. Args: - request: (StorageBucketAccessControlsListRequest) input message + request: (StorageDefaultObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControls) The response message. + (ObjectAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates an ACL entry on the specified bucket. This method supports patch semantics. + """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. Args: - request: (BucketAccessControl) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) def Update(self, request, global_params=None): - """Updates an ACL entry on the specified bucket. + """Updates a default object ACL entry on the specified bucket. Args: - request: (BucketAccessControl) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) - class BucketsService(base_api.BaseApiService): - """Service class for the buckets resource.""" + class BucketAccessControlsService(base_api.BaseApiService): + """Service class for the bucketAccessControls resource.""" + + _NAME = u'bucketAccessControls' def __init__(self, client): - super(StorageV1.BucketsService, self).__init__(client) - self.__configs = { + super(StorageV1.BucketAccessControlsService, self).__init__(client) + self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.buckets.delete', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}', + method_id=u'storage.bucketAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'StorageBucketsDeleteRequest', - response_type_name=u'StorageBucketsDeleteResponse', + request_type_name=u'StorageBucketAccessControlsDeleteRequest', + response_type_name=u'StorageBucketAccessControlsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.buckets.get', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}', + method_id=u'storage.bucketAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'StorageBucketsGetRequest', - response_type_name=u'Bucket', + request_type_name=u'StorageBucketAccessControlsGetRequest', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.buckets.insert', - ordered_params=[u'project'], - path_params=[], - query_params=[u'predefinedAcl', u'project', u'projection'], - relative_path=u'b', - request_field=u'bucket', - request_type_name=u'StorageBucketsInsertRequest', - response_type_name=u'Bucket', + method_id=u'storage.bucketAccessControls.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.buckets.list', - ordered_params=[u'project'], - path_params=[], - query_params=[u'maxResults', u'pageToken', u'project', u'projection'], - relative_path=u'b', + method_id=u'storage.bucketAccessControls.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', request_field='', - request_type_name=u'StorageBucketsListRequest', - response_type_name=u'Buckets', + request_type_name=u'StorageBucketAccessControlsListRequest', + response_type_name=u'BucketAccessControls', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.buckets.patch', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsPatchRequest', - response_type_name=u'Bucket', + method_id=u'storage.bucketAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.buckets.update', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsUpdateRequest', - response_type_name=u'Bucket', + method_id=u'storage.bucketAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), } - self.__upload_configs = { + self._upload_configs = { } - def GetMethodConfig(self, method): - return self.__configs.get(method) - - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) - def Delete(self, request, global_params=None): - """Permanently deletes an empty bucket. + """Permanently deletes the ACL entry for the specified entity on the specified bucket. Args: - request: (StorageBucketsDeleteRequest) input message + request: (StorageBucketAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageBucketsDeleteResponse) The response message. + (StorageBucketAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) def Get(self, request, global_params=None): - """Returns metadata for the specified bucket. + """Returns the ACL entry for the specified entity on the specified bucket. Args: - request: (StorageBucketsGetRequest) input message + request: (StorageBucketAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) def Insert(self, request, global_params=None): - """Creates a new bucket. + """Creates a new ACL entry on the specified bucket. Args: - request: (StorageBucketsInsertRequest) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves a list of buckets for a given project. + """Retrieves ACL entries on the specified bucket. Args: - request: (StorageBucketsListRequest) input message + request: (StorageBucketAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Buckets) The response message. + (BucketAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates a bucket. This method supports patch semantics. + """Updates an ACL entry on the specified bucket. This method supports patch semantics. Args: - request: (StorageBucketsPatchRequest) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) def Update(self, request, global_params=None): - """Updates a bucket. + """Updates an ACL entry on the specified bucket. Args: - request: (StorageBucketsUpdateRequest) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( @@ -374,9 +368,11 @@ class StorageV1(base_api.BaseApiClient): class ChannelsService(base_api.BaseApiService): """Service class for the channels resource.""" + _NAME = u'channels' + def __init__(self, client): super(StorageV1.ChannelsService, self).__init__(client) - self.__configs = { + self._method_configs = { 'Stop': base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.channels.stop', @@ -391,15 +387,9 @@ class StorageV1(base_api.BaseApiClient): ), } - self.__upload_configs = { + self._upload_configs = { } - def GetMethodConfig(self, method): - return self.__configs.get(method) - - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) - def Stop(self, request, global_params=None): """Stop watching resources through this channel. @@ -413,179 +403,279 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) - class DefaultObjectAccessControlsService(base_api.BaseApiService): - """Service class for the defaultObjectAccessControls resource.""" + class ObjectsService(base_api.BaseApiService): + """Service class for the objects resource.""" + + _NAME = u'objects' def __init__(self, client): - super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) - self.__configs = { + super(StorageV1.ObjectsService, self).__init__(client) + self._method_configs = { + 'Compose': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.compose', + ordered_params=[u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], + relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', + request_field=u'composeRequest', + request_type_name=u'StorageObjectsComposeRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'Copy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.copy', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsCopyRequest', + response_type_name=u'Object', + supports_download=True, + ), 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.defaultObjectAccessControls.delete', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + method_id=u'storage.objects.delete', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/o/{object}', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', - response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', + request_type_name=u'StorageObjectsDeleteRequest', + response_type_name=u'StorageObjectsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.get', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + method_id=u'storage.objects.get', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}/o/{object}', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, + request_type_name=u'StorageObjectsGetRequest', + response_type_name=u'Object', + supports_download=True, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.defaultObjectAccessControls.insert', + method_id=u'storage.objects.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', - supports_download=False, + query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o', + request_field=u'object', + request_type_name=u'StorageObjectsInsertRequest', + response_type_name=u'Object', + supports_download=True, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.list', + method_id=u'storage.objects.list', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/defaultObjectAcl', + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsListRequest', - response_type_name=u'ObjectAccessControls', + request_type_name=u'StorageObjectsListRequest', + response_type_name=u'Objects', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.defaultObjectAccessControls.patch', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', + method_id=u'storage.objects.patch', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsPatchRequest', + response_type_name=u'Object', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.defaultObjectAccessControls.update', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', + method_id=u'storage.objects.update', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsUpdateRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'WatchAll': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.watchAll', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o/watch', + request_field=u'channel', + request_type_name=u'StorageObjectsWatchAllRequest', + response_type_name=u'Channel', supports_download=False, ), } - self.__upload_configs = { + self._upload_configs = { + 'Insert': base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload/storage/v1/b/{bucket}/o', + simple_multipart=True, + simple_path=u'/upload/storage/v1/b/{bucket}/o', + ), } - def GetMethodConfig(self, method): - return self.__configs.get(method) + def Compose(self, request, global_params=None, download=None): + """Concatenates a list of existing objects into a new object in the same bucket. + + Args: + request: (StorageObjectsComposeRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Compose') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def Copy(self, request, global_params=None, download=None): + """Copies an object to a specified location. Optionally overrides metadata. - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) + Args: + request: (StorageObjectsCopyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Copy') + return self._RunMethod( + config, request, global_params=global_params, + download=download) def Delete(self, request, global_params=None): - """Permanently deletes the default object ACL entry for the specified entity on the specified bucket. + """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. Args: - request: (StorageDefaultObjectAccessControlsDeleteRequest) input message + request: (StorageObjectsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageDefaultObjectAccessControlsDeleteResponse) The response message. + (StorageObjectsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) - def Get(self, request, global_params=None): - """Returns the default object ACL entry for the specified entity on the specified bucket. + def Get(self, request, global_params=None, download=None): + """Retrieves an object or its metadata. Args: - request: (StorageDefaultObjectAccessControlsGetRequest) input message + request: (StorageObjectsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. Returns: - (ObjectAccessControl) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( - config, request, global_params=global_params) + config, request, global_params=global_params, + download=download) - def Insert(self, request, global_params=None): - """Creates a new default object ACL entry on the specified bucket. + def Insert(self, request, global_params=None, upload=None, download=None): + """Stores a new object and metadata. Args: - request: (ObjectAccessControl) input message + request: (StorageObjectsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments + upload: (Upload, default: None) If present, upload + this stream with the request. + download: (Download, default: None) If present, download + data from the request via this stream. Returns: - (ObjectAccessControl) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Insert') + upload_config = self.GetUploadConfig('Insert') return self._RunMethod( - config, request, global_params=global_params) + config, request, global_params=global_params, + upload=upload, upload_config=upload_config, + download=download) def List(self, request, global_params=None): - """Retrieves default object ACL entries on the specified bucket. + """Retrieves a list of objects matching the criteria. Args: - request: (StorageDefaultObjectAccessControlsListRequest) input message + request: (StorageObjectsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControls) The response message. + (Objects) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. + """Updates an object's metadata. This method supports patch semantics. Args: - request: (ObjectAccessControl) input message + request: (StorageObjectsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) - def Update(self, request, global_params=None): - """Updates a default object ACL entry on the specified bucket. + def Update(self, request, global_params=None, download=None): + """Updates an object's metadata. + + Args: + request: (StorageObjectsUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def WatchAll(self, request, global_params=None): + """Watch for changes on all objects in a bucket. Args: - request: (ObjectAccessControl) input message + request: (StorageObjectsWatchAllRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (Channel) The response message. """ - config = self.GetMethodConfig('Update') + config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) class ObjectAccessControlsService(base_api.BaseApiService): """Service class for the objectAccessControls resource.""" + _NAME = u'objectAccessControls' + def __init__(self, client): super(StorageV1.ObjectAccessControlsService, self).__init__(client) - self.__configs = { + self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.objectAccessControls.delete', @@ -660,15 +750,9 @@ class StorageV1(base_api.BaseApiClient): ), } - self.__upload_configs = { + self._upload_configs = { } - def GetMethodConfig(self, method): - return self.__configs.get(method) - - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) - def Delete(self, request, global_params=None): """Permanently deletes the ACL entry for the specified entity on the specified object. @@ -747,271 +831,165 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) - class ObjectsService(base_api.BaseApiService): - """Service class for the objects resource.""" + class BucketsService(base_api.BaseApiService): + """Service class for the buckets resource.""" + + _NAME = u'buckets' def __init__(self, client): - super(StorageV1.ObjectsService, self).__init__(client) - self.__configs = { - 'Compose': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.compose', - ordered_params=[u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], - relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', - request_field=u'composeRequest', - request_type_name=u'StorageObjectsComposeRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'Copy': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.copy', - ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], - relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', - request_field=u'object', - request_type_name=u'StorageObjectsCopyRequest', - response_type_name=u'Object', - supports_download=True, - ), + super(StorageV1.BucketsService, self).__init__(client) + self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.objects.delete', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/o/{object}', + method_id=u'storage.buckets.delete', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}', request_field='', - request_type_name=u'StorageObjectsDeleteRequest', - response_type_name=u'StorageObjectsDeleteResponse', + request_type_name=u'StorageBucketsDeleteRequest', + response_type_name=u'StorageBucketsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.objects.get', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}/o/{object}', + method_id=u'storage.buckets.get', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}', request_field='', - request_type_name=u'StorageObjectsGetRequest', - response_type_name=u'Object', - supports_download=True, + request_type_name=u'StorageBucketsGetRequest', + response_type_name=u'Bucket', + supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.objects.insert', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o', - request_field=u'object', - request_type_name=u'StorageObjectsInsertRequest', - response_type_name=u'Object', - supports_download=True, + method_id=u'storage.buckets.insert', + ordered_params=[u'project'], + path_params=[], + query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection'], + relative_path=u'b', + request_field=u'bucket', + request_type_name=u'StorageBucketsInsertRequest', + response_type_name=u'Bucket', + supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.objects.list', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o', + method_id=u'storage.buckets.list', + ordered_params=[u'project'], + path_params=[], + query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection'], + relative_path=u'b', request_field='', - request_type_name=u'StorageObjectsListRequest', - response_type_name=u'Objects', + request_type_name=u'StorageBucketsListRequest', + response_type_name=u'Buckets', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.objects.patch', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsPatchRequest', - response_type_name=u'Object', + method_id=u'storage.buckets.patch', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsPatchRequest', + response_type_name=u'Bucket', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.objects.update', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsUpdateRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'WatchAll': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.watchAll', + method_id=u'storage.buckets.update', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o/watch', - request_field=u'channel', - request_type_name=u'StorageObjectsWatchAllRequest', - response_type_name=u'Channel', + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsUpdateRequest', + response_type_name=u'Bucket', supports_download=False, ), } - self.__upload_configs = { - 'Insert': base_api.ApiUploadInfo( - accept=['*/*'], - max_size=None, - resumable_multipart=True, - resumable_path=u'/resumable/upload/storage/v1/b/{bucket}/o', - simple_multipart=True, - simple_path=u'/upload/storage/v1/b/{bucket}/o', - ), + self._upload_configs = { } - def GetMethodConfig(self, method): - return self.__configs.get(method) - - def GetMethodUploadConfig(self, method): - return self.__upload_configs.get(method) - - def Compose(self, request, global_params=None, download=None): - """Concatenates a list of existing objects into a new object in the same bucket. - - Args: - request: (StorageObjectsComposeRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. - Returns: - (Object) The response message. - """ - config = self.GetMethodConfig('Compose') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - - def Copy(self, request, global_params=None, download=None): - """Copies an object to a specified location. Optionally overrides metadata. - - Args: - request: (StorageObjectsCopyRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. - Returns: - (Object) The response message. - """ - config = self.GetMethodConfig('Copy') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - def Delete(self, request, global_params=None): - """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. + """Permanently deletes an empty bucket. Args: - request: (StorageObjectsDeleteRequest) input message + request: (StorageBucketsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageObjectsDeleteResponse) The response message. + (StorageBucketsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) - def Get(self, request, global_params=None, download=None): - """Retrieves an object or its metadata. + def Get(self, request, global_params=None): + """Returns metadata for the specified bucket. Args: - request: (StorageObjectsGetRequest) input message + request: (StorageBucketsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( - config, request, global_params=global_params, - download=download) + config, request, global_params=global_params) - def Insert(self, request, global_params=None, upload=None, download=None): - """Stores a new object and metadata. + def Insert(self, request, global_params=None): + """Creates a new bucket. Args: - request: (StorageObjectsInsertRequest) input message + request: (StorageBucketsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments - upload: (Upload, default: None) If present, upload - this stream with the request. - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Insert') - upload_config = self.GetMethodUploadConfig('Insert') return self._RunMethod( - config, request, global_params=global_params, - upload=upload, upload_config=upload_config, - download=download) + config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves a list of objects matching the criteria. + """Retrieves a list of buckets for a given project. Args: - request: (StorageObjectsListRequest) input message + request: (StorageBucketsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Objects) The response message. + (Buckets) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates an object's metadata. This method supports patch semantics. + """Updates a bucket. This method supports patch semantics. Args: - request: (StorageObjectsPatchRequest) input message + request: (StorageBucketsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Object) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) - def Update(self, request, global_params=None, download=None): - """Updates an object's metadata. + def Update(self, request, global_params=None): + """Updates a bucket. Args: - request: (StorageObjectsUpdateRequest) input message + request: (StorageBucketsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Update') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - - def WatchAll(self, request, global_params=None): - """Watch for changes on all objects in a bucket. - - Args: - request: (StorageObjectsWatchAllRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - Returns: - (Channel) The response message. - """ - config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py index 764fcd8..0e7b585 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -4,6 +4,7 @@ Lets you store and retrieve potentially-large, immutable data objects. """ from apitools.base.py import encoding +from apitools.base.py import extra_types from protorpc import message_types from protorpc import messages @@ -129,7 +130,7 @@ class Bucket(messages.Message): """ age = messages.IntegerField(1, variant=messages.Variant.INT32) - createdBefore = message_types.DateTimeField(2) + createdBefore = extra_types.DateField(2) isLive = messages.BooleanField(3) numNewerVersions = messages.IntegerField(4, variant=messages.Variant.INT32) @@ -733,6 +734,8 @@ class StorageBucketsInsertRequest(messages.Message): Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. + PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of + default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. @@ -740,6 +743,8 @@ class StorageBucketsInsertRequest(messages.Message): Fields: bucket: A Bucket resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it @@ -766,6 +771,30 @@ class StorageBucketsInsertRequest(messages.Message): publicRead = 3 publicReadWrite = 4 + class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + """Apply a predefined set of default object access controls to this + bucket. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + class ProjectionValueValuesEnum(messages.Enum): """Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to @@ -780,8 +809,9 @@ class StorageBucketsInsertRequest(messages.Message): bucket = messages.MessageField('Bucket', 1) predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 2) - project = messages.StringField(3, required=True) - projection = messages.EnumField('ProjectionValueValuesEnum', 4) + predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 3) + project = messages.StringField(4, required=True) + projection = messages.EnumField('ProjectionValueValuesEnum', 5) class StorageBucketsListRequest(messages.Message): @@ -794,6 +824,7 @@ class StorageBucketsListRequest(messages.Message): maxResults: Maximum number of buckets to return. pageToken: A previously-returned page token representing part of the larger set of results to view. + prefix: Filter results to buckets whose names begin with this prefix. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl. """ @@ -810,8 +841,9 @@ class StorageBucketsListRequest(messages.Message): maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) pageToken = messages.StringField(2) - project = messages.StringField(3, required=True) - projection = messages.EnumField('ProjectionValueValuesEnum', 4) + prefix = messages.StringField(3) + project = messages.StringField(4, required=True) + projection = messages.EnumField('ProjectionValueValuesEnum', 5) class StorageBucketsPatchRequest(messages.Message): @@ -820,6 +852,8 @@ class StorageBucketsPatchRequest(messages.Message): Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. + PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of + default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: @@ -831,6 +865,8 @@ class StorageBucketsPatchRequest(messages.Message): conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. projection: Set of properties to return. Defaults to full. """ @@ -854,6 +890,30 @@ class StorageBucketsPatchRequest(messages.Message): publicRead = 3 publicReadWrite = 4 + class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + """Apply a predefined set of default object access controls to this + bucket. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + class ProjectionValueValuesEnum(messages.Enum): """Set of properties to return. Defaults to full. @@ -869,7 +929,8 @@ class StorageBucketsPatchRequest(messages.Message): ifMetagenerationMatch = messages.IntegerField(3) ifMetagenerationNotMatch = messages.IntegerField(4) predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) - projection = messages.EnumField('ProjectionValueValuesEnum', 6) + predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) + projection = messages.EnumField('ProjectionValueValuesEnum', 7) class StorageBucketsUpdateRequest(messages.Message): @@ -878,6 +939,8 @@ class StorageBucketsUpdateRequest(messages.Message): Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. + PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of + default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: @@ -889,6 +952,8 @@ class StorageBucketsUpdateRequest(messages.Message): conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. projection: Set of properties to return. Defaults to full. """ @@ -912,6 +977,30 @@ class StorageBucketsUpdateRequest(messages.Message): publicRead = 3 publicReadWrite = 4 + class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + """Apply a predefined set of default object access controls to this + bucket. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + class ProjectionValueValuesEnum(messages.Enum): """Set of properties to return. Defaults to full. @@ -927,7 +1016,8 @@ class StorageBucketsUpdateRequest(messages.Message): ifMetagenerationMatch = messages.IntegerField(3) ifMetagenerationNotMatch = messages.IntegerField(4) predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) - projection = messages.EnumField('ProjectionValueValuesEnum', 6) + predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) + projection = messages.EnumField('ProjectionValueValuesEnum', 7) class StorageChannelsStopResponse(messages.Message): -- GitLab From cb4dce456dfccbc5c8bc235ca99e4a11f99282e8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 17 Feb 2015 17:00:51 -0800 Subject: [PATCH 062/295] Update README. --- README.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 4b84393..7674e1d 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,10 @@ google-apitools ``google-apitools`` is a collection of utilities to make it easier to build client-side tools, especially those that talk to Google APIs. +**NOTE**: This library is stable, but in maintenance mode, and not under +active development. However, any bugs or security issues will be fixed +promptly. + Installing as a library ----------------------- @@ -35,15 +39,3 @@ and the ``nose`` testrunner:: Then run the tests:: $ nosetests - -Current status --------------- - -There are a few imminent large changes: - -- finish the protorpc -> proto2 transition -- switch from httplib2 to requests -- better retry support -- R client library generation -- optional support for `dict -> dict` as the signature on client methods, - doing the proto conversion (and validation!) under the hood. -- GitLab From 7bb660ad5c65f8ccff44f17b48eba069e196d2c0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 17 Feb 2015 17:11:30 -0800 Subject: [PATCH 063/295] Temporary fix for ez_setup issue. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index e9d2654..c1ddbfd 100644 --- a/tox.ini +++ b/tox.ini @@ -4,5 +4,6 @@ envlist = py27 [testenv] deps = nose commands = + pip install ez_setup # temporary pip install google-apitools[testing] nosetests -- GitLab From 5e4d64cb21efe2eb7f728356980c8c6fd9d6820b Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 18 Feb 2015 09:12:42 -0800 Subject: [PATCH 064/295] Add tox envs. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c1ddbfd..46f0867 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27 +envlist = py26,py27,pypy [testenv] deps = nose -- GitLab From 75f5f5aef30e315e45c0c211aff90c60da37524d Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 20 Feb 2015 17:42:16 -0800 Subject: [PATCH 065/295] Drop a now-unnecessary ez_setup install. --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 46f0867..1c99527 100644 --- a/tox.ini +++ b/tox.ini @@ -4,6 +4,5 @@ envlist = py26,py27,pypy [testenv] deps = nose commands = - pip install ez_setup # temporary pip install google-apitools[testing] nosetests -- GitLab From e43ac390b92210d168f631cb17b6bda9aab76520 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 1 Mar 2015 01:24:32 -0800 Subject: [PATCH 066/295] Massive update from internal, and merge gsutil code. This finishes the merge of the internal and gsutil branches of apitools. Phew. --- apitools/base/py/app2.py | 6 +- apitools/base/py/base_api.py | 44 ++- apitools/base/py/base_api_test.py | 55 +++- apitools/base/py/base_cli.py | 1 - apitools/base/py/batch.py | 3 +- apitools/base/py/buffered_stream.py | 57 ++++ apitools/base/py/buffered_stream_test.py | 57 ++++ apitools/base/py/credentials_lib.py | 247 ++++++++++++++-- apitools/base/py/credentials_lib_test.py | 24 +- apitools/base/py/encoding.py | 158 ++++++++++- apitools/base/py/encoding_test.py | 88 +++++- apitools/base/py/exceptions.py | 25 ++ apitools/base/py/extra_types.py | 4 +- apitools/base/py/http_wrapper.py | 287 +++++++++++++++---- apitools/base/py/list_pager.py | 15 +- apitools/base/py/stream_slice.py | 51 ++++ apitools/base/py/stream_slice_test.py | 55 ++++ apitools/base/py/transfer.py | 342 +++++++++++++++++------ apitools/base/py/transfer_test.py | 68 +++++ apitools/base/py/util.py | 53 +++- apitools/base/py/util_test.py | 190 +++++++++++++ apitools/gen/client_generation_test.py | 8 +- apitools/gen/command_registry.py | 2 + apitools/gen/extended_descriptor.py | 65 ++++- apitools/gen/gen_client.py | 14 +- apitools/gen/gen_client_lib.py | 23 +- apitools/gen/message_registry.py | 72 +++-- apitools/gen/service_registry.py | 16 +- apitools/gen/util.py | 26 +- 29 files changed, 1794 insertions(+), 262 deletions(-) create mode 100644 apitools/base/py/buffered_stream.py create mode 100644 apitools/base/py/buffered_stream_test.py create mode 100644 apitools/base/py/stream_slice.py create mode 100644 apitools/base/py/stream_slice_test.py create mode 100644 apitools/base/py/transfer_test.py create mode 100644 apitools/base/py/util_test.py diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 5301874..cff5437 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -10,10 +10,11 @@ import sys import traceback import types +import six + from google.apputils import app from google.apputils import appcommands import gflags as flags -import six __all__ = [ 'NewCmd', @@ -134,7 +135,8 @@ class NewCmd(appcommands.Cmd): def _HandleError(self, e): message = self._FormatError(e) - print('Exception raised in %s operation: %s' % (self._command_name, message)) + print('Exception raised in %s operation: %s' % ( + self._command_name, message)) return 1 def _IsDebuggableException(self, e): diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index e88bb80..73ecbb0 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -8,11 +8,13 @@ import pprint import urllib import urlparse + from protorpc import message_types from protorpc import messages import six from six.moves import http_client + from apitools.base.py import credentials_lib from apitools.base.py import encoding from apitools.base.py import exceptions @@ -351,7 +353,8 @@ class BaseApiClient(object): if http_request.body: # TODO(craigcitro): Make this safe to print in the case of # non-printable body characters. - logging.info('Body:\n%s', http_request.body) + logging.info('Body:\n%s', + http_request.loggable_body or http_request.body) else: logging.info('Body: (none)') return http_request @@ -430,21 +433,32 @@ class BaseApiService(object): setattr(result, field.name, value) return result + def __EncodePrettyPrint(self, query_info): + # The prettyPrint flag needs custom encoding: it should be encoded + # as 0 if False, and ignored otherwise (True is the default). + if not query_info.pop('prettyPrint', True): + query_info['prettyPrint'] = 0 + return query_info + def __ConstructQueryParams(self, query_params, request, global_params): """Construct a dictionary of query parameters for this request.""" + # First, handle the global params. global_params = self.__CombineGlobalParams( global_params, self.__client.global_params) - query_info = dict((field.name, getattr(global_params, field.name)) - for field in self.__client.params_type.all_fields()) + global_param_names = util.MapParamNames( + [x.name for x in self.__client.params_type.all_fields()], + self.__client.params_type) + query_info = dict((param, getattr(global_params, param)) + for param in global_param_names) + # Next, add the query params. + query_param_names = util.MapParamNames(query_params, type(request)) query_info.update( - (param, getattr(request, param, None)) for param in query_params) - query_info = dict((k, v) for k, v in query_info.items() + (param, getattr(request, param, None)) for param in query_param_names) + query_info = dict((k, v) for k, v in six.iteritems(query_info) if v is not None) - # The prettyPrint flag needs custom encoding: it should be encoded - # as 0 if False, and ignored otherwise (True is the default). - if not query_info.pop('prettyPrint', True): - query_info['prettyPrint'] = 0 - for k, v in query_info.items(): + query_info = self.__EncodePrettyPrint(query_info) + query_info = util.MapRequestParams(query_info, type(request)) + for k, v in six.iteritems(query_info): if isinstance(v, six.text_type): query_info[k] = v.encode('utf8') elif isinstance(v, str): @@ -455,8 +469,11 @@ class BaseApiService(object): def __ConstructRelativePath(self, method_config, request, relative_path=None): """Determine the relative path for request.""" + python_param_names = util.MapParamNames( + method_config.path_params, type(request)) params = dict([(param, getattr(request, param, None)) - for param in method_config.path_params]) + for param in python_param_names]) + params = util.MapRequestParams(params, type(request)) return util.ExpandRelativePath(method_config, params, relative_path=relative_path) @@ -577,8 +594,11 @@ class BaseApiService(object): if upload is not None: http_response = upload.InitializeUpload(http_request, client=self.client) if http_response is None: + http = self.__client.http + if upload and upload.bytes_http: + http = upload.bytes_http http_response = http_wrapper.MakeRequest( - self.__client.http, http_request, retries=self.__client.num_retries) + http, http_request, retries=self.__client.num_retries) return self.ProcessHttpResponse(method_config, http_response) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index f6993ff..7e9ce10 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -4,13 +4,14 @@ import datetime import sys import urllib +import urlparse from protorpc import message_types from protorpc import messages - import unittest2 from apitools.base.py import base_api +from apitools.base.py import encoding from apitools.base.py import http_wrapper @@ -22,6 +23,22 @@ class MessageWithTime(messages.Message): timestamp = message_types.DateTimeField(1) +class MessageWithRemappings(messages.Message): + + class AnEnum(messages.Enum): + value_one = 1 + value_two = 2 + + str_field = messages.StringField(1) + enum_field = messages.EnumField('AnEnum', 2) + + +encoding.AddCustomJsonFieldMapping( + MessageWithRemappings, 'str_field', 'remapped_field') +encoding.AddCustomJsonEnumMapping( + MessageWithRemappings.AnEnum, 'value_one', 'ONE/TWO') + + class StandardQueryParameters(messages.Message): field = messages.StringField(1) prettyPrint = messages.BooleanField(5, default=True) # pylint: disable=invalid-name @@ -103,7 +120,7 @@ class BaseApiTest(unittest2.TestCase): request_type_name='MessageWithTime', query_params=['timestamp']) service = FakeService() request = MessageWithTime( - timestamp=datetime.datetime(2014, 10, 7, 12, 53, 13)) + timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) http_request = service.PrepareHttpRequest(method_config, request) url_timestamp = urllib.quote(request.timestamp.isoformat()) @@ -111,10 +128,10 @@ class BaseApiTest(unittest2.TestCase): def testPrettyPrintEncoding(self): method_config = base_api.ApiMethodInfo( - request_type_name='MessageWithTime', query_params=['timestamp']) + request_type_name='MessageWithTime', query_params=['timestamp']) service = FakeService() request = MessageWithTime( - timestamp=datetime.datetime(2014, 10, 07, 12, 53, 13)) + timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) global_params = StandardQueryParameters() http_request = service.PrepareHttpRequest(method_config, request, @@ -126,6 +143,36 @@ class BaseApiTest(unittest2.TestCase): global_params=global_params) self.assertTrue('prettyPrint=0' in http_request.url) + def testQueryRemapping(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithRemappings', + query_params=['remapped_field', 'enum_field']) + request = MessageWithRemappings( + str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one) + http_request = FakeService().PrepareHttpRequest(method_config, request) + result_params = urlparse.parse_qs( + urlparse.urlparse(http_request.url).query) + expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'} + self.assertTrue(expected_params, result_params) + + def testPathRemapping(self): + method_config = base_api.ApiMethodInfo( + relative_path='parameters/{remapped_field}/remap/{enum_field}', + request_type_name='MessageWithRemappings', + path_params=['remapped_field', 'enum_field']) + request = MessageWithRemappings( + str_field='gonna', enum_field=MessageWithRemappings.AnEnum.value_one) + service = FakeService() + expected_url = service.client.url + 'parameters/gonna/remap/ONE%2FTWO' + http_request = service.PrepareHttpRequest(method_config, request) + self.assertEqual(expected_url, http_request.url) + + method_config.relative_path = ( + 'parameters/{+remapped_field}/remap/{+enum_field}') + expected_url = service.client.url + 'parameters/gonna/remap/ONE/TWO' + http_request = service.PrepareHttpRequest(method_config, request) + self.assertEqual(expected_url, http_request.url) + if __name__ == '__main__': unittest2.main() diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index d55394b..399c8a9 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """Base script for generated CLI.""" - import atexit import code import logging diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index bd07cd0..e098282 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -13,7 +13,6 @@ import urllib import urlparse import uuid -from six.moves import range from six.moves import http_client from apitools.base.py import exceptions @@ -281,7 +280,7 @@ class BatchHttpRequest(object): msg = mime_nonmultipart.MIMENonMultipart(major, minor) # MIMENonMultipart adds its own Content-Type header. - # Keep all of the other headers in headers. + # Keep all of the other headers in `request.headers`. for key, value in request.headers.items(): if key == 'content-type': continue diff --git a/apitools/base/py/buffered_stream.py b/apitools/base/py/buffered_stream.py new file mode 100644 index 0000000..f1d84fe --- /dev/null +++ b/apitools/base/py/buffered_stream.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Small helper class to provide a small slice of a stream. + +This class reads ahead to detect if we are at the end of the stream. +""" + +from apitools.base.py import exceptions + + +# TODO(user): Consider replacing this with a StringIO. +class BufferedStream(object): + """Buffers a stream, reading ahead to determine if we're at the end.""" + + def __init__(self, stream, start, size): + self.__stream = stream + self.__start_pos = start + self.__buffer_pos = 0 + self.__buffered_data = self.__stream.read(size) + self.__stream_at_end = len(self.__buffered_data) < size + self.__end_pos = self.__start_pos + len(self.__buffered_data) + + def __str__(self): + return ('Buffered stream %s from position %s-%s with %s ' + 'bytes remaining' % (self.__stream, self.__start_pos, + self.__end_pos, self._bytes_remaining)) + + def __len__(self): + return len(self.__buffered_data) + + @property + def stream_exhausted(self): + return self.__stream_at_end + + @property + def stream_end_position(self): + return self.__end_pos + + @property + def _bytes_remaining(self): + return len(self.__buffered_data) - self.__buffer_pos + + def read(self, size=None): # pylint: disable=invalid-name + """Reads from the buffer.""" + if size is None or size < 0: + raise exceptions.NotYetImplementedError( + 'Illegal read of size %s requested on BufferedStream. ' + 'Wrapped stream %s is at position %s-%s, ' + '%s bytes remaining.' % + (size, self.__stream, self.__start_pos, self.__end_pos, + self._bytes_remaining)) + + data = '' + if self._bytes_remaining: + size = min(size, self._bytes_remaining) + data = self.__buffered_data[self.__buffer_pos:self.__buffer_pos + size] + self.__buffer_pos += size + return data diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py new file mode 100644 index 0000000..c8e3b3a --- /dev/null +++ b/apitools/base/py/buffered_stream_test.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Tests for stream_slice.""" + +import string +import StringIO + +import unittest2 + +from apitools.base.py import buffered_stream +from apitools.base.py import exceptions + + +class BufferedStreamTest(unittest2.TestCase): + + def setUp(self): + self.stream = StringIO.StringIO(string.letters) + self.value = self.stream.getvalue() + self.stream.seek(0) + + def testEmptyBuffer(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 0) + self.assertEqual('', bs.read(0)) + self.assertEqual(0, bs.stream_end_position) + + def testOffsetStream(self): + bs = buffered_stream.BufferedStream(self.stream, 50, 100) + self.assertEqual(len(self.value), len(bs)) + self.assertEqual(self.value, bs.read(len(self.value))) + self.assertEqual(50 + len(self.value), bs.stream_end_position) + + def testUnexhaustedStream(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 50) + self.assertEqual(50, bs.stream_end_position) + self.assertEqual(False, bs.stream_exhausted) + self.assertEqual(self.value[0:50], bs.read(50)) + self.assertEqual(False, bs.stream_exhausted) + self.assertEqual('', bs.read(0)) + self.assertEqual('', bs.read(100)) + + def testExhaustedStream(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 100) + self.assertEqual(len(self.value), bs.stream_end_position) + self.assertEqual(True, bs.stream_exhausted) + self.assertEqual(self.value, bs.read(100)) + self.assertEqual('', bs.read(0)) + self.assertEqual('', bs.read(100)) + + def testArbitraryLengthRead(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 20) + with self.assertRaises(exceptions.NotYetImplementedError): + bs.read() + with self.assertRaises(exceptions.NotYetImplementedError): + bs.read(size=-1) + + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index d08fcd9..6f4dcd3 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -2,34 +2,39 @@ """Common credentials classes and constructors.""" from __future__ import print_function +import datetime import json -import logging import os +import urllib import urllib2 - import httplib2 import oauth2client import oauth2client.client import oauth2client.gce +import oauth2client.locked_file import oauth2client.multistore_file -import oauth2client.tools +import oauth2client.tools # for flag declarations from six.moves import http_client -try: - from gflags import FLAGS -except ImportError: - FLAGS = None - +import logging from apitools.base.py import exceptions from apitools.base.py import util +try: + import gflags as flags + FLAGS = flags.FLAGS +except ImportError: + FLAGS = None + + __all__ = [ 'CredentialsFromFile', 'GaeAssertionCredentials', 'GceAssertionCredentials', 'GetCredentials', + 'GetUserinfo', 'ServiceAccountCredentials', 'ServiceAccountCredentialsFromFile', ] @@ -83,17 +88,131 @@ def ServiceAccountCredentials(service_account_name, private_key, scopes): service_account_name, private_key, scopes) +def _EnsureFileExists(filename): + """Touches a file; returns False on error, True on success.""" + if not os.path.exists(filename): + old_umask = os.umask(0o177) + try: + open(filename, 'a+b').close() + except OSError: + return False + finally: + os.umask(old_umask) + return True + + +def _OpenNoProxy(request): + """Wrapper around urllib2.open that ignores proxies.""" + opener = urllib2.build_opener(urllib2.ProxyHandler({})) + return opener.open(request) + + # TODO(craigcitro): We override to add some utility code, and to -# update the old refresh implementation. Either push this code into -# oauth2client or drop oauth2client. +# update the old refresh implementation. Push this code into +# oauth2client. class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """Assertion credentials for GCE instances.""" def __init__(self, scopes=None, service_account_name='default', **kwds): + """Initializes the credentials instance. + + Args: + scopes: The scopes to get. If None, whatever scopes that are available + to the instance are used. + service_account_name: The service account to retrieve the scopes from. + **kwds: Additional keyword args. + """ + # If there is a connectivity issue with the metadata server, + # detection calls may fail even if we've already successfully identified + # these scopes in the same execution. However, the available scopes don't + # change once an instance is created, so there is no reason to perform + # more than one query. + # + # TODO(craigcitro): Move this into oauth2client. + self.__service_account_name = service_account_name + cache_filename = None + cached_scopes = None + if 'cache_filename' in kwds: + cache_filename = kwds['cache_filename'] + cached_scopes = self._CheckCacheFileForMatch(cache_filename, scopes) + + scopes = cached_scopes or self._ScopesFromMetadataServer(scopes) + + if cache_filename and not cached_scopes: + self._WriteCacheFile(cache_filename, scopes) + + super(GceAssertionCredentials, self).__init__(scopes, **kwds) + + @classmethod + def Get(cls, *args, **kwds): + try: + return cls(*args, **kwds) + except exceptions.Error: + return None + + def _CheckCacheFileForMatch(self, cache_filename, scopes): + """Checks the cache file to see if it matches the given credentials. + + Args: + cache_filename: Cache filename to check. + scopes: Scopes for the desired credentials. + + Returns: + List of scopes (if cache matches) or None. + """ + creds = { # Credentials metadata dict. + 'scopes': sorted(list(scopes)) if scopes else None, + 'svc_acct_name': self.__service_account_name} + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + cached_creds_str = locked_file.file_handle().read() + if cached_creds_str: + # Cached credentials metadata dict. + cached_creds = json.loads(cached_creds_str) + if (creds['svc_acct_name'] == cached_creds['svc_acct_name'] and + (creds['scopes'] is None or + creds['scopes'] == cached_creds['scopes'])): + scopes = cached_creds['scopes'] + finally: + locked_file.unlock_and_close() + return scopes + + def _WriteCacheFile(self, cache_filename, scopes): + """Writes the credential metadata to the cache file. + + This does not save the credentials themselves (CredentialStore class + optionally handles that after this class is initialized). + + Args: + cache_filename: Cache filename to check. + scopes: Scopes for the desired credentials. + """ + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + if locked_file.is_locked(): + creds = { # Credentials metadata dict. + 'scopes': sorted(list(scopes)), + 'svc_acct_name': self.__service_account_name} + locked_file.file_handle().write(json.dumps(creds, encoding='ascii')) + # If it's not locked, the locking process will write the same + # data to the file, so just continue. + finally: + locked_file.unlock_and_close() + + def _ScopesFromMetadataServer(self, scopes): if not util.DetectGce(): raise exceptions.ResourceUnavailableError( 'GCE credentials requested outside a GCE instance') - self.__service_account_name = service_account_name + if not self.GetServiceAccount(self.__service_account_name): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested but service account %s does not exist.' % + self.__service_account_name) if scopes: scope_ls = util.NormalizeScopes(scopes) instance_scopes = self.GetInstanceScopes() @@ -103,14 +222,21 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): sorted(list(scope_ls - instance_scopes)),)) else: scopes = self.GetInstanceScopes() - super(GceAssertionCredentials, self).__init__(scopes, **kwds) + return scopes - @classmethod - def Get(cls, *args, **kwds): + def GetServiceAccount(self, account): + account_uri = ( + 'http://metadata.google.internal/computeMetadata/' + 'v1/instance/service-accounts') + additional_headers = {'X-Google-Metadata-Request': 'True'} + request = urllib2.Request(account_uri, headers=additional_headers) try: - return cls(*args, **kwds) - except exceptions.Error: - return None + response = _OpenNoProxy(request) + except urllib2.URLError as e: + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + response_lines = [line.rstrip('/\n\r') for line in response.readlines()] + return account in response_lines def GetInstanceScopes(self): # Extra header requirement can be found here: @@ -121,7 +247,7 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): additional_headers = {'X-Google-Metadata-Request': 'True'} request = urllib2.Request(scopes_uri, headers=additional_headers) try: - response = urllib2.urlopen(request) + response = _OpenNoProxy(request) except urllib2.URLError as e: raise exceptions.CommunicationError( 'Could not reach metadata service: %s' % e.reason) @@ -130,23 +256,64 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): def _refresh(self, do_request): """Refresh self.access_token. + This function replaces AppAssertionCredentials._refresh, which does not use + the credential store and is therefore poorly suited for multi-threaded + scenarios. + Args: do_request: A function matching httplib2.Http.request's signature. """ + # pylint: disable=protected-access + oauth2client.client.OAuth2Credentials._refresh(self, do_request) + # pylint: enable=protected-access + + def _do_refresh_request(self, unused_http_request): + """Refresh self.access_token by querying the metadata server. + + If self.store is initialized, store acquired credentials there. + """ token_uri = ( - 'http://metadata.google.internal/computeMetadata/v1beta1/instance/' + 'http://metadata.google.internal/computeMetadata/v1/instance/' 'service-accounts/%s/token') % self.__service_account_name extra_headers = {'X-Google-Metadata-Request': 'True'} - response, content = do_request(token_uri, headers=extra_headers) - if response.status != http_client.OK: - raise exceptions.CredentialsError( - 'Error refreshing credentials: %s' % content) + request = urllib2.Request(token_uri, headers=extra_headers) + try: + content = _OpenNoProxy(request).read() + except urllib2.URLError as e: + self.invalid = True + if self.store: + self.store.locked_put(self) + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) try: credential_info = json.loads(content) except ValueError: raise exceptions.CredentialsError( - 'Invalid credentials response: %s' % content) + 'Invalid credentials response: uri %s' % token_uri) + self.access_token = credential_info['access_token'] + if 'expires_in' in credential_info: + self.token_expiry = ( + datetime.timedelta(seconds=int(credential_info['expires_in'])) + + datetime.datetime.utcnow()) + else: + self.token_expiry = None + self.invalid = False + if self.store: + self.store.locked_put(self) + + @classmethod + def from_json(cls, json_data): + data = json.loads(json_data) + credentials = GceAssertionCredentials(scopes=[data['scope']]) + if 'access_token' in data: + credentials.access_token = data['access_token'] + if 'token_expiry' in data: + credentials.token_expiry = datetime.datetime.strptime( + data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) + if 'invalid' in data: + credentials.invalid = data['invalid'] + return credentials # TODO(craigcitro): Currently, we can't even *load* @@ -208,7 +375,9 @@ def CredentialsFromFile(path, client_info): # retry loop, they can ^C. try: flow = oauth2client.client.OAuth2WebServerFlow(**client_info) - credentials = oauth2client.tools.run(flow, credential_store) + # We delay this import because it's rarely needed and takes a long time. + from oauth2client import tools + credentials = tools.run(flow, credential_store) break except (oauth2client.client.FlowExchangeError, SystemExit) as e: # Here SystemExit is "no credential at all", and the @@ -220,3 +389,31 @@ def CredentialsFromFile(path, client_info): raise exceptions.CredentialsError( 'Communication error creating credentials: %s' % e) return credentials + + +# TODO(craigcitro): Push this into oauth2client. +def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name + """Get the userinfo associated with the given credentials. + + This is dependent on the token having either the userinfo.email or + userinfo.profile scope for the given token. + + Args: + credentials: (oauth2client.client.Credentials) incoming credentials + http: (httplib2.Http, optional) http instance to use + + Returns: + The email address for this token, or None if the required scopes + aren't available. + """ + http = http or httplib2.Http() + url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' + query_args = {'access_token': credentials.access_token} + url = '?'.join((url_root, urllib.urlencode(query_args))) + # We ignore communication woes here (i.e. SSL errors, socket + # timeout), as handling these should be done in a common location. + response, content = http.request(url) + if response.status == http_client.BAD_REQUEST: + credentials.refresh(http) + response, content = http.request(url) + return json.loads(content or '{}') # Save ourselves from an empty reply. diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 2ad5fc8..e445df5 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python + import re import StringIO -import urllib2 import mock from six.moves import http_client @@ -34,15 +34,29 @@ class CredentialsLibTest(unittest2.TestCase): if service_account_name is not None: kwargs['service_account_name'] = service_account_name service_account_name = service_account_name or 'default' - with mock.patch.object(urllib2, 'urlopen', autospec=True) as urllib_mock: - urllib_mock.return_value = StringIO.StringIO(''.join(scopes)) - with mock.patch.object(util, 'DetectGce', autospec=True) as mock_util: - mock_util.return_value = True + + def MockMetadataCalls(request): + request_url = request.get_full_url() + if request_url.endswith('scopes'): + return StringIO.StringIO(''.join(scopes)) + elif request_url.endswith('service-accounts'): + return StringIO.StringIO(service_account_name) + elif request_url.endswith( + '/service-accounts/%s/token' % service_account_name): + return StringIO.StringIO('{"access_token": "token"}') + self.fail('Unexpected HTTP request to %s' % request_url) + + with mock.patch.object(credentials_lib, '_OpenNoProxy', + side_effect=MockMetadataCalls, + autospec=True) as opener_mock: + with mock.patch.object(util, 'DetectGce', autospec=True) as mock_detect: + mock_detect.return_value = True validator = CreateUriValidator( re.compile(r'.*/%s/.*' % service_account_name), content='{"access_token": "token"}') credentials = credentials_lib.GceAssertionCredentials(scopes, **kwargs) self.assertIsNone(credentials._refresh(validator)) + self.assertEqual(3, opener_mock.call_count) def testGceServiceAccounts(self): self._GetServiceCreds() diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index bb15392..489cef8 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -7,6 +7,7 @@ import datetime import json import logging + from protorpc import message_types from protorpc import messages from protorpc import protojson @@ -23,6 +24,10 @@ __all__ = [ 'PyValueToMessage', 'MessageToPyValue', 'MessageToRepr', + 'GetCustomJsonFieldMapping', + 'AddCustomJsonFieldMapping', + 'GetCustomJsonEnumMapping', + 'AddCustomJsonEnumMapping', ] @@ -255,8 +260,9 @@ class _ProtoJsonApiTools(protojson.ProtoJson): # remove this later. old_level = logging.getLogger().level logging.getLogger().setLevel(logging.ERROR) + result = _DecodeCustomFieldNames(message_type, encoded_message) result = super(_ProtoJsonApiTools, self).decode_message( - message_type, encoded_message) + message_type, result) logging.getLogger().setLevel(old_level) result = _ProcessUnknownEnums(result, encoded_message) result = _ProcessUnknownMessages(result, encoded_message) @@ -280,6 +286,7 @@ class _ProtoJsonApiTools(protojson.ProtoJson): if isinstance(field, messages.MessageField): field_value = self.decode_message(field.message_type, json.dumps(value)) elif isinstance(field, messages.EnumField): + value = GetCustomJsonEnumMapping(field.type, json_name=value) or value try: field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) except messages.DecodeError: @@ -296,7 +303,8 @@ class _ProtoJsonApiTools(protojson.ProtoJson): if type(message) in _CUSTOM_MESSAGE_CODECS: return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) message = _EncodeUnknownFields(message) - return super(_ProtoJsonApiTools, self).encode_message(message) + result = super(_ProtoJsonApiTools, self).encode_message(message) + return _EncodeCustomFieldNames(message, result) def encode_field(self, field, value): """Encode the given value as JSON. @@ -313,6 +321,15 @@ class _ProtoJsonApiTools(protojson.ProtoJson): value = result.value if result.complete: return value + if isinstance(field, messages.EnumField): + if field.repeated: + remapped_value = [GetCustomJsonEnumMapping( + field.type, python_name=e.name) or e.name for e in value] + else: + remapped_value = GetCustomJsonEnumMapping( + field.type, python_name=value.name) + if remapped_value: + return remapped_value if (isinstance(field, messages.MessageField) and not isinstance(field, message_types.DateTimeField)): value = json.loads(self.encode_message(value)) @@ -350,7 +367,7 @@ def _DecodeUnknownMessages(message, encoded_message, pair_type): field_type = pair_type.value.type new_values = [] all_field_names = [x.name for x in message.all_fields()] - for name, value_dict in encoded_message.items(): + for name, value_dict in six.iteritems(encoded_message): if name in all_field_names: continue value = PyValueToMessage(field_type, value_dict) @@ -475,7 +492,7 @@ def _ProcessUnknownMessages(message, encoded_message): decoded_message = json.loads(encoded_message) message_fields = [x.name for x in message.all_fields()] + list( message.all_unrecognized_fields()) - missing_fields = [x for x in decoded_message.keys() + missing_fields = [x for x in six.iterkeys(decoded_message) if x not in message_fields] for field_name in missing_fields: message.set_unrecognized_field(field_name, decoded_message[field_name], @@ -484,3 +501,136 @@ def _ProcessUnknownMessages(message, encoded_message): RegisterFieldTypeCodec(_SafeEncodeBytes, _SafeDecodeBytes)(messages.BytesField) + + +# Note that these could share a dictionary, since they're keyed by +# distinct types, but it's not really worth it. +_JSON_ENUM_MAPPINGS = {} +_JSON_FIELD_MAPPINGS = {} + + +def AddCustomJsonEnumMapping(enum_type, python_name, json_name): + """Add a custom wire encoding for a given enum value. + + This is primarily used in generated code, to handle enum values + which happen to be Python keywords. + + Args: + enum_type: (messages.Enum) An enum type + python_name: (basestring) Python name for this value. + json_name: (basestring) JSON name to be used on the wire. + """ + if not issubclass(enum_type, messages.Enum): + raise exceptions.TypecheckError( + 'Cannot set JSON enum mapping for non-enum "%s"' % enum_type) + enum_name = enum_type.definition_name() + if python_name not in enum_type.names(): + raise exceptions.InvalidDataError( + 'Enum value %s not a value for type %s' % (python_name, enum_type)) + field_mappings = _JSON_ENUM_MAPPINGS.setdefault(enum_name, {}) + _CheckForExistingMappings('enum', enum_type, python_name, json_name) + field_mappings[python_name] = json_name + + +def AddCustomJsonFieldMapping(message_type, python_name, json_name): + """Add a custom wire encoding for a given message field. + + This is primarily used in generated code, to handle enum values + which happen to be Python keywords. + + Args: + message_type: (messages.Message) A message type + python_name: (basestring) Python name for this value. + json_name: (basestring) JSON name to be used on the wire. + """ + if not issubclass(message_type, messages.Message): + raise exceptions.TypecheckError( + 'Cannot set JSON field mapping for non-message "%s"' % message_type) + message_name = message_type.definition_name() + try: + _ = message_type.field_by_name(python_name) + except KeyError: + raise exceptions.InvalidDataError( + 'Field %s not recognized for type %s' % (python_name, message_type)) + field_mappings = _JSON_FIELD_MAPPINGS.setdefault(message_name, {}) + _CheckForExistingMappings('field', message_type, python_name, json_name) + field_mappings[python_name] = json_name + + +def GetCustomJsonEnumMapping(enum_type, python_name=None, json_name=None): + """Return the appropriate remapping for the given enum, or None.""" + return _FetchRemapping(enum_type.definition_name(), 'enum', + python_name=python_name, json_name=json_name, + mappings=_JSON_ENUM_MAPPINGS) + + +def GetCustomJsonFieldMapping(message_type, python_name=None, json_name=None): + """Return the appropriate remapping for the given field, or None.""" + return _FetchRemapping(message_type.definition_name(), 'field', + python_name=python_name, json_name=json_name, + mappings=_JSON_FIELD_MAPPINGS) + + +def _FetchRemapping(type_name, mapping_type, python_name=None, json_name=None, + mappings=None): + """Common code for fetching a key or value from a remapping dict.""" + if python_name and json_name: + raise exceptions.InvalidDataError( + 'Cannot specify both python_name and json_name for %s remapping' % ( + mapping_type,)) + if not (python_name or json_name): + raise exceptions.InvalidDataError( + 'Must specify either python_name or json_name for %s remapping' % ( + mapping_type,)) + field_remappings = mappings.get(type_name, {}) + if field_remappings: + if python_name: + return field_remappings.get(python_name) + elif json_name: + if json_name in list(field_remappings.values()): + return [k for k in field_remappings + if field_remappings[k] == json_name][0] + return None + + +def _CheckForExistingMappings(mapping_type, message_type, + python_name, json_name): + """Validate that no mappings exist for the given values.""" + if mapping_type == 'field': + getter = GetCustomJsonFieldMapping + elif mapping_type == 'enum': + getter = GetCustomJsonEnumMapping + remapping = getter(message_type, python_name=python_name) + if remapping is not None: + raise exceptions.InvalidDataError( + 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( + mapping_type, python_name, remapping)) + remapping = getter(message_type, json_name=json_name) + if remapping is not None: + raise exceptions.InvalidDataError( + 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( + mapping_type, json_name, remapping)) + + +def _EncodeCustomFieldNames(message, encoded_value): + message_name = type(message).definition_name() + field_remappings = list(_JSON_FIELD_MAPPINGS.get(message_name, {}).items()) + if field_remappings: + decoded_value = json.loads(encoded_value) + for python_name, json_name in field_remappings: + if python_name in encoded_value: + decoded_value[json_name] = decoded_value.pop(python_name) + encoded_value = json.dumps(decoded_value) + return encoded_value + + +def _DecodeCustomFieldNames(message_type, encoded_message): + message_name = message_type.definition_name() + field_remappings = _JSON_FIELD_MAPPINGS.get(message_name, {}) + if field_remappings: + decoded_message = json.loads(encoded_message) + for python_name, json_name in list(field_remappings.items()): + if json_name in decoded_message: + decoded_message[python_name] = decoded_message.pop(json_name) + encoded_message = json.dumps(decoded_message) + return encoded_message diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 77224d6..81b02c0 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -11,6 +11,7 @@ from protorpc import util import unittest2 from apitools.base.py import encoding +from apitools.base.py import exceptions class SimpleMessage(messages.Message): @@ -74,6 +75,29 @@ class ExtraNestedMessage(messages.Message): nested = messages.MessageField(HasNestedMessage, 1) +class MessageWithRemappings(messages.Message): + + class SomeEnum(messages.Enum): + enum_value = 1 + second_value = 2 + + enum_field = messages.EnumField(SomeEnum, 1) + double_encoding = messages.EnumField(SomeEnum, 2) + another_field = messages.StringField(3) + repeated_enum = messages.EnumField(SomeEnum, 4, repeated=True) + repeated_field = messages.StringField(5, repeated=True) + + +encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, + 'enum_value', 'wire_name') +encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'double_encoding', 'doubleEncoding') +encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'another_field', 'anotherField') +encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'repeated_field', 'repeatedField') + + class EncodingTest(unittest2.TestCase): def testCopyProtoMessage(self): @@ -94,9 +118,7 @@ class EncodingTest(unittest2.TestCase): self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, b64_msg)) self.assertEqual(urlsafe_b64_msg, encoding.MessageToJson(msg)) - enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % { - 'b': urlsafe_b64_str, - } + enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % {'b': urlsafe_b64_str} rep_msg = BytesMessage(repfield=[data, data]) self.assertEqual(rep_msg, encoding.JsonToMessage(BytesMessage, enc_rep_msg)) self.assertEqual(enc_rep_msg, encoding.MessageToJson(rep_msg)) @@ -231,6 +253,66 @@ class EncodingTest(unittest2.TestCase): '{"timefield": "2014-07-02T23:33:25.541000+00:00"}', encoding.MessageToJson(msg)) + def testEnumRemapping(self): + msg = MessageWithRemappings( + enum_field=MessageWithRemappings.SomeEnum.enum_value) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"enum_field": "wire_name"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testRepeatedEnumRemapping(self): + msg = MessageWithRemappings( + repeated_enum=[ + MessageWithRemappings.SomeEnum.enum_value, + MessageWithRemappings.SomeEnum.second_value, + ]) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"repeated_enum": ["wire_name", "second_value"]}', + json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testFieldRemapping(self): + msg = MessageWithRemappings(another_field='abc') + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"anotherField": "abc"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testRepeatedFieldRemapping(self): + msg = MessageWithRemappings(repeated_field=['abc', 'def']) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"repeatedField": ["abc", "def"]}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testMultipleRemapping(self): + msg = MessageWithRemappings( + double_encoding=MessageWithRemappings.SomeEnum.enum_value) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"doubleEncoding": "wire_name"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testNoRepeatedRemapping(self): + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonFieldMapping, + MessageWithRemappings, 'double_encoding', 'something_else') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonFieldMapping, + MessageWithRemappings, 'enum_field', 'anotherField') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonEnumMapping, + MessageWithRemappings.SomeEnum, 'enum_value', 'another_name') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonEnumMapping, + MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') + def testMessageToRepr(self): # pylint:disable=bad-whitespace, Using the same string returned by # MessageToRepr, with the module names fixed. diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index 55faa49..9af1f37 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -88,10 +88,35 @@ class TransferError(CommunicationError): """Errors related to transfers.""" +class TransferRetryError(TransferError): + """Retryable errors related to transfers.""" + + class TransferInvalidError(TransferError): """The given transfer is invalid.""" +class RequestError(CommunicationError): + """The request was not successful.""" + + +class RetryAfterError(HttpError): + """The response contained a retry-after header.""" + + def __init__(self, response, content, url, retry_after): + super(RetryAfterError, self).__init__(response, content, url) + self.retry_after = int(retry_after) + + @classmethod + def FromResponse(cls, http_response): + return cls(http_response.info, http_response.content, + http_response.request_url, http_response.retry_after) + + +class BadStatusCodeError(HttpError): + """The request completed but returned a bad status code.""" + + class NotYetImplementedError(GeneratedClientError): """This functionality is not yet implemented.""" diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index b848ade..62ae7b5 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -122,11 +122,11 @@ def _PythonValueToJsonObject(py_value): return JsonObject( properties=[ JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) - for key, value in py_value.items()]) + for key, value in six.iteritems(py_value)]) def _PythonValueToJsonArray(py_value): - return JsonArray(entries=[_PythonValueToJsonValue(val) for val in py_value]) + return JsonArray(entries=list(map(_PythonValueToJsonValue, py_value))) class JsonValue(messages.Message): diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 41bdcd9..f91c361 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -6,22 +6,28 @@ currently httplib2. """ import collections +import contextlib import logging import socket import time import httplib2 +import six from six.moves import http_client -from six.moves import range -from six.moves.urllib.parse import urlsplit +from six.moves.urllib import parse from apitools.base.py import exceptions from apitools.base.py import util __all__ = [ + 'CheckResponse', 'GetHttp', + 'HandleExceptionsAndRebuildHttpConnections', 'MakeRequest', + 'RebuildHttpConnections', 'Request', + 'Response', + 'RethrowExceptionHandler', ] @@ -36,6 +42,57 @@ _REDIRECT_STATUS_CODES = ( RESUME_INCOMPLETE, ) +# http: An httplib2.Http instance. +# http_request: A http_wrapper.Request. +# exc: Exception being raised. +# num_retries: Number of retries consumed; used for exponential backoff. +ExceptionRetryArgs = collections.namedtuple( + 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries']) + + +@contextlib.contextmanager +def _Httplib2Debuglevel(http_request, level, http=None): + """Temporarily change the value of httplib2.debuglevel, if necessary. + + If http_request has a `loggable_body` distinct from `body`, then we + need to prevent httplib2 from logging the full body. This sets + httplib2.debuglevel for the duration of the `with` block; however, + that alone won't change the value of existing HTTP connections. If + an httplib2.Http object is provided, we'll also change the level on + any cached connections attached to it. + + Args: + http_request: a Request we're logging. + level: (int) the debuglevel for logging. + http: (optional) an httplib2.Http whose connections we should + set the debuglevel on. + + Yields: + None. + """ + if http_request.loggable_body is None: + yield + return + old_level = httplib2.debuglevel + http_levels = {} + httplib2.debuglevel = level + if http is not None: + for connection_key, connection in six.iteritems(http.connections): + # httplib2 stores two kinds of values in this dict, connection + # classes and instances. Since the connection types are all + # old-style classes, we can't easily distinguish by connection + # type -- so instead we use the key pattern. + if ':' not in connection_key: + continue + http_levels[connection_key] = connection.debuglevel + connection.set_debuglevel(level) + yield + httplib2.debuglevel = old_level + if http is not None: + for connection_key, old_level in six.iteritems(http_levels): + if connection_key in http.connections: + http.connections[connection_key].set_debuglevel(old_level) + class Request(object): """Class encapsulating the data for an HTTP request.""" @@ -45,8 +102,20 @@ class Request(object): self.http_method = http_method self.headers = headers or {} self.__body = None + self.__loggable_body = None self.body = body + @property + def loggable_body(self): + return self.__loggable_body + + @loggable_body.setter + def loggable_body(self, value): + if self.body is None: + raise exceptions.RequestError( + 'Cannot set loggable body on request with no body') + self.__loggable_body = value + @property def body(self): return self.__body @@ -58,6 +127,9 @@ class Request(object): self.headers['content-length'] = str(len(self.__body)) else: self.headers.pop('content-length', None) + # This line ensures we don't try to print large requests. + if not isinstance(value, six.string_types): + self.loggable_body = '' # Note: currently the order of fields here is important, since we want @@ -68,6 +140,18 @@ class Response(collections.namedtuple( __slots__ = () def __len__(self): + return self.length + + @property + def length(self): + """Return the length of this response. + + We expose this as an attribute since using len() directly can fail + for responses larger than sys.maxint. + + Returns: + Response length (as int or long) + """ def ProcessContentRange(content_range): _, _, range_spec = content_range.partition(' ') byte_range, _, _ = range_spec.partition('/') @@ -100,84 +184,173 @@ class Response(collections.namedtuple( 'location' in self.info) -def MakeRequest(http, http_request, retries=5, redirections=5): +def CheckResponse(response): + if response is None: + # Caller shouldn't call us if the response is None, but handle anyway. + raise exceptions.RequestError('Request to url %s did not return a response.' + % response.request_url) + elif (response.status_code >= 500 or + response.status_code == TOO_MANY_REQUESTS): + raise exceptions.BadStatusCodeError.FromResponse(response) + elif response.status_code == http_client.UNAUTHORIZED: + # Sometimes we get a 401 after a connection break. + # TODO(craigcitro): this shouldn't be a retryable exception, but + # for now we retry. + raise exceptions.BadStatusCodeError.FromResponse(response) + elif response.retry_after: + raise exceptions.RetryAfterError.FromResponse(response) + + +def RebuildHttpConnections(http): + """Rebuilds all http connections in the httplib2.Http instance. + + httplib2 overloads the map in http.connections to contain two different + types of values: + { scheme string: connection class } and + { scheme + authority string : actual http connection } + Here we remove all of the entries for actual connections so that on the + next request httplib2 will rebuild them from the connection types. + + Args: + http: An httplib2.Http instance. + """ + if getattr(http, 'connections', None): + for conn_key in list(http.connections.keys()): + if ':' in conn_key: + del http.connections[conn_key] + + +def RethrowExceptionHandler(*unused_args): + raise + + +def HandleExceptionsAndRebuildHttpConnections(retry_args): + """Exception handler for http failures. + + This catches known failures and rebuilds the underlying HTTP connections. + + Args: + retry_args: An ExceptionRetryArgs tuple. + """ + # If the server indicates how long to wait, use that value. Otherwise, + # calculate the wait time on our own. + retry_after = None + + # Transport failures + if isinstance(retry_args.exc, http_client.BadStatusLine): + logging.debug('Caught BadStatusLine from httplib, retrying: %s', + retry_args.exc) + elif isinstance(retry_args.exc, socket.error): + logging.debug('Caught socket error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, socket.gaierror): + logging.debug('Caught socket address error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, httplib2.ServerNotFoundError): + logging.debug('Caught server not found error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, ValueError): + # oauth2client tries to JSON-decode the response, which can result + # in a ValueError if the response was invalid. Until that is fixed in + # oauth2client, need to handle it here. + logging.debug('Response content was invalid (%s), retrying', + retry_args.exc) + elif isinstance(retry_args.exc, exceptions.RequestError): + logging.debug('Request returned no response, retrying') + # API-level failures + elif isinstance(retry_args.exc, exceptions.BadStatusCodeError): + logging.debug('Response returned status %s, retrying', + retry_args.exc.status_code) + elif isinstance(retry_args.exc, exceptions.RetryAfterError): + logging.debug('Response returned a retry-after header, retrying') + retry_after = retry_args.exc.retry_after + else: + raise + RebuildHttpConnections(retry_args.http) + logging.debug('Retrying request to url %s after exception %s', + retry_args.http_request.url, retry_args.exc) + time.sleep(retry_after or util.CalculateWaitForRetry(retry_args.num_retries)) + + +def MakeRequest(http, http_request, retries=7, redirections=5, + retry_func=HandleExceptionsAndRebuildHttpConnections, + check_response_func=CheckResponse): + """Send http_request via the given http, performing error/retry handling. + + Args: + http: An httplib2.Http instance, or a http multiplexer that delegates to + an underlying http, for example, HTTPMultiplexer. + http_request: A Request to send. + retries: (int, default 5) Number of retries to attempt on 5XX replies. + redirections: (int, default 5) Number of redirects to follow. + retry_func: Function to handle retries on exceptions. Arguments are + (Httplib2.Http, Request, Exception, int num_retries). + check_response_func: Function to validate the HTTP response. Arguments are + (Response, response content, url). + + Raises: + InvalidDataFromServerError: if there is no response after retries. + + Returns: + A Response object. + """ + retry = 0 + while True: + try: + return _MakeRequestNoRetry(http, http_request, redirections=redirections, + check_response_func=check_response_func) + # retry_func will consume the exception types it handles and raise. + # pylint: disable=broad-except + except Exception as e: + retry += 1 + if retry >= retries: + raise + else: + retry_func(ExceptionRetryArgs(http, http_request, e, retry)) + + +def _MakeRequestNoRetry(http, http_request, redirections=5, + check_response_func=CheckResponse): """Send http_request via the given http. This wrapper exists to handle translation between the plain httplib2 request/response types and the Request and Response types above. - This will also be the hook for error/retry handling. Args: http: An httplib2.Http instance, or a http multiplexer that delegates to an underlying http, for example, HTTPMultiplexer. http_request: A Request to send. - retries: (int, default 5) Number of retries to attempt on 5XX replies. redirections: (int, default 5) Number of redirects to follow. + check_response_func: Function to validate the HTTP response. Arguments are + (Response, response content, url). Returns: A Response object. Raises: - InvalidDataFromServerError: if there is no response after retries. + RequestError if no response could be parsed. """ - response = None - exc = None connection_type = None # Handle overrides for connection types. This is used if the caller # wants control over the underlying connection for managing callbacks # or hash digestion. if getattr(http, 'connections', None): - url_scheme = urlsplit(http_request.url).scheme + url_scheme = parse.urlsplit(http_request.url).scheme if url_scheme and url_scheme in http.connections: connection_type = http.connections[url_scheme] - for retry in range(retries + 1): - # Note that the str() calls here are important for working around - # some funny business with message construction and unicode in - # httplib itself. See, eg, - # http://bugs.python.org/issue11898 - info = None - try: - info, content = http.request( - str(http_request.url), method=str(http_request.http_method), - body=http_request.body, headers=http_request.headers, - redirections=redirections, connection_type=connection_type) - except http_client.BadStatusLine as e: - logging.error('Caught BadStatusLine from httplib, retrying: %s', e) - exc = e - except socket.error as e: - if http_request.http_method != 'GET': - raise - logging.error('Caught socket error, retrying: %s', e) - exc = e - except http_client.IncompleteRead as e: - if http_request.http_method != 'GET': - raise - logging.error('Caught IncompleteRead error, retrying: %s', e) - exc = e - if info is not None: - response = Response(info, content, http_request.url) - if (response.status_code < 500 and - response.status_code != TOO_MANY_REQUESTS and - not response.retry_after): - break - logging.info('Retrying request to url <%s> after status code %s.', - response.request_url, response.status_code) - elif isinstance(exc, http_client.IncompleteRead): - logging.info('Retrying request to url <%s> after incomplete read.', - str(http_request.url)) - else: - logging.info('Retrying request to url <%s> after connection break.', - str(http_request.url)) - # TODO(craigcitro): Make this timeout configurable. - if response: - time.sleep(response.retry_after or util.CalculateWaitForRetry(retry)) - else: - time.sleep(util.CalculateWaitForRetry(retry)) - if response is None: - raise exceptions.InvalidDataFromServerError( - 'HTTP error on final retry: %s' % exc) + + # Custom printing only at debuglevel 4 + new_debuglevel = 4 if httplib2.debuglevel == 4 else 0 + with _Httplib2Debuglevel(http_request, new_debuglevel, http=http): + info, content = http.request( + str(http_request.url), method=str(http_request.http_method), + body=http_request.body, headers=http_request.headers, + redirections=redirections, connection_type=connection_type) + + if info is None: + raise exceptions.RequestError() + + response = Response(info, content, http_request.url) + check_response_func(response) return response -def GetHttp(): - return httplib2.Http() +def GetHttp(**kwds): + return httplib2.Http(**kwds) diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 0509b83..8b3d936 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -10,7 +10,9 @@ __all__ = [ def YieldFromList( service, request, limit=None, batch_size=100, - method='List', field='items', predicate=None): + method='List', field='items', predicate=None, + current_token_attribute='pageToken', + next_token_attribute='nextPageToken'): """Make a series of List requests, keeping track of page tokens. Args: @@ -24,6 +26,10 @@ def YieldFromList( method: str, The name of the method used to fetch resources. field: str, The field in the response that will be a list of items. predicate: lambda, A function that returns true for items to be yielded. + current_token_attribute: str, The name of the attribute in a request message + holding the page token for the page being requested. + next_token_attribute: str, The name of the attribute in a response message + holding the page token for the next page. Yields: protorpc.message.Message, The resources listed by the service. @@ -36,7 +42,7 @@ def YieldFromList( response = getattr(service, method)(request) items = getattr(response, field) if predicate: - items = [item for item in items if predicate(item)] + items = list(filter(predicate, items)) for item in items: yield item if limit is None: @@ -44,6 +50,7 @@ def YieldFromList( limit -= 1 if not limit: return - request.pageToken = response.nextPageToken - if not request.pageToken: + token = getattr(response, next_token_attribute) + if not token: return + setattr(request, current_token_attribute, token) diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py new file mode 100644 index 0000000..58d2390 --- /dev/null +++ b/apitools/base/py/stream_slice.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""Small helper class to provide a small slice of a stream.""" + +from apitools.base.py import exceptions + + +class StreamSlice(object): + """Provides a slice-like object for streams.""" + + def __init__(self, stream, max_bytes): + self.__stream = stream + self.__remaining_bytes = max_bytes + self.__max_bytes = max_bytes + + def __str__(self): + return 'Slice of stream %s with %s/%s bytes not yet read' % ( + self.__stream, self.__remaining_bytes, self.__max_bytes) + + def __len__(self): + return self.__max_bytes + + def read(self, size=None): # pylint: disable=missing-docstring + """Read at most size bytes from this slice. + + Compared to other streams, there is one case where we may + unexpectedly raise an exception on read: if the underlying stream + is exhausted (i.e. returns no bytes on read), and the size of this + slice indicates we should still be able to read more bytes, we + raise exceptions.StreamExhausted. + + Args: + size: If provided, read no more than size bytes from the stream. + + Returns: + The bytes read from this slice. + + Raises: + exceptions.StreamExhausted + + """ + if size is not None: + read_size = min(size, self.__remaining_bytes) + else: + read_size = self.__remaining_bytes + data = self.__stream.read(read_size) + if read_size > 0 and not data: + raise exceptions.StreamExhausted( + 'Not enough bytes in stream; expected %d, exhausted after %d' % ( + self.__max_bytes, self.__max_bytes - self.__remaining_bytes)) + self.__remaining_bytes -= len(data) + return data diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py new file mode 100644 index 0000000..900ba81 --- /dev/null +++ b/apitools/base/py/stream_slice_test.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Tests for stream_slice.""" + +import string +import StringIO + +import unittest2 + +from apitools.base.py import exceptions +from apitools.base.py import stream_slice + + +class StreamSliceTest(unittest2.TestCase): + + def setUp(self): + self.stream = StringIO.StringIO(string.letters) + self.value = self.stream.getvalue() + self.stream.seek(0) + + def testSimpleSlice(self): + ss = stream_slice.StreamSlice(self.stream, 10) + self.assertEqual('', ss.read(0)) + self.assertEqual(self.value[0:3], ss.read(3)) + self.assertIn('7/10', str(ss)) + self.assertEqual(self.value[3:10], ss.read()) + self.assertEqual('', ss.read()) + self.assertEqual('', ss.read(10)) + self.assertEqual(10, self.stream.tell()) + + def testEmptySlice(self): + ss = stream_slice.StreamSlice(self.stream, 0) + self.assertEqual('', ss.read(5)) + self.assertEqual('', ss.read()) + self.assertEqual(0, self.stream.tell()) + + def testOffsetStream(self): + self.stream.seek(26) + ss = stream_slice.StreamSlice(self.stream, 26) + self.assertEqual(self.value[26:36], ss.read(10)) + self.assertEqual(self.value[36:], ss.read()) + self.assertEqual('', ss.read()) + + def testTooShortStream(self): + ss = stream_slice.StreamSlice(self.stream, 1000) + self.assertEqual(self.value, ss.read()) + self.assertEqual('', ss.read(0)) + with self.assertRaises(exceptions.StreamExhausted) as e: + ss.read() + with self.assertRaises(exceptions.StreamExhausted) as e: + ss.read(10) + self.assertIn('exhausted after %d' % len(self.value), str(e.exception)) + + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 4a1412a..73445bc 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -9,35 +9,46 @@ import io import json import mimetypes import os +import StringIO import threading +import six from six.moves import http_client +from apitools.base.py import buffered_stream from apitools.base.py import exceptions from apitools.base.py import http_wrapper +from apitools.base.py import stream_slice from apitools.base.py import util __all__ = [ 'Download', 'Upload', + 'RESUMABLE_UPLOAD', + 'SIMPLE_UPLOAD', ] _RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 -_SIMPLE_UPLOAD = 'simple' -_RESUMABLE_UPLOAD = 'resumable' +SIMPLE_UPLOAD = 'simple' +RESUMABLE_UPLOAD = 'resumable' class _Transfer(object): """Generic bits common to Uploads and Downloads.""" def __init__(self, stream, close_stream=False, chunksize=None, - auto_transfer=True, http=None): + auto_transfer=True, http=None, num_retries=5): self.__bytes_http = None self.__close_stream = close_stream self.__http = http self.__stream = stream self.__url = None + self.__num_retries = 5 + # Let the @property do validation + self.num_retries = num_retries + + self.retry_func = http_wrapper.HandleExceptionsAndRebuildHttpConnections self.auto_transfer = auto_transfer self.chunksize = chunksize or 1048576 @@ -60,6 +71,18 @@ class _Transfer(object): def bytes_http(self, value): self.__bytes_http = value + @property + def num_retries(self): + return self.__num_retries + + @num_retries.setter + def num_retries(self, value): + util.Typecheck(value, six.integer_types) + if value < 0: + raise exceptions.InvalidDataError( + 'Cannot have negative value for num_retries') + self.__num_retries = value + @property def stream(self): return self.__stream @@ -131,31 +154,39 @@ class Download(_Transfer): 'auto_transfer', 'progress', 'total_size', 'url')) def __init__(self, *args, **kwds): + total_size = kwds.pop('total_size', None) super(Download, self).__init__(*args, **kwds) self.__initial_response = None self.__progress = 0 - self.__total_size = None + self.__total_size = total_size + self.__encoding = None @property def progress(self): return self.__progress + @property + def encoding(self): + return self.__encoding + @classmethod - def FromFile(cls, filename, overwrite=False, auto_transfer=True): + def FromFile(cls, filename, overwrite=False, auto_transfer=True, **kwds): """Create a new download object from a filename.""" path = os.path.expanduser(filename) if os.path.exists(path) and not overwrite: raise exceptions.InvalidUserInputError( 'File %s exists and overwrite not specified' % path) - return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer) + return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer, + **kwds) @classmethod - def FromStream(cls, stream, auto_transfer=True): + def FromStream(cls, stream, auto_transfer=True, total_size=None, **kwds): """Create a new Download object from a stream.""" - return cls(stream, auto_transfer=auto_transfer) + return cls(stream, auto_transfer=auto_transfer, total_size=total_size, + **kwds) @classmethod - def FromData(cls, stream, json_data, http=None, auto_transfer=None): + def FromData(cls, stream, json_data, http=None, auto_transfer=None, **kwds): """Create a new Download object from a stream and serialized data.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) @@ -163,7 +194,7 @@ class Download(_Transfer): raise exceptions.InvalidDataError( 'Invalid serialization data, missing keys: %s' % ( ', '.join(missing_keys))) - download = cls.FromStream(stream) + download = cls.FromStream(stream, **kwds) if auto_transfer is not None: download.auto_transfer = auto_transfer else: @@ -196,6 +227,10 @@ class Download(_Transfer): def ConfigureRequest(self, http_request, url_builder): url_builder.query_params['alt'] = 'media' + # TODO(craigcitro): We need to send range requests because by + # default httplib2 stores entire reponses in memory. Override + # httplib2's download method (as gsutil does) so that this is not + # necessary. http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) def __SetTotal(self, info): @@ -226,12 +261,14 @@ class Download(_Transfer): http = http or client.http if client is not None: http_request.url = client.FinalizeTransferUrl(http_request.url) - response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) - if response.status_code not in self._ACCEPTABLE_STATUSES: - raise exceptions.HttpError.FromResponse(response) - self.__initial_response = response - self.__SetTotal(response.info) - url = response.info.get('content-location', response.request_url) + url = http_request.url + if self.auto_transfer: + response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) + if response.status_code not in self._ACCEPTABLE_STATUSES: + raise exceptions.HttpError.FromResponse(response) + self.__initial_response = response + self.__SetTotal(response.info) + url = response.info.get('content-location', response.request_url) if client is not None: url = client.FinalizeTransferUrl(url) self._Initialize(http, url) @@ -245,7 +282,7 @@ class Download(_Transfer): if 'content-range' in response.info: print('Received %s' % response.info['content-range']) else: - print('Received %d bytes' % len(response)) + print('Received %d bytes' % response.length) @staticmethod def _CompletePrinter(*unused_args): @@ -280,20 +317,33 @@ class Download(_Transfer): def __GetChunk(self, start, end=None, additional_headers=None): """Retrieve a chunk, and return the full response.""" self.EnsureInitialized() - end_byte = min(end or start + self.chunksize, self.total_size) + end_byte = end + if self.total_size and end: + end_byte = min(end, self.total_size) request = http_wrapper.Request(url=self.url) self.__SetRangeHeader(request, start, end=end_byte) if additional_headers is not None: request.headers.update(additional_headers) - return http_wrapper.MakeRequest(self.bytes_http, request) + return http_wrapper.MakeRequest( + self.bytes_http, request, retry_func=self.retry_func, + retries=self.num_retries) def __ProcessResponse(self, response): """Process this response (by updating self and writing to self.stream).""" if response.status_code not in self._ACCEPTABLE_STATUSES: - raise exceptions.TransferInvalidError(response.content) + # We distinguish errors that mean we made a mistake in setting + # up the transfer versus something we should attempt again. + if response.status_code in (http_client.FORBIDDEN, http_client.NOT_FOUND): + raise exceptions.HttpError.FromResponse(response) + else: + raise exceptions.TransferRetryError(response.content) if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): self.stream.write(response.content) - self.__progress += len(response) + self.__progress += response.length + if response.info and 'content-encoding' in response.info: + # TODO(craigcitro): Handle the case where this changes over a + # download. + self.__encoding = response.info['content-encoding'] elif response.status_code == http_client.NO_CONTENT: # It's important to write something to the stream for the case # of a 0-byte download to a file, as otherwise python won't @@ -322,15 +372,23 @@ class Download(_Transfer): None. Streams bytes into self.stream. """ self.EnsureInitialized() - progress, end = self.__NormalizeStartEnd(start, end) - while progress < end: - chunk_end = min(progress + self.chunksize, end) - response = self.__GetChunk(progress, end=chunk_end, + progress_end_normalized = False + if self.total_size is not None: + progress, end = self.__NormalizeStartEnd(start, end) + progress_end_normalized = True + else: + progress = start + while not progress_end_normalized or progress < end: + response = self.__GetChunk(progress, end=end, additional_headers=additional_headers) + if not progress_end_normalized: + self.__SetTotal(response.info) + progress, end = self.__NormalizeStartEnd(start, end) + progress_end_normalized = True response = self.__ProcessResponse(response) - progress += len(response) + progress += response.length if not response: - raise exceptions.TransferInvalidError( + raise exceptions.TransferRetryError( 'Zero bytes unexpectedly returned in download response') def StreamInChunks(self, callback=None, finish_callback=None, @@ -371,11 +429,13 @@ class Upload(_Transfer): 'auto_transfer', 'mime_type', 'total_size', 'url')) def __init__(self, stream, mime_type, total_size=None, http=None, - close_stream=False, chunksize=None, auto_transfer=True): + close_stream=False, chunksize=None, auto_transfer=True, + **kwds): super(Upload, self).__init__( stream, close_stream=close_stream, chunksize=chunksize, - auto_transfer=auto_transfer, http=http) + auto_transfer=auto_transfer, http=http, **kwds) self.__complete = False + self.__final_response = None self.__mime_type = mime_type self.__progress = 0 self.__server_chunk_granularity = None @@ -388,7 +448,7 @@ class Upload(_Transfer): return self.__progress @classmethod - def FromFile(cls, filename, mime_type=None, auto_transfer=True): + def FromFile(cls, filename, mime_type=None, auto_transfer=True, **kwds): """Create a new Upload object from a filename.""" path = os.path.expanduser(filename) if not os.path.exists(path): @@ -400,19 +460,20 @@ class Upload(_Transfer): 'Could not guess mime type for %s' % path) size = os.stat(path).st_size return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, - auto_transfer=auto_transfer) + auto_transfer=auto_transfer, **kwds) @classmethod - def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True): + def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True, + **kwds): """Create a new Upload object from a stream.""" if mime_type is None: raise exceptions.InvalidUserInputError( 'No mime_type specified for stream') return cls(stream, mime_type, total_size=total_size, close_stream=False, - auto_transfer=auto_transfer) + auto_transfer=auto_transfer, **kwds) @classmethod - def FromData(cls, stream, json_data, http, auto_transfer=None): + def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds): """Create a new Upload of stream from serialized json_data using http.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) @@ -420,8 +481,11 @@ class Upload(_Transfer): raise exceptions.InvalidDataError( 'Invalid serialization data, missing keys: %s' % ( ', '.join(missing_keys))) + if 'total_size' in kwds: + raise exceptions.InvalidUserInputError( + 'Cannot override total_size on serialized Upload') upload = cls.FromStream(stream, info['mime_type'], - total_size=info.get('total_size')) + total_size=info.get('total_size'), **kwds) if isinstance(stream, io.IOBase) and not stream.seekable(): raise exceptions.InvalidUserInputError( 'Cannot restart resumable upload on non-seekable stream') @@ -429,9 +493,9 @@ class Upload(_Transfer): upload.auto_transfer = auto_transfer else: upload.auto_transfer = info['auto_transfer'] - upload.strategy = _RESUMABLE_UPLOAD + upload.strategy = RESUMABLE_UPLOAD upload._Initialize(http, info['url']) # pylint: disable=protected-access - upload._RefreshResumableUploadState() # pylint: disable=protected-access + upload.RefreshResumableUploadState() upload.EnsureInitialized() if upload.auto_transfer: upload.StreamInChunks() @@ -440,7 +504,7 @@ class Upload(_Transfer): @property def serialization_data(self): self.EnsureInitialized() - if self.strategy != _RESUMABLE_UPLOAD: + if self.strategy != RESUMABLE_UPLOAD: raise exceptions.InvalidDataError( 'Serialization only supported for resumable uploads') return { @@ -471,7 +535,7 @@ class Upload(_Transfer): @strategy.setter def strategy(self, value): - if value not in (_SIMPLE_UPLOAD, _RESUMABLE_UPLOAD): + if value not in (SIMPLE_UPLOAD, RESUMABLE_UPLOAD): raise exceptions.UserError(( 'Invalid value "%s" for upload strategy, must be one of ' '"simple" or "resumable".') % value) @@ -503,14 +567,14 @@ class Upload(_Transfer): """ if self.strategy is not None: return - strategy = _SIMPLE_UPLOAD + strategy = SIMPLE_UPLOAD if (self.total_size is not None and self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): - strategy = _RESUMABLE_UPLOAD + strategy = RESUMABLE_UPLOAD if http_request.body and not upload_config.simple_multipart: - strategy = _RESUMABLE_UPLOAD + strategy = RESUMABLE_UPLOAD if not upload_config.simple_path: - strategy = _RESUMABLE_UPLOAD + strategy = RESUMABLE_UPLOAD self.strategy = strategy def ConfigureRequest(self, upload_config, http_request, url_builder): @@ -528,7 +592,7 @@ class Upload(_Transfer): self.mime_type, upload_config.accept)) self.__SetDefaultUploadStrategy(upload_config, http_request) - if self.strategy == _SIMPLE_UPLOAD: + if self.strategy == SIMPLE_UPLOAD: url_builder.relative_path = upload_config.simple_path if http_request.body: url_builder.query_params['uploadType'] = 'multipart' @@ -545,6 +609,7 @@ class Upload(_Transfer): """Configure http_request as a simple request for this upload.""" http_request.headers['content-type'] = self.mime_type http_request.body = self.stream.read() + http_request.loggable_body = '' def __ConfigureMultipartRequest(self, http_request): """Configure http_request as a multipart request for this upload.""" @@ -567,7 +632,7 @@ class Upload(_Transfer): # encode the body: note that we can't use `as_string`, because # it plays games with `From ` lines. - fp = io.StringIO() + fp = StringIO.StringIO() g = email_generator.Generator(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() @@ -576,24 +641,37 @@ class Upload(_Transfer): http_request.headers['content-type'] = ( 'multipart/related; boundary=%r' % multipart_boundary) + body_components = http_request.body.split(multipart_boundary) + headers, _, _ = body_components[-2].partition('\n\n') + body_components[-2] = '\n\n'.join([headers, '\n\n--']) + http_request.loggable_body = multipart_boundary.join(body_components) + def __ConfigureResumableRequest(self, http_request): http_request.headers['X-Upload-Content-Type'] = self.mime_type if self.total_size is not None: http_request.headers['X-Upload-Content-Length'] = str(self.total_size) - def _RefreshResumableUploadState(self): - """Talk to the server and refresh the state of this resumable upload.""" - if self.strategy != _RESUMABLE_UPLOAD: + def RefreshResumableUploadState(self): + """Talk to the server and refresh the state of this resumable upload. + + Returns: + Response if the upload is complete. + """ + if self.strategy != RESUMABLE_UPLOAD: return self.EnsureInitialized() refresh_request = http_wrapper.Request( url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) refresh_response = http_wrapper.MakeRequest( - self.http, refresh_request, redirections=0) - range_header = refresh_response.info.get( - 'Range', refresh_response.info.get('range')) + self.http, refresh_request, redirections=0, retries=self.num_retries) + range_header = self._GetRangeHeaderFromResponse(refresh_response) if refresh_response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True + self.__progress = self.total_size + self.stream.seek(self.progress) + # If we're finished, the refresh response will contain the metadata + # originally requested. Cache it so it can be returned in StreamInChunks. + self.__final_response = refresh_response elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: if range_header is None: self.__progress = 0 @@ -603,6 +681,9 @@ class Upload(_Transfer): else: raise exceptions.HttpError.FromResponse(refresh_response) + def _GetRangeHeaderFromResponse(self, response): + return response.info.get('Range', response.info.get('range')) + def InitializeUpload(self, http_request, http=None, client=None): """Initialize this upload from the given http_request.""" if self.strategy is None: @@ -610,22 +691,19 @@ class Upload(_Transfer): 'No upload strategy set; did you call ConfigureRequest?') if http is None and client is None: raise exceptions.UserError('Must provide client or http.') - if self.strategy != _RESUMABLE_UPLOAD: + if self.strategy != RESUMABLE_UPLOAD: return - if self.total_size is None: - raise exceptions.InvalidUserInputError( - 'Cannot stream upload without total size') http = http or client.http if client is not None: http_request.url = client.FinalizeTransferUrl(http_request.url) self.EnsureUninitialized() - http_response = http_wrapper.MakeRequest(http, http_request) + http_response = http_wrapper.MakeRequest(http, http_request, + retries=self.num_retries) if http_response.status_code != http_client.OK: raise exceptions.HttpError.FromResponse(http_response) self.__server_chunk_granularity = http_response.info.get( 'X-Goog-Upload-Chunk-Granularity') - self.__ValidateChunksize() url = http_response.info['location'] if client is not None: url = client.FinalizeTransferUrl(url) @@ -658,23 +736,23 @@ class Upload(_Transfer): def _CompletePrinter(*unused_args): print('Upload complete') - def StreamInChunks(self, callback=None, finish_callback=None, - additional_headers=None): - """Send this (resumable) upload in chunks.""" - if self.strategy != _RESUMABLE_UPLOAD: + def __StreamMedia(self, callback=None, finish_callback=None, + additional_headers=None, use_chunks=True): + """Helper function for StreamMedia / StreamInChunks.""" + if self.strategy != RESUMABLE_UPLOAD: raise exceptions.InvalidUserInputError( 'Cannot stream non-resumable upload') - if self.total_size is None: - raise exceptions.InvalidUserInputError( - 'Cannot stream upload without total size') callback = callback or self._ArgPrinter finish_callback = finish_callback or self._CompletePrinter - response = None - self.__ValidateChunksize(self.chunksize) + # final_response is set if we resumed an already-completed upload. + response = self.__final_response + send_func = self.__SendChunk if use_chunks else self.__SendMediaBody + if use_chunks: + self.__ValidateChunksize(self.chunksize) self.EnsureInitialized() while not self.complete: - response = self.__SendChunk(self.stream.tell(), - additional_headers=additional_headers) + response = send_func(self.stream.tell(), + additional_headers=additional_headers) if response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True break @@ -685,33 +763,119 @@ class Upload(_Transfer): 'Failed to transfer all bytes in chunk, upload paused at byte ' '%d' % self.progress) self._ExecuteCallback(callback, response) + if self.__complete: + # TODO(craigcitro): Decide how to handle errors in the + # non-seekable case. + current_pos = self.stream.tell() + self.stream.seek(0, os.SEEK_END) + end_pos = self.stream.tell() + self.stream.seek(current_pos) + if current_pos != end_pos: + raise exceptions.TransferInvalidError( + 'Upload complete with %s additional bytes left in stream' % + (int(end_pos) - int(current_pos))) self._ExecuteCallback(finish_callback, response) return response - def __SendChunk(self, start, additional_headers=None, data=None): - """Send the specified chunk.""" - self.EnsureInitialized() - if data is None: - data = self.stream.read(self.chunksize) - end = start + len(data) + def StreamMedia(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this resumable upload in a single request. - request = http_wrapper.Request(url=self.url, http_method='PUT', body=data) - request.headers['Content-Type'] = self.mime_type - if data: - request.headers['Content-Range'] = 'bytes %s-%s/%s' % ( - start, end - 1, self.total_size) - if additional_headers: - request.headers.update(additional_headers) + Args: + callback: Progress callback function with inputs + (http_wrapper.Response, transfer.Upload) + finish_callback: Final callback function with inputs + (http_wrapper.Response, transfer.Upload) + additional_headers: Dict of headers to include with the upload + http_wrapper.Request. + + Returns: + http_wrapper.Response of final response. + """ + return self.__StreamMedia( + callback=callback, finish_callback=finish_callback, + additional_headers=additional_headers, use_chunks=False) - response = http_wrapper.MakeRequest(self.bytes_http, request) + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this (resumable) upload in chunks.""" + return self.__StreamMedia( + callback=callback, finish_callback=finish_callback, + additional_headers=additional_headers) + + def __SendMediaRequest(self, request, end): + """Helper function to make the request for SendMediaBody & SendChunk.""" + response = http_wrapper.MakeRequest( + self.bytes_http, request, retry_func=self.retry_func, + retries=self.num_retries) if response.status_code not in (http_client.OK, http_client.CREATED, http_wrapper.RESUME_INCOMPLETE): + # We want to reset our state to wherever the server left us + # before this failed request, and then raise. + self.RefreshResumableUploadState() raise exceptions.HttpError.FromResponse(response) - if response.status_code in (http_client.OK, http_client.CREATED): - return response - # TODO(craigcitro): Add retries on no progress? - last_byte = self.__GetLastByte(response.info['range']) - if last_byte + 1 != end: - new_start = last_byte + 1 - start - response = self.__SendChunk(last_byte + 1, data=data[new_start:]) + if response.status_code == http_wrapper.RESUME_INCOMPLETE: + last_byte = self.__GetLastByte( + self._GetRangeHeaderFromResponse(response)) + if last_byte + 1 != end: + self.stream.seek(last_byte) return response + + def __SendMediaBody(self, start, additional_headers=None): + """Send the entire media stream in a single request.""" + self.EnsureInitialized() + if self.total_size is None: + raise exceptions.TransferInvalidError( + 'Total size must be known for SendMediaBody') + body_stream = stream_slice.StreamSlice(self.stream, self.total_size - start) + + request = http_wrapper.Request(url=self.url, http_method='PUT', + body=body_stream) + request.headers['Content-Type'] = self.mime_type + if start == self.total_size: + # End of an upload with 0 bytes left to send; just finalize. + range_string = 'bytes */%s' % self.total_size + else: + range_string = 'bytes %s-%s/%s' % (start, self.total_size - 1, + self.total_size) + + request.headers['Content-Range'] = range_string + if additional_headers: + request.headers.update(additional_headers) + + return self.__SendMediaRequest(request, self.total_size) + + def __SendChunk(self, start, additional_headers=None): + """Send the specified chunk.""" + self.EnsureInitialized() + if self.total_size is None: + # For the streaming resumable case, we need to detect when we're at the + # end of the stream. + body_stream = buffered_stream.BufferedStream( + self.stream, start, self.chunksize) + end = body_stream.stream_end_position + if body_stream.stream_exhausted: + self.__total_size = end + else: + end = min(start + self.chunksize, self.total_size) + body_stream = stream_slice.StreamSlice(self.stream, end - start) + # TODO(craigcitro): Think about clearer errors on "no data in + # stream". + request = http_wrapper.Request(url=self.url, http_method='PUT', + body=body_stream) + request.headers['Content-Type'] = self.mime_type + if self.total_size is None: + # Streaming resumable upload case, unknown total size. + range_string = 'bytes %s-%s/*' % (start, end - 1) + elif end == start: + # End of an upload with 0 bytes left to send; just finalize. + range_string = 'bytes */%s' % self.total_size + else: + # Normal resumable upload case with known sizes. + range_string = 'bytes %s-%s/%s' % (start, end - 1, self.total_size) + + request.headers['Content-Range'] = range_string + if additional_headers: + request.headers.update(additional_headers) + + return self.__SendMediaRequest(request, end) diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py new file mode 100644 index 0000000..d1bc7c8 --- /dev/null +++ b/apitools/base/py/transfer_test.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + + +import StringIO + +import unittest2 + +from apitools.base.py import base_api +from apitools.base.py import http_wrapper +from apitools.base.py import transfer + + +class TransferTest(unittest2.TestCase): + + def testFromEncoding(self): + """Test a specific corner case in multipart encoding. + + Python's mime module by default encodes lines that start with + "From " as ">From ", which we need to make sure we don't run afoul + of when sending content that isn't intended to be so encoded. This + test calls out that we get this right. We test for both the + multipart and non-multipart case. + """ + multipart_body = '{"body_field_one": 7}' + upload_contents = 'line one\nFrom \nline two' + upload_config = base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload', + simple_multipart=True, + simple_path=u'/upload', + ) + url_builder = base_api._UrlBuilder('http://www.uploads.com') + + # Test multipart: having a body argument in http_request forces + # multipart here. + upload = transfer.Upload.FromStream( + StringIO.StringIO(upload_contents), + 'text/plain', + total_size=len(upload_contents)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}, + body=multipart_body) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual(url_builder.query_params['uploadType'], 'multipart') + rewritten_upload_contents = '\n'.join( + http_request.body.split('--')[2].splitlines()[1:]) + self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) + + # Test non-multipart (aka media): no body argument means this is + # sent as media. + upload = transfer.Upload.FromStream( + StringIO.StringIO(upload_contents), + 'text/plain', + total_size=len(upload_contents)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual(url_builder.query_params['uploadType'], 'media') + rewritten_upload_contents = http_request.body + self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) + + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 1ef0b46..5158bfa 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -5,12 +5,14 @@ import collections import os import random +from protorpc import messages import six from six.moves import http_client -from six.moves.urllib.error import URLError -from six.moves.urllib.parse import quote -from six.moves.urllib.request import urlopen +import six.moves.urllib.error as urllib_error +import six.moves.urllib.parse as urllib_parse +import six.moves.urllib.request as urllib_request +from apitools.base.py import encoding from apitools.base.py import exceptions __all__ = [ @@ -45,8 +47,9 @@ def DetectGce(): True iff we're running on a GCE instance. """ try: - o = urlopen('http://metadata.google.internal') - except URLError: + o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open( + urllib_request.Request('http://metadata.google.internal')) + except urllib_error.URLError: return False return (o.getcode() == http_client.OK and o.headers.get('metadata-flavor') == 'Google') @@ -104,7 +107,8 @@ def ExpandRelativePath(method_config, params, relative_path=None): if not isinstance(value, six.string_types): value = str(value) path = path.replace(param_template, - quote(value.encode('utf_8'), reserved_chars)) + urllib_parse.quote(value.encode('utf_8'), + reserved_chars)) except TypeError as e: raise exceptions.InvalidUserInputError( 'Error setting required parameter %s to value %s: %s' % ( @@ -165,3 +169,40 @@ def AcceptableMimeType(accept_patterns, mime_type): in zip(pattern.split('/'), mime_type.split('/'))) return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) + + +def MapParamNames(params, request_type): + """Reverse parameter remappings for URL construction.""" + return [encoding.GetCustomJsonFieldMapping(request_type, json_name=p) or p + for p in params] + + +def MapRequestParams(params, request_type): + """Perform any renames/remappings needed for URL construction. + + Currently, we have several ways to customize JSON encoding, in + particular of field names and enums. This works fine for JSON + bodies, but also needs to be applied for path and query parameters + in the URL. + + This function takes a dictionary from param names to values, and + performs any registered mappings. We also need the request type (to + look up the mappings). + + Args: + params: (dict) Map from param names to values + request_type: (protorpc.messages.Message) request type for this API call + + Returns: + A new dict of the same size, with all registered mappings applied. + """ + new_params = dict(params) + for param_name, value in params.items(): + field_remapping = encoding.GetCustomJsonFieldMapping( + request_type, python_name=param_name) + if field_remapping is not None: + new_params[field_remapping] = new_params.pop(param_name) + if isinstance(value, messages.Enum): + new_params[param_name] = encoding.GetCustomJsonEnumMapping( + type(value), python_name=str(value)) or str(value) + return new_params diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py new file mode 100644 index 0000000..025626c --- /dev/null +++ b/apitools/base/py/util_test.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python + + +from protorpc import messages +import unittest2 + +from apitools.base.py import encoding +from apitools.base.py import exceptions +from apitools.base.py import util + + +class MockedMethodConfig(object): + + def __init__(self, relative_path, path_params): + self.relative_path = relative_path + self.path_params = path_params + + +class MessageWithRemappings(messages.Message): + + class AnEnum(messages.Enum): + value_one = 1 + value_two = 2 + + str_field = messages.StringField(1) + enum_field = messages.EnumField('AnEnum', 2) + + +encoding.AddCustomJsonFieldMapping( + MessageWithRemappings, 'str_field', 'path_field') +encoding.AddCustomJsonEnumMapping( + MessageWithRemappings.AnEnum, 'value_one', 'ONE') + + +class UtilTest(unittest2.TestCase): + + def testExpand(self): + method_config_xy = MockedMethodConfig(relative_path='{x}/y/{z}', + path_params=['x', 'z']) + self.assertEquals( + util.ExpandRelativePath(method_config_xy, {'x': '1', 'z': '2'}), + '1/y/2') + self.assertEquals( + util.ExpandRelativePath( + method_config_xy, + {'x': '1', 'z': '2'}, + relative_path='{x}/y/{z}/q'), + '1/y/2/q') + + def testReservedExpansion(self): + method_config_reserved = MockedMethodConfig(relative_path='{+x}/baz', + path_params=['x']) + self.assertEquals('foo/:bar:/baz', util.ExpandRelativePath( + method_config_reserved, {'x': 'foo/:bar:'})) + method_config_no_reserved = MockedMethodConfig(relative_path='{x}/baz', + path_params=['x']) + self.assertEquals('foo%2F%3Abar%3A/baz', util.ExpandRelativePath( + method_config_no_reserved, {'x': 'foo/:bar:'})) + + def testCalculateWaitForRetry(self): + self.assertTrue(util.CalculateWaitForRetry(1) in range(1, 4)) + self.assertTrue(util.CalculateWaitForRetry(2) in range(2, 7)) + self.assertTrue(util.CalculateWaitForRetry(3) in range(4, 13)) + self.assertTrue(util.CalculateWaitForRetry(4) in range(8, 25)) + + self.assertEquals(10, util.CalculateWaitForRetry(5, max_wait=10)) + + self.assertGreater(util.CalculateWaitForRetry(0), 0) + + def testTypecheck(self): + + class Class1(object): + pass + + class Class2(object): + pass + + class Class3(object): + pass + + instance_of_class1 = Class1() + + self.assertEquals( + instance_of_class1, util.Typecheck(instance_of_class1, Class1)) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, ((Class1, Class2), Class3))) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, (Class1, (Class2, Class3)))) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, Class1, 'message')) + + self.assertEquals( + instance_of_class1, + util.Typecheck( + instance_of_class1, ((Class1, Class2), Class3), 'message')) + + self.assertEquals( + instance_of_class1, + util.Typecheck( + instance_of_class1, (Class1, (Class2, Class3)), 'message')) + + try: + util.Typecheck(instance_of_class1, Class2) + self.fail('Type mismatch not detected when called with 2 arguments') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, (Class2, Class3)) + self.fail( + 'Type mismatch not detected when called with 2 arguments including ' + 'type tuple') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, Class2, 'message') + self.fail('Type mismatch not detected when called with 3 arguments') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, (Class2, Class3), 'message') + self.fail( + 'Type mismatch not detected when called with 3 arguments including ' + 'type tuple') + except exceptions.TypecheckError: + pass # expected + + def testAcceptableMimeType(self): + valid_pairs = ( + ('*', 'text/plain'), + ('*/*', 'text/plain'), + ('text/*', 'text/plain'), + ('*/plain', 'text/plain'), + ('text/plain', 'text/plain'), + ) + + for accept, mime_type in valid_pairs: + self.assertTrue(util.AcceptableMimeType([accept], mime_type)) + + invalid_pairs = ( + ('text/*', 'application/json'), + ('text/plain', 'application/json'), + ) + + for accept, mime_type in invalid_pairs: + self.assertFalse(util.AcceptableMimeType([accept], mime_type)) + + self.assertTrue(util.AcceptableMimeType(['application/json', '*/*'], + 'text/plain')) + self.assertFalse(util.AcceptableMimeType(['application/json', 'img/*'], + 'text/plain')) + + def testUnsupportedMimeType(self): + self.assertRaises( + exceptions.GeneratedClientError, + util.AcceptableMimeType, ['text/html;q=0.9'], 'text/html') + + def testMapRequestParams(self): + params = { + 'str_field': 'foo', + 'enum_field': MessageWithRemappings.AnEnum.value_one, + } + remapped_params = { + 'path_field': 'foo', + 'enum_field': 'ONE', + } + self.assertEqual(remapped_params, + util.MapRequestParams(params, MessageWithRemappings)) + + params['enum_field'] = MessageWithRemappings.AnEnum.value_two + remapped_params['enum_field'] = 'value_two' + self.assertEqual(remapped_params, + util.MapRequestParams(params, MessageWithRemappings)) + + def testMapParamNames(self): + params = ['path_field', 'enum_field'] + remapped_params = ['str_field', 'enum_field'] + self.assertEqual(remapped_params, + util.MapParamNames(params, MessageWithRemappings)) + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 292d443..602296d 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -8,7 +8,9 @@ import shutil import subprocess import sys import tempfile -import unittest + + +import unittest2 _API_LIST = [ 'drive.v2', @@ -30,7 +32,7 @@ def TempDir(): shutil.rmtree(path) -class ClientGenerationTest(unittest.TestCase): +class ClientGenerationTest(unittest2.TestCase): def setUp(self): super(ClientGenerationTest, self).setUp() @@ -73,4 +75,4 @@ class ClientGenerationTest(unittest.TestCase): if __name__ == '__main__': - unittest.main() + unittest2.main() diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 675d5fa..bb3f75a 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -425,6 +425,8 @@ class CommandRegistry(object): """Write a simple CLI (currently just a stub).""" printer('#!/usr/bin/env python') printer('"""CLI for %s, version %s."""', self.__package, self.__version) + printer('# NOTE: This file is autogenerated and should not be edited by ' + 'hand.') # TODO(craigcitro): Add a build stamp, along with some other # information. printer() diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 66afd73..d10532b 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -16,7 +16,7 @@ import abc import operator import textwrap -from protorpc import descriptor +from protorpc import descriptor as protorpc_descriptor from protorpc import message_types from protorpc import messages import six @@ -46,12 +46,20 @@ class ExtendedEnumDescriptor(messages.Message): values: Values defined by Enum class. description: Description of this enum class. full_name: Fully qualified name of this enum class. + enum_mappings: Mappings from python to JSON names for enum values. """ + + class JsonEnumMapping(messages.Message): + """Mapping from a python name to the wire name for an enum.""" + python_name = messages.StringField(1) + json_name = messages.StringField(2) + name = messages.StringField(1) values = messages.MessageField(ExtendedEnumValueDescriptor, 2, repeated=True) description = messages.StringField(100) full_name = messages.StringField(101) + enum_mappings = messages.MessageField('JsonEnumMapping', 102, repeated=True) class ExtendedFieldDescriptor(messages.Message): @@ -62,7 +70,8 @@ class ExtendedFieldDescriptor(messages.Message): name: The name of this field. description: Description of this field. """ - field_descriptor = messages.MessageField(descriptor.FieldDescriptor, 100) + field_descriptor = messages.MessageField( + protorpc_descriptor.FieldDescriptor, 100) # We duplicate the names for easier bookkeeping. name = messages.StringField(101) description = messages.StringField(102) @@ -82,7 +91,14 @@ class ExtendedMessageDescriptor(messages.Message): Printed in the given order from top to bottom (so the last entry is the innermost decorator). alias_for: This type is just an alias for the named type. + field_mappings: Mappings from python to json field names. """ + + class JsonFieldMapping(messages.Message): + """Mapping from a python name to the wire name for a field.""" + python_name = messages.StringField(1) + json_name = messages.StringField(2) + name = messages.StringField(1) fields = messages.MessageField(ExtendedFieldDescriptor, 2, repeated=True) message_types = messages.MessageField( @@ -93,6 +109,7 @@ class ExtendedMessageDescriptor(messages.Message): full_name = messages.StringField(101) decorators = messages.StringField(102, repeated=True) alias_for = messages.StringField(103) + field_mappings = messages.MessageField('JsonFieldMapping', 104, repeated=True) class ExtendedFileDescriptor(messages.Message): @@ -121,6 +138,11 @@ def _WriteFile(file_descriptor, package, version, proto_printer): proto_printer.PrintPreamble(package, version, file_descriptor) _PrintEnums(proto_printer, file_descriptor.enum_types) _PrintMessages(proto_printer, file_descriptor.message_types) + custom_json_mappings = _FetchCustomMappings(file_descriptor.enum_types) + custom_json_mappings.extend( + _FetchCustomMappings(file_descriptor.message_types)) + for mapping in custom_json_mappings: + proto_printer.PrintCustomJsonMapping(mapping) def WriteMessagesFile(file_descriptor, package, version, printer): @@ -149,6 +171,31 @@ def PrintIndentedDescriptions(printer, ls, name, prefix=''): printer(line) +def _FetchCustomMappings(descriptor_ls): + """Find and return all custom mappings for descriptors in descriptor_ls.""" + custom_mappings = [] + for descriptor in descriptor_ls: + if isinstance(descriptor, ExtendedEnumDescriptor): + custom_mappings.extend( + _FormatCustomJsonMapping('Enum', m, descriptor) + for m in descriptor.enum_mappings) + elif isinstance(descriptor, ExtendedMessageDescriptor): + custom_mappings.extend( + _FormatCustomJsonMapping('Field', m, descriptor) + for m in descriptor.field_mappings) + custom_mappings.extend(_FetchCustomMappings(descriptor.enum_types)) + custom_mappings.extend(_FetchCustomMappings(descriptor.message_types)) + return custom_mappings + + +def _FormatCustomJsonMapping(mapping_type, mapping, descriptor): + return '\n'.join(( + 'encoding.AddCustomJson%sMapping(' % mapping_type, + " %s, '%s', '%s')" % (descriptor.full_name, mapping.python_name, + mapping.json_name) + )) + + def _EmptyMessage(message_type): return not any((message_type.enum_types, message_type.message_types, @@ -205,6 +252,8 @@ class _Proto2Printer(ProtoPrinter): def PrintPreamble(self, package, version, file_descriptor): self.__printer('// Generated message classes for %s version %s.', package, version) + self.__printer('// NOTE: This file is autogenerated and should not be ' + 'edited by hand.') description_lines = textwrap.wrap(file_descriptor.description, 75) if description_lines: self.__printer('//') @@ -270,6 +319,9 @@ class _Proto2Printer(ProtoPrinter): self.__PrintFields(message_type.fields) self.__printer('}') + def PrintCustomJsonMapping(self, mapping_lines): + raise NotImplementedError('Custom JSON encoding not supported for proto2') + class _ProtoRpcPrinter(ProtoPrinter): """Printer for ProtoRPC definitions.""" @@ -323,6 +375,8 @@ class _ProtoRpcPrinter(ProtoPrinter): for line in textwrap.wrap(file_descriptor.description, 78): self.__printer(line) self.__printer('"""') + self.__printer('# NOTE: This file is autogenerated and should not be ' + 'edited by hand.') self.__printer() self.__PrintAdditionalImports(file_descriptor.additional_imports) self.__printer() @@ -370,6 +424,9 @@ class _ProtoRpcPrinter(ProtoPrinter): _PrintFields(message_type.fields, self.__printer) self.__PrintClassSeparator() + def PrintCustomJsonMapping(self, mapping): + self.__printer(mapping) + def _PrintEnums(proto_printer, enum_types): """Print all enums to the given proto_printer.""" @@ -416,9 +473,9 @@ def _PrintFields(fields, printer): if field_type in (messages.EnumField, messages.MessageField): printed_field_info['type_format'] = "'%s', " % field.type_name - if field.label == descriptor.FieldDescriptor.Label.REQUIRED: + if field.label == protorpc_descriptor.FieldDescriptor.Label.REQUIRED: printed_field_info['label_format'] = ', required=True' - elif field.label == descriptor.FieldDescriptor.Label.REPEATED: + elif field.label == protorpc_descriptor.FieldDescriptor.Label.REPEATED: printed_field_info['label_format'] = ', repeated=True' if field_type.DEFAULT_VARIANT != field.variant: diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 96326a3..d64855b 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -67,6 +67,10 @@ flags.DEFINE_string( 'User agent for the generated client. Defaults to -generated/0.1.') flags.DEFINE_boolean( 'generate_cli', True, 'If True, a CLI is also generated.') +flags.DEFINE_list( + 'unelidable_request_methods', [], + 'Full method IDs of methods for which we should NOT try to elide ' + 'the request type. (Should be a comma-separated list.)') flags.DEFINE_boolean( 'experimental_capitalize_enums', False, @@ -129,9 +133,11 @@ def _GetCodegenFromFlags(): if not client_id: logging.warning('No client ID supplied') + client_id = '' if not client_secret: logging.warning('No client secret supplied') + client_secret = '' client_info = util.ClientInfo.Create( discovery_doc, FLAGS.scope, client_id, client_secret, @@ -141,12 +147,14 @@ def _GetCodegenFromFlags(): raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') - root_package = FLAGS.root_package or util.GetPackage(outdir) + + root_package = FLAGS.root_package or util.GetPackage(outdir) # pylint: disable=line-too-long return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, base_package=FLAGS.base_package, generate_cli=FLAGS.generate_cli, - use_proto2=FLAGS.experimental_proto2_output) + use_proto2=FLAGS.experimental_proto2_output, + unelidable_request_methods=FLAGS.unelidable_request_methods) # TODO(craigcitro): Delete this if we don't need this functionality. @@ -178,7 +186,7 @@ def _WriteGeneratedFiles(codegen): if FLAGS.generate_cli: with open(codegen.client_info.cli_file_name, 'w') as out: codegen.WriteCli(out) - os.chmod(codegen.client_info.cli_file_name, 0o755) + os.chmod(codegen.client_info.cli_file_name, 0755) def _WriteInit(codegen): diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 3e5b656..f1fe36a 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -5,9 +5,11 @@ Relevant links: https://developers.google.com/discovery/v1/reference/apis#resource """ -import logging +import json import urlparse +import six + from apitools.base.py import base_cli from apitools.gen import command_registry from apitools.gen import message_registry @@ -37,8 +39,6 @@ def _ComputePaths(package, version, discovery_doc): discovery_doc['rootUrl'], discovery_doc['servicePath']) api_path_component = '/'.join((package, version, '')) if api_path_component not in full_path: - logging.warning('Could not find path "%s" in API path "%s"', - api_path_component, full_path) return full_path, '' prefix, _, suffix = full_path.rpartition(api_path_component) return prefix + api_path_component, suffix @@ -48,12 +48,14 @@ class DescriptorGenerator(object): """Code generator for a given discovery document.""" def __init__(self, discovery_doc, client_info, names, root_package, outdir, - base_package, generate_cli=False, use_proto2=False): + base_package, generate_cli=False, use_proto2=False, + unelidable_request_methods=None): self.__discovery_doc = discovery_doc self.__client_info = client_info self.__outdir = outdir self.__use_proto2 = use_proto2 - self.__description = self.__discovery_doc.get('description', '') + self.__description = util.CleanDescription( + self.__discovery_doc.get('description', '')) self.__package = self.__client_info.package self.__version = self.__client_info.version self.__generate_cli = generate_cli @@ -71,7 +73,7 @@ class DescriptorGenerator(object): self.__client_info, self.__names, self.__description, self.__root_package, self.__base_files_package) schemas = self.__discovery_doc.get('schemas', {}) - for schema_name, schema in schemas.items(): + for schema_name, schema in six.iteritems(schemas): self.__message_registry.AddDescriptorFromSchema(schema_name, schema) # We need to add one more message type for the global parameters. @@ -80,6 +82,10 @@ class DescriptorGenerator(object): self.__message_registry.AddDescriptorFromSchema( standard_query_schema['id'], standard_query_schema) + # Now that we know all the messages, we need to correct some + # fields from MessageFields to EnumFields. + self.__message_registry.FixupMessageFields() + self.__command_registry = command_registry.CommandRegistry( self.__package, self.__version, self.__client_info, self.__message_registry, self.__root_package, self.__base_files_package, @@ -96,9 +102,10 @@ class DescriptorGenerator(object): self.__base_path, self.__names, self.__root_package, - self.__base_files_package) + self.__base_files_package, + unelidable_request_methods or []) services = self.__discovery_doc.get('resources', {}) - for service_name, methods in services.items(): + for service_name, methods in sorted(six.iteritems(services)): self.__services_registry.AddServiceFromResource(service_name, methods) # We might also have top-level methods. api_methods = self.__discovery_doc.get('methods', []) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index c93d995..970a0dc 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -10,6 +10,7 @@ from protorpc import messages import six from apitools.gen import extended_descriptor +from apitools.gen import util TypeInfo = collections.namedtuple('TypeInfo', ('type_name', 'variant')) @@ -63,7 +64,7 @@ class MessageRegistry(object): self.__names = names self.__client_info = client_info self.__package = client_info.package - self.__description = description + self.__description = util.CleanDescription(description) self.__root_package_dir = root_package_dir self.__base_files_package = base_files_package self.__file_descriptor = extended_descriptor.ExtendedFileDescriptor( @@ -110,7 +111,7 @@ class MessageRegistry(object): raise ValueError('Malformed MessageRegistry: %s' % mysteries) def __ComputeFullName(self, name): - return '.'.join([six.text_type(x) for x in self.__current_path + [name]]) + return '.'.join(map(six.text_type, self.__current_path[:] + [name])) def __AddImport(self, new_import): if new_import not in self.__file_descriptor.additional_imports: @@ -176,14 +177,20 @@ class MessageRegistry(object): """Add a new EnumDescriptor named name with the given enum values.""" message = extended_descriptor.ExtendedEnumDescriptor() message.name = self.__names.ClassName(name) - message.description = description + message.description = util.CleanDescription(description) self.__DeclareDescriptor(message.name) for index, (enum_name, enum_description) in enumerate( zip(enum_values, enum_descriptions)): enum_value = extended_descriptor.ExtendedEnumValueDescriptor() enum_value.name = self.__names.NormalizeEnumName(enum_name) + if enum_value.name != enum_name: + message.enum_mappings.append( + extended_descriptor.ExtendedEnumDescriptor.JsonEnumMapping( + python_name=enum_value.name, json_name=enum_name)) + self.__AddImport('from %s import encoding' % self.__base_files_package) enum_value.number = index - enum_value.description = enum_description or '' + enum_value.description = util.CleanDescription( + enum_description or '') message.values.append(enum_value) self.__RegisterDescriptor(message) @@ -202,7 +209,8 @@ class MessageRegistry(object): additional_properties_info = schema['additionalProperties'] entries_type_name = self.__AddAdditionalPropertyType( message.name, additional_properties_info) - description = additional_properties_info.get('description') + description = util.CleanDescription( + additional_properties_info.get('description')) if description is None: description = 'Additional properties of type %s' % message.name attrs = { @@ -235,14 +243,20 @@ class MessageRegistry(object): 'Cannot create message descriptors for type %s', schema.get('type')) message = extended_descriptor.ExtendedMessageDescriptor() message.name = self.__names.ClassName(schema['id']) - message.description = schema.get( - 'description', 'A %s object.' % message.name) + message.description = util.CleanDescription(schema.get( + 'description', 'A %s object.' % message.name)) self.__DeclareDescriptor(message.name) with self.__DescriptorEnv(message): properties = schema.get('properties', {}) - for index, (name, attrs) in enumerate(sorted(properties.items())): + for index, (name, attrs) in enumerate(sorted(six.iteritems(properties))): field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) message.fields.append(field) + if field.name != name: + message.field_mappings.append( + extended_descriptor.ExtendedMessageDescriptor.JsonFieldMapping( + python_name=field.name, json_name=name)) + self.__AddImport( + 'from %s import encoding' % self.__base_files_package) if 'additionalProperties' in schema: self.__AddAdditionalProperties(message, schema, properties) self.__RegisterDescriptor(message) @@ -309,8 +323,8 @@ class MessageRegistry(object): field.default_value = default extended_field = extended_descriptor.ExtendedFieldDescriptor() extended_field.name = field.name - extended_field.description = attrs.get('description', 'A %s attribute.' % ( - field.type_name,)) + extended_field.description = util.CleanDescription( + attrs.get('description', 'A %s attribute.' % field.type_name)) extended_field.field_descriptor = field return extended_field @@ -325,17 +339,19 @@ class MessageRegistry(object): return descriptor.FieldDescriptor.Label.OPTIONAL def __DeclareEnum(self, enum_name, attrs): - description = attrs.get('description', '') + description = util.CleanDescription(attrs.get('description', '')) + enum_values = attrs['enum'] + enum_descriptions = attrs.get('enumDescriptions', [''] * len(enum_values)) self.AddEnumDescriptor(enum_name, description, - attrs['enum'], attrs['enumDescriptions']) + enum_values, enum_descriptions) self.__AddIfUnknown(enum_name) return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) def __AddIfUnknown(self, type_name): type_name = self.__names.ClassName(type_name) full_type_name = self.__ComputeFullName(type_name) - if (full_type_name not in self.__message_registry.keys() and - type_name not in self.__message_registry.keys()): + if (full_type_name not in six.iterkeys(self.__message_registry) and + type_name not in six.iterkeys(self.__message_registry)): self.__unknown_types.add(type_name) def __GetTypeInfo(self, attrs, name_hint): @@ -348,6 +364,9 @@ class MessageRegistry(object): if type_ref: self.__AddIfUnknown(type_ref) + # We don't actually know this is a message -- it might be an + # enum. However, we can't check that until we've created all the + # types, so we come back and fix this up later. return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) if 'enum' in attrs: @@ -356,15 +375,19 @@ class MessageRegistry(object): if 'format' in attrs: type_info = self.PRIMITIVE_FORMAT_MAP.get(attrs['format']) + if type_info is None: + # If we don't recognize the format, the spec says we fall back + # to just using the type name. + if type_name in self.PRIMITIVE_TYPE_INFO_MAP: + return self.PRIMITIVE_TYPE_INFO_MAP[type_name] + raise ValueError('Unknown type/format "%s"/"%s"' % ( + attrs['format'], type_name)) if (type_info.type_name.startswith('protorpc.message_types.') or type_info.type_name.startswith('message_types.')): self.__AddImport('from protorpc import message_types') if type_info.type_name.startswith('extra_types.'): self.__AddImport( 'from %s import extra_types' % self.__base_files_package) - if type_info is None: - raise ValueError('Unknown format %s for type %s' % ( - attrs['format'], type_name)) return type_info if type_name in self.PRIMITIVE_TYPE_INFO_MAP: @@ -400,3 +423,18 @@ class MessageRegistry(object): return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) raise ValueError('Unknown type: %s' % type_name) + + def FixupMessageFields(self): + for message_type in self.file_descriptor.message_types: + self._FixupMessage(message_type) + + def _FixupMessage(self, message_type): + with self.__DescriptorEnv(message_type): + for field in message_type.fields: + if field.field_descriptor.variant == messages.Variant.MESSAGE: + field_type_name = field.field_descriptor.type_name + field_type = self.LookupDescriptor(field_type_name) + if isinstance(field_type, extended_descriptor.ExtendedEnumDescriptor): + field.field_descriptor.variant = messages.Variant.ENUM + for submessage_type in message_type.message_types: + self._FixupMessage(submessage_type) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 0942ccb..7aca0fe 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -7,6 +7,7 @@ import re import textwrap from apitools.base.py import base_api +from apitools.gen import util # We're a code generator. I don't care. # pylint:disable=too-many-statements @@ -19,7 +20,8 @@ class ServiceRegistry(object): def __init__(self, client_info, message_registry, command_registry, base_url, base_path, names, - root_package_dir, base_files_package): + root_package_dir, base_files_package, + unelidable_request_methods): self.__client_info = client_info self.__package = client_info.package self.__names = names @@ -30,6 +32,7 @@ class ServiceRegistry(object): self.__base_path = base_path self.__root_package_dir = root_package_dir self.__base_files_package = base_files_package + self.__unelidable_request_methods = unelidable_request_methods self.__all_scopes = set(self.__client_info.scopes) def Validate(self): @@ -46,7 +49,7 @@ class ServiceRegistry(object): def __PrintDocstring(self, printer, method_info, method_name, name): """Print a docstring for a service method.""" if method_info.description: - description = method_info.description + description = util.CleanDescription(method_info.description) first_line, newline, remaining = method_info.description.partition( '\n') if not first_line.endswith('.'): @@ -174,6 +177,8 @@ class ServiceRegistry(object): client_info = self.__client_info printer('"""Generated client library for %s version %s."""', client_info.package, client_info.version) + printer('# NOTE: This file is autogenerated and should not be edited by ' + 'hand.') printer('from %s import base_api', self.__base_files_package) import_prefix = '' printer('%simport %s as messages', import_prefix, @@ -209,7 +214,7 @@ class ServiceRegistry(object): printer(' credentials_args=credentials_args,') printer(' default_global_params=default_global_params,') printer(' additional_http_headers=additional_http_headers)') - for name in self.__service_method_info_map: + for name in self.__service_method_info_map.keys(): printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) for name, method_info_map in self.__service_method_info_map.items(): @@ -268,6 +273,8 @@ class ServiceRegistry(object): """Determine if this method needs a new request type created.""" if not request_type: return True + if method_description.get('id', '') in self.__unelidable_request_methods: + return True message = self.__message_registry.LookupDescriptorOrDie(request_type) if message is None: return True @@ -329,7 +336,8 @@ class ServiceRegistry(object): relative_path=relative_path, method_id=method_id, http_method=method_description['httpMethod'], - description=method_description.get('description', ''), + description=util.CleanDescription( + method_description.get('description', '')), query_params=[], path_params=[], ordered_params=method_description.get('parameterOrder', []), diff --git a/apitools/gen/util.py b/apitools/gen/util.py index c3b6dc3..1b7abe3 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -10,8 +10,9 @@ import logging import os import re import urllib2 +import urlparse -from six.moves import range +import six @@ -70,8 +71,12 @@ class Names(object): name = re.sub('[^_A-Za-z0-9]', '_', name) if name[0].isdigit(): name = '_%s' % name - while name in keyword.kwlist: + while keyword.iskeyword(name): name = '%s_' % name + # If we end up with __ as a prefix, we'll run afoul of python + # field renaming, so we manually correct for it. + if name.startswith('__'): + name = 'f%s' % name return name @staticmethod @@ -89,8 +94,8 @@ class Names(object): def NormalizeEnumName(self, enum_name): if self.__capitalize_enums: - return enum_name.upper() - return enum_name + enum_name = enum_name.upper() + return self.CleanName(enum_name) def ClassName(self, name, separator='_'): """Generate a valid class name from name.""" @@ -165,9 +170,9 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( 'user_agent': user_agent, 'api_key': api_key, } - client_info['client_class_name'] = '%s%s' % ( - names.ClassName(client_info['package']), - names.ClassName(client_info['version'])) + client_class_name = ''.join( + map(names.ClassName, (client_info['package'], client_info['version']))) + client_info['client_class_name'] = client_class_name return cls(**client_info) @property @@ -216,6 +221,13 @@ def GetPackage(path): return '.'.join(path_components) +def CleanDescription(description): + """Return a version of description safe for printing in a docstring.""" + if not isinstance(description, six.string_types): + return description + return description.replace('"""', '" " "') + + class SimplePrettyPrinter(object): """Simple pretty-printer that supports an indent contextmanager.""" -- GitLab From cd9d221b938bd4fb38413d5780a7c146bb966f3c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 1 Mar 2015 01:25:40 -0800 Subject: [PATCH 067/295] Update version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4e825f9..3af84ba 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.3' +_APITOOLS_VERSION = '0.4' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 3c522a48a0b1a18d7d1a172664d35e3c77706fd2 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 1 Mar 2015 22:47:05 -0800 Subject: [PATCH 068/295] Several small py3-related cleanups. I made a quick pass at low-hanging py3 compatibility bits; got a few, but there's still real work to be done dealing with flags/appcommands. Also swapped out the existing uses of `six.iterkeys` and `six.iteritems`, since they were all over small python objects anyway. --- apitools/base/py/base_api.py | 25 ++++++++++++------------- apitools/base/py/credentials_lib.py | 19 +++++++++---------- apitools/base/py/encoding.py | 4 ++-- apitools/base/py/extra_types.py | 2 +- apitools/base/py/http_wrapper.py | 4 ++-- apitools/gen/client_generation_test.py | 2 +- apitools/gen/gen_client.py | 2 +- apitools/gen/gen_client_lib.py | 6 ++---- apitools/gen/message_registry.py | 6 +++--- 9 files changed, 33 insertions(+), 37 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 73ecbb0..1954b8b 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -5,14 +5,13 @@ import contextlib import datetime import logging import pprint -import urllib -import urlparse from protorpc import message_types from protorpc import messages import six from six.moves import http_client +from six.moves import urllib from apitools.base.py import credentials_lib @@ -129,12 +128,12 @@ class _UrlBuilder(object): """Convenient container for url data.""" def __init__(self, base_url, relative_path=None, query_params=None): - components = urlparse.urlsplit(urlparse.urljoin( + components = urllib.parse.urlsplit(urllib.parse.urljoin( base_url, relative_path or '')) if components.fragment: raise exceptions.ConfigurationValueError( 'Unexpected url fragment: %s' % components.fragment) - self.query_params = urlparse.parse_qs(components.query or '') + self.query_params = urllib.parse.parse_qs(components.query or '') if query_params is not None: self.query_params.update(query_params) self.__scheme = components.scheme @@ -143,20 +142,20 @@ class _UrlBuilder(object): @classmethod def FromUrl(cls, url): - urlparts = urlparse.urlsplit(url) - query_params = urlparse.parse_qs(urlparts.query) - base_url = urlparse.urlunsplit(( + urlparts = urllib.parse.urlsplit(url) + query_params = urllib.parse.parse_qs(urlparts.query) + base_url = urllib.parse.urlunsplit(( urlparts.scheme, urlparts.netloc, '', None, None)) relative_path = urlparts.path or '' return cls(base_url, relative_path=relative_path, query_params=query_params) @property def base_url(self): - return urlparse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) + return urllib.parse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) @base_url.setter def base_url(self, value): - components = urlparse.urlsplit(value) + components = urllib.parse.urlsplit(value) if components.path or components.query or components.fragment: raise exceptions.ConfigurationValueError('Invalid base url: %s' % value) self.__scheme = components.scheme @@ -168,14 +167,14 @@ class _UrlBuilder(object): # non-ASCII, we may silently fail to encode correctly. We should # figure out who is responsible for owning the object -> str # conversion. - return urllib.urlencode(self.query_params, doseq=True) + return urllib.parse.urlencode(self.query_params, doseq=True) @property def url(self): if '{' in self.relative_path or '}' in self.relative_path: raise exceptions.ConfigurationValueError( 'Cannot create url with relative path %s' % self.relative_path) - return urlparse.urlunsplit(( + return urllib.parse.urlunsplit(( self.__scheme, self.__netloc, self.relative_path, self.query, '')) @@ -454,11 +453,11 @@ class BaseApiService(object): query_param_names = util.MapParamNames(query_params, type(request)) query_info.update( (param, getattr(request, param, None)) for param in query_param_names) - query_info = dict((k, v) for k, v in six.iteritems(query_info) + query_info = dict((k, v) for k, v in query_info.items() if v is not None) query_info = self.__EncodePrettyPrint(query_info) query_info = util.MapRequestParams(query_info, type(request)) - for k, v in six.iteritems(query_info): + for k, v in query_info.items(): if isinstance(v, six.text_type): query_info[k] = v.encode('utf8') elif isinstance(v, str): diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 6f4dcd3..5307203 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -5,8 +5,6 @@ from __future__ import print_function import datetime import json import os -import urllib -import urllib2 import httplib2 import oauth2client @@ -16,6 +14,7 @@ import oauth2client.locked_file import oauth2client.multistore_file import oauth2client.tools # for flag declarations from six.moves import http_client +from six.moves import urllib import logging @@ -103,7 +102,7 @@ def _EnsureFileExists(filename): def _OpenNoProxy(request): """Wrapper around urllib2.open that ignores proxies.""" - opener = urllib2.build_opener(urllib2.ProxyHandler({})) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) return opener.open(request) @@ -229,10 +228,10 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'http://metadata.google.internal/computeMetadata/' 'v1/instance/service-accounts') additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib2.Request(account_uri, headers=additional_headers) + request = urllib.request.Request(account_uri, headers=additional_headers) try: response = _OpenNoProxy(request) - except urllib2.URLError as e: + except urllib.error.URLError as e: raise exceptions.CommunicationError( 'Could not reach metadata service: %s' % e.reason) response_lines = [line.rstrip('/\n\r') for line in response.readlines()] @@ -245,10 +244,10 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'http://metadata.google.internal/computeMetadata/v1/instance/' 'service-accounts/%s/scopes') % self.__service_account_name additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib2.Request(scopes_uri, headers=additional_headers) + request = urllib.request.Request(scopes_uri, headers=additional_headers) try: response = _OpenNoProxy(request) - except urllib2.URLError as e: + except urllib.error.URLError as e: raise exceptions.CommunicationError( 'Could not reach metadata service: %s' % e.reason) return util.NormalizeScopes(scope.strip() for scope in response.readlines()) @@ -276,10 +275,10 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'http://metadata.google.internal/computeMetadata/v1/instance/' 'service-accounts/%s/token') % self.__service_account_name extra_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib2.Request(token_uri, headers=extra_headers) + request = urllib.request.Request(token_uri, headers=extra_headers) try: content = _OpenNoProxy(request).read() - except urllib2.URLError as e: + except urllib.error.URLError as e: self.invalid = True if self.store: self.store.locked_put(self) @@ -409,7 +408,7 @@ def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name http = http or httplib2.Http() url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' query_args = {'access_token': credentials.access_token} - url = '?'.join((url_root, urllib.urlencode(query_args))) + url = '?'.join((url_root, urllib.parse.urlencode(query_args))) # We ignore communication woes here (i.e. SSL errors, socket # timeout), as handling these should be done in a common location. response, content = http.request(url) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 489cef8..a8d3c89 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -367,7 +367,7 @@ def _DecodeUnknownMessages(message, encoded_message, pair_type): field_type = pair_type.value.type new_values = [] all_field_names = [x.name for x in message.all_fields()] - for name, value_dict in six.iteritems(encoded_message): + for name, value_dict in encoded_message.items(): if name in all_field_names: continue value = PyValueToMessage(field_type, value_dict) @@ -492,7 +492,7 @@ def _ProcessUnknownMessages(message, encoded_message): decoded_message = json.loads(encoded_message) message_fields = [x.name for x in message.all_fields()] + list( message.all_unrecognized_fields()) - missing_fields = [x for x in six.iterkeys(decoded_message) + missing_fields = [x for x in decoded_message.keys() if x not in message_fields] for field_name in missing_fields: message.set_unrecognized_field(field_name, decoded_message[field_name], diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 62ae7b5..8b29c76 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -122,7 +122,7 @@ def _PythonValueToJsonObject(py_value): return JsonObject( properties=[ JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) - for key, value in six.iteritems(py_value)]) + for key, value in py_value.items()]) def _PythonValueToJsonArray(py_value): diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index f91c361..a7d9cff 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -77,7 +77,7 @@ def _Httplib2Debuglevel(http_request, level, http=None): http_levels = {} httplib2.debuglevel = level if http is not None: - for connection_key, connection in six.iteritems(http.connections): + for connection_key, connection in http.connections.items(): # httplib2 stores two kinds of values in this dict, connection # classes and instances. Since the connection types are all # old-style classes, we can't easily distinguish by connection @@ -89,7 +89,7 @@ def _Httplib2Debuglevel(http_request, level, http=None): yield httplib2.debuglevel = old_level if http is not None: - for connection_key, old_level in six.iteritems(http_levels): + for connection_key, old_level in http_levels.items(): if connection_key in http.connections: http.connections[connection_key].set_debuglevel(old_level) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 602296d..87bba67 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -39,7 +39,7 @@ class ClientGenerationTest(unittest2.TestCase): self.gen_client_binary = 'gen_client' def testGeneration(self): - if sys.version < (2, 7): + if sys.version_info < (2, 7): # TODO(craigcitro): Make apitools codegen support python 2.6. # Maybe. # diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index d64855b..95e947e 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -186,7 +186,7 @@ def _WriteGeneratedFiles(codegen): if FLAGS.generate_cli: with open(codegen.client_info.cli_file_name, 'w') as out: codegen.WriteCli(out) - os.chmod(codegen.client_info.cli_file_name, 0755) + os.chmod(codegen.client_info.cli_file_name, 0o755) def _WriteInit(codegen): diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index f1fe36a..e9370bc 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -8,8 +8,6 @@ Relevant links: import json import urlparse -import six - from apitools.base.py import base_cli from apitools.gen import command_registry from apitools.gen import message_registry @@ -73,7 +71,7 @@ class DescriptorGenerator(object): self.__client_info, self.__names, self.__description, self.__root_package, self.__base_files_package) schemas = self.__discovery_doc.get('schemas', {}) - for schema_name, schema in six.iteritems(schemas): + for schema_name, schema in schemas.items(): self.__message_registry.AddDescriptorFromSchema(schema_name, schema) # We need to add one more message type for the global parameters. @@ -105,7 +103,7 @@ class DescriptorGenerator(object): self.__base_files_package, unelidable_request_methods or []) services = self.__discovery_doc.get('resources', {}) - for service_name, methods in sorted(six.iteritems(services)): + for service_name, methods in sorted(services.items()): self.__services_registry.AddServiceFromResource(service_name, methods) # We might also have top-level methods. api_methods = self.__discovery_doc.get('methods', []) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 970a0dc..76c5911 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -248,7 +248,7 @@ class MessageRegistry(object): self.__DeclareDescriptor(message.name) with self.__DescriptorEnv(message): properties = schema.get('properties', {}) - for index, (name, attrs) in enumerate(sorted(six.iteritems(properties))): + for index, (name, attrs) in enumerate(sorted(properties.items())): field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) message.fields.append(field) if field.name != name: @@ -350,8 +350,8 @@ class MessageRegistry(object): def __AddIfUnknown(self, type_name): type_name = self.__names.ClassName(type_name) full_type_name = self.__ComputeFullName(type_name) - if (full_type_name not in six.iterkeys(self.__message_registry) and - type_name not in six.iterkeys(self.__message_registry)): + if (full_type_name not in self.__message_registry.keys() and + type_name not in self.__message_registry.keys()): self.__unknown_types.add(type_name) def __GetTypeInfo(self, attrs, name_hint): -- GitLab From 79f3c87d332e55fc3eb056b4f641076848149615 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 9 Mar 2015 11:52:14 -0700 Subject: [PATCH 069/295] Fix OverflowError for uploads >4GiB on 32-bit OSes. (Originally by thobrla@.) --- apitools/base/py/http_wrapper.py | 7 +++++-- apitools/base/py/http_wrapper_test.py | 28 +++++++++++++++++++++++++++ apitools/base/py/stream_slice.py | 10 ++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 apitools/base/py/http_wrapper_test.py diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index a7d9cff..ebe80a9 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -122,13 +122,16 @@ class Request(object): @body.setter def body(self, value): + """Sets the request body; handles logging and length measurement.""" self.__body = value if value is not None: - self.headers['content-length'] = str(len(self.__body)) + # Avoid calling len() which cannot exceed 4GiB in 32-bit python. + body_length = getattr(self.__body, 'length', None) or len(self.__body) + self.headers['content-length'] = str(body_length) else: self.headers.pop('content-length', None) # This line ensures we don't try to print large requests. - if not isinstance(value, six.string_types): + if not isinstance(value, basestring): self.loggable_body = '' diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py new file mode 100644 index 0000000..f99eea1 --- /dev/null +++ b/apitools/base/py/http_wrapper_test.py @@ -0,0 +1,28 @@ +"""Tests for http_wrapper.""" + +import unittest2 + +from apitools.base.py import http_wrapper + + +class RaisesExceptionOnLen(object): + """Supports length property but raises if __len__ is used.""" + + def __len__(self): + raise Exception('len() called unnecessarily') + + def length(self): + return 1 + + +class HttpWrapperTest(unittest2.TestCase): + + def testRequestBodyUsesLengthProperty(self): + http_wrapper.Request(body=RaisesExceptionOnLen()) + + def testRequestBodyWithLen(self): + http_wrapper.Request(body='burrito') + + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py index 58d2390..7c06b4b 100644 --- a/apitools/base/py/stream_slice.py +++ b/apitools/base/py/stream_slice.py @@ -19,6 +19,16 @@ class StreamSlice(object): def __len__(self): return self.__max_bytes + def __nonzero__(self): + # For 32-bit python2.x, len() cannot exceed a 32-bit number; avoid + # accidental len() calls from httplib in the form of "if this_object:". + return self.__max_bytes + + @property + def length(self): + # For 32-bit python2.x, len() cannot exceed a 32-bit number. + return self.__max_bytes + def read(self, size=None): # pylint: disable=missing-docstring """Read at most size bytes from this slice. -- GitLab From e2f7c74c508d0e4a3f5005530b7dcd27ac782131 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 10 Mar 2015 14:55:59 -0700 Subject: [PATCH 070/295] Pull stream buffers into string objects to work around httplib2 issue. This works around https://code.google.com/p/httplib2/issues/detail?id=176 which can cause data corruption in a stream. This would eventually result in a mismatching hash on the final object or a 410 from the service when a non-idempotent write was issued. (Patch originally by thobrla, see https://github.com/GoogleCloudPlatform/gsutil/commit/af195c87e19570831c4d9197810e8be0be2d85aa) --- apitools/base/py/transfer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 73445bc..a2b7da0 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -848,6 +848,7 @@ class Upload(_Transfer): def __SendChunk(self, start, additional_headers=None): """Send the specified chunk.""" self.EnsureInitialized() + no_log_body = self.total_size is None if self.total_size is None: # For the streaming resumable case, we need to detect when we're at the # end of the stream. @@ -856,6 +857,12 @@ class Upload(_Transfer): end = body_stream.stream_end_position if body_stream.stream_exhausted: self.__total_size = end + # TODO: Here, change body_stream from a stream to a string object, + # which means reading a chunk into memory. This works around + # https://code.google.com/p/httplib2/issues/detail?id=176 which can + # cause httplib2 to skip bytes on 401's for file objects. + # Rework this solution to be more general. + body_stream = body_stream.read(self.chunksize) else: end = min(start + self.chunksize, self.total_size) body_stream = stream_slice.StreamSlice(self.stream, end - start) @@ -864,6 +871,10 @@ class Upload(_Transfer): request = http_wrapper.Request(url=self.url, http_method='PUT', body=body_stream) request.headers['Content-Type'] = self.mime_type + if no_log_body: + # Disable logging of streaming body. + # TODO: Remove no_log_body and rework as part of a larger logs refactor. + request.loggable_body = '' if self.total_size is None: # Streaming resumable upload case, unknown total size. range_string = 'bytes %s-%s/*' % (start, end - 1) -- GitLab From 3ea053daf8108da4d2f8d08243bad9c53656c375 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 10 Mar 2015 14:59:26 -0700 Subject: [PATCH 071/295] Change 32-bit large file fix to use bool instead of long. (Patch originally by thobrla@, see https://github.com/GoogleCloudPlatform/gsutil/commit/edf1c888e96048d65d8d643e2131cc144b5a7bb6) --- apitools/base/py/stream_slice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py index 7c06b4b..9901b81 100644 --- a/apitools/base/py/stream_slice.py +++ b/apitools/base/py/stream_slice.py @@ -22,7 +22,7 @@ class StreamSlice(object): def __nonzero__(self): # For 32-bit python2.x, len() cannot exceed a 32-bit number; avoid # accidental len() calls from httplib in the form of "if this_object:". - return self.__max_bytes + return bool(self.__max_bytes) @property def length(self): -- GitLab From b5d4d5b9250c821b0228635b419880e4087d3bfc Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 10:59:28 -0700 Subject: [PATCH 072/295] Adding PyPI badge to GitHub repo. --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 7674e1d..15e7e4f 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,7 @@ google-apitools =============== -.. image:: https://travis-ci.org/craigcitro/apitools.png?branch=master - :target: https://travis-ci.org/craigcitro/apitools +|pypi| |build| ``google-apitools`` is a collection of utilities to make it easier to build client-side tools, especially those that talk to Google APIs. @@ -39,3 +38,8 @@ and the ``nose`` testrunner:: Then run the tests:: $ nosetests + +.. |build| image:: https://travis-ci.org/google/apitools.svg?branch=master + :target: https://travis-ci.org/google/apitools +.. |pypi| image:: https://img.shields.io/pypi/v/google-apitools.svg + :target: https://pypi.python.org/pypi/google-apitools -- GitLab From ab9ec67338f1b90ba8d4175c76e02e6e1a1a8662 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 14:21:40 -0700 Subject: [PATCH 073/295] Adding Python 3.4 to Travis and tox config. --- .travis.yml | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b9a9a4a..36a15ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python env: - TOX_ENV=py27 - TOX_ENV=pypy + - TOX_ENV=py34 install: - pip install tox - pip install . --allow-external argparse diff --git a/tox.ini b/tox.ini index 1c99527..2bd03ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pypy +envlist = py26,py27,pypy,py33,py34 [testenv] deps = nose -- GitLab From 158cc5bc2fa09d66518622855a618f474732f73b Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Tue, 24 Mar 2015 14:45:02 -0700 Subject: [PATCH 074/295] Include socket timeout in http_wrapper retryable errors Reviewed at https://codereview.appspot.com/219290043/ --- apitools/base/py/http_wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index ebe80a9..7579d9c 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -247,6 +247,8 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): logging.debug('Caught socket error, retrying: %s', retry_args.exc) elif isinstance(retry_args.exc, socket.gaierror): logging.debug('Caught socket address error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, socket.timeout): + logging.debug('Caught socket timeout error, retrying: %s', retry_args.exc) elif isinstance(retry_args.exc, httplib2.ServerNotFoundError): logging.debug('Caught server not found error, retrying: %s', retry_args.exc) elif isinstance(retry_args.exc, ValueError): -- GitLab From 179640d05b1675157ac8f730154ef99868b5c4e4 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 15:09:13 -0700 Subject: [PATCH 075/295] Removing Python2 only imports StringIO and urlparse. --- apitools/base/py/base_api_test.py | 7 ++++--- apitools/base/py/batch.py | 10 +++++----- apitools/base/py/buffered_stream_test.py | 4 ++-- apitools/base/py/credentials_lib_test.py | 8 ++++---- apitools/base/py/stream_slice_test.py | 4 ++-- apitools/base/py/transfer.py | 3 +-- apitools/base/py/transfer_test.py | 6 +++--- apitools/gen/client_generation_test.py | 13 +++++++------ apitools/gen/gen_client_lib.py | 4 ++-- apitools/gen/util.py | 1 - samples/storage_sample/downloads_test.py | 6 +++--- samples/storage_sample/uploads_test.py | 9 +++++---- 12 files changed, 38 insertions(+), 37 deletions(-) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 7e9ce10..52c3e1d 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -4,7 +4,8 @@ import datetime import sys import urllib -import urlparse + +from six.moves import urllib_parse from protorpc import message_types from protorpc import messages @@ -150,8 +151,8 @@ class BaseApiTest(unittest2.TestCase): request = MessageWithRemappings( str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one) http_request = FakeService().PrepareHttpRequest(method_config, request) - result_params = urlparse.parse_qs( - urlparse.urlparse(http_request.url).query) + result_params = urllib_parse.parse_qs( + urllib_parse.urlparse(http_request.url).query) expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'} self.assertTrue(expected_params, result_params) diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index e098282..89c962f 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -7,13 +7,13 @@ import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart import email.parser as email_parser import itertools -import StringIO +import io import time import urllib -import urlparse import uuid from six.moves import http_client +from six.moves import urllib_parse from apitools.base.py import exceptions from apitools.base.py import http_wrapper @@ -271,8 +271,8 @@ class BatchHttpRequest(object): The request as a string in application/http format. """ # Construct status line - parsed = urlparse.urlsplit(request.url) - request_line = urlparse.urlunsplit( + parsed = urllib_parse.urlsplit(request.url) + request_line = urllib_parse.urlunsplit( (None, None, parsed.path, parsed.query, None)) status_line = request.http_method + ' ' + request_line + ' HTTP/1.1\n' major, minor = request.headers.get( @@ -293,7 +293,7 @@ class BatchHttpRequest(object): msg.set_payload(request.body) # Serialize the mime message. - str_io = StringIO.StringIO() + str_io = io.StringIO() # maxheaderlen=0 means don't line wrap headers. gen = generator.Generator(str_io, maxheaderlen=0) gen.flatten(msg, unixfrom=False) diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py index c8e3b3a..0ee69f6 100644 --- a/apitools/base/py/buffered_stream_test.py +++ b/apitools/base/py/buffered_stream_test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python """Tests for stream_slice.""" +import io import string -import StringIO import unittest2 @@ -13,7 +13,7 @@ from apitools.base.py import exceptions class BufferedStreamTest(unittest2.TestCase): def setUp(self): - self.stream = StringIO.StringIO(string.letters) + self.stream = io.BytesIO(string.ascii_letters) self.value = self.stream.getvalue() self.stream.seek(0) diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index e445df5..810f4ce 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python +import io import re -import StringIO import mock from six.moves import http_client @@ -38,12 +38,12 @@ class CredentialsLibTest(unittest2.TestCase): def MockMetadataCalls(request): request_url = request.get_full_url() if request_url.endswith('scopes'): - return StringIO.StringIO(''.join(scopes)) + return io.BytesIO(''.join(scopes)) elif request_url.endswith('service-accounts'): - return StringIO.StringIO(service_account_name) + return io.BytesIO(service_account_name) elif request_url.endswith( '/service-accounts/%s/token' % service_account_name): - return StringIO.StringIO('{"access_token": "token"}') + return io.BytesIO('{"access_token": "token"}') self.fail('Unexpected HTTP request to %s' % request_url) with mock.patch.object(credentials_lib, '_OpenNoProxy', diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py index 900ba81..31ba5c8 100644 --- a/apitools/base/py/stream_slice_test.py +++ b/apitools/base/py/stream_slice_test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python """Tests for stream_slice.""" +import io import string -import StringIO import unittest2 @@ -13,7 +13,7 @@ from apitools.base.py import stream_slice class StreamSliceTest(unittest2.TestCase): def setUp(self): - self.stream = StringIO.StringIO(string.letters) + self.stream = io.BytesIO(string.ascii_letters) self.value = self.stream.getvalue() self.stream.seek(0) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index a2b7da0..6ba40e4 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -9,7 +9,6 @@ import io import json import mimetypes import os -import StringIO import threading import six @@ -632,7 +631,7 @@ class Upload(_Transfer): # encode the body: note that we can't use `as_string`, because # it plays games with `From ` lines. - fp = StringIO.StringIO() + fp = io.BytesIO() g = email_generator.Generator(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index d1bc7c8..594f823 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import StringIO +import io import unittest2 @@ -36,7 +36,7 @@ class TransferTest(unittest2.TestCase): # Test multipart: having a body argument in http_request forces # multipart here. upload = transfer.Upload.FromStream( - StringIO.StringIO(upload_contents), + io.BytesIO(upload_contents), 'text/plain', total_size=len(upload_contents)) http_request = http_wrapper.Request( @@ -52,7 +52,7 @@ class TransferTest(unittest2.TestCase): # Test non-multipart (aka media): no body argument means this is # sent as media. upload = transfer.Upload.FromStream( - StringIO.StringIO(upload_contents), + io.BytesIO(upload_contents), 'text/plain', total_size=len(upload_contents)) http_request = http_wrapper.Request( diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 87bba67..0974316 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -38,13 +38,14 @@ class ClientGenerationTest(unittest2.TestCase): super(ClientGenerationTest, self).setUp() self.gen_client_binary = 'gen_client' + # TODO(craigcitro): Make apitools codegen support python 2.6. + # Maybe. + # + # unittest in 2.6 doesn't have skipIf. + @unittest2.skipUnless(sys.version_info[0] == 2 and + sys.version_info[1] == 7, + 'Only runs in Python 2.7') def testGeneration(self): - if sys.version_info < (2, 7): - # TODO(craigcitro): Make apitools codegen support python 2.6. - # Maybe. - # - # unittest in 2.6 doesn't have skipIf. - return for api in _API_LIST: with TempDir(): args = [ diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index e9370bc..732bf38 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -6,7 +6,7 @@ Relevant links: """ import json -import urlparse +from six.moves import urllib_parse from apitools.base.py import base_cli from apitools.gen import command_registry @@ -33,7 +33,7 @@ def _StandardQueryParametersSchema(discovery_doc): def _ComputePaths(package, version, discovery_doc): - full_path = urlparse.urljoin( + full_path = urllib_parse.urljoin( discovery_doc['rootUrl'], discovery_doc['servicePath']) api_path_component = '/'.join((package, version, '')) if api_path_component not in full_path: diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 1b7abe3..c6f4afb 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -10,7 +10,6 @@ import logging import os import re import urllib2 -import urlparse import six diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index e8e52c3..e9ce36a 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -4,10 +4,10 @@ These tests exercise most of the corner cases for upload/download of files in apitools, via GCS. There are no performance tests here yet. """ +import io import json import os import pkgutil -import StringIO import unittest import apitools.base.py as apitools_base @@ -29,7 +29,7 @@ class DownloadsTest(unittest.TestCase): self.__ResetDownload() def __ResetDownload(self, auto_transfer=False): - self.__buffer = StringIO.StringIO() + self.__buffer = io.StringIO() self.__download = storage.Download.FromStream( self.__buffer, auto_transfer=auto_transfer) @@ -157,7 +157,7 @@ class DownloadsTest(unittest.TestCase): request = storage.StorageObjectsGetRequest( bucket=self._DEFAULT_BUCKET, object=object_name) response = self.__client.objects.Get(request) - self.__buffer = StringIO.StringIO() + self.__buffer = io.StringIO() download_data = json.dumps({ 'auto_transfer': False, 'progress': 0, diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index 6d01828..63f4f69 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -4,11 +4,11 @@ These tests exercise most of the corner cases for upload/download of files in apitools, via GCS. There are no performance tests here yet. """ +import io import json import os import random import string -import StringIO import unittest import apitools.base.py as apitools_base @@ -37,8 +37,8 @@ class UploadsTest(unittest.TestCase): def __ResetUpload(self, size, auto_transfer=True): self.__content = ''.join( - random.choice(string.letters) for _ in xrange(size)) - self.__buffer = StringIO.StringIO(self.__content) + random.choice(string.ascii_letters) for _ in xrange(size)) + self.__buffer = io.StringIO(self.__content) self.__upload = storage.Upload.FromStream( self.__buffer, 'text/plain', auto_transfer=auto_transfer) @@ -100,7 +100,8 @@ class UploadsTest(unittest.TestCase): self.assertEqual(size, response.size) def testBreakAndResumeUpload(self): - filename = 'ten_meg_file_' + ''.join(random.sample(string.letters, 5)) + filename = ('ten_meg_file_' + + ''.join(random.sample(string.ascii_letters, 5))) size = 10 << 20 self.__ResetUpload(size, auto_transfer=False) self.__upload.strategy = 'resumable' -- GitLab From b92b57f2200a4f6f25ec134632627aed9f2ca32b Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 15:15:53 -0700 Subject: [PATCH 076/295] Removing use of basestring and import of urllib (for Python3). --- apitools/base/py/base_api_test.py | 3 +-- apitools/base/py/batch.py | 5 ++--- apitools/base/py/encoding.py | 8 ++++---- apitools/base/py/http_wrapper.py | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 52c3e1d..6a663c7 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -3,7 +3,6 @@ import datetime import sys -import urllib from six.moves import urllib_parse @@ -124,7 +123,7 @@ class BaseApiTest(unittest2.TestCase): timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) http_request = service.PrepareHttpRequest(method_config, request) - url_timestamp = urllib.quote(request.timestamp.isoformat()) + url_timestamp = urllib_parse.quote(request.timestamp.isoformat()) self.assertTrue(http_request.url.endswith(url_timestamp)) def testPrettyPrintEncoding(self): diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 89c962f..48d0616 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -9,7 +9,6 @@ import email.parser as email_parser import itertools import io import time -import urllib import uuid from six.moves import http_client @@ -235,7 +234,7 @@ class BatchHttpRequest(object): the value because Content-ID headers are supposed to be universally unique. """ - return '<%s+%s>' % (self.__base_id, urllib.quote(request_id)) + return '<%s+%s>' % (self.__base_id, urllib_parse.quote(request_id)) @staticmethod def _ConvertHeaderToId(header): @@ -259,7 +258,7 @@ class BatchHttpRequest(object): raise exceptions.BatchError('Invalid value for Content-ID: %s' % header) _, request_id = header[1:-1].rsplit('+', 1) - return urllib.unquote(request_id) + return urllib_parse.unquote(request_id) def _SerializeRequest(self, request): """Convert a http_wrapper.Request object into a string. diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index a8d3c89..abd2e0b 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -517,8 +517,8 @@ def AddCustomJsonEnumMapping(enum_type, python_name, json_name): Args: enum_type: (messages.Enum) An enum type - python_name: (basestring) Python name for this value. - json_name: (basestring) JSON name to be used on the wire. + python_name: (string) Python name for this value. + json_name: (string) JSON name to be used on the wire. """ if not issubclass(enum_type, messages.Enum): raise exceptions.TypecheckError( @@ -540,8 +540,8 @@ def AddCustomJsonFieldMapping(message_type, python_name, json_name): Args: message_type: (messages.Message) A message type - python_name: (basestring) Python name for this value. - json_name: (basestring) JSON name to be used on the wire. + python_name: (string) Python name for this value. + json_name: (string) JSON name to be used on the wire. """ if not issubclass(message_type, messages.Message): raise exceptions.TypecheckError( diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index ebe80a9..1df1d02 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -131,7 +131,7 @@ class Request(object): else: self.headers.pop('content-length', None) # This line ensures we don't try to print large requests. - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): self.loggable_body = '' -- GitLab From 0ad6a8248841e4c5f496793225a332d2cc4a8b14 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 15:45:07 -0700 Subject: [PATCH 077/295] Updating string/bytes I/O so tests pass in Python3. --- apitools/base/py/buffered_stream_test.py | 4 ++-- apitools/base/py/credentials_lib_test.py | 8 ++++---- apitools/base/py/stream_slice_test.py | 4 ++-- apitools/base/py/transfer.py | 2 +- apitools/base/py/transfer_test.py | 6 +++--- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py index 0ee69f6..89c9af7 100644 --- a/apitools/base/py/buffered_stream_test.py +++ b/apitools/base/py/buffered_stream_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """Tests for stream_slice.""" -import io +import six import string import unittest2 @@ -13,7 +13,7 @@ from apitools.base.py import exceptions class BufferedStreamTest(unittest2.TestCase): def setUp(self): - self.stream = io.BytesIO(string.ascii_letters) + self.stream = six.StringIO(string.ascii_letters) self.value = self.stream.getvalue() self.stream.seek(0) diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 810f4ce..313ac1a 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -import io import re +import six import mock from six.moves import http_client @@ -38,12 +38,12 @@ class CredentialsLibTest(unittest2.TestCase): def MockMetadataCalls(request): request_url = request.get_full_url() if request_url.endswith('scopes'): - return io.BytesIO(''.join(scopes)) + return six.StringIO(''.join(scopes)) elif request_url.endswith('service-accounts'): - return io.BytesIO(service_account_name) + return six.StringIO(service_account_name) elif request_url.endswith( '/service-accounts/%s/token' % service_account_name): - return io.BytesIO('{"access_token": "token"}') + return six.StringIO('{"access_token": "token"}') self.fail('Unexpected HTTP request to %s' % request_url) with mock.patch.object(credentials_lib, '_OpenNoProxy', diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py index 31ba5c8..e544952 100644 --- a/apitools/base/py/stream_slice_test.py +++ b/apitools/base/py/stream_slice_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """Tests for stream_slice.""" -import io +import six import string import unittest2 @@ -13,7 +13,7 @@ from apitools.base.py import stream_slice class StreamSliceTest(unittest2.TestCase): def setUp(self): - self.stream = io.BytesIO(string.ascii_letters) + self.stream = six.StringIO(string.ascii_letters) self.value = self.stream.getvalue() self.stream.seek(0) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 6ba40e4..ca862b8 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -631,7 +631,7 @@ class Upload(_Transfer): # encode the body: note that we can't use `as_string`, because # it plays games with `From ` lines. - fp = io.BytesIO() + fp = six.StringIO() # email: cStringIO in Py2 / io.StringIO in Py3. g = email_generator.Generator(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 594f823..2926593 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -import io +import six import unittest2 @@ -36,7 +36,7 @@ class TransferTest(unittest2.TestCase): # Test multipart: having a body argument in http_request forces # multipart here. upload = transfer.Upload.FromStream( - io.BytesIO(upload_contents), + six.StringIO(upload_contents), 'text/plain', total_size=len(upload_contents)) http_request = http_wrapper.Request( @@ -52,7 +52,7 @@ class TransferTest(unittest2.TestCase): # Test non-multipart (aka media): no body argument means this is # sent as media. upload = transfer.Upload.FromStream( - io.BytesIO(upload_contents), + six.StringIO(upload_contents), 'text/plain', total_size=len(upload_contents)) http_request = http_wrapper.Request( -- GitLab From 1ec0080e9fd20394640b5dff23ea631d215671b6 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 16:08:51 -0700 Subject: [PATCH 078/295] Removing [testing] optional tests from Python 3. --- tox.ini | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tox.ini b/tox.ini index 2bd03ef..bbe6411 100644 --- a/tox.ini +++ b/tox.ini @@ -6,3 +6,19 @@ deps = nose commands = pip install google-apitools[testing] nosetests + +[testenv:py33] +basepython = python3.3 +deps = + mock + nose + unittest2 +commands = nosetests + +[testenv:py34] +basepython = python3.4 +deps = + mock + nose + unittest2 +commands = nosetests -- GitLab From 97dc3c3fc4bf6fc387cd7f42b77a1025d844fee1 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 16:36:16 -0700 Subject: [PATCH 079/295] Python 3 fixes and comment clean-up. - Ditching xrange (gone in Py3) - Removing docstring from a unittest. For nosetests reporting, unittest cases should not have docstrings. - Clarifying comment about use of six.StringIO with email library. --- apitools/base/py/transfer.py | 8 +++++--- apitools/base/py/transfer_test.py | 13 ++++++------- samples/storage_sample/uploads_test.py | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index ca862b8..a152945 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -629,9 +629,11 @@ class Upload(_Transfer): msg.set_payload(self.stream.read()) msg_root.attach(msg) - # encode the body: note that we can't use `as_string`, because - # it plays games with `From ` lines. - fp = six.StringIO() # email: cStringIO in Py2 / io.StringIO in Py3. + # NOTE: We encode the body, but can't use `email.message.Message.as_string` + # because it prepends `> ` to `From ` lines. + # NOTE: We must use six.StringIO() instead of io.StringIO() since the + # `email` library uses cStringIO in Py2 and io.StringIO in Py3. + fp = six.StringIO() g = email_generator.Generator(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 2926593..1e12de0 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -13,14 +13,13 @@ from apitools.base.py import transfer class TransferTest(unittest2.TestCase): def testFromEncoding(self): - """Test a specific corner case in multipart encoding. + # Test a specific corner case in multipart encoding. - Python's mime module by default encodes lines that start with - "From " as ">From ", which we need to make sure we don't run afoul - of when sending content that isn't intended to be so encoded. This - test calls out that we get this right. We test for both the - multipart and non-multipart case. - """ + # Python's mime module by default encodes lines that start with + # "From " as ">From ", which we need to make sure we don't run afoul + # of when sending content that isn't intended to be so encoded. This + # test calls out that we get this right. We test for both the + # multipart and non-multipart case. multipart_body = '{"body_field_one": 7}' upload_contents = 'line one\nFrom \nline two' upload_config = base_api.ApiUploadInfo( diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index 63f4f69..9c9dcf8 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -37,7 +37,7 @@ class UploadsTest(unittest.TestCase): def __ResetUpload(self, size, auto_transfer=True): self.__content = ''.join( - random.choice(string.ascii_letters) for _ in xrange(size)) + random.choice(string.ascii_letters) for _ in range(size)) self.__buffer = io.StringIO(self.__content) self.__upload = storage.Upload.FromStream( self.__buffer, 'text/plain', auto_transfer=auto_transfer) -- GitLab From 9a0e3ca2d2db3590387f4c4a120128a0a8dbba66 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 24 Mar 2015 18:15:51 -0700 Subject: [PATCH 080/295] Adding tox test coverage rule. - Adding badge in README - Adding after_success rule to run test coverage in Travis --- .coveragerc | 8 ++++++++ .gitignore | 4 ++++ .travis.yml | 2 ++ README.rst | 4 +++- tox.ini | 22 ++++++++++++++++++++++ 5 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..6339b4e --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[report] +omit = + */demo/* +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ diff --git a/.gitignore b/.gitignore index c044de5..2254491 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,9 @@ build/ dist/ distribute-* + # Test files .tox/ +.coverage +coverage.xml +nosetests.xml diff --git a/.travis.yml b/.travis.yml index 36a15ac..f9d3d31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,3 +7,5 @@ install: - pip install tox - pip install . --allow-external argparse script: tox -e $TOX_ENV +after_success: + - tox -e coveralls diff --git a/README.rst b/README.rst index 15e7e4f..3c8e901 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ google-apitools =============== -|pypi| |build| +|pypi| |build| |coverage| ``google-apitools`` is a collection of utilities to make it easier to build client-side tools, especially those that talk to Google APIs. @@ -43,3 +43,5 @@ Then run the tests:: :target: https://travis-ci.org/google/apitools .. |pypi| image:: https://img.shields.io/pypi/v/google-apitools.svg :target: https://pypi.python.org/pypi/google-apitools +.. |coverage| image:: https://coveralls.io/repos/google/apitools/badge.png?branch=master + :target: https://coveralls.io/r/google/apitools?branch=master diff --git a/tox.ini b/tox.ini index bbe6411..0001af4 100644 --- a/tox.ini +++ b/tox.ini @@ -22,3 +22,25 @@ deps = nose unittest2 commands = nosetests + +[testenv:cover] +basepython = + python2.7 +commands = + nosetests --with-xunit --with-xcoverage --cover-package=apitools --nocapture --cover-erase --cover-tests --cover-branches +deps = + google-apputils + mock + nose + unittest2 + coverage + nosexcover + +[testenv:coveralls] +basepython = {[testenv:cover]basepython} +commands = + {[testenv:cover]commands} + coveralls +deps = + {[testenv:cover]deps} + coveralls -- GitLab From 660d8734d6e2f1e72276baf1e014f97fad09b39b Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 26 Mar 2015 23:45:34 -0700 Subject: [PATCH 081/295] Special encode 'pp' query parameter. --- apitools/base/py/base_api.py | 4 ++++ apitools/base/py/base_api_test.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 1954b8b..15e42ac 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -437,6 +437,10 @@ class BaseApiService(object): # as 0 if False, and ignored otherwise (True is the default). if not query_info.pop('prettyPrint', True): query_info['prettyPrint'] = 0 + # The One Platform equivalent of prettyPrint is pp, which also needs + # custom encoding. + if not query_info.pop('pp', True): + query_info['pp'] = 0 return query_info def __ConstructQueryParams(self, query_params, request, global_params): diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 6a663c7..f2c438c 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -42,6 +42,7 @@ encoding.AddCustomJsonEnumMapping( class StandardQueryParameters(messages.Message): field = messages.StringField(1) prettyPrint = messages.BooleanField(5, default=True) # pylint: disable=invalid-name + pp = messages.BooleanField(6, default=True) class FakeCredentials(object): @@ -137,11 +138,15 @@ class BaseApiTest(unittest2.TestCase): http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) self.assertFalse('prettyPrint' in http_request.url) + self.assertFalse('pp' in http_request.url) global_params.prettyPrint = False # pylint: disable=invalid-name + global_params.pp = False + http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) self.assertTrue('prettyPrint=0' in http_request.url) + self.assertTrue('pp=0' in http_request.url) def testQueryRemapping(self): method_config = base_api.ApiMethodInfo( -- GitLab From 8c08107904235ab54c767ddd5f508419d6a61573 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 26 Mar 2015 23:47:48 -0700 Subject: [PATCH 082/295] Fix for client generation and orderedParams. Forces client generation to not trust orderedParams, and actually check for the required attribute in the parameter description. --- apitools/gen/service_registry.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 7aca0fe..63185c8 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -332,6 +332,10 @@ class ServiceRegistry(object): relative_path = self.__names.NormalizeRelativePath( ''.join((self.__base_path, method_description['path']))) method_id = method_description['id'] + ordered_params = [] + for param_name in method_description.get('parameterOrder', []): + if method_description['parameters'][param_name].get('required', False): + ordered_params.append(param_name) method_info = base_api.ApiMethodInfo( relative_path=relative_path, method_id=method_id, @@ -340,7 +344,7 @@ class ServiceRegistry(object): method_description.get('description', '')), query_params=[], path_params=[], - ordered_params=method_description.get('parameterOrder', []), + ordered_params=ordered_params, request_type_name=self.__names.ClassName(request), response_type_name=self.__names.ClassName(response), request_field=request_field, -- GitLab From 45321b23d6d02270f79ea4559d30bc08318f480c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 26 Mar 2015 23:52:25 -0700 Subject: [PATCH 083/295] Wire in a flag for service accounts in generated CLIs. --- apitools/gen/command_registry.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index bb3f75a..4e8cf2e 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -293,6 +293,11 @@ class CommandRegistry(object): printer("'add_header', [],") printer("'Additional http headers (as key=value strings). Can be '") printer("'specified multiple times.')") + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'service_account_json_keyfile', '',") + printer("'Filename for a JSON service account key downloaded from '") + printer("'the Developer Console.')") for flag_info in self.__global_flags: self.__PrintFlag(printer, flag_info) printer() @@ -328,12 +333,18 @@ class CommandRegistry(object): 'FLAGS.api_endpoint)') printer("additional_http_headers = dict(x.split('=', 1) for x in " "FLAGS.add_header)") + printer('credentials_args = {') + with printer.Indent(' '): + printer("'service_account_json_keyfile': os.path.expanduser(" + 'FLAGS.service_account_json_keyfile)') + printer('}') printer('try:') with printer.Indent(): printer('client = client_lib.%s(', self.__client_info.client_class_name) with printer.Indent(indent=' '): printer('api_endpoint, log_request=log_request,') printer('log_response=log_response,') + printer('credentials_args=credentials_args,') printer('additional_http_headers=additional_http_headers)') printer('except apitools_base.CredentialsError as e:') with printer.Indent(): @@ -431,6 +442,7 @@ class CommandRegistry(object): # information. printer() printer('import code') + printer('import os') printer('import platform') printer('import sys') printer() -- GitLab From 9853b2c74695055f9cc73a34bbb23d0b249052bc Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 26 Mar 2015 23:55:55 -0700 Subject: [PATCH 084/295] Better error messages around service accounts. --- apitools/base/py/credentials_lib.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 5307203..4ecae2f 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -48,6 +48,10 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, api_key=None, client=None): """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) + if ((service_account_name and not service_account_keyfile) or + (service_account_keyfile and not service_account_name)): + raise exceptions.CredentialsError( + 'Service account name or keyfile provided without the other') # TODO(craigcitro): Error checking. client_info = { 'client_id': client_id, @@ -55,7 +59,7 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), 'user_agent': user_agent or '%s-generated/0.1' % package_name, } - if service_account_name is not None: + if service_account_name: credentials = ServiceAccountCredentialsFromFile( service_account_name, service_account_keyfile, scopes) if credentials is not None: -- GitLab From e73706cee4a5e5bdd97ab640bc730067e49ccd39 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 27 Mar 2015 00:04:30 -0700 Subject: [PATCH 085/295] Update apitools to handle JSON service accounts. This adds support for the new JSON format for service accounts (which removes the need for a second argument when specifying a service account). --- apitools/base/py/credentials_lib.py | 37 ++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 4ecae2f..9ad9677 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -12,6 +12,7 @@ import oauth2client.client import oauth2client.gce import oauth2client.locked_file import oauth2client.multistore_file +import oauth2client.service_account import oauth2client.tools # for flag declarations from six.moves import http_client from six.moves import urllib @@ -45,6 +46,7 @@ __all__ = [ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, credentials_filename=None, service_account_name=None, service_account_keyfile=None, + service_account_json_keyfile=None, api_key=None, client=None): """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) @@ -59,9 +61,28 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), 'user_agent': user_agent or '%s-generated/0.1' % package_name, } - if service_account_name: + service_account_kwargs = { + 'user_agent': client_info['user_agent'], + } + if service_account_json_keyfile: + with open(service_account_json_keyfile) as keyfile: + service_account_info = json.load(keyfile) + if service_account_info.get('type') != oauth2client.client.SERVICE_ACCOUNT: + raise exceptions.CredentialsError( + 'Invalid service account credentials: %s' % ( + service_account_json_keyfile,)) + credentials = oauth2client.service_account._ServiceAccountCredentials( # pylint: disable=protected-access + service_account_id=service_account_info['client_id'], + service_account_email=service_account_info['client_email'], + private_key_id=service_account_info['private_key_id'], + private_key_pkcs8_text=service_account_info['private_key'], + scopes=scopes, + **service_account_kwargs) + return credentials + if service_account_name is not None: credentials = ServiceAccountCredentialsFromFile( - service_account_name, service_account_keyfile, scopes) + service_account_name, service_account_keyfile, scopes, + service_account_kwargs=service_account_kwargs) if credentials is not None: return credentials credentials = GaeAssertionCredentials.Get(scopes) @@ -79,16 +100,20 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, def ServiceAccountCredentialsFromFile( - service_account_name, private_key_filename, scopes): + service_account_name, private_key_filename, scopes, + service_account_kwargs=None): with open(private_key_filename) as key_file: return ServiceAccountCredentials( - service_account_name, key_file.read(), scopes) + service_account_name, key_file.read(), scopes, + service_account_kwargs=service_account_kwargs) -def ServiceAccountCredentials(service_account_name, private_key, scopes): +def ServiceAccountCredentials(service_account_name, private_key, scopes, + service_account_kwargs=None): + service_account_kwargs = service_account_kwargs or {} scopes = util.NormalizeScopes(scopes) return oauth2client.client.SignedJwtAssertionCredentials( - service_account_name, private_key, scopes) + service_account_name, private_key, scopes, **service_account_kwargs) def _EnsureFileExists(filename): -- GitLab From c431900c80ec2ffc41809a3a8fb5603a3854291d Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 27 Mar 2015 00:09:27 -0700 Subject: [PATCH 086/295] Update version for release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3af84ba..f12ef44 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4' +_APITOOLS_VERSION = '0.4.1' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From b3947f2ff6eafe22f04bfe026a0d3af0570c4264 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 31 Mar 2015 13:09:30 -0700 Subject: [PATCH 087/295] Fixing non-deterministic dict. failure in extra_types_test. Failure occurs when comparing encoded strings via json.dumps and apitools.base.py.encoding.MessageToJson. The MessageToJson function has non-deterministic results, as does json.dumps. --- apitools/base/py/extra_types_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 457c606..13cd439 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -125,12 +125,12 @@ class ExtraTypesTest(unittest2.TestCase): datetime.date(1980, 10, 24), datetime.date(1981, 1, 19), ]) - json_msg = json.dumps({ - 'start_date': '1752-09-09', 'all_dates': [ - '1979-05-06', '1980-10-24', '1981-01-19', - ]}) - self.assertEqual(json_msg, encoding.MessageToJson(msg)) - self.assertEqual(msg, encoding.JsonToMessage(DateMsg, json_msg)) + msg_dict = { + 'start_date': '1752-09-09', + 'all_dates': ['1979-05-06', '1980-10-24', '1981-01-19'], + } + self.assertEqual(msg_dict, json.loads(encoding.MessageToJson(msg))) + self.assertEqual(msg, encoding.JsonToMessage(DateMsg, json.dumps(msg_dict))) def testInt64(self): # Testing roundtrip of type 'long' -- GitLab From 11bd9cceba275301d953d95acac96b0ca79061c6 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Wed, 8 Apr 2015 10:12:48 -0700 Subject: [PATCH 088/295] Upgrading oauth2client.tools.run to run_flow. Allowing fallback to FLAGS within this library without having to rely on gflags within `oauth2client`. --- apitools/base/py/credentials_lib.py | 23 ++++++++++++++--- apitools/base/py/credentials_lib_test.py | 33 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 9ad9677..e69e84a 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -2,6 +2,7 @@ """Common credentials classes and constructors.""" from __future__ import print_function +import argparse import datetime import json import os @@ -13,7 +14,7 @@ import oauth2client.gce import oauth2client.locked_file import oauth2client.multistore_file import oauth2client.service_account -import oauth2client.tools # for flag declarations +from oauth2client import tools # for gflags declarations from six.moves import http_client from six.moves import urllib @@ -384,6 +385,21 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): self.access_token = token +def _GetRunFlowFlags(): + parser = argparse.ArgumentParser(parents=[tools.argparser]) + # Get command line argparse flags. + flags = parser.parse_args() + + # Allow `gflags` and `argparse` to be used side-by-side. + if hasattr(FLAGS, 'auth_host_name'): + flags.auth_host_name = FLAGS.auth_host_name + if hasattr(FLAGS, 'auth_host_port'): + flags.auth_host_port = FLAGS.auth_host_port + if hasattr(FLAGS, 'auth_local_webserver'): + flags.noauth_local_webserver = (not FLAGS.auth_local_webserver) + return flags + + # TODO(craigcitro): Switch this from taking a path to taking a stream. def CredentialsFromFile(path, client_info): """Read credentials from a file.""" @@ -403,9 +419,8 @@ def CredentialsFromFile(path, client_info): # retry loop, they can ^C. try: flow = oauth2client.client.OAuth2WebServerFlow(**client_info) - # We delay this import because it's rarely needed and takes a long time. - from oauth2client import tools - credentials = tools.run(flow, credential_store) + flags = _GetRunFlowFlags() + credentials = tools.run_flow(flow, credential_store, flags) break except (oauth2client.client.FlowExchangeError, SystemExit) as e: # Here SystemExit is "no credential at all", and the diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 313ac1a..f7be04a 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -63,5 +63,38 @@ class CredentialsLibTest(unittest2.TestCase): self._GetServiceCreds(service_account_name='my_service_account') +class TestGetRunFlowFlags(unittest2.TestCase): + + def setUp(self): + self._flags_actual = credentials_lib.FLAGS + + def tearDown(self): + credentials_lib.FLAGS = self._flags_actual + + def test_with_gflags(self): + HOST = object() + PORT = object() + + class MockFlags(object): + auth_host_name = HOST + auth_host_port = PORT + auth_local_webserver = False + + credentials_lib.FLAGS = MockFlags + flags = credentials_lib._GetRunFlowFlags() + self.assertEqual(flags.auth_host_name, HOST) + self.assertEqual(flags.auth_host_port, PORT) + self.assertEqual(flags.logging_level, 'ERROR') + self.assertEqual(flags.noauth_local_webserver, True) + + def test_without_gflags(self): + credentials_lib.FLAGS = None + flags = credentials_lib._GetRunFlowFlags() + self.assertEqual(flags.auth_host_name, 'localhost') + self.assertEqual(flags.auth_host_port, [8080, 8090]) + self.assertEqual(flags.logging_level, 'ERROR') + self.assertEqual(flags.noauth_local_webserver, False) + + if __name__ == '__main__': unittest2.main() -- GitLab From a6feb8a17c6d9e8735d878c1935f489604f2b953 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 8 Apr 2015 16:01:35 -0700 Subject: [PATCH 089/295] Fix a bug in download streaming. Currently apitools has a bug where downloads with (1) no total size and (2) auto_transfer off will only stream one chunk and quit. (Test added internally, but internal tests still aren't sync'd.) There's still more to be done around normalizing use of chunksize in downloads. --- apitools/base/py/transfer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index a152945..e8d4f2e 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -404,6 +404,8 @@ class Download(_Transfer): else: response = self.__GetChunk(self.progress, additional_headers=additional_headers) + if self.total_size is None: + self.__SetTotal(response.info) response = self.__ProcessResponse(response) self._ExecuteCallback(callback, response) if (response.status_code == http_client.OK or -- GitLab From 25aeb184e6b80cc6fa6117d540d0b9ce1d917fa0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 9 Apr 2015 00:26:18 -0700 Subject: [PATCH 090/295] Update version for 0.4.2 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f12ef44..6861969 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.1' +_APITOOLS_VERSION = '0.4.2' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 4a14d2e73dd4e1933de5c876a9bf2f4fad5487cf Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 10 Apr 2015 00:36:17 -0700 Subject: [PATCH 091/295] Rename LICENSE.txt to LICENSE (for easier internal mirroring). --- LICENSE.txt => LICENSE | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.txt => LICENSE (100%) diff --git a/LICENSE.txt b/LICENSE similarity index 100% rename from LICENSE.txt rename to LICENSE -- GitLab From f92dfb0b597a7acb4b5caaadc41feef5fcb89600 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 15 Apr 2015 16:28:02 -0700 Subject: [PATCH 092/295] Update version for 0.4.3 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6861969..d9ba756 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.2' +_APITOOLS_VERSION = '0.4.3' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 4569fa53aae7ba5539b54a549c9704cb417e926e Mon Sep 17 00:00:00 2001 From: Michal Witkowski Date: Thu, 16 Apr 2015 17:44:05 +0100 Subject: [PATCH 093/295] Dont check end of stream for non-seekable --- apitools/base/py/transfer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index e8d4f2e..83db282 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -766,9 +766,7 @@ class Upload(_Transfer): 'Failed to transfer all bytes in chunk, upload paused at byte ' '%d' % self.progress) self._ExecuteCallback(callback, response) - if self.__complete: - # TODO(craigcitro): Decide how to handle errors in the - # non-seekable case. + if self.__complete and hasattr(self.stream, 'seek'): current_pos = self.stream.tell() self.stream.seek(0, os.SEEK_END) end_pos = self.stream.tell() -- GitLab From b61e7ff921391cce4c3e1510eafacc05c82f9729 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 21 Apr 2015 13:34:35 -0700 Subject: [PATCH 094/295] Catch a handful of httplib exceptions and retry. --- apitools/base/py/http_wrapper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 4c34a2b..8267d0c 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -240,9 +240,11 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): retry_after = None # Transport failures - if isinstance(retry_args.exc, http_client.BadStatusLine): - logging.debug('Caught BadStatusLine from httplib, retrying: %s', - retry_args.exc) + if isinstance(retry_args.exc, (http_client.BadStatusLine, + http_client.IncompleteRead, + http_client.ResponseNotReady)): + logging.debug('Caught HTTP error %s, retrying: %s', + type(retry_args.exc).__name__, retry_args.exc) elif isinstance(retry_args.exc, socket.error): logging.debug('Caught socket error, retrying: %s', retry_args.exc) elif isinstance(retry_args.exc, socket.gaierror): -- GitLab From 17da00a2bf7eb965df3065bb1195df4b5a67b538 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 29 Apr 2015 12:15:10 -0700 Subject: [PATCH 095/295] Update a typo in a flag description. Fixes #20. --- apitools/gen/gen_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 95e947e..0473c9f 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -21,7 +21,7 @@ flags.DEFINE_string( '--discovery_url.') flags.DEFINE_string( 'discovery_url', '', - 'URL (or "name/version") of the discovery document to use. ' + 'URL (or "name.version") of the discovery document to use. ' 'Mutually exclusive with --infile.') flags.DEFINE_string( -- GitLab From 64992dacc5194534ad3f263bee1300c5772f3cbc Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 30 Apr 2015 01:21:12 -0700 Subject: [PATCH 096/295] Change callback defaults for transfers. This makes several changes to the way we use callbacks in media transfers: * allow callbacks to be attached to a transfer, not just be provided as an argument to streaming functions, * switch the defaults to "don't print", * expose the "print progress" defaults as public functions instead of protected static methods, and * teach the generated CLIs to print by default. --- apitools/base/py/transfer.py | 65 ++++++++++++++++++++------------ apitools/gen/command_registry.py | 12 +++++- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 83db282..dc983b3 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -25,6 +25,10 @@ __all__ = [ 'Upload', 'RESUMABLE_UPLOAD', 'SIMPLE_UPLOAD', + 'DownloadProgressPrinter', + 'DownloadCompletePrinter', + 'UploadProgressPrinter', + 'UploadCompletePrinter', ] _RESUMABLE_UPLOAD_THRESHOLD = 5 << 20 @@ -32,6 +36,29 @@ SIMPLE_UPLOAD = 'simple' RESUMABLE_UPLOAD = 'resumable' +def DownloadProgressPrinter(response, download): + """Print download progress based on response.""" + if 'content-range' in response.info: + print('Received %s' % response.info['content-range']) + else: + print('Received %d bytes' % response.length) + + +def DownloadCompletePrinter(response, download): + """Print information about a completed download.""" + print('Download complete') + + +def UploadProgressPrinter(response, upload): + """Print upload progress based on response.""" + print('Sent %s' % response.info['range']) + + +def UploadCompletePrinter(response, upload): + """Print information about a completed upload.""" + print('Upload complete') + + class _Transfer(object): """Generic bits common to Uploads and Downloads.""" @@ -152,14 +179,18 @@ class Download(_Transfer): _REQUIRED_SERIALIZATION_KEYS = set(( 'auto_transfer', 'progress', 'total_size', 'url')) - def __init__(self, *args, **kwds): + def __init__(self, stream, progress_callback=None, finish_callback=None, + **kwds): total_size = kwds.pop('total_size', None) - super(Download, self).__init__(*args, **kwds) + super(Download, self).__init__(stream, **kwds) self.__initial_response = None self.__progress = 0 self.__total_size = total_size self.__encoding = None + self.progress_callback = progress_callback + self.finish_callback = finish_callback + @property def progress(self): return self.__progress @@ -276,17 +307,6 @@ class Download(_Transfer): if self.auto_transfer: self.StreamInChunks() - @staticmethod - def _ArgPrinter(response, unused_download): - if 'content-range' in response.info: - print('Received %s' % response.info['content-range']) - else: - print('Received %d bytes' % response.length) - - @staticmethod - def _CompletePrinter(*unused_args): - print('Download complete') - def __NormalizeStartEnd(self, start, end=None): if end is not None: if start < 0: @@ -393,8 +413,8 @@ class Download(_Transfer): def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=None): """Stream the entire download.""" - callback = callback or self._ArgPrinter - finish_callback = finish_callback or self._CompletePrinter + callback = callback or self.progress_callback + finish_callback = finish_callback or self.finish_callback self.EnsureInitialized() while True: @@ -431,6 +451,7 @@ class Upload(_Transfer): def __init__(self, stream, mime_type, total_size=None, http=None, close_stream=False, chunksize=None, auto_transfer=True, + progress_callback=None, finish_callback=None, **kwds): super(Upload, self).__init__( stream, close_stream=close_stream, chunksize=chunksize, @@ -442,6 +463,8 @@ class Upload(_Transfer): self.__server_chunk_granularity = None self.__strategy = None + self.progress_callback = progress_callback + self.finish_callback = finish_callback self.total_size = total_size @property @@ -731,22 +754,14 @@ class Upload(_Transfer): 'Server requires chunksize to be a multiple of %d', self.__server_chunk_granularity) - @staticmethod - def _ArgPrinter(response, unused_upload): - print('Sent %s' % response.info['range']) - - @staticmethod - def _CompletePrinter(*unused_args): - print('Upload complete') - def __StreamMedia(self, callback=None, finish_callback=None, additional_headers=None, use_chunks=True): """Helper function for StreamMedia / StreamInChunks.""" if self.strategy != RESUMABLE_UPLOAD: raise exceptions.InvalidUserInputError( 'Cannot stream non-resumable upload') - callback = callback or self._ArgPrinter - finish_callback = finish_callback or self._CompletePrinter + callback = callback or self.progress_callback + finish_callback = finish_callback or self.finish_callback # final_response is set if we resumed an already-completed upload. response = self.__final_response send_func = self.__SendChunk if use_chunks else self.__SendMediaBody diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 4e8cf2e..93c5d53 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -541,14 +541,22 @@ class CommandRegistry(object): printer('if FLAGS.upload_filename:') with printer.Indent(): printer('upload = apitools_base.Upload.FromFile(') - printer(' FLAGS.upload_filename, FLAGS.upload_mime_type)') + printer(' FLAGS.upload_filename, FLAGS.upload_mime_type,') + printer(' progress_callback=' + 'apitools_base.UploadProgressPrinter,') + printer(' finish_callback=' + 'apitools_base.UploadCompletePrinter)') if command_info.has_download: call_args.append('download=download') printer('download = None') printer('if FLAGS.download_filename:') with printer.Indent(): printer('download = apitools_base.Download.FromFile(' - 'FLAGS.download_filename, overwrite=FLAGS.overwrite)') + 'FLAGS.download_filename, overwrite=FLAGS.overwrite,') + printer(' progress_callback=' + 'apitools_base.DownloadProgressPrinter,') + printer(' finish_callback=' + 'apitools_base.DownloadCompletePrinter)') printer('result = client.%s(', command_info.client_method_path) with printer.Indent(indent=' '): printer('%s)', ', '.join(call_args)) -- GitLab From 4d8a287639a6921d18b58ddf53599c87077ce059 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 30 Apr 2015 11:24:14 -0700 Subject: [PATCH 097/295] Update whitespace to pep8 via autopep8. This change just runs `autopep8 -i` over all non-generated python code, via `autopep8 -i $(find apitools -name '*py')`. --- apitools/base/py/__init__.py | 1 - apitools/base/py/app2.py | 619 ++++---- apitools/base/py/base_api.py | 1115 +++++++------- apitools/base/py/base_api_test.py | 262 ++-- apitools/base/py/base_cli.py | 171 +-- apitools/base/py/batch.py | 783 +++++----- apitools/base/py/buffered_stream.py | 94 +- apitools/base/py/buffered_stream_test.py | 80 +- apitools/base/py/credentials_lib.py | 752 +++++----- apitools/base/py/credentials_lib_test.py | 139 +- apitools/base/py/encoding.py | 952 ++++++------ apitools/base/py/encoding_test.py | 551 +++---- apitools/base/py/exceptions.py | 119 +- apitools/base/py/extra_types.py | 288 ++-- apitools/base/py/extra_types_test.py | 313 ++-- apitools/base/py/http_wrapper.py | 555 +++---- apitools/base/py/http_wrapper_test.py | 21 +- apitools/base/py/list_pager.py | 84 +- apitools/base/py/stream_slice.py | 109 +- apitools/base/py/stream_slice_test.py | 76 +- apitools/base/py/transfer.py | 1700 +++++++++++----------- apitools/base/py/transfer_test.py | 94 +- apitools/base/py/util.py | 302 ++-- apitools/base/py/util_test.py | 322 ++-- apitools/gen/client_generation_test.py | 93 +- apitools/gen/command_registry.py | 1047 ++++++------- apitools/gen/extended_descriptor.py | 841 +++++------ apitools/gen/gen_client.py | 240 +-- apitools/gen/gen_client_lib.py | 301 ++-- apitools/gen/message_registry.py | 845 +++++------ apitools/gen/service_registry.py | 842 +++++------ apitools/gen/util.py | 503 +++---- samples/storage_sample/downloads_test.py | 311 ++-- samples/storage_sample/uploads_test.py | 243 ++-- 34 files changed, 7480 insertions(+), 7288 deletions(-) diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py index cbf7f86..53b34d0 100644 --- a/apitools/base/py/__init__.py +++ b/apitools/base/py/__init__.py @@ -12,4 +12,3 @@ from apitools.base.py.http_wrapper import * from apitools.base.py.list_pager import * from apitools.base.py.transfer import * from apitools.base.py.util import * - diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index cff5437..629a476 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -31,321 +31,326 @@ FLAGS = flags.FLAGS def _SafeMakeAscii(s): - if isinstance(s, six.text_type): - return s.encode('ascii') - elif isinstance(s, str): - return s.decode('ascii') - else: - return six.text_type(s).encode('ascii', 'backslashreplace') + if isinstance(s, six.text_type): + return s.encode('ascii') + elif isinstance(s, str): + return s.decode('ascii') + else: + return six.text_type(s).encode('ascii', 'backslashreplace') class NewCmd(appcommands.Cmd): - """Featureful extension of appcommands.Cmd.""" - - def __init__(self, name, flag_values): - super(NewCmd, self).__init__(name, flag_values) - run_with_args = getattr(self, 'RunWithArgs', None) - self._new_style = isinstance(run_with_args, types.MethodType) - if self._new_style: - func = run_with_args.__func__ - - argspec = inspect.getargspec(func) - if argspec.args and argspec.args[0] == 'self': - argspec = argspec._replace( # pylint: disable=protected-access - args=argspec.args[1:]) - self._argspec = argspec - # TODO(craigcitro): Do we really want to support all this - # nonsense? - self._star_args = self._argspec.varargs is not None - self._star_kwds = self._argspec.keywords is not None - self._max_args = len(self._argspec.args or ()) - self._min_args = self._max_args - len(self._argspec.defaults or ()) - if self._star_args: - self._max_args = sys.maxint - - self._debug_mode = FLAGS.debug_mode - self.surface_in_shell = True - self.__doc__ = self.RunWithArgs.__doc__ - - def __getattr__(self, name): - if name in self._command_flags: - return self._command_flags[name].value - return super(NewCmd, self).__getattribute__(name) - - def _GetFlag(self, flagname): - if flagname in self._command_flags: - return self._command_flags[flagname] - else: - return None - - def Run(self, argv): - """Run this command. - - If self is a new-style command, we set up arguments and call - self.RunWithArgs, gracefully handling exceptions. If not, we - simply call self.Run(argv). - - Args: - argv: List of arguments as strings. - - Returns: - 0 on success, nonzero on failure. - """ - if not self._new_style: - return super(NewCmd, self).Run(argv) - - # TODO(craigcitro): We need to save and restore flags each time so - # that we can per-command flags in the REPL. - args = argv[1:] - fail = None - if len(args) < self._min_args: - fail = 'Not enough positional args; found %d, expected at least %d' % ( - len(args), self._min_args) - if len(args) > self._max_args: - fail = 'Too many positional args; found %d, expected at most %d' % ( - len(args), self._max_args) - if fail: - print(fail) - if self.usage: - print('Usage: %s' % (self.usage,)) - return 1 - - if self._debug_mode: - return self.RunDebug(args, {}) - else: - return self.RunSafely(args, {}) - - def RunCmdLoop(self, argv): - """Hook for use in cmd.Cmd-based command shells.""" - try: - args = shlex.split(argv) - except ValueError as e: - raise SyntaxError(self.EncodeForPrinting(e)) - return self.Run([self._command_name] + args) - - @staticmethod - def EncodeForPrinting(s): - """Safely encode a string as the encoding for sys.stdout.""" - encoding = sys.stdout.encoding or 'ascii' - return six.text_type(s).encode(encoding, 'backslashreplace') - - def _FormatError(self, e): - """Hook for subclasses to modify how error messages are printed.""" - return _SafeMakeAscii(e) - - def _HandleError(self, e): - message = self._FormatError(e) - print('Exception raised in %s operation: %s' % ( - self._command_name, message)) - return 1 - - def _IsDebuggableException(self, e): - """Hook for subclasses to skip debugging on certain exceptions.""" - return not isinstance(e, app.UsageError) - - def RunDebug(self, args, kwds): - """Run this command in debug mode.""" - try: - return_value = self.RunWithArgs(*args, **kwds) - except BaseException as e: - # Don't break into the debugger for expected exceptions. - if not self._IsDebuggableException(e): - return self._HandleError(e) - print() - print('****************************************************') - print('** Unexpected Exception raised in execution! **') - if FLAGS.headless: - print('** --headless mode enabled, exiting. **') - print('** See STDERR for traceback. **') - else: - print('** --debug_mode enabled, starting pdb. **') - print('****************************************************') - print() - traceback.print_exc() - print() - if not FLAGS.headless: - pdb.post_mortem() - return 1 - return return_value - - def RunSafely(self, args, kwds): - """Run this command, turning exceptions into print statements.""" - try: - return_value = self.RunWithArgs(*args, **kwds) - except BaseException as e: - return self._HandleError(e) - return return_value + + """Featureful extension of appcommands.Cmd.""" + + def __init__(self, name, flag_values): + super(NewCmd, self).__init__(name, flag_values) + run_with_args = getattr(self, 'RunWithArgs', None) + self._new_style = isinstance(run_with_args, types.MethodType) + if self._new_style: + func = run_with_args.__func__ + + argspec = inspect.getargspec(func) + if argspec.args and argspec.args[0] == 'self': + argspec = argspec._replace( # pylint: disable=protected-access + args=argspec.args[1:]) + self._argspec = argspec + # TODO(craigcitro): Do we really want to support all this + # nonsense? + self._star_args = self._argspec.varargs is not None + self._star_kwds = self._argspec.keywords is not None + self._max_args = len(self._argspec.args or ()) + self._min_args = self._max_args - len(self._argspec.defaults or ()) + if self._star_args: + self._max_args = sys.maxint + + self._debug_mode = FLAGS.debug_mode + self.surface_in_shell = True + self.__doc__ = self.RunWithArgs.__doc__ + + def __getattr__(self, name): + if name in self._command_flags: + return self._command_flags[name].value + return super(NewCmd, self).__getattribute__(name) + + def _GetFlag(self, flagname): + if flagname in self._command_flags: + return self._command_flags[flagname] + else: + return None + + def Run(self, argv): + """Run this command. + + If self is a new-style command, we set up arguments and call + self.RunWithArgs, gracefully handling exceptions. If not, we + simply call self.Run(argv). + + Args: + argv: List of arguments as strings. + + Returns: + 0 on success, nonzero on failure. + """ + if not self._new_style: + return super(NewCmd, self).Run(argv) + + # TODO(craigcitro): We need to save and restore flags each time so + # that we can per-command flags in the REPL. + args = argv[1:] + fail = None + if len(args) < self._min_args: + fail = 'Not enough positional args; found %d, expected at least %d' % ( + len(args), self._min_args) + if len(args) > self._max_args: + fail = 'Too many positional args; found %d, expected at most %d' % ( + len(args), self._max_args) + if fail: + print(fail) + if self.usage: + print('Usage: %s' % (self.usage,)) + return 1 + + if self._debug_mode: + return self.RunDebug(args, {}) + else: + return self.RunSafely(args, {}) + + def RunCmdLoop(self, argv): + """Hook for use in cmd.Cmd-based command shells.""" + try: + args = shlex.split(argv) + except ValueError as e: + raise SyntaxError(self.EncodeForPrinting(e)) + return self.Run([self._command_name] + args) + + @staticmethod + def EncodeForPrinting(s): + """Safely encode a string as the encoding for sys.stdout.""" + encoding = sys.stdout.encoding or 'ascii' + return six.text_type(s).encode(encoding, 'backslashreplace') + + def _FormatError(self, e): + """Hook for subclasses to modify how error messages are printed.""" + return _SafeMakeAscii(e) + + def _HandleError(self, e): + message = self._FormatError(e) + print('Exception raised in %s operation: %s' % ( + self._command_name, message)) + return 1 + + def _IsDebuggableException(self, e): + """Hook for subclasses to skip debugging on certain exceptions.""" + return not isinstance(e, app.UsageError) + + def RunDebug(self, args, kwds): + """Run this command in debug mode.""" + try: + return_value = self.RunWithArgs(*args, **kwds) + except BaseException as e: + # Don't break into the debugger for expected exceptions. + if not self._IsDebuggableException(e): + return self._HandleError(e) + print() + print('****************************************************') + print('** Unexpected Exception raised in execution! **') + if FLAGS.headless: + print('** --headless mode enabled, exiting. **') + print('** See STDERR for traceback. **') + else: + print('** --debug_mode enabled, starting pdb. **') + print('****************************************************') + print() + traceback.print_exc() + print() + if not FLAGS.headless: + pdb.post_mortem() + return 1 + return return_value + + def RunSafely(self, args, kwds): + """Run this command, turning exceptions into print statements.""" + try: + return_value = self.RunWithArgs(*args, **kwds) + except BaseException as e: + return self._HandleError(e) + return return_value class CommandLoop(cmd.Cmd): - """Instance of cmd.Cmd built to work with NewCmd.""" - - class TerminateSignal(Exception): - """Exception type used for signaling loop completion.""" - - def __init__(self, commands, prompt): - cmd.Cmd.__init__(self) - self._commands = {'help': commands['help']} - self._special_command_names = ['help', 'repl', 'EOF'] - for name, command in commands.items(): - if (name not in self._special_command_names and - isinstance(command, NewCmd) and - command.surface_in_shell): - self._commands[name] = command - setattr(self, 'do_%s' % (name,), command.RunCmdLoop) - self._default_prompt = prompt - self._set_prompt() - self._last_return_code = 0 - - @property - def last_return_code(self): - return self._last_return_code - - def _set_prompt(self): - self.prompt = self._default_prompt - - def do_EOF(self, *unused_args): - """Terminate the running command loop. - - This function raises an exception to avoid the need to do - potentially-error-prone string parsing inside onecmd. - - Args: - *unused_args: unused. - - Returns: - Never returns. - - Raises: - CommandLoop.TerminateSignal: always. - """ - raise CommandLoop.TerminateSignal() - - def postloop(self): - print('Goodbye.') - - def completedefault(self, unused_text, line, unused_begidx, unused_endidx): - if not line: - return [] - else: - command_name = line.partition(' ')[0].lower() - usage = '' - if command_name in self._commands: - usage = self._commands[command_name].usage - if usage: - print() - print(usage) - print('%s%s' % (self.prompt, line), end=' ') - return [] - - def emptyline(self): - print('Available commands:', end=' ') - print(' '.join(list(self._commands))) - - def precmd(self, line): - """Preprocess the shell input.""" - if line == 'EOF': - return line - if line.startswith('exit') or line.startswith('quit'): - return 'EOF' - words = line.strip().split() - if len(words) == 1 and words[0] not in ['help', 'ls', 'version']: - return 'help %s' % (line.strip(),) - return line - - def onecmd(self, line): - """Process a single command. - - Runs a single command, and stores the return code in - self._last_return_code. Always returns False unless the command - was EOF. - - Args: - line: (str) Command line to process. - - Returns: - A bool signaling whether or not the command loop should terminate. - """ - try: - self._last_return_code = cmd.Cmd.onecmd(self, line) - except CommandLoop.TerminateSignal: - return True - except BaseException as e: - name = line.split(' ')[0] - print('Error running %s:' % name) - print(e) - self._last_return_code = 1 - return False - - def get_names(self): - names = dir(self) - commands = (name for name in self._commands - if name not in self._special_command_names) - names.extend('do_%s' % (name,) for name in commands) - names.remove('do_EOF') - return names - - def do_help(self, command_name): - """Print the help for command_name (if present) or general help.""" - - # TODO(craigcitro): Add command-specific flags. - def FormatOneCmd(name, command, command_names): - indent_size = appcommands.GetMaxCommandLength() + 3 - if len(command_names) > 1: - indent = ' ' * indent_size - command_help = flags.TextWrap( - command.CommandGetHelp('', cmd_names=command_names), - indent=indent, - firstline_indent='') - first_help_line, _, rest = command_help.partition('\n') - first_line = '%-*s%s' % (indent_size, name + ':', first_help_line) - return '\n'.join((first_line, rest)) - else: - default_indent = ' ' - return '\n' + flags.TextWrap( - command.CommandGetHelp('', cmd_names=command_names), - indent=default_indent, - firstline_indent=default_indent) + '\n' - - if not command_name: - print('\nHelp for commands:\n') - command_names = list(self._commands) - print('\n\n'.join( - FormatOneCmd(name, command, command_names) - for name, command in self._commands.items() - if name not in self._special_command_names)) - print() - elif command_name in self._commands: - print(FormatOneCmd(command_name, self._commands[command_name], - command_names=[command_name])) - return 0 - - def postcmd(self, stop, line): - return bool(stop) or line == 'EOF' + + """Instance of cmd.Cmd built to work with NewCmd.""" + + class TerminateSignal(Exception): + + """Exception type used for signaling loop completion.""" + + def __init__(self, commands, prompt): + cmd.Cmd.__init__(self) + self._commands = {'help': commands['help']} + self._special_command_names = ['help', 'repl', 'EOF'] + for name, command in commands.items(): + if (name not in self._special_command_names and + isinstance(command, NewCmd) and + command.surface_in_shell): + self._commands[name] = command + setattr(self, 'do_%s' % (name,), command.RunCmdLoop) + self._default_prompt = prompt + self._set_prompt() + self._last_return_code = 0 + + @property + def last_return_code(self): + return self._last_return_code + + def _set_prompt(self): + self.prompt = self._default_prompt + + def do_EOF(self, *unused_args): + """Terminate the running command loop. + + This function raises an exception to avoid the need to do + potentially-error-prone string parsing inside onecmd. + + Args: + *unused_args: unused. + + Returns: + Never returns. + + Raises: + CommandLoop.TerminateSignal: always. + """ + raise CommandLoop.TerminateSignal() + + def postloop(self): + print('Goodbye.') + + def completedefault(self, unused_text, line, unused_begidx, unused_endidx): + if not line: + return [] + else: + command_name = line.partition(' ')[0].lower() + usage = '' + if command_name in self._commands: + usage = self._commands[command_name].usage + if usage: + print() + print(usage) + print('%s%s' % (self.prompt, line), end=' ') + return [] + + def emptyline(self): + print('Available commands:', end=' ') + print(' '.join(list(self._commands))) + + def precmd(self, line): + """Preprocess the shell input.""" + if line == 'EOF': + return line + if line.startswith('exit') or line.startswith('quit'): + return 'EOF' + words = line.strip().split() + if len(words) == 1 and words[0] not in ['help', 'ls', 'version']: + return 'help %s' % (line.strip(),) + return line + + def onecmd(self, line): + """Process a single command. + + Runs a single command, and stores the return code in + self._last_return_code. Always returns False unless the command + was EOF. + + Args: + line: (str) Command line to process. + + Returns: + A bool signaling whether or not the command loop should terminate. + """ + try: + self._last_return_code = cmd.Cmd.onecmd(self, line) + except CommandLoop.TerminateSignal: + return True + except BaseException as e: + name = line.split(' ')[0] + print('Error running %s:' % name) + print(e) + self._last_return_code = 1 + return False + + def get_names(self): + names = dir(self) + commands = (name for name in self._commands + if name not in self._special_command_names) + names.extend('do_%s' % (name,) for name in commands) + names.remove('do_EOF') + return names + + def do_help(self, command_name): + """Print the help for command_name (if present) or general help.""" + + # TODO(craigcitro): Add command-specific flags. + def FormatOneCmd(name, command, command_names): + indent_size = appcommands.GetMaxCommandLength() + 3 + if len(command_names) > 1: + indent = ' ' * indent_size + command_help = flags.TextWrap( + command.CommandGetHelp('', cmd_names=command_names), + indent=indent, + firstline_indent='') + first_help_line, _, rest = command_help.partition('\n') + first_line = '%-*s%s' % (indent_size, + name + ':', first_help_line) + return '\n'.join((first_line, rest)) + else: + default_indent = ' ' + return '\n' + flags.TextWrap( + command.CommandGetHelp('', cmd_names=command_names), + indent=default_indent, + firstline_indent=default_indent) + '\n' + + if not command_name: + print('\nHelp for commands:\n') + command_names = list(self._commands) + print('\n\n'.join( + FormatOneCmd(name, command, command_names) + for name, command in self._commands.items() + if name not in self._special_command_names)) + print() + elif command_name in self._commands: + print(FormatOneCmd(command_name, self._commands[command_name], + command_names=[command_name])) + return 0 + + def postcmd(self, stop, line): + return bool(stop) or line == 'EOF' # pylint: enable=g-bad-name class Repl(NewCmd): - """Start an interactive session.""" - PROMPT = '> ' - - def __init__(self, name, fv): - super(Repl, self).__init__(name, fv) - self.surface_in_shell = False - flags.DEFINE_string( - 'prompt', '', - 'Prompt to use for interactive shell.', - flag_values=fv) - - def RunWithArgs(self): + """Start an interactive session.""" - prompt = FLAGS.prompt or self.PROMPT - repl = CommandLoop(appcommands.GetCommandList(), prompt=prompt) - print('Welcome! (Type help for more information.)') - while True: - try: - repl.cmdloop() - break - except KeyboardInterrupt: - print() - return repl.last_return_code + PROMPT = '> ' + + def __init__(self, name, fv): + super(Repl, self).__init__(name, fv) + self.surface_in_shell = False + flags.DEFINE_string( + 'prompt', '', + 'Prompt to use for interactive shell.', + flag_values=fv) + + def RunWithArgs(self): + """Start an interactive session.""" + prompt = FLAGS.prompt or self.PROMPT + repl = CommandLoop(appcommands.GetCommandList(), prompt=prompt) + print('Welcome! (Type help for more information.)') + while True: + try: + repl.cmdloop() + break + except KeyboardInterrupt: + print() + return repl.last_return_code diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 15e42ac..c6d7f9d 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -36,577 +36,588 @@ _MAX_URL_LENGTH = 2048 class ApiUploadInfo(messages.Message): - """Media upload information for a method. - - Fields: - accept: (repeated) MIME Media Ranges for acceptable media uploads - to this method. - max_size: (integer) Maximum size of a media upload, such as 3MB - or 1TB (converted to an integer). - resumable_path: Path to use for resumable uploads. - resumable_multipart: (boolean) Whether or not the resumable endpoint - supports multipart uploads. - simple_path: Path to use for simple uploads. - simple_multipart: (boolean) Whether or not the simple endpoint - supports multipart uploads. - """ - accept = messages.StringField(1, repeated=True) - max_size = messages.IntegerField(2) - resumable_path = messages.StringField(3) - resumable_multipart = messages.BooleanField(4) - simple_path = messages.StringField(5) - simple_multipart = messages.BooleanField(6) + + """Media upload information for a method. + + Fields: + accept: (repeated) MIME Media Ranges for acceptable media uploads + to this method. + max_size: (integer) Maximum size of a media upload, such as 3MB + or 1TB (converted to an integer). + resumable_path: Path to use for resumable uploads. + resumable_multipart: (boolean) Whether or not the resumable endpoint + supports multipart uploads. + simple_path: Path to use for simple uploads. + simple_multipart: (boolean) Whether or not the simple endpoint + supports multipart uploads. + """ + accept = messages.StringField(1, repeated=True) + max_size = messages.IntegerField(2) + resumable_path = messages.StringField(3) + resumable_multipart = messages.BooleanField(4) + simple_path = messages.StringField(5) + simple_multipart = messages.BooleanField(6) class ApiMethodInfo(messages.Message): - """Configuration info for an API method. - - All fields are strings unless noted otherwise. - - Fields: - relative_path: Relative path for this method. - method_id: ID for this method. - http_method: HTTP verb to use for this method. - path_params: (repeated) path parameters for this method. - query_params: (repeated) query parameters for this method. - ordered_params: (repeated) ordered list of parameters for - this method. - description: description of this method. - request_type_name: name of the request type. - response_type_name: name of the response type. - request_field: if not null, the field to pass as the body - of this POST request. may also be the REQUEST_IS_BODY - value below to indicate the whole message is the body. - upload_config: (ApiUploadInfo) Information about the upload - configuration supported by this method. - supports_download: (boolean) If True, this method supports - downloading the request via the `alt=media` query - parameter. - """ - - relative_path = messages.StringField(1) - method_id = messages.StringField(2) - http_method = messages.StringField(3) - path_params = messages.StringField(4, repeated=True) - query_params = messages.StringField(5, repeated=True) - ordered_params = messages.StringField(6, repeated=True) - description = messages.StringField(7) - request_type_name = messages.StringField(8) - response_type_name = messages.StringField(9) - request_field = messages.StringField(10, default='') - upload_config = messages.MessageField(ApiUploadInfo, 11) - supports_download = messages.BooleanField(12, default=False) + + """Configuration info for an API method. + + All fields are strings unless noted otherwise. + + Fields: + relative_path: Relative path for this method. + method_id: ID for this method. + http_method: HTTP verb to use for this method. + path_params: (repeated) path parameters for this method. + query_params: (repeated) query parameters for this method. + ordered_params: (repeated) ordered list of parameters for + this method. + description: description of this method. + request_type_name: name of the request type. + response_type_name: name of the response type. + request_field: if not null, the field to pass as the body + of this POST request. may also be the REQUEST_IS_BODY + value below to indicate the whole message is the body. + upload_config: (ApiUploadInfo) Information about the upload + configuration supported by this method. + supports_download: (boolean) If True, this method supports + downloading the request via the `alt=media` query + parameter. + """ + + relative_path = messages.StringField(1) + method_id = messages.StringField(2) + http_method = messages.StringField(3) + path_params = messages.StringField(4, repeated=True) + query_params = messages.StringField(5, repeated=True) + ordered_params = messages.StringField(6, repeated=True) + description = messages.StringField(7) + request_type_name = messages.StringField(8) + response_type_name = messages.StringField(9) + request_field = messages.StringField(10, default='') + upload_config = messages.MessageField(ApiUploadInfo, 11) + supports_download = messages.BooleanField(12, default=False) REQUEST_IS_BODY = '' def _LoadClass(name, messages_module): - if name.startswith('message_types.'): - _, _, classname = name.partition('.') - return getattr(message_types, classname) - elif '.' not in name: - return getattr(messages_module, name) - else: - raise exceptions.GeneratedClientError('Unknown class %s' % name) + if name.startswith('message_types.'): + _, _, classname = name.partition('.') + return getattr(message_types, classname) + elif '.' not in name: + return getattr(messages_module, name) + else: + raise exceptions.GeneratedClientError('Unknown class %s' % name) def _RequireClassAttrs(obj, attrs): - for attr in attrs: - attr_name = attr.upper() - if not hasattr(obj, '%s' % attr_name) or not getattr(obj, attr_name): - msg = 'No %s specified for object of class %s.' % ( - attr_name, type(obj).__name__) - raise exceptions.GeneratedClientError(msg) + for attr in attrs: + attr_name = attr.upper() + if not hasattr(obj, '%s' % attr_name) or not getattr(obj, attr_name): + msg = 'No %s specified for object of class %s.' % ( + attr_name, type(obj).__name__) + raise exceptions.GeneratedClientError(msg) def NormalizeApiEndpoint(api_endpoint): - if not api_endpoint.endswith('/'): - api_endpoint += '/' - return api_endpoint + if not api_endpoint.endswith('/'): + api_endpoint += '/' + return api_endpoint class _UrlBuilder(object): - """Convenient container for url data.""" - - def __init__(self, base_url, relative_path=None, query_params=None): - components = urllib.parse.urlsplit(urllib.parse.urljoin( - base_url, relative_path or '')) - if components.fragment: - raise exceptions.ConfigurationValueError( - 'Unexpected url fragment: %s' % components.fragment) - self.query_params = urllib.parse.parse_qs(components.query or '') - if query_params is not None: - self.query_params.update(query_params) - self.__scheme = components.scheme - self.__netloc = components.netloc - self.relative_path = components.path or '' - - @classmethod - def FromUrl(cls, url): - urlparts = urllib.parse.urlsplit(url) - query_params = urllib.parse.parse_qs(urlparts.query) - base_url = urllib.parse.urlunsplit(( - urlparts.scheme, urlparts.netloc, '', None, None)) - relative_path = urlparts.path or '' - return cls(base_url, relative_path=relative_path, query_params=query_params) - - @property - def base_url(self): - return urllib.parse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) - - @base_url.setter - def base_url(self, value): - components = urllib.parse.urlsplit(value) - if components.path or components.query or components.fragment: - raise exceptions.ConfigurationValueError('Invalid base url: %s' % value) - self.__scheme = components.scheme - self.__netloc = components.netloc - - @property - def query(self): - # TODO(craigcitro): In the case that some of the query params are - # non-ASCII, we may silently fail to encode correctly. We should - # figure out who is responsible for owning the object -> str - # conversion. - return urllib.parse.urlencode(self.query_params, doseq=True) - - @property - def url(self): - if '{' in self.relative_path or '}' in self.relative_path: - raise exceptions.ConfigurationValueError( - 'Cannot create url with relative path %s' % self.relative_path) - return urllib.parse.urlunsplit(( - self.__scheme, self.__netloc, self.relative_path, self.query, '')) + + """Convenient container for url data.""" + + def __init__(self, base_url, relative_path=None, query_params=None): + components = urllib.parse.urlsplit(urllib.parse.urljoin( + base_url, relative_path or '')) + if components.fragment: + raise exceptions.ConfigurationValueError( + 'Unexpected url fragment: %s' % components.fragment) + self.query_params = urllib.parse.parse_qs(components.query or '') + if query_params is not None: + self.query_params.update(query_params) + self.__scheme = components.scheme + self.__netloc = components.netloc + self.relative_path = components.path or '' + + @classmethod + def FromUrl(cls, url): + urlparts = urllib.parse.urlsplit(url) + query_params = urllib.parse.parse_qs(urlparts.query) + base_url = urllib.parse.urlunsplit(( + urlparts.scheme, urlparts.netloc, '', None, None)) + relative_path = urlparts.path or '' + return cls(base_url, relative_path=relative_path, query_params=query_params) + + @property + def base_url(self): + return urllib.parse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) + + @base_url.setter + def base_url(self, value): + components = urllib.parse.urlsplit(value) + if components.path or components.query or components.fragment: + raise exceptions.ConfigurationValueError( + 'Invalid base url: %s' % value) + self.__scheme = components.scheme + self.__netloc = components.netloc + + @property + def query(self): + # TODO(craigcitro): In the case that some of the query params are + # non-ASCII, we may silently fail to encode correctly. We should + # figure out who is responsible for owning the object -> str + # conversion. + return urllib.parse.urlencode(self.query_params, doseq=True) + + @property + def url(self): + if '{' in self.relative_path or '}' in self.relative_path: + raise exceptions.ConfigurationValueError( + 'Cannot create url with relative path %s' % self.relative_path) + return urllib.parse.urlunsplit(( + self.__scheme, self.__netloc, self.relative_path, self.query, '')) class BaseApiClient(object): - """Base class for client libraries.""" - MESSAGES_MODULE = None - - _API_KEY = '' - _CLIENT_ID = '' - _CLIENT_SECRET = '' - _PACKAGE = '' - _SCOPES = [] - _USER_AGENT = '' - - def __init__(self, url, credentials=None, get_credentials=True, http=None, - model=None, log_request=False, log_response=False, num_retries=5, - credentials_args=None, default_global_params=None, - additional_http_headers=None): - _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) - if default_global_params is not None: - util.Typecheck(default_global_params, self.params_type) - self.__default_global_params = default_global_params - self.log_request = log_request - self.log_response = log_response - self.__num_retries = 5 - # We let the @property machinery below do our validation. - self.num_retries = num_retries - self._credentials = credentials - if get_credentials and not credentials: - credentials_args = credentials_args or {} - self._SetCredentials(**credentials_args) - self._url = NormalizeApiEndpoint(url) - self._http = http or http_wrapper.GetHttp() - # Note that "no credentials" is totally possible. - if self._credentials is not None: - self._http = self._credentials.authorize(self._http) - # TODO(craigcitro): Remove this field when we switch to proto2. - self.__include_fields = None - - self.additional_http_headers = additional_http_headers or {} - - # TODO(craigcitro): Finish deprecating these fields. - _ = model - - self.__response_type_model = 'proto' - - def _SetCredentials(self, **kwds): - """Fetch credentials, and set them for this client. - - Note that we can't simply return credentials, since creating them - may involve side-effecting self. - - Args: - **kwds: Additional keyword arguments are passed on to GetCredentials. - - Returns: - None. Sets self._credentials. - """ - args = { - 'api_key': self._API_KEY, - 'client': self, - 'client_id': self._CLIENT_ID, - 'client_secret': self._CLIENT_SECRET, - 'package_name': self._PACKAGE, - 'scopes': self._SCOPES, - 'user_agent': self._USER_AGENT, - } - args.update(kwds) - # TODO(craigcitro): It's a bit dangerous to pass this - # still-half-initialized self into this method, but we might need - # to set attributes on it associated with our credentials. - # Consider another way around this (maybe a callback?) and whether - # or not it's worth it. - self._credentials = credentials_lib.GetCredentials(**args) - - @classmethod - def ClientInfo(cls): - return { - 'client_id': cls._CLIENT_ID, - 'client_secret': cls._CLIENT_SECRET, - 'scope': ' '.join(sorted(util.NormalizeScopes(cls._SCOPES))), - 'user_agent': cls._USER_AGENT, - } - - @property - def base_model_class(self): - return None - - @property - def http(self): - return self._http - - @property - def url(self): - return self._url - - @classmethod - def GetScopes(cls): - return cls._SCOPES - - @property - def params_type(self): - return _LoadClass('StandardQueryParameters', self.MESSAGES_MODULE) - - @property - def user_agent(self): - return self._USER_AGENT - - @property - def _default_global_params(self): - if self.__default_global_params is None: - self.__default_global_params = self.params_type() - return self.__default_global_params - - def AddGlobalParam(self, name, value): - params = self._default_global_params - setattr(params, name, value) - - @property - def global_params(self): - return encoding.CopyProtoMessage(self._default_global_params) - - @contextlib.contextmanager - def IncludeFields(self, include_fields): - self.__include_fields = include_fields - yield - self.__include_fields = None - - @property - def response_type_model(self): - return self.__response_type_model - - @contextlib.contextmanager - def JsonResponseModel(self): - """In this context, return raw JSON instead of proto.""" - old_model = self.response_type_model - self.__response_type_model = 'json' - yield - self.__response_type_model = old_model - - @property - def num_retries(self): - return self.__num_retries - - @num_retries.setter - def num_retries(self, value): - util.Typecheck(value, six.integer_types) - if value < 0: - raise exceptions.InvalidDataError( - 'Cannot have negative value for num_retries') - self.__num_retries = value - - @contextlib.contextmanager - def WithRetries(self, num_retries): - old_num_retries = self.num_retries - self.num_retries = num_retries - yield - self.num_retries = old_num_retries - - def ProcessRequest(self, method_config, request): - """Hook for pre-processing of requests.""" - if self.log_request: - logging.info( - 'Calling method %s with %s: %s', method_config.method_id, - method_config.request_type_name, request) - return request - - def ProcessHttpRequest(self, http_request): - """Hook for pre-processing of http requests.""" - http_request.headers.update(self.additional_http_headers) - if self.log_request: - logging.info('Making http %s to %s', - http_request.http_method, http_request.url) - logging.info('Headers: %s', pprint.pformat(http_request.headers)) - if http_request.body: - # TODO(craigcitro): Make this safe to print in the case of - # non-printable body characters. - logging.info('Body:\n%s', - http_request.loggable_body or http_request.body) - else: - logging.info('Body: (none)') - return http_request - - def ProcessResponse(self, method_config, response): - if self.log_response: - logging.info('Response of type %s: %s', - method_config.response_type_name, response) - return response - - # TODO(craigcitro): Decide where these two functions should live. - def SerializeMessage(self, message): - return encoding.MessageToJson(message, include_fields=self.__include_fields) - - def DeserializeMessage(self, response_type, data): - """Deserialize the given data as method_config.response_type.""" - try: - message = encoding.JsonToMessage(response_type, data) - except (exceptions.InvalidDataFromServerError, - messages.ValidationError) as e: - raise exceptions.InvalidDataFromServerError( - 'Error decoding response "%s" as type %s: %s' % ( - data, response_type.__name__, e)) - return message - - def FinalizeTransferUrl(self, url): - """Modify the url for a given transfer, based on auth and version.""" - url_builder = _UrlBuilder.FromUrl(url) - if self.global_params.key: - url_builder.query_params['key'] = self.global_params.key - return url_builder.url + + """Base class for client libraries.""" + MESSAGES_MODULE = None + + _API_KEY = '' + _CLIENT_ID = '' + _CLIENT_SECRET = '' + _PACKAGE = '' + _SCOPES = [] + _USER_AGENT = '' + + def __init__(self, url, credentials=None, get_credentials=True, http=None, + model=None, log_request=False, log_response=False, num_retries=5, + credentials_args=None, default_global_params=None, + additional_http_headers=None): + _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) + if default_global_params is not None: + util.Typecheck(default_global_params, self.params_type) + self.__default_global_params = default_global_params + self.log_request = log_request + self.log_response = log_response + self.__num_retries = 5 + # We let the @property machinery below do our validation. + self.num_retries = num_retries + self._credentials = credentials + if get_credentials and not credentials: + credentials_args = credentials_args or {} + self._SetCredentials(**credentials_args) + self._url = NormalizeApiEndpoint(url) + self._http = http or http_wrapper.GetHttp() + # Note that "no credentials" is totally possible. + if self._credentials is not None: + self._http = self._credentials.authorize(self._http) + # TODO(craigcitro): Remove this field when we switch to proto2. + self.__include_fields = None + + self.additional_http_headers = additional_http_headers or {} + + # TODO(craigcitro): Finish deprecating these fields. + _ = model + + self.__response_type_model = 'proto' + + def _SetCredentials(self, **kwds): + """Fetch credentials, and set them for this client. + + Note that we can't simply return credentials, since creating them + may involve side-effecting self. + + Args: + **kwds: Additional keyword arguments are passed on to GetCredentials. + + Returns: + None. Sets self._credentials. + """ + args = { + 'api_key': self._API_KEY, + 'client': self, + 'client_id': self._CLIENT_ID, + 'client_secret': self._CLIENT_SECRET, + 'package_name': self._PACKAGE, + 'scopes': self._SCOPES, + 'user_agent': self._USER_AGENT, + } + args.update(kwds) + # TODO(craigcitro): It's a bit dangerous to pass this + # still-half-initialized self into this method, but we might need + # to set attributes on it associated with our credentials. + # Consider another way around this (maybe a callback?) and whether + # or not it's worth it. + self._credentials = credentials_lib.GetCredentials(**args) + + @classmethod + def ClientInfo(cls): + return { + 'client_id': cls._CLIENT_ID, + 'client_secret': cls._CLIENT_SECRET, + 'scope': ' '.join(sorted(util.NormalizeScopes(cls._SCOPES))), + 'user_agent': cls._USER_AGENT, + } + + @property + def base_model_class(self): + return None + + @property + def http(self): + return self._http + + @property + def url(self): + return self._url + + @classmethod + def GetScopes(cls): + return cls._SCOPES + + @property + def params_type(self): + return _LoadClass('StandardQueryParameters', self.MESSAGES_MODULE) + + @property + def user_agent(self): + return self._USER_AGENT + + @property + def _default_global_params(self): + if self.__default_global_params is None: + self.__default_global_params = self.params_type() + return self.__default_global_params + + def AddGlobalParam(self, name, value): + params = self._default_global_params + setattr(params, name, value) + + @property + def global_params(self): + return encoding.CopyProtoMessage(self._default_global_params) + + @contextlib.contextmanager + def IncludeFields(self, include_fields): + self.__include_fields = include_fields + yield + self.__include_fields = None + + @property + def response_type_model(self): + return self.__response_type_model + + @contextlib.contextmanager + def JsonResponseModel(self): + """In this context, return raw JSON instead of proto.""" + old_model = self.response_type_model + self.__response_type_model = 'json' + yield + self.__response_type_model = old_model + + @property + def num_retries(self): + return self.__num_retries + + @num_retries.setter + def num_retries(self, value): + util.Typecheck(value, six.integer_types) + if value < 0: + raise exceptions.InvalidDataError( + 'Cannot have negative value for num_retries') + self.__num_retries = value + + @contextlib.contextmanager + def WithRetries(self, num_retries): + old_num_retries = self.num_retries + self.num_retries = num_retries + yield + self.num_retries = old_num_retries + + def ProcessRequest(self, method_config, request): + """Hook for pre-processing of requests.""" + if self.log_request: + logging.info( + 'Calling method %s with %s: %s', method_config.method_id, + method_config.request_type_name, request) + return request + + def ProcessHttpRequest(self, http_request): + """Hook for pre-processing of http requests.""" + http_request.headers.update(self.additional_http_headers) + if self.log_request: + logging.info('Making http %s to %s', + http_request.http_method, http_request.url) + logging.info('Headers: %s', pprint.pformat(http_request.headers)) + if http_request.body: + # TODO(craigcitro): Make this safe to print in the case of + # non-printable body characters. + logging.info('Body:\n%s', + http_request.loggable_body or http_request.body) + else: + logging.info('Body: (none)') + return http_request + + def ProcessResponse(self, method_config, response): + if self.log_response: + logging.info('Response of type %s: %s', + method_config.response_type_name, response) + return response + + # TODO(craigcitro): Decide where these two functions should live. + def SerializeMessage(self, message): + return encoding.MessageToJson(message, include_fields=self.__include_fields) + + def DeserializeMessage(self, response_type, data): + """Deserialize the given data as method_config.response_type.""" + try: + message = encoding.JsonToMessage(response_type, data) + except (exceptions.InvalidDataFromServerError, + messages.ValidationError) as e: + raise exceptions.InvalidDataFromServerError( + 'Error decoding response "%s" as type %s: %s' % ( + data, response_type.__name__, e)) + return message + + def FinalizeTransferUrl(self, url): + """Modify the url for a given transfer, based on auth and version.""" + url_builder = _UrlBuilder.FromUrl(url) + if self.global_params.key: + url_builder.query_params['key'] = self.global_params.key + return url_builder.url class BaseApiService(object): - """Base class for generated API services.""" - - def __init__(self, client): - self.__client = client - self._method_configs = {} - self._upload_configs = {} - - @property - def _client(self): - return self.__client - - @property - def client(self): - return self.__client - - def GetMethodConfig(self, method): - return self._method_configs[method] - - def GetUploadConfig(self, method): - return self._upload_configs.get(method) - - def GetRequestType(self, method): - method_config = self.GetMethodConfig(method) - return getattr(self.client.MESSAGES_MODULE, - method_config.request_type_name) - - def GetResponseType(self, method): - method_config = self.GetMethodConfig(method) - return getattr(self.client.MESSAGES_MODULE, - method_config.response_type_name) - - def __CombineGlobalParams(self, global_params, default_params): - """Combine the given params with the defaults.""" - util.Typecheck(global_params, (type(None), self.__client.params_type)) - result = self.__client.params_type() - global_params = global_params or self.__client.params_type() - for field in result.all_fields(): - value = global_params.get_assigned_value(field.name) - if value is None: - value = default_params.get_assigned_value(field.name) - if value not in (None, [], ()): - setattr(result, field.name, value) - return result - - def __EncodePrettyPrint(self, query_info): - # The prettyPrint flag needs custom encoding: it should be encoded - # as 0 if False, and ignored otherwise (True is the default). - if not query_info.pop('prettyPrint', True): - query_info['prettyPrint'] = 0 - # The One Platform equivalent of prettyPrint is pp, which also needs - # custom encoding. - if not query_info.pop('pp', True): - query_info['pp'] = 0 - return query_info - - def __ConstructQueryParams(self, query_params, request, global_params): - """Construct a dictionary of query parameters for this request.""" - # First, handle the global params. - global_params = self.__CombineGlobalParams( - global_params, self.__client.global_params) - global_param_names = util.MapParamNames( - [x.name for x in self.__client.params_type.all_fields()], - self.__client.params_type) - query_info = dict((param, getattr(global_params, param)) - for param in global_param_names) - # Next, add the query params. - query_param_names = util.MapParamNames(query_params, type(request)) - query_info.update( - (param, getattr(request, param, None)) for param in query_param_names) - query_info = dict((k, v) for k, v in query_info.items() - if v is not None) - query_info = self.__EncodePrettyPrint(query_info) - query_info = util.MapRequestParams(query_info, type(request)) - for k, v in query_info.items(): - if isinstance(v, six.text_type): - query_info[k] = v.encode('utf8') - elif isinstance(v, str): - query_info[k] = v.decode('utf8') - elif isinstance(v, datetime.datetime): - query_info[k] = v.isoformat() - return query_info - - def __ConstructRelativePath(self, method_config, request, relative_path=None): - """Determine the relative path for request.""" - python_param_names = util.MapParamNames( - method_config.path_params, type(request)) - params = dict([(param, getattr(request, param, None)) - for param in python_param_names]) - params = util.MapRequestParams(params, type(request)) - return util.ExpandRelativePath(method_config, params, - relative_path=relative_path) - - def __FinalizeRequest(self, http_request, url_builder): - """Make any final general adjustments to the request.""" - if (http_request.http_method == 'GET' and - len(http_request.url) > _MAX_URL_LENGTH): - http_request.http_method = 'POST' - http_request.headers['x-http-method-override'] = 'GET' - http_request.headers['content-type'] = 'application/x-www-form-urlencoded' - http_request.body = url_builder.query - url_builder.query_params = {} - http_request.url = url_builder.url - - def __ProcessHttpResponse(self, method_config, http_response): - """Process the given http response.""" - if http_response.status_code not in (http_client.OK, - http_client.NO_CONTENT): - raise exceptions.HttpError.FromResponse(http_response) - if http_response.status_code == http_client.NO_CONTENT: - # TODO(craigcitro): Find out why _replace doesn't seem to work here. - http_response = http_wrapper.Response( - info=http_response.info, content='{}', - request_url=http_response.request_url) - if self.__client.response_type_model == 'json': - return http_response.content - else: - response_type = _LoadClass( - method_config.response_type_name, self.__client.MESSAGES_MODULE) - return self.__client.DeserializeMessage( - response_type, http_response.content) - - def __SetBaseHeaders(self, http_request, client): - """Fill in the basic headers on http_request.""" - # TODO(craigcitro): Make the default a little better here, and - # include the apitools version. - user_agent = client.user_agent or 'apitools-client/1.0' - http_request.headers['user-agent'] = user_agent - http_request.headers['accept'] = 'application/json' - http_request.headers['accept-encoding'] = 'gzip, deflate' - - def __SetBody(self, http_request, method_config, request, upload): - """Fill in the body on http_request.""" - if not method_config.request_field: - return - - request_type = _LoadClass( - method_config.request_type_name, self.__client.MESSAGES_MODULE) - if method_config.request_field == REQUEST_IS_BODY: - body_value = request - body_type = request_type - else: - body_value = getattr(request, method_config.request_field) - body_field = request_type.field_by_name(method_config.request_field) - util.Typecheck(body_field, messages.MessageField) - body_type = body_field.type - - if upload and not body_value: - # We're going to fill in the body later. - return - util.Typecheck(body_value, body_type) - http_request.headers['content-type'] = 'application/json' - http_request.body = self.__client.SerializeMessage(body_value) - - def PrepareHttpRequest(self, method_config, request, global_params=None, - upload=None, upload_config=None, download=None): - """Prepares an HTTP request to be sent.""" - request_type = _LoadClass( - method_config.request_type_name, self.__client.MESSAGES_MODULE) - util.Typecheck(request, request_type) - request = self.__client.ProcessRequest(method_config, request) - - http_request = http_wrapper.Request(http_method=method_config.http_method) - self.__SetBaseHeaders(http_request, self.__client) - self.__SetBody(http_request, method_config, request, upload) - - url_builder = _UrlBuilder( - self.__client.url, relative_path=method_config.relative_path) - url_builder.query_params = self.__ConstructQueryParams( - method_config.query_params, request, global_params) - - # It's important that upload and download go before we fill in the - # relative path, so that they can replace it. - if upload is not None: - upload.ConfigureRequest(upload_config, http_request, url_builder) - if download is not None: - download.ConfigureRequest(http_request, url_builder) - - url_builder.relative_path = self.__ConstructRelativePath( - method_config, request, relative_path=url_builder.relative_path) - self.__FinalizeRequest(http_request, url_builder) - - return self.__client.ProcessHttpRequest(http_request) - - def _RunMethod(self, method_config, request, global_params=None, - upload=None, upload_config=None, download=None): - """Call this method with request.""" - if upload is not None and download is not None: - # TODO(craigcitro): This just involves refactoring the logic - # below into callbacks that we can pass around; in particular, - # the order should be that the upload gets the initial request, - # and then passes its reply to a download if one exists, and - # then that goes to ProcessResponse and is returned. - raise exceptions.NotYetImplementedError( - 'Cannot yet use both upload and download at once') - - http_request = self.PrepareHttpRequest( - method_config, request, global_params, upload, upload_config, download) - - # TODO(craigcitro): Make num_retries customizable on Transfer - # objects, and pass in self.__client.num_retries when initializing - # an upload or download. - if download is not None: - download.InitializeDownload(http_request, client=self.client) - return - - http_response = None - if upload is not None: - http_response = upload.InitializeUpload(http_request, client=self.client) - if http_response is None: - http = self.__client.http - if upload and upload.bytes_http: - http = upload.bytes_http - http_response = http_wrapper.MakeRequest( - http, http_request, retries=self.__client.num_retries) - - return self.ProcessHttpResponse(method_config, http_response) - - def ProcessHttpResponse(self, method_config, http_response): - """Convert an HTTP response to the expected message type.""" - return self.__client.ProcessResponse( - method_config, - self.__ProcessHttpResponse(method_config, http_response)) + + """Base class for generated API services.""" + + def __init__(self, client): + self.__client = client + self._method_configs = {} + self._upload_configs = {} + + @property + def _client(self): + return self.__client + + @property + def client(self): + return self.__client + + def GetMethodConfig(self, method): + return self._method_configs[method] + + def GetUploadConfig(self, method): + return self._upload_configs.get(method) + + def GetRequestType(self, method): + method_config = self.GetMethodConfig(method) + return getattr(self.client.MESSAGES_MODULE, + method_config.request_type_name) + + def GetResponseType(self, method): + method_config = self.GetMethodConfig(method) + return getattr(self.client.MESSAGES_MODULE, + method_config.response_type_name) + + def __CombineGlobalParams(self, global_params, default_params): + """Combine the given params with the defaults.""" + util.Typecheck(global_params, (type(None), self.__client.params_type)) + result = self.__client.params_type() + global_params = global_params or self.__client.params_type() + for field in result.all_fields(): + value = global_params.get_assigned_value(field.name) + if value is None: + value = default_params.get_assigned_value(field.name) + if value not in (None, [], ()): + setattr(result, field.name, value) + return result + + def __EncodePrettyPrint(self, query_info): + # The prettyPrint flag needs custom encoding: it should be encoded + # as 0 if False, and ignored otherwise (True is the default). + if not query_info.pop('prettyPrint', True): + query_info['prettyPrint'] = 0 + # The One Platform equivalent of prettyPrint is pp, which also needs + # custom encoding. + if not query_info.pop('pp', True): + query_info['pp'] = 0 + return query_info + + def __ConstructQueryParams(self, query_params, request, global_params): + """Construct a dictionary of query parameters for this request.""" + # First, handle the global params. + global_params = self.__CombineGlobalParams( + global_params, self.__client.global_params) + global_param_names = util.MapParamNames( + [x.name for x in self.__client.params_type.all_fields()], + self.__client.params_type) + query_info = dict((param, getattr(global_params, param)) + for param in global_param_names) + # Next, add the query params. + query_param_names = util.MapParamNames(query_params, type(request)) + query_info.update( + (param, getattr(request, param, None)) for param in query_param_names) + query_info = dict((k, v) for k, v in query_info.items() + if v is not None) + query_info = self.__EncodePrettyPrint(query_info) + query_info = util.MapRequestParams(query_info, type(request)) + for k, v in query_info.items(): + if isinstance(v, six.text_type): + query_info[k] = v.encode('utf8') + elif isinstance(v, str): + query_info[k] = v.decode('utf8') + elif isinstance(v, datetime.datetime): + query_info[k] = v.isoformat() + return query_info + + def __ConstructRelativePath(self, method_config, request, relative_path=None): + """Determine the relative path for request.""" + python_param_names = util.MapParamNames( + method_config.path_params, type(request)) + params = dict([(param, getattr(request, param, None)) + for param in python_param_names]) + params = util.MapRequestParams(params, type(request)) + return util.ExpandRelativePath(method_config, params, + relative_path=relative_path) + + def __FinalizeRequest(self, http_request, url_builder): + """Make any final general adjustments to the request.""" + if (http_request.http_method == 'GET' and + len(http_request.url) > _MAX_URL_LENGTH): + http_request.http_method = 'POST' + http_request.headers['x-http-method-override'] = 'GET' + http_request.headers[ + 'content-type'] = 'application/x-www-form-urlencoded' + http_request.body = url_builder.query + url_builder.query_params = {} + http_request.url = url_builder.url + + def __ProcessHttpResponse(self, method_config, http_response): + """Process the given http response.""" + if http_response.status_code not in (http_client.OK, + http_client.NO_CONTENT): + raise exceptions.HttpError.FromResponse(http_response) + if http_response.status_code == http_client.NO_CONTENT: + # TODO(craigcitro): Find out why _replace doesn't seem to work + # here. + http_response = http_wrapper.Response( + info=http_response.info, content='{}', + request_url=http_response.request_url) + if self.__client.response_type_model == 'json': + return http_response.content + else: + response_type = _LoadClass( + method_config.response_type_name, self.__client.MESSAGES_MODULE) + return self.__client.DeserializeMessage( + response_type, http_response.content) + + def __SetBaseHeaders(self, http_request, client): + """Fill in the basic headers on http_request.""" + # TODO(craigcitro): Make the default a little better here, and + # include the apitools version. + user_agent = client.user_agent or 'apitools-client/1.0' + http_request.headers['user-agent'] = user_agent + http_request.headers['accept'] = 'application/json' + http_request.headers['accept-encoding'] = 'gzip, deflate' + + def __SetBody(self, http_request, method_config, request, upload): + """Fill in the body on http_request.""" + if not method_config.request_field: + return + + request_type = _LoadClass( + method_config.request_type_name, self.__client.MESSAGES_MODULE) + if method_config.request_field == REQUEST_IS_BODY: + body_value = request + body_type = request_type + else: + body_value = getattr(request, method_config.request_field) + body_field = request_type.field_by_name( + method_config.request_field) + util.Typecheck(body_field, messages.MessageField) + body_type = body_field.type + + if upload and not body_value: + # We're going to fill in the body later. + return + util.Typecheck(body_value, body_type) + http_request.headers['content-type'] = 'application/json' + http_request.body = self.__client.SerializeMessage(body_value) + + def PrepareHttpRequest(self, method_config, request, global_params=None, + upload=None, upload_config=None, download=None): + """Prepares an HTTP request to be sent.""" + request_type = _LoadClass( + method_config.request_type_name, self.__client.MESSAGES_MODULE) + util.Typecheck(request, request_type) + request = self.__client.ProcessRequest(method_config, request) + + http_request = http_wrapper.Request( + http_method=method_config.http_method) + self.__SetBaseHeaders(http_request, self.__client) + self.__SetBody(http_request, method_config, request, upload) + + url_builder = _UrlBuilder( + self.__client.url, relative_path=method_config.relative_path) + url_builder.query_params = self.__ConstructQueryParams( + method_config.query_params, request, global_params) + + # It's important that upload and download go before we fill in the + # relative path, so that they can replace it. + if upload is not None: + upload.ConfigureRequest(upload_config, http_request, url_builder) + if download is not None: + download.ConfigureRequest(http_request, url_builder) + + url_builder.relative_path = self.__ConstructRelativePath( + method_config, request, relative_path=url_builder.relative_path) + self.__FinalizeRequest(http_request, url_builder) + + return self.__client.ProcessHttpRequest(http_request) + + def _RunMethod(self, method_config, request, global_params=None, + upload=None, upload_config=None, download=None): + """Call this method with request.""" + if upload is not None and download is not None: + # TODO(craigcitro): This just involves refactoring the logic + # below into callbacks that we can pass around; in particular, + # the order should be that the upload gets the initial request, + # and then passes its reply to a download if one exists, and + # then that goes to ProcessResponse and is returned. + raise exceptions.NotYetImplementedError( + 'Cannot yet use both upload and download at once') + + http_request = self.PrepareHttpRequest( + method_config, request, global_params, upload, upload_config, download) + + # TODO(craigcitro): Make num_retries customizable on Transfer + # objects, and pass in self.__client.num_retries when initializing + # an upload or download. + if download is not None: + download.InitializeDownload(http_request, client=self.client) + return + + http_response = None + if upload is not None: + http_response = upload.InitializeUpload( + http_request, client=self.client) + if http_response is None: + http = self.__client.http + if upload and upload.bytes_http: + http = upload.bytes_http + http_response = http_wrapper.MakeRequest( + http, http_request, retries=self.__client.num_retries) + + return self.ProcessHttpResponse(method_config, http_response) + + def ProcessHttpResponse(self, method_config, http_response): + """Convert an HTTP response to the expected message type.""" + return self.__client.ProcessResponse( + method_config, + self.__ProcessHttpResponse(method_config, http_response)) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index f2c438c..8bd665f 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -16,21 +16,21 @@ from apitools.base.py import http_wrapper class SimpleMessage(messages.Message): - field = messages.StringField(1) + field = messages.StringField(1) class MessageWithTime(messages.Message): - timestamp = message_types.DateTimeField(1) + timestamp = message_types.DateTimeField(1) class MessageWithRemappings(messages.Message): - class AnEnum(messages.Enum): - value_one = 1 - value_two = 2 + class AnEnum(messages.Enum): + value_one = 1 + value_two = 2 - str_field = messages.StringField(1) - enum_field = messages.EnumField('AnEnum', 2) + str_field = messages.StringField(1) + enum_field = messages.EnumField('AnEnum', 2) encoding.AddCustomJsonFieldMapping( @@ -40,144 +40,146 @@ encoding.AddCustomJsonEnumMapping( class StandardQueryParameters(messages.Message): - field = messages.StringField(1) - prettyPrint = messages.BooleanField(5, default=True) # pylint: disable=invalid-name - pp = messages.BooleanField(6, default=True) + field = messages.StringField(1) + prettyPrint = messages.BooleanField( + 5, default=True) # pylint: disable=invalid-name + pp = messages.BooleanField(6, default=True) class FakeCredentials(object): - def authorize(self, _): # pylint: disable=invalid-name - return None + def authorize(self, _): # pylint: disable=invalid-name + return None class FakeClient(base_api.BaseApiClient): - MESSAGES_MODULE = sys.modules[__name__] - _PACKAGE = 'package' - _SCOPES = ['scope1'] - _CLIENT_ID = 'client_id' - _CLIENT_SECRET = 'client_secret' + MESSAGES_MODULE = sys.modules[__name__] + _PACKAGE = 'package' + _SCOPES = ['scope1'] + _CLIENT_ID = 'client_id' + _CLIENT_SECRET = 'client_secret' class FakeService(base_api.BaseApiService): - def __init__(self, client=None): - client = client or FakeClient( - 'http://www.example.com/', credentials=FakeCredentials()) - super(FakeService, self).__init__(client) + def __init__(self, client=None): + client = client or FakeClient( + 'http://www.example.com/', credentials=FakeCredentials()) + super(FakeService, self).__init__(client) class BaseApiTest(unittest2.TestCase): - def __GetFakeClient(self): - return FakeClient('', credentials=FakeCredentials()) - - def testUrlNormalization(self): - client = FakeClient('http://www.googleapis.com', get_credentials=False) - self.assertTrue(client.url.endswith('/')) - - def testNoCredentials(self): - client = FakeClient('', get_credentials=False) - self.assertIsNotNone(client) - self.assertIsNone(client._credentials) - - def testIncludeEmptyFieldsClient(self): - msg = SimpleMessage() - client = self.__GetFakeClient() - self.assertEqual('{}', client.SerializeMessage(msg)) - with client.IncludeFields(('field',)): - self.assertEqual('{"field": null}', client.SerializeMessage(msg)) - - def testJsonResponse(self): - method_config = base_api.ApiMethodInfo(response_type_name='SimpleMessage') - service = FakeService() - http_response = http_wrapper.Response( - info={'status': '200'}, content='{"field": "abc"}', - request_url='http://www.google.com') - response_message = SimpleMessage(field='abc') - self.assertEqual(response_message, service.ProcessHttpResponse( - method_config, http_response)) - with service.client.JsonResponseModel(): - self.assertEqual(http_response.content, service.ProcessHttpResponse( - method_config, http_response)) - - def testAdditionalHeaders(self): - additional_headers = {'Request-Is-Awesome': '1'} - client = self.__GetFakeClient() - - # No headers to start - http_request = http_wrapper.Request('http://www.example.com') - new_request = client.ProcessHttpRequest(http_request) - self.assertFalse('Request-Is-Awesome' in new_request.headers) - - # Add a new header and ensure it's added to the request. - client.additional_http_headers = additional_headers - http_request = http_wrapper.Request('http://www.example.com') - new_request = client.ProcessHttpRequest(http_request) - self.assertTrue('Request-Is-Awesome' in new_request.headers) - - def testQueryEncoding(self): - method_config = base_api.ApiMethodInfo( - request_type_name='MessageWithTime', query_params=['timestamp']) - service = FakeService() - request = MessageWithTime( - timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) - http_request = service.PrepareHttpRequest(method_config, request) - - url_timestamp = urllib_parse.quote(request.timestamp.isoformat()) - self.assertTrue(http_request.url.endswith(url_timestamp)) - - def testPrettyPrintEncoding(self): - method_config = base_api.ApiMethodInfo( - request_type_name='MessageWithTime', query_params=['timestamp']) - service = FakeService() - request = MessageWithTime( - timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) - - global_params = StandardQueryParameters() - http_request = service.PrepareHttpRequest(method_config, request, - global_params=global_params) - self.assertFalse('prettyPrint' in http_request.url) - self.assertFalse('pp' in http_request.url) - - global_params.prettyPrint = False # pylint: disable=invalid-name - global_params.pp = False - - http_request = service.PrepareHttpRequest(method_config, request, - global_params=global_params) - self.assertTrue('prettyPrint=0' in http_request.url) - self.assertTrue('pp=0' in http_request.url) - - def testQueryRemapping(self): - method_config = base_api.ApiMethodInfo( - request_type_name='MessageWithRemappings', - query_params=['remapped_field', 'enum_field']) - request = MessageWithRemappings( - str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one) - http_request = FakeService().PrepareHttpRequest(method_config, request) - result_params = urllib_parse.parse_qs( - urllib_parse.urlparse(http_request.url).query) - expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'} - self.assertTrue(expected_params, result_params) - - def testPathRemapping(self): - method_config = base_api.ApiMethodInfo( - relative_path='parameters/{remapped_field}/remap/{enum_field}', - request_type_name='MessageWithRemappings', - path_params=['remapped_field', 'enum_field']) - request = MessageWithRemappings( - str_field='gonna', enum_field=MessageWithRemappings.AnEnum.value_one) - service = FakeService() - expected_url = service.client.url + 'parameters/gonna/remap/ONE%2FTWO' - http_request = service.PrepareHttpRequest(method_config, request) - self.assertEqual(expected_url, http_request.url) - - method_config.relative_path = ( - 'parameters/{+remapped_field}/remap/{+enum_field}') - expected_url = service.client.url + 'parameters/gonna/remap/ONE/TWO' - http_request = service.PrepareHttpRequest(method_config, request) - self.assertEqual(expected_url, http_request.url) + def __GetFakeClient(self): + return FakeClient('', credentials=FakeCredentials()) + + def testUrlNormalization(self): + client = FakeClient('http://www.googleapis.com', get_credentials=False) + self.assertTrue(client.url.endswith('/')) + + def testNoCredentials(self): + client = FakeClient('', get_credentials=False) + self.assertIsNotNone(client) + self.assertIsNone(client._credentials) + + def testIncludeEmptyFieldsClient(self): + msg = SimpleMessage() + client = self.__GetFakeClient() + self.assertEqual('{}', client.SerializeMessage(msg)) + with client.IncludeFields(('field',)): + self.assertEqual('{"field": null}', client.SerializeMessage(msg)) + + def testJsonResponse(self): + method_config = base_api.ApiMethodInfo( + response_type_name='SimpleMessage') + service = FakeService() + http_response = http_wrapper.Response( + info={'status': '200'}, content='{"field": "abc"}', + request_url='http://www.google.com') + response_message = SimpleMessage(field='abc') + self.assertEqual(response_message, service.ProcessHttpResponse( + method_config, http_response)) + with service.client.JsonResponseModel(): + self.assertEqual(http_response.content, service.ProcessHttpResponse( + method_config, http_response)) + + def testAdditionalHeaders(self): + additional_headers = {'Request-Is-Awesome': '1'} + client = self.__GetFakeClient() + + # No headers to start + http_request = http_wrapper.Request('http://www.example.com') + new_request = client.ProcessHttpRequest(http_request) + self.assertFalse('Request-Is-Awesome' in new_request.headers) + + # Add a new header and ensure it's added to the request. + client.additional_http_headers = additional_headers + http_request = http_wrapper.Request('http://www.example.com') + new_request = client.ProcessHttpRequest(http_request) + self.assertTrue('Request-Is-Awesome' in new_request.headers) + + def testQueryEncoding(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithTime', query_params=['timestamp']) + service = FakeService() + request = MessageWithTime( + timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) + http_request = service.PrepareHttpRequest(method_config, request) + + url_timestamp = urllib_parse.quote(request.timestamp.isoformat()) + self.assertTrue(http_request.url.endswith(url_timestamp)) + + def testPrettyPrintEncoding(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithTime', query_params=['timestamp']) + service = FakeService() + request = MessageWithTime( + timestamp=datetime.datetime(2014, 10, 0o7, 12, 53, 13)) + + global_params = StandardQueryParameters() + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + self.assertFalse('prettyPrint' in http_request.url) + self.assertFalse('pp' in http_request.url) + + global_params.prettyPrint = False # pylint: disable=invalid-name + global_params.pp = False + + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + self.assertTrue('prettyPrint=0' in http_request.url) + self.assertTrue('pp=0' in http_request.url) + + def testQueryRemapping(self): + method_config = base_api.ApiMethodInfo( + request_type_name='MessageWithRemappings', + query_params=['remapped_field', 'enum_field']) + request = MessageWithRemappings( + str_field='foo', enum_field=MessageWithRemappings.AnEnum.value_one) + http_request = FakeService().PrepareHttpRequest(method_config, request) + result_params = urllib_parse.parse_qs( + urllib_parse.urlparse(http_request.url).query) + expected_params = {'enum_field': 'ONE%2FTWO', 'remapped_field': 'foo'} + self.assertTrue(expected_params, result_params) + + def testPathRemapping(self): + method_config = base_api.ApiMethodInfo( + relative_path='parameters/{remapped_field}/remap/{enum_field}', + request_type_name='MessageWithRemappings', + path_params=['remapped_field', 'enum_field']) + request = MessageWithRemappings( + str_field='gonna', enum_field=MessageWithRemappings.AnEnum.value_one) + service = FakeService() + expected_url = service.client.url + 'parameters/gonna/remap/ONE%2FTWO' + http_request = service.PrepareHttpRequest(method_config, request) + self.assertEqual(expected_url, http_request.url) + + method_config.relative_path = ( + 'parameters/{+remapped_field}/remap/{+enum_field}') + expected_url = service.client.url + 'parameters/gonna/remap/ONE/TWO' + http_request = service.PrepareHttpRequest(method_config, request) + self.assertEqual(expected_url, http_request.url) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index 399c8a9..70872b8 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -36,27 +36,27 @@ _OUTPUT_FORMATTER_MAP = { def DeclareBaseFlags(): - """Declare base flags for all CLIs.""" - # TODO(craigcitro): FlagValidators? - global _BASE_FLAGS_DECLARED - if _BASE_FLAGS_DECLARED: - return - flags.DEFINE_boolean( - 'log_request', False, - 'Log requests.') - flags.DEFINE_boolean( - 'log_response', False, - 'Log responses.') - flags.DEFINE_boolean( - 'log_request_response', False, - 'Log requests and responses.') - flags.DEFINE_enum( - 'output_format', - 'protorpc', - _OUTPUT_FORMATTER_MAP.keys(), - 'Display format for results.') - - _BASE_FLAGS_DECLARED = True + """Declare base flags for all CLIs.""" + # TODO(craigcitro): FlagValidators? + global _BASE_FLAGS_DECLARED + if _BASE_FLAGS_DECLARED: + return + flags.DEFINE_boolean( + 'log_request', False, + 'Log requests.') + flags.DEFINE_boolean( + 'log_response', False, + 'Log responses.') + flags.DEFINE_boolean( + 'log_request_response', False, + 'Log requests and responses.') + flags.DEFINE_enum( + 'output_format', + 'protorpc', + _OUTPUT_FORMATTER_MAP.keys(), + 'Display format for results.') + + _BASE_FLAGS_DECLARED = True # NOTE: This is specified here so that it can be read by other files # without depending on the flag to be registered. @@ -67,84 +67,85 @@ FLAGS = flags.FLAGS def SetupLogger(): - if FLAGS.log_request or FLAGS.log_response or FLAGS.log_request_response: - logging.basicConfig() - logging.getLogger().setLevel(logging.INFO) + if FLAGS.log_request or FLAGS.log_response or FLAGS.log_request_response: + logging.basicConfig() + logging.getLogger().setLevel(logging.INFO) def FormatOutput(message, output_format=None): - """Convert the output to the user-specified format.""" - output_format = output_format or FLAGS.output_format - formatter = _OUTPUT_FORMATTER_MAP.get(FLAGS.output_format) - if formatter is None: - raise exceptions.UserError('Unknown output format: %s' % output_format) - return formatter(message) + """Convert the output to the user-specified format.""" + output_format = output_format or FLAGS.output_format + formatter = _OUTPUT_FORMATTER_MAP.get(FLAGS.output_format) + if formatter is None: + raise exceptions.UserError('Unknown output format: %s' % output_format) + return formatter(message) class _SmartCompleter(rlcompleter.Completer): - def _callable_postfix(self, val, word): - if ('(' in readline.get_line_buffer() or - not callable(val)): - return word - else: - return word + '(' + def _callable_postfix(self, val, word): + if ('(' in readline.get_line_buffer() or + not callable(val)): + return word + else: + return word + '(' - def complete(self, text, state): - if not readline.get_line_buffer().strip(): - if not state: - return ' ' - else: - return None - return rlcompleter.Completer.complete(self, text, state) + def complete(self, text, state): + if not readline.get_line_buffer().strip(): + if not state: + return ' ' + else: + return None + return rlcompleter.Completer.complete(self, text, state) class ConsoleWithReadline(code.InteractiveConsole): - """InteractiveConsole with readline, tab completion, and history.""" - - def __init__(self, env, filename='', histfile=None): - new_locals = dict(env) - new_locals.update({ - '_SmartCompleter': _SmartCompleter, - 'readline': readline, - 'rlcompleter': rlcompleter, - }) - code.InteractiveConsole.__init__(self, new_locals, filename) - readline.parse_and_bind('tab: complete') - readline.set_completer(_SmartCompleter(new_locals).complete) - if histfile is not None: - histfile = os.path.expanduser(histfile) - if os.path.exists(histfile): - readline.read_history_file(histfile) - atexit.register(lambda: readline.write_history_file(histfile)) + + """InteractiveConsole with readline, tab completion, and history.""" + + def __init__(self, env, filename='', histfile=None): + new_locals = dict(env) + new_locals.update({ + '_SmartCompleter': _SmartCompleter, + 'readline': readline, + 'rlcompleter': rlcompleter, + }) + code.InteractiveConsole.__init__(self, new_locals, filename) + readline.parse_and_bind('tab: complete') + readline.set_completer(_SmartCompleter(new_locals).complete) + if histfile is not None: + histfile = os.path.expanduser(histfile) + if os.path.exists(histfile): + readline.read_history_file(histfile) + atexit.register(lambda: readline.write_history_file(histfile)) def run_main(): - """Function to be used as setuptools script entry point. - - Appcommands assumes that it always runs as __main__, but launching - via a setuptools-generated entry_point breaks this rule. We do some - trickery here to make sure that appcommands and flags find their - state where they expect to by faking ourselves as __main__. - """ - - # Put the flags for this module somewhere the flags module will look - # for them. - # pylint: disable=protected-access - new_name = flags._GetMainModule() - sys.modules[new_name] = sys.modules['__main__'] - for flag in FLAGS.FlagsByModuleDict().get(__name__, []): - FLAGS._RegisterFlagByModule(new_name, flag) - for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): - FLAGS._RegisterKeyFlagForModule(new_name, key_flag) - # pylint: enable=protected-access - - # Now set __main__ appropriately so that appcommands will be - # happy. - sys.modules['__main__'] = sys.modules[__name__] - appcommands.Run() - sys.modules['__main__'] = sys.modules.pop(new_name) + """Function to be used as setuptools script entry point. + + Appcommands assumes that it always runs as __main__, but launching + via a setuptools-generated entry_point breaks this rule. We do some + trickery here to make sure that appcommands and flags find their + state where they expect to by faking ourselves as __main__. + """ + + # Put the flags for this module somewhere the flags module will look + # for them. + # pylint: disable=protected-access + new_name = flags._GetMainModule() + sys.modules[new_name] = sys.modules['__main__'] + for flag in FLAGS.FlagsByModuleDict().get(__name__, []): + FLAGS._RegisterFlagByModule(new_name, flag) + for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): + FLAGS._RegisterKeyFlagForModule(new_name, key_flag) + # pylint: enable=protected-access + + # Now set __main__ appropriately so that appcommands will be + # happy. + sys.modules['__main__'] = sys.modules[__name__] + appcommands.Run() + sys.modules['__main__'] = sys.modules.pop(new_name) if __name__ == '__main__': - appcommands.Run() + appcommands.Run() diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 48d0616..1fbf1bd 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -23,419 +23,428 @@ __all__ = [ class RequestResponseAndHandler(collections.namedtuple( - 'RequestResponseAndHandler', ['request', 'response', 'handler'])): - """Container for data related to completing an HTTP request. + 'RequestResponseAndHandler', ['request', 'response', 'handler'])): - This contains an HTTP request, its response, and a callback for handling - the response from the server. + """Container for data related to completing an HTTP request. - Attributes: - request: An http_wrapper.Request object representing the HTTP request. - response: The http_wrapper.Response object returned from the server. - handler: A callback function accepting two arguments, response - and exception. Response is an http_wrapper.Response object, and - exception is an apiclient.errors.HttpError object if an error - occurred, or otherwise None. - """ - - -class BatchApiRequest(object): - - class ApiCall(object): - """Holds request and response information for each request. - - ApiCalls are ultimately exposed to the client once the HTTP batch request - has been completed. + This contains an HTTP request, its response, and a callback for handling + the response from the server. Attributes: - http_request: A client-supplied http_wrapper.Request to be - submitted to the server. - response: A http_wrapper.Response object given by the server as a - response to the user request, or None if an error occurred. - exception: An apiclient.errors.HttpError object if an error - occurred, or None. + request: An http_wrapper.Request object representing the HTTP request. + response: The http_wrapper.Response object returned from the server. + handler: A callback function accepting two arguments, response + and exception. Response is an http_wrapper.Response object, and + exception is an apiclient.errors.HttpError object if an error + occurred, or otherwise None. """ - def __init__(self, request, retryable_codes, service, method_config): - """Initialize an individual API request. - - Args: - request: An http_wrapper.Request object. - retryable_codes: A list of integer HTTP codes that can be retried. - service: A service inheriting from base_api.BaseApiService. - method_config: Method config for the desired API request. - """ - self.__retryable_codes = list( - set(retryable_codes + [http_client.UNAUTHORIZED])) - self.__http_response = None - self.__service = service - self.__method_config = method_config - - self.http_request = request - # TODO(user): Add some validation to these fields. - self.__response = None - self.__exception = None - - @property - def is_error(self): - return self.exception is not None - - @property - def response(self): - return self.__response - - @property - def exception(self): - return self.__exception - - @property - def authorization_failed(self): - return (self.__http_response and ( - self.__http_response.status_code == http_client.UNAUTHORIZED)) - - @property - def terminal_state(self): - return (self.__http_response and ( - self.__http_response.status_code not in self.__retryable_codes)) - - def HandleResponse(self, http_response, exception): - """Handles an incoming http response to the request in http_request. - - This is intended to be used as a callback function for - BatchHttpRequest.Add. - - Args: - http_response: Deserialized http_wrapper.Response object. - exception: apiclient.errors.HttpError object if an error occurred. - """ - self.__http_response = http_response - self.__exception = exception - if self.terminal_state and not self.__exception: - self.__response = self.__service.ProcessHttpResponse( - self.__method_config, self.__http_response) - - def __init__(self, batch_url=None, retryable_codes=None): - """Initialize a batch API request object. - - Args: - batch_url: Base URL for batch API calls. - retryable_codes: A list of integer HTTP codes that can be retried. - """ - self.api_requests = [] - self.retryable_codes = retryable_codes or [] - self.batch_url = batch_url or 'https://www.googleapis.com/batch' - - def Add(self, service, method, request, global_params=None): - """Add a request to the batch. - - Args: - service: A class inheriting base_api.BaseApiService. - method: A string indicated desired method from the service. See - the example in the class docstring. - request: An input message appropriate for the specified service.method. - global_params: Optional additional parameters to pass into - method.PrepareHttpRequest. - - Returns: - None - """ - # Retrieve the configs for the desired method and service. - method_config = service.GetMethodConfig(method) - upload_config = service.GetUploadConfig(method) - - # Prepare the HTTP Request. - http_request = service.PrepareHttpRequest( - method_config, request, global_params=global_params, - upload_config=upload_config) - - # Create the request and add it to our master list. - api_request = self.ApiCall( - http_request, self.retryable_codes, service, method_config) - self.api_requests.append(api_request) - - def Execute(self, http, sleep_between_polls=5, max_retries=5): - """Execute all of the requests in the batch. - - Args: - http: httplib2.Http object for use in the request. - sleep_between_polls: Integer number of seconds to sleep between polls. - max_retries: Max retries. Any requests that have not succeeded by - this number of retries simply report the last response or - exception, whatever it happened to be. - - Returns: - List of ApiCalls. - """ - requests = [request for request in self.api_requests if not - request.terminal_state] - - for attempt in range(max_retries): - if attempt: - time.sleep(sleep_between_polls) - - # Create a batch_http_request object and populate it with incomplete - # requests. - batch_http_request = BatchHttpRequest(batch_url=self.batch_url) - for request in requests: - batch_http_request.Add(request.http_request, request.HandleResponse) - batch_http_request.Execute(http) - # Collect retryable requests. - requests = [request for request in self.api_requests if not - request.terminal_state] - - if (any(request.authorization_failed for request in requests) - and hasattr(http.request, 'credentials')): - http.request.credentials.refresh(http) - - if not requests: - break +class BatchApiRequest(object): - return self.api_requests + class ApiCall(object): + + """Holds request and response information for each request. + + ApiCalls are ultimately exposed to the client once the HTTP batch request + has been completed. + + Attributes: + http_request: A client-supplied http_wrapper.Request to be + submitted to the server. + response: A http_wrapper.Response object given by the server as a + response to the user request, or None if an error occurred. + exception: An apiclient.errors.HttpError object if an error + occurred, or None. + """ + + def __init__(self, request, retryable_codes, service, method_config): + """Initialize an individual API request. + + Args: + request: An http_wrapper.Request object. + retryable_codes: A list of integer HTTP codes that can be retried. + service: A service inheriting from base_api.BaseApiService. + method_config: Method config for the desired API request. + """ + self.__retryable_codes = list( + set(retryable_codes + [http_client.UNAUTHORIZED])) + self.__http_response = None + self.__service = service + self.__method_config = method_config + + self.http_request = request + # TODO(user): Add some validation to these fields. + self.__response = None + self.__exception = None + + @property + def is_error(self): + return self.exception is not None + + @property + def response(self): + return self.__response + + @property + def exception(self): + return self.__exception + + @property + def authorization_failed(self): + return (self.__http_response and ( + self.__http_response.status_code == http_client.UNAUTHORIZED)) + + @property + def terminal_state(self): + return (self.__http_response and ( + self.__http_response.status_code not in self.__retryable_codes)) + + def HandleResponse(self, http_response, exception): + """Handles an incoming http response to the request in http_request. + + This is intended to be used as a callback function for + BatchHttpRequest.Add. + + Args: + http_response: Deserialized http_wrapper.Response object. + exception: apiclient.errors.HttpError object if an error occurred. + """ + self.__http_response = http_response + self.__exception = exception + if self.terminal_state and not self.__exception: + self.__response = self.__service.ProcessHttpResponse( + self.__method_config, self.__http_response) + + def __init__(self, batch_url=None, retryable_codes=None): + """Initialize a batch API request object. + + Args: + batch_url: Base URL for batch API calls. + retryable_codes: A list of integer HTTP codes that can be retried. + """ + self.api_requests = [] + self.retryable_codes = retryable_codes or [] + self.batch_url = batch_url or 'https://www.googleapis.com/batch' + + def Add(self, service, method, request, global_params=None): + """Add a request to the batch. + + Args: + service: A class inheriting base_api.BaseApiService. + method: A string indicated desired method from the service. See + the example in the class docstring. + request: An input message appropriate for the specified service.method. + global_params: Optional additional parameters to pass into + method.PrepareHttpRequest. + + Returns: + None + """ + # Retrieve the configs for the desired method and service. + method_config = service.GetMethodConfig(method) + upload_config = service.GetUploadConfig(method) + + # Prepare the HTTP Request. + http_request = service.PrepareHttpRequest( + method_config, request, global_params=global_params, + upload_config=upload_config) + + # Create the request and add it to our master list. + api_request = self.ApiCall( + http_request, self.retryable_codes, service, method_config) + self.api_requests.append(api_request) + + def Execute(self, http, sleep_between_polls=5, max_retries=5): + """Execute all of the requests in the batch. + + Args: + http: httplib2.Http object for use in the request. + sleep_between_polls: Integer number of seconds to sleep between polls. + max_retries: Max retries. Any requests that have not succeeded by + this number of retries simply report the last response or + exception, whatever it happened to be. + + Returns: + List of ApiCalls. + """ + requests = [request for request in self.api_requests if not + request.terminal_state] + + for attempt in range(max_retries): + if attempt: + time.sleep(sleep_between_polls) + + # Create a batch_http_request object and populate it with incomplete + # requests. + batch_http_request = BatchHttpRequest(batch_url=self.batch_url) + for request in requests: + batch_http_request.Add( + request.http_request, request.HandleResponse) + batch_http_request.Execute(http) + + # Collect retryable requests. + requests = [request for request in self.api_requests if not + request.terminal_state] + + if (any(request.authorization_failed for request in requests) + and hasattr(http.request, 'credentials')): + http.request.credentials.refresh(http) + + if not requests: + break + + return self.api_requests class BatchHttpRequest(object): - """Batches multiple http_wrapper.Request objects into a single request.""" - - def __init__(self, batch_url, callback=None): - """Constructor for a BatchHttpRequest. - - Args: - batch_url: URL to send batch requests to. - callback: A callback to be called for each response, of the - form callback(response, exception). The first parameter is - the deserialized Response object. The second is an - apiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no error occurred. - """ - # Endpoint to which these requests are sent. - self.__batch_url = batch_url - - # Global callback to be called for each individual response in the batch. - self.__callback = callback - - # List of requests, responses and handlers. - self.__request_response_handlers = {} - - # The last auto generated id. - self.__last_auto_id = itertools.count() - - # Unique ID on which to base the Content-ID headers. - self.__base_id = uuid.uuid4() - - def _ConvertIdToHeader(self, request_id): - """Convert an id to a Content-ID header value. - - Args: - request_id: String identifier for a individual request. - - Returns: - A Content-ID header with the id_ encoded into it. A UUID is prepended to - the value because Content-ID headers are supposed to be universally - unique. - """ - return '<%s+%s>' % (self.__base_id, urllib_parse.quote(request_id)) - - @staticmethod - def _ConvertHeaderToId(header): - """Convert a Content-ID header value to an id. - - Presumes the Content-ID header conforms to the format that - _ConvertIdToHeader() returns. - - Args: - header: A string indicating the Content-ID header value. - - Returns: - The extracted id value. - Raises: - BatchError if the header is not in the expected format. - """ - if not (header.startswith('<') or header.endswith('>')): - raise exceptions.BatchError('Invalid value for Content-ID: %s' % header) - if '+' not in header: - raise exceptions.BatchError('Invalid value for Content-ID: %s' % header) - _, request_id = header[1:-1].rsplit('+', 1) - - return urllib_parse.unquote(request_id) - - def _SerializeRequest(self, request): - """Convert a http_wrapper.Request object into a string. - - Args: - request: A http_wrapper.Request to serialize. - - Returns: - The request as a string in application/http format. - """ - # Construct status line - parsed = urllib_parse.urlsplit(request.url) - request_line = urllib_parse.urlunsplit( - (None, None, parsed.path, parsed.query, None)) - status_line = request.http_method + ' ' + request_line + ' HTTP/1.1\n' - major, minor = request.headers.get( - 'content-type', 'application/json').split('/') - msg = mime_nonmultipart.MIMENonMultipart(major, minor) - - # MIMENonMultipart adds its own Content-Type header. - # Keep all of the other headers in `request.headers`. - for key, value in request.headers.items(): - if key == 'content-type': - continue - msg[key] = value - - msg['Host'] = parsed.netloc - msg.set_unixfrom(None) - - if request.body is not None: - msg.set_payload(request.body) - - # Serialize the mime message. - str_io = io.StringIO() - # maxheaderlen=0 means don't line wrap headers. - gen = generator.Generator(str_io, maxheaderlen=0) - gen.flatten(msg, unixfrom=False) - body = str_io.getvalue() - - # Strip off the \n\n that the MIME lib tacks onto the end of the payload. - if request.body is None: - body = body[:-2] - - return status_line.encode('utf-8') + body - - def _DeserializeResponse(self, payload): - """Convert string into Response and content. - - Args: - payload: Header and body string to be deserialized. - - Returns: - A Response object - """ - # Strip off the status line. - status_line, payload = payload.split('\n', 1) - _, status, _ = status_line.split(' ', 2) - - # Parse the rest of the response. - parser = email_parser.Parser() - msg = parser.parsestr(payload) - - # Get the headers. - info = dict(msg) - info['status'] = status - - # Create Response from the parsed headers. - content = msg.get_payload() - - return http_wrapper.Response(info, content, self.__batch_url) - - def _NewId(self): - """Create a new id. - - Auto incrementing number that avoids conflicts with ids already used. + """Batches multiple http_wrapper.Request objects into a single request.""" + + def __init__(self, batch_url, callback=None): + """Constructor for a BatchHttpRequest. + + Args: + batch_url: URL to send batch requests to. + callback: A callback to be called for each response, of the + form callback(response, exception). The first parameter is + the deserialized Response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no error occurred. + """ + # Endpoint to which these requests are sent. + self.__batch_url = batch_url + + # Global callback to be called for each individual response in the + # batch. + self.__callback = callback + + # List of requests, responses and handlers. + self.__request_response_handlers = {} + + # The last auto generated id. + self.__last_auto_id = itertools.count() + + # Unique ID on which to base the Content-ID headers. + self.__base_id = uuid.uuid4() + + def _ConvertIdToHeader(self, request_id): + """Convert an id to a Content-ID header value. + + Args: + request_id: String identifier for a individual request. + + Returns: + A Content-ID header with the id_ encoded into it. A UUID is prepended to + the value because Content-ID headers are supposed to be universally + unique. + """ + return '<%s+%s>' % (self.__base_id, urllib_parse.quote(request_id)) + + @staticmethod + def _ConvertHeaderToId(header): + """Convert a Content-ID header value to an id. + + Presumes the Content-ID header conforms to the format that + _ConvertIdToHeader() returns. + + Args: + header: A string indicating the Content-ID header value. + + Returns: + The extracted id value. + + Raises: + BatchError if the header is not in the expected format. + """ + if not (header.startswith('<') or header.endswith('>')): + raise exceptions.BatchError( + 'Invalid value for Content-ID: %s' % header) + if '+' not in header: + raise exceptions.BatchError( + 'Invalid value for Content-ID: %s' % header) + _, request_id = header[1:-1].rsplit('+', 1) + + return urllib_parse.unquote(request_id) + + def _SerializeRequest(self, request): + """Convert a http_wrapper.Request object into a string. + + Args: + request: A http_wrapper.Request to serialize. + + Returns: + The request as a string in application/http format. + """ + # Construct status line + parsed = urllib_parse.urlsplit(request.url) + request_line = urllib_parse.urlunsplit( + (None, None, parsed.path, parsed.query, None)) + status_line = request.http_method + ' ' + request_line + ' HTTP/1.1\n' + major, minor = request.headers.get( + 'content-type', 'application/json').split('/') + msg = mime_nonmultipart.MIMENonMultipart(major, minor) + + # MIMENonMultipart adds its own Content-Type header. + # Keep all of the other headers in `request.headers`. + for key, value in request.headers.items(): + if key == 'content-type': + continue + msg[key] = value + + msg['Host'] = parsed.netloc + msg.set_unixfrom(None) + + if request.body is not None: + msg.set_payload(request.body) + + # Serialize the mime message. + str_io = io.StringIO() + # maxheaderlen=0 means don't line wrap headers. + gen = generator.Generator(str_io, maxheaderlen=0) + gen.flatten(msg, unixfrom=False) + body = str_io.getvalue() + + # Strip off the \n\n that the MIME lib tacks onto the end of the + # payload. + if request.body is None: + body = body[:-2] + + return status_line.encode('utf-8') + body + + def _DeserializeResponse(self, payload): + """Convert string into Response and content. + + Args: + payload: Header and body string to be deserialized. + + Returns: + A Response object + """ + # Strip off the status line. + status_line, payload = payload.split('\n', 1) + _, status, _ = status_line.split(' ', 2) + + # Parse the rest of the response. + parser = email_parser.Parser() + msg = parser.parsestr(payload) + + # Get the headers. + info = dict(msg) + info['status'] = status + + # Create Response from the parsed headers. + content = msg.get_payload() + + return http_wrapper.Response(info, content, self.__batch_url) + + def _NewId(self): + """Create a new id. + + Auto incrementing number that avoids conflicts with ids already used. + + Returns: + A new unique id string. + """ + return str(next(self.__last_auto_id)) + + def Add(self, request, callback=None): + """Add a new request. + + Args: + request: A http_wrapper.Request to add to the batch. + callback: A callback to be called for this response, of the + form callback(response, exception). The first parameter is the + deserialized response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no errors occurred. + + Returns: + None + """ + self.__request_response_handlers[self._NewId()] = RequestResponseAndHandler( + request, None, callback) + + def _Execute(self, http): + """Serialize batch request, send to server, process response. + + Args: + http: A httplib2.Http object to be used to make the request with. + + Raises: + httplib2.HttpLib2Error if a transport error has occured. + apiclient.errors.BatchError if the response is the wrong format. + """ + message = mime_multipart.MIMEMultipart('mixed') + # Message should not write out its own headers. + setattr(message, '_write_headers', lambda self: None) + + # Add all the individual requests. + for key in self.__request_response_handlers: + msg = mime_nonmultipart.MIMENonMultipart('application', 'http') + msg['Content-Transfer-Encoding'] = 'binary' + msg['Content-ID'] = self._ConvertIdToHeader(key) - Returns: - A new unique id string. - """ - return str(next(self.__last_auto_id)) - - def Add(self, request, callback=None): - """Add a new request. + body = self._SerializeRequest( + self.__request_response_handlers[key].request) + msg.set_payload(body) + message.attach(msg) - Args: - request: A http_wrapper.Request to add to the batch. - callback: A callback to be called for this response, of the - form callback(response, exception). The first parameter is the - deserialized response object. The second is an - apiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no errors occurred. - - Returns: - None - """ - self.__request_response_handlers[self._NewId()] = RequestResponseAndHandler( - request, None, callback) + request = http_wrapper.Request(self.__batch_url, 'POST') + request.body = message.as_string() + request.headers['content-type'] = ( + 'multipart/mixed; boundary="%s"') % message.get_boundary() - def _Execute(self, http): - """Serialize batch request, send to server, process response. - - Args: - http: A httplib2.Http object to be used to make the request with. - - Raises: - httplib2.HttpLib2Error if a transport error has occured. - apiclient.errors.BatchError if the response is the wrong format. - """ - message = mime_multipart.MIMEMultipart('mixed') - # Message should not write out its own headers. - setattr(message, '_write_headers', lambda self: None) + response = http_wrapper.MakeRequest(http, request) - # Add all the individual requests. - for key in self.__request_response_handlers: - msg = mime_nonmultipart.MIMENonMultipart('application', 'http') - msg['Content-Transfer-Encoding'] = 'binary' - msg['Content-ID'] = self._ConvertIdToHeader(key) + if response.status_code >= 300: + raise exceptions.HttpError.FromResponse(response) - body = self._SerializeRequest( - self.__request_response_handlers[key].request) - msg.set_payload(body) - message.attach(msg) + # Prepend with a content-type header so Parser can handle it. + header = 'content-type: %s\r\n\r\n' % response.info['content-type'] - request = http_wrapper.Request(self.__batch_url, 'POST') - request.body = message.as_string() - request.headers['content-type'] = ( - 'multipart/mixed; boundary="%s"') % message.get_boundary() + parser = email_parser.Parser() + mime_response = parser.parsestr(header + response.content) - response = http_wrapper.MakeRequest(http, request) + if not mime_response.is_multipart(): + raise exceptions.BatchError( + 'Response not in multipart/mixed format.') - if response.status_code >= 300: - raise exceptions.HttpError.FromResponse(response) + for part in mime_response.get_payload(): + request_id = self._ConvertHeaderToId(part['Content-ID']) + response = self._DeserializeResponse(part.get_payload()) - # Prepend with a content-type header so Parser can handle it. - header = 'content-type: %s\r\n\r\n' % response.info['content-type'] + # Disable protected access because namedtuple._replace(...) + # is not actually meant to be protected. + self.__request_response_handlers[request_id] = ( + self.__request_response_handlers[request_id]._replace( # pylint: disable=protected-access + response=response)) - parser = email_parser.Parser() - mime_response = parser.parsestr(header + response.content) + def Execute(self, http): + """Execute all the requests as a single batched HTTP request. - if not mime_response.is_multipart(): - raise exceptions.BatchError('Response not in multipart/mixed format.') + Args: + http: A httplib2.Http object to be used with the request. - for part in mime_response.get_payload(): - request_id = self._ConvertHeaderToId(part['Content-ID']) - response = self._DeserializeResponse(part.get_payload()) - - # Disable protected access because namedtuple._replace(...) - # is not actually meant to be protected. - self.__request_response_handlers[request_id] = ( - self.__request_response_handlers[request_id]._replace( # pylint: disable=protected-access - response=response)) - - def Execute(self, http): - """Execute all the requests as a single batched HTTP request. - - Args: - http: A httplib2.Http object to be used with the request. - - Returns: - None - - Raises: - BatchError if the response is the wrong format. - """ + Returns: + None - self._Execute(http) + Raises: + BatchError if the response is the wrong format. + """ - for key in self.__request_response_handlers: - response = self.__request_response_handlers[key].response - callback = self.__request_response_handlers[key].handler + self._Execute(http) - exception = None + for key in self.__request_response_handlers: + response = self.__request_response_handlers[key].response + callback = self.__request_response_handlers[key].handler - if response.status_code >= 300: - exception = exceptions.HttpError.FromResponse(response) + exception = None - if callback is not None: - callback(response, exception) - if self.__callback is not None: - self.__callback(response, exception) + if response.status_code >= 300: + exception = exceptions.HttpError.FromResponse(response) + + if callback is not None: + callback(response, exception) + if self.__callback is not None: + self.__callback(response, exception) diff --git a/apitools/base/py/buffered_stream.py b/apitools/base/py/buffered_stream.py index f1d84fe..bda7e65 100644 --- a/apitools/base/py/buffered_stream.py +++ b/apitools/base/py/buffered_stream.py @@ -9,49 +9,51 @@ from apitools.base.py import exceptions # TODO(user): Consider replacing this with a StringIO. class BufferedStream(object): - """Buffers a stream, reading ahead to determine if we're at the end.""" - - def __init__(self, stream, start, size): - self.__stream = stream - self.__start_pos = start - self.__buffer_pos = 0 - self.__buffered_data = self.__stream.read(size) - self.__stream_at_end = len(self.__buffered_data) < size - self.__end_pos = self.__start_pos + len(self.__buffered_data) - - def __str__(self): - return ('Buffered stream %s from position %s-%s with %s ' - 'bytes remaining' % (self.__stream, self.__start_pos, - self.__end_pos, self._bytes_remaining)) - - def __len__(self): - return len(self.__buffered_data) - - @property - def stream_exhausted(self): - return self.__stream_at_end - - @property - def stream_end_position(self): - return self.__end_pos - - @property - def _bytes_remaining(self): - return len(self.__buffered_data) - self.__buffer_pos - - def read(self, size=None): # pylint: disable=invalid-name - """Reads from the buffer.""" - if size is None or size < 0: - raise exceptions.NotYetImplementedError( - 'Illegal read of size %s requested on BufferedStream. ' - 'Wrapped stream %s is at position %s-%s, ' - '%s bytes remaining.' % - (size, self.__stream, self.__start_pos, self.__end_pos, - self._bytes_remaining)) - - data = '' - if self._bytes_remaining: - size = min(size, self._bytes_remaining) - data = self.__buffered_data[self.__buffer_pos:self.__buffer_pos + size] - self.__buffer_pos += size - return data + + """Buffers a stream, reading ahead to determine if we're at the end.""" + + def __init__(self, stream, start, size): + self.__stream = stream + self.__start_pos = start + self.__buffer_pos = 0 + self.__buffered_data = self.__stream.read(size) + self.__stream_at_end = len(self.__buffered_data) < size + self.__end_pos = self.__start_pos + len(self.__buffered_data) + + def __str__(self): + return ('Buffered stream %s from position %s-%s with %s ' + 'bytes remaining' % (self.__stream, self.__start_pos, + self.__end_pos, self._bytes_remaining)) + + def __len__(self): + return len(self.__buffered_data) + + @property + def stream_exhausted(self): + return self.__stream_at_end + + @property + def stream_end_position(self): + return self.__end_pos + + @property + def _bytes_remaining(self): + return len(self.__buffered_data) - self.__buffer_pos + + def read(self, size=None): # pylint: disable=invalid-name + """Reads from the buffer.""" + if size is None or size < 0: + raise exceptions.NotYetImplementedError( + 'Illegal read of size %s requested on BufferedStream. ' + 'Wrapped stream %s is at position %s-%s, ' + '%s bytes remaining.' % + (size, self.__stream, self.__start_pos, self.__end_pos, + self._bytes_remaining)) + + data = '' + if self._bytes_remaining: + size = min(size, self._bytes_remaining) + data = self.__buffered_data[ + self.__buffer_pos:self.__buffer_pos + size] + self.__buffer_pos += size + return data diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py index 89c9af7..02f515f 100644 --- a/apitools/base/py/buffered_stream_test.py +++ b/apitools/base/py/buffered_stream_test.py @@ -12,46 +12,46 @@ from apitools.base.py import exceptions class BufferedStreamTest(unittest2.TestCase): - def setUp(self): - self.stream = six.StringIO(string.ascii_letters) - self.value = self.stream.getvalue() - self.stream.seek(0) - - def testEmptyBuffer(self): - bs = buffered_stream.BufferedStream(self.stream, 0, 0) - self.assertEqual('', bs.read(0)) - self.assertEqual(0, bs.stream_end_position) - - def testOffsetStream(self): - bs = buffered_stream.BufferedStream(self.stream, 50, 100) - self.assertEqual(len(self.value), len(bs)) - self.assertEqual(self.value, bs.read(len(self.value))) - self.assertEqual(50 + len(self.value), bs.stream_end_position) - - def testUnexhaustedStream(self): - bs = buffered_stream.BufferedStream(self.stream, 0, 50) - self.assertEqual(50, bs.stream_end_position) - self.assertEqual(False, bs.stream_exhausted) - self.assertEqual(self.value[0:50], bs.read(50)) - self.assertEqual(False, bs.stream_exhausted) - self.assertEqual('', bs.read(0)) - self.assertEqual('', bs.read(100)) - - def testExhaustedStream(self): - bs = buffered_stream.BufferedStream(self.stream, 0, 100) - self.assertEqual(len(self.value), bs.stream_end_position) - self.assertEqual(True, bs.stream_exhausted) - self.assertEqual(self.value, bs.read(100)) - self.assertEqual('', bs.read(0)) - self.assertEqual('', bs.read(100)) - - def testArbitraryLengthRead(self): - bs = buffered_stream.BufferedStream(self.stream, 0, 20) - with self.assertRaises(exceptions.NotYetImplementedError): - bs.read() - with self.assertRaises(exceptions.NotYetImplementedError): - bs.read(size=-1) + def setUp(self): + self.stream = six.StringIO(string.ascii_letters) + self.value = self.stream.getvalue() + self.stream.seek(0) + + def testEmptyBuffer(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 0) + self.assertEqual('', bs.read(0)) + self.assertEqual(0, bs.stream_end_position) + + def testOffsetStream(self): + bs = buffered_stream.BufferedStream(self.stream, 50, 100) + self.assertEqual(len(self.value), len(bs)) + self.assertEqual(self.value, bs.read(len(self.value))) + self.assertEqual(50 + len(self.value), bs.stream_end_position) + + def testUnexhaustedStream(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 50) + self.assertEqual(50, bs.stream_end_position) + self.assertEqual(False, bs.stream_exhausted) + self.assertEqual(self.value[0:50], bs.read(50)) + self.assertEqual(False, bs.stream_exhausted) + self.assertEqual('', bs.read(0)) + self.assertEqual('', bs.read(100)) + + def testExhaustedStream(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 100) + self.assertEqual(len(self.value), bs.stream_end_position) + self.assertEqual(True, bs.stream_exhausted) + self.assertEqual(self.value, bs.read(100)) + self.assertEqual('', bs.read(0)) + self.assertEqual('', bs.read(100)) + + def testArbitraryLengthRead(self): + bs = buffered_stream.BufferedStream(self.stream, 0, 20) + with self.assertRaises(exceptions.NotYetImplementedError): + bs.read() + with self.assertRaises(exceptions.NotYetImplementedError): + bs.read(size=-1) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index e69e84a..61a13a5 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -24,10 +24,10 @@ from apitools.base.py import exceptions from apitools.base.py import util try: - import gflags as flags - FLAGS = flags.FLAGS + import gflags as flags + FLAGS = flags.FLAGS except ImportError: - FLAGS = None + FLAGS = None __all__ = [ @@ -41,7 +41,6 @@ __all__ = [ ] - # TODO(craigcitro): Expose the extra args here somewhere higher up, # possibly as flags in the generated CLI. def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, @@ -49,300 +48,306 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, service_account_name=None, service_account_keyfile=None, service_account_json_keyfile=None, api_key=None, client=None): - """Attempt to get credentials, using an oauth dance as the last resort.""" - scopes = util.NormalizeScopes(scopes) - if ((service_account_name and not service_account_keyfile) or - (service_account_keyfile and not service_account_name)): - raise exceptions.CredentialsError( - 'Service account name or keyfile provided without the other') - # TODO(craigcitro): Error checking. - client_info = { - 'client_id': client_id, - 'client_secret': client_secret, - 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), - 'user_agent': user_agent or '%s-generated/0.1' % package_name, - } - service_account_kwargs = { - 'user_agent': client_info['user_agent'], - } - if service_account_json_keyfile: - with open(service_account_json_keyfile) as keyfile: - service_account_info = json.load(keyfile) - if service_account_info.get('type') != oauth2client.client.SERVICE_ACCOUNT: - raise exceptions.CredentialsError( - 'Invalid service account credentials: %s' % ( - service_account_json_keyfile,)) - credentials = oauth2client.service_account._ServiceAccountCredentials( # pylint: disable=protected-access - service_account_id=service_account_info['client_id'], - service_account_email=service_account_info['client_email'], - private_key_id=service_account_info['private_key_id'], - private_key_pkcs8_text=service_account_info['private_key'], - scopes=scopes, - **service_account_kwargs) - return credentials - if service_account_name is not None: - credentials = ServiceAccountCredentialsFromFile( - service_account_name, service_account_keyfile, scopes, - service_account_kwargs=service_account_kwargs) + """Attempt to get credentials, using an oauth dance as the last resort.""" + scopes = util.NormalizeScopes(scopes) + if ((service_account_name and not service_account_keyfile) or + (service_account_keyfile and not service_account_name)): + raise exceptions.CredentialsError( + 'Service account name or keyfile provided without the other') + # TODO(craigcitro): Error checking. + client_info = { + 'client_id': client_id, + 'client_secret': client_secret, + 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), + 'user_agent': user_agent or '%s-generated/0.1' % package_name, + } + service_account_kwargs = { + 'user_agent': client_info['user_agent'], + } + if service_account_json_keyfile: + with open(service_account_json_keyfile) as keyfile: + service_account_info = json.load(keyfile) + if service_account_info.get('type') != oauth2client.client.SERVICE_ACCOUNT: + raise exceptions.CredentialsError( + 'Invalid service account credentials: %s' % ( + service_account_json_keyfile,)) + credentials = oauth2client.service_account._ServiceAccountCredentials( # pylint: disable=protected-access + service_account_id=service_account_info['client_id'], + service_account_email=service_account_info['client_email'], + private_key_id=service_account_info['private_key_id'], + private_key_pkcs8_text=service_account_info['private_key'], + scopes=scopes, + **service_account_kwargs) + return credentials + if service_account_name is not None: + credentials = ServiceAccountCredentialsFromFile( + service_account_name, service_account_keyfile, scopes, + service_account_kwargs=service_account_kwargs) + if credentials is not None: + return credentials + credentials = GaeAssertionCredentials.Get(scopes) if credentials is not None: - return credentials - credentials = GaeAssertionCredentials.Get(scopes) - if credentials is not None: - return credentials - credentials = GceAssertionCredentials.Get(scopes) - if credentials is not None: - return credentials - credentials_filename = credentials_filename or os.path.expanduser( - '~/.apitools.token') - credentials = CredentialsFromFile(credentials_filename, client_info) - if credentials is not None: - return credentials - raise exceptions.CredentialsError('Could not create valid credentials') + return credentials + credentials = GceAssertionCredentials.Get(scopes) + if credentials is not None: + return credentials + credentials_filename = credentials_filename or os.path.expanduser( + '~/.apitools.token') + credentials = CredentialsFromFile(credentials_filename, client_info) + if credentials is not None: + return credentials + raise exceptions.CredentialsError('Could not create valid credentials') def ServiceAccountCredentialsFromFile( - service_account_name, private_key_filename, scopes, - service_account_kwargs=None): - with open(private_key_filename) as key_file: - return ServiceAccountCredentials( - service_account_name, key_file.read(), scopes, - service_account_kwargs=service_account_kwargs) + service_account_name, private_key_filename, scopes, + service_account_kwargs=None): + with open(private_key_filename) as key_file: + return ServiceAccountCredentials( + service_account_name, key_file.read(), scopes, + service_account_kwargs=service_account_kwargs) def ServiceAccountCredentials(service_account_name, private_key, scopes, service_account_kwargs=None): - service_account_kwargs = service_account_kwargs or {} - scopes = util.NormalizeScopes(scopes) - return oauth2client.client.SignedJwtAssertionCredentials( - service_account_name, private_key, scopes, **service_account_kwargs) + service_account_kwargs = service_account_kwargs or {} + scopes = util.NormalizeScopes(scopes) + return oauth2client.client.SignedJwtAssertionCredentials( + service_account_name, private_key, scopes, **service_account_kwargs) def _EnsureFileExists(filename): - """Touches a file; returns False on error, True on success.""" - if not os.path.exists(filename): - old_umask = os.umask(0o177) - try: - open(filename, 'a+b').close() - except OSError: - return False - finally: - os.umask(old_umask) - return True + """Touches a file; returns False on error, True on success.""" + if not os.path.exists(filename): + old_umask = os.umask(0o177) + try: + open(filename, 'a+b').close() + except OSError: + return False + finally: + os.umask(old_umask) + return True def _OpenNoProxy(request): - """Wrapper around urllib2.open that ignores proxies.""" - opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - return opener.open(request) + """Wrapper around urllib2.open that ignores proxies.""" + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + return opener.open(request) # TODO(craigcitro): We override to add some utility code, and to # update the old refresh implementation. Push this code into # oauth2client. class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): - """Assertion credentials for GCE instances.""" - def __init__(self, scopes=None, service_account_name='default', **kwds): - """Initializes the credentials instance. - - Args: - scopes: The scopes to get. If None, whatever scopes that are available - to the instance are used. - service_account_name: The service account to retrieve the scopes from. - **kwds: Additional keyword args. - """ - # If there is a connectivity issue with the metadata server, - # detection calls may fail even if we've already successfully identified - # these scopes in the same execution. However, the available scopes don't - # change once an instance is created, so there is no reason to perform - # more than one query. - # - # TODO(craigcitro): Move this into oauth2client. - self.__service_account_name = service_account_name - cache_filename = None - cached_scopes = None - if 'cache_filename' in kwds: - cache_filename = kwds['cache_filename'] - cached_scopes = self._CheckCacheFileForMatch(cache_filename, scopes) - - scopes = cached_scopes or self._ScopesFromMetadataServer(scopes) - - if cache_filename and not cached_scopes: - self._WriteCacheFile(cache_filename, scopes) - - super(GceAssertionCredentials, self).__init__(scopes, **kwds) - - @classmethod - def Get(cls, *args, **kwds): - try: - return cls(*args, **kwds) - except exceptions.Error: - return None - - def _CheckCacheFileForMatch(self, cache_filename, scopes): - """Checks the cache file to see if it matches the given credentials. - - Args: - cache_filename: Cache filename to check. - scopes: Scopes for the desired credentials. - - Returns: - List of scopes (if cache matches) or None. - """ - creds = { # Credentials metadata dict. - 'scopes': sorted(list(scopes)) if scopes else None, - 'svc_acct_name': self.__service_account_name} - if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( - cache_filename, 'r+b', 'rb') - try: - locked_file.open_and_lock() - cached_creds_str = locked_file.file_handle().read() - if cached_creds_str: - # Cached credentials metadata dict. - cached_creds = json.loads(cached_creds_str) - if (creds['svc_acct_name'] == cached_creds['svc_acct_name'] and - (creds['scopes'] is None or - creds['scopes'] == cached_creds['scopes'])): - scopes = cached_creds['scopes'] - finally: - locked_file.unlock_and_close() - return scopes - - def _WriteCacheFile(self, cache_filename, scopes): - """Writes the credential metadata to the cache file. - - This does not save the credentials themselves (CredentialStore class - optionally handles that after this class is initialized). - - Args: - cache_filename: Cache filename to check. - scopes: Scopes for the desired credentials. - """ - if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( - cache_filename, 'r+b', 'rb') - try: - locked_file.open_and_lock() - if locked_file.is_locked(): - creds = { # Credentials metadata dict. - 'scopes': sorted(list(scopes)), - 'svc_acct_name': self.__service_account_name} - locked_file.file_handle().write(json.dumps(creds, encoding='ascii')) - # If it's not locked, the locking process will write the same - # data to the file, so just continue. - finally: - locked_file.unlock_and_close() - - def _ScopesFromMetadataServer(self, scopes): - if not util.DetectGce(): - raise exceptions.ResourceUnavailableError( - 'GCE credentials requested outside a GCE instance') - if not self.GetServiceAccount(self.__service_account_name): - raise exceptions.ResourceUnavailableError( - 'GCE credentials requested but service account %s does not exist.' % - self.__service_account_name) - if scopes: - scope_ls = util.NormalizeScopes(scopes) - instance_scopes = self.GetInstanceScopes() - if scope_ls > instance_scopes: - raise exceptions.CredentialsError( - 'Instance did not have access to scopes %s' % ( - sorted(list(scope_ls - instance_scopes)),)) - else: - scopes = self.GetInstanceScopes() - return scopes - - def GetServiceAccount(self, account): - account_uri = ( - 'http://metadata.google.internal/computeMetadata/' - 'v1/instance/service-accounts') - additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request(account_uri, headers=additional_headers) - try: - response = _OpenNoProxy(request) - except urllib.error.URLError as e: - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) - response_lines = [line.rstrip('/\n\r') for line in response.readlines()] - return account in response_lines - - def GetInstanceScopes(self): - # Extra header requirement can be found here: - # https://developers.google.com/compute/docs/metadata - scopes_uri = ( - 'http://metadata.google.internal/computeMetadata/v1/instance/' - 'service-accounts/%s/scopes') % self.__service_account_name - additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request(scopes_uri, headers=additional_headers) - try: - response = _OpenNoProxy(request) - except urllib.error.URLError as e: - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) - return util.NormalizeScopes(scope.strip() for scope in response.readlines()) - - def _refresh(self, do_request): - """Refresh self.access_token. - - This function replaces AppAssertionCredentials._refresh, which does not use - the credential store and is therefore poorly suited for multi-threaded - scenarios. - - Args: - do_request: A function matching httplib2.Http.request's signature. - """ - # pylint: disable=protected-access - oauth2client.client.OAuth2Credentials._refresh(self, do_request) - # pylint: enable=protected-access - - def _do_refresh_request(self, unused_http_request): - """Refresh self.access_token by querying the metadata server. - - If self.store is initialized, store acquired credentials there. - """ - token_uri = ( - 'http://metadata.google.internal/computeMetadata/v1/instance/' - 'service-accounts/%s/token') % self.__service_account_name - extra_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request(token_uri, headers=extra_headers) - try: - content = _OpenNoProxy(request).read() - except urllib.error.URLError as e: - self.invalid = True - if self.store: - self.store.locked_put(self) - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) - try: - credential_info = json.loads(content) - except ValueError: - raise exceptions.CredentialsError( - 'Invalid credentials response: uri %s' % token_uri) - - self.access_token = credential_info['access_token'] - if 'expires_in' in credential_info: - self.token_expiry = ( - datetime.timedelta(seconds=int(credential_info['expires_in'])) + - datetime.datetime.utcnow()) - else: - self.token_expiry = None - self.invalid = False - if self.store: - self.store.locked_put(self) - - @classmethod - def from_json(cls, json_data): - data = json.loads(json_data) - credentials = GceAssertionCredentials(scopes=[data['scope']]) - if 'access_token' in data: - credentials.access_token = data['access_token'] - if 'token_expiry' in data: - credentials.token_expiry = datetime.datetime.strptime( - data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) - if 'invalid' in data: - credentials.invalid = data['invalid'] - return credentials + """Assertion credentials for GCE instances.""" + + def __init__(self, scopes=None, service_account_name='default', **kwds): + """Initializes the credentials instance. + + Args: + scopes: The scopes to get. If None, whatever scopes that are available + to the instance are used. + service_account_name: The service account to retrieve the scopes from. + **kwds: Additional keyword args. + """ + # If there is a connectivity issue with the metadata server, + # detection calls may fail even if we've already successfully identified + # these scopes in the same execution. However, the available scopes don't + # change once an instance is created, so there is no reason to perform + # more than one query. + # + # TODO(craigcitro): Move this into oauth2client. + self.__service_account_name = service_account_name + cache_filename = None + cached_scopes = None + if 'cache_filename' in kwds: + cache_filename = kwds['cache_filename'] + cached_scopes = self._CheckCacheFileForMatch( + cache_filename, scopes) + + scopes = cached_scopes or self._ScopesFromMetadataServer(scopes) + + if cache_filename and not cached_scopes: + self._WriteCacheFile(cache_filename, scopes) + + super(GceAssertionCredentials, self).__init__(scopes, **kwds) + + @classmethod + def Get(cls, *args, **kwds): + try: + return cls(*args, **kwds) + except exceptions.Error: + return None + + def _CheckCacheFileForMatch(self, cache_filename, scopes): + """Checks the cache file to see if it matches the given credentials. + + Args: + cache_filename: Cache filename to check. + scopes: Scopes for the desired credentials. + + Returns: + List of scopes (if cache matches) or None. + """ + creds = { # Credentials metadata dict. + 'scopes': sorted(list(scopes)) if scopes else None, + 'svc_acct_name': self.__service_account_name} + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + cached_creds_str = locked_file.file_handle().read() + if cached_creds_str: + # Cached credentials metadata dict. + cached_creds = json.loads(cached_creds_str) + if (creds['svc_acct_name'] == cached_creds['svc_acct_name'] and + (creds['scopes'] is None or + creds['scopes'] == cached_creds['scopes'])): + scopes = cached_creds['scopes'] + finally: + locked_file.unlock_and_close() + return scopes + + def _WriteCacheFile(self, cache_filename, scopes): + """Writes the credential metadata to the cache file. + + This does not save the credentials themselves (CredentialStore class + optionally handles that after this class is initialized). + + Args: + cache_filename: Cache filename to check. + scopes: Scopes for the desired credentials. + """ + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + if locked_file.is_locked(): + creds = { # Credentials metadata dict. + 'scopes': sorted(list(scopes)), + 'svc_acct_name': self.__service_account_name} + locked_file.file_handle().write( + json.dumps(creds, encoding='ascii')) + # If it's not locked, the locking process will write the same + # data to the file, so just continue. + finally: + locked_file.unlock_and_close() + + def _ScopesFromMetadataServer(self, scopes): + if not util.DetectGce(): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested outside a GCE instance') + if not self.GetServiceAccount(self.__service_account_name): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested but service account %s does not exist.' % + self.__service_account_name) + if scopes: + scope_ls = util.NormalizeScopes(scopes) + instance_scopes = self.GetInstanceScopes() + if scope_ls > instance_scopes: + raise exceptions.CredentialsError( + 'Instance did not have access to scopes %s' % ( + sorted(list(scope_ls - instance_scopes)),)) + else: + scopes = self.GetInstanceScopes() + return scopes + + def GetServiceAccount(self, account): + account_uri = ( + 'http://metadata.google.internal/computeMetadata/' + 'v1/instance/service-accounts') + additional_headers = {'X-Google-Metadata-Request': 'True'} + request = urllib.request.Request( + account_uri, headers=additional_headers) + try: + response = _OpenNoProxy(request) + except urllib.error.URLError as e: + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + response_lines = [line.rstrip('/\n\r') + for line in response.readlines()] + return account in response_lines + + def GetInstanceScopes(self): + # Extra header requirement can be found here: + # https://developers.google.com/compute/docs/metadata + scopes_uri = ( + 'http://metadata.google.internal/computeMetadata/v1/instance/' + 'service-accounts/%s/scopes') % self.__service_account_name + additional_headers = {'X-Google-Metadata-Request': 'True'} + request = urllib.request.Request( + scopes_uri, headers=additional_headers) + try: + response = _OpenNoProxy(request) + except urllib.error.URLError as e: + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + return util.NormalizeScopes(scope.strip() for scope in response.readlines()) + + def _refresh(self, do_request): + """Refresh self.access_token. + + This function replaces AppAssertionCredentials._refresh, which does not use + the credential store and is therefore poorly suited for multi-threaded + scenarios. + + Args: + do_request: A function matching httplib2.Http.request's signature. + """ + # pylint: disable=protected-access + oauth2client.client.OAuth2Credentials._refresh(self, do_request) + # pylint: enable=protected-access + + def _do_refresh_request(self, unused_http_request): + """Refresh self.access_token by querying the metadata server. + + If self.store is initialized, store acquired credentials there. + """ + token_uri = ( + 'http://metadata.google.internal/computeMetadata/v1/instance/' + 'service-accounts/%s/token') % self.__service_account_name + extra_headers = {'X-Google-Metadata-Request': 'True'} + request = urllib.request.Request(token_uri, headers=extra_headers) + try: + content = _OpenNoProxy(request).read() + except urllib.error.URLError as e: + self.invalid = True + if self.store: + self.store.locked_put(self) + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + try: + credential_info = json.loads(content) + except ValueError: + raise exceptions.CredentialsError( + 'Invalid credentials response: uri %s' % token_uri) + + self.access_token = credential_info['access_token'] + if 'expires_in' in credential_info: + self.token_expiry = ( + datetime.timedelta(seconds=int(credential_info['expires_in'])) + + datetime.datetime.utcnow()) + else: + self.token_expiry = None + self.invalid = False + if self.store: + self.store.locked_put(self) + + @classmethod + def from_json(cls, json_data): + data = json.loads(json_data) + credentials = GceAssertionCredentials(scopes=[data['scope']]) + if 'access_token' in data: + credentials.access_token = data['access_token'] + if 'token_expiry' in data: + credentials.token_expiry = datetime.datetime.strptime( + data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) + if 'invalid' in data: + credentials.invalid = data['invalid'] + return credentials # TODO(craigcitro): Currently, we can't even *load* @@ -350,113 +355,114 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): # it handles imports. Fix that by splitting that module into # GAE-specific and GAE-independent bits, and guarding imports. class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): - """Assertion credentials for Google App Engine apps.""" - - def __init__(self, scopes, **kwds): - if not util.DetectGae(): - raise exceptions.ResourceUnavailableError( - 'GCE credentials requested outside a GCE instance') - self._scopes = list(util.NormalizeScopes(scopes)) - super(GaeAssertionCredentials, self).__init__(None, **kwds) - - @classmethod - def Get(cls, *args, **kwds): - try: - return cls(*args, **kwds) - except exceptions.Error: - return None - - @classmethod - def from_json(cls, json_data): - data = json.loads(json_data) - return GaeAssertionCredentials(data['_scopes']) - - def _refresh(self, _): - """Refresh self.access_token. - Args: - _: (ignored) A function matching httplib2.Http.request's signature. - """ - from google.appengine.api import app_identity - try: - token, _ = app_identity.get_access_token(self._scopes) - except app_identity.Error as e: - raise exceptions.CredentialsError(str(e)) - self.access_token = token + """Assertion credentials for Google App Engine apps.""" + + def __init__(self, scopes, **kwds): + if not util.DetectGae(): + raise exceptions.ResourceUnavailableError( + 'GCE credentials requested outside a GCE instance') + self._scopes = list(util.NormalizeScopes(scopes)) + super(GaeAssertionCredentials, self).__init__(None, **kwds) + + @classmethod + def Get(cls, *args, **kwds): + try: + return cls(*args, **kwds) + except exceptions.Error: + return None + + @classmethod + def from_json(cls, json_data): + data = json.loads(json_data) + return GaeAssertionCredentials(data['_scopes']) + + def _refresh(self, _): + """Refresh self.access_token. + + Args: + _: (ignored) A function matching httplib2.Http.request's signature. + """ + from google.appengine.api import app_identity + try: + token, _ = app_identity.get_access_token(self._scopes) + except app_identity.Error as e: + raise exceptions.CredentialsError(str(e)) + self.access_token = token def _GetRunFlowFlags(): - parser = argparse.ArgumentParser(parents=[tools.argparser]) - # Get command line argparse flags. - flags = parser.parse_args() + parser = argparse.ArgumentParser(parents=[tools.argparser]) + # Get command line argparse flags. + flags = parser.parse_args() - # Allow `gflags` and `argparse` to be used side-by-side. - if hasattr(FLAGS, 'auth_host_name'): - flags.auth_host_name = FLAGS.auth_host_name - if hasattr(FLAGS, 'auth_host_port'): - flags.auth_host_port = FLAGS.auth_host_port - if hasattr(FLAGS, 'auth_local_webserver'): - flags.noauth_local_webserver = (not FLAGS.auth_local_webserver) - return flags + # Allow `gflags` and `argparse` to be used side-by-side. + if hasattr(FLAGS, 'auth_host_name'): + flags.auth_host_name = FLAGS.auth_host_name + if hasattr(FLAGS, 'auth_host_port'): + flags.auth_host_port = FLAGS.auth_host_port + if hasattr(FLAGS, 'auth_local_webserver'): + flags.noauth_local_webserver = (not FLAGS.auth_local_webserver) + return flags # TODO(craigcitro): Switch this from taking a path to taking a stream. def CredentialsFromFile(path, client_info): - """Read credentials from a file.""" - credential_store = oauth2client.multistore_file.get_credential_storage( - path, - client_info['client_id'], - client_info['user_agent'], - client_info['scope']) - if hasattr(FLAGS, 'auth_local_webserver'): - FLAGS.auth_local_webserver = False - credentials = credential_store.get() - if credentials is None or credentials.invalid: - print('Generating new OAuth credentials ...') - while True: - # If authorization fails, we want to retry, rather than let this - # cascade up and get caught elsewhere. If users want out of the - # retry loop, they can ^C. - try: - flow = oauth2client.client.OAuth2WebServerFlow(**client_info) - flags = _GetRunFlowFlags() - credentials = tools.run_flow(flow, credential_store, flags) - break - except (oauth2client.client.FlowExchangeError, SystemExit) as e: - # Here SystemExit is "no credential at all", and the - # FlowExchangeError is "invalid" -- usually because you reused - # a token. - print('Invalid authorization: %s' % (e,)) - except httplib2.HttpLib2Error as e: - print('Communication error: %s' % (e,)) - raise exceptions.CredentialsError( - 'Communication error creating credentials: %s' % e) - return credentials + """Read credentials from a file.""" + credential_store = oauth2client.multistore_file.get_credential_storage( + path, + client_info['client_id'], + client_info['user_agent'], + client_info['scope']) + if hasattr(FLAGS, 'auth_local_webserver'): + FLAGS.auth_local_webserver = False + credentials = credential_store.get() + if credentials is None or credentials.invalid: + print('Generating new OAuth credentials ...') + while True: + # If authorization fails, we want to retry, rather than let this + # cascade up and get caught elsewhere. If users want out of the + # retry loop, they can ^C. + try: + flow = oauth2client.client.OAuth2WebServerFlow(**client_info) + flags = _GetRunFlowFlags() + credentials = tools.run_flow(flow, credential_store, flags) + break + except (oauth2client.client.FlowExchangeError, SystemExit) as e: + # Here SystemExit is "no credential at all", and the + # FlowExchangeError is "invalid" -- usually because you reused + # a token. + print('Invalid authorization: %s' % (e,)) + except httplib2.HttpLib2Error as e: + print('Communication error: %s' % (e,)) + raise exceptions.CredentialsError( + 'Communication error creating credentials: %s' % e) + return credentials # TODO(craigcitro): Push this into oauth2client. def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name - """Get the userinfo associated with the given credentials. - - This is dependent on the token having either the userinfo.email or - userinfo.profile scope for the given token. - - Args: - credentials: (oauth2client.client.Credentials) incoming credentials - http: (httplib2.Http, optional) http instance to use - - Returns: - The email address for this token, or None if the required scopes - aren't available. - """ - http = http or httplib2.Http() - url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' - query_args = {'access_token': credentials.access_token} - url = '?'.join((url_root, urllib.parse.urlencode(query_args))) - # We ignore communication woes here (i.e. SSL errors, socket - # timeout), as handling these should be done in a common location. - response, content = http.request(url) - if response.status == http_client.BAD_REQUEST: - credentials.refresh(http) + """Get the userinfo associated with the given credentials. + + This is dependent on the token having either the userinfo.email or + userinfo.profile scope for the given token. + + Args: + credentials: (oauth2client.client.Credentials) incoming credentials + http: (httplib2.Http, optional) http instance to use + + Returns: + The email address for this token, or None if the required scopes + aren't available. + """ + http = http or httplib2.Http() + url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' + query_args = {'access_token': credentials.access_token} + url = '?'.join((url_root, urllib.parse.urlencode(query_args))) + # We ignore communication woes here (i.e. SSL errors, socket + # timeout), as handling these should be done in a common location. response, content = http.request(url) - return json.loads(content or '{}') # Save ourselves from an empty reply. + if response.status == http_client.BAD_REQUEST: + credentials.refresh(http) + response, content = http.request(url) + return json.loads(content or '{}') # Save ourselves from an empty reply. diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index f7be04a..af099a3 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -13,88 +13,89 @@ from apitools.base.py import util def CreateUriValidator(uri_regexp, content=''): - def CheckUri(uri, headers=None): - if 'X-Google-Metadata-Request' not in headers: - raise ValueError('Missing required header') - if uri_regexp.match(uri): - message = content - status = http_client.OK - else: - message = 'Expected uri matching pattern %s' % uri_regexp.pattern - status = http_client.BAD_REQUEST - return type('HttpResponse', (object,), {'status': status})(), message - return CheckUri + def CheckUri(uri, headers=None): + if 'X-Google-Metadata-Request' not in headers: + raise ValueError('Missing required header') + if uri_regexp.match(uri): + message = content + status = http_client.OK + else: + message = 'Expected uri matching pattern %s' % uri_regexp.pattern + status = http_client.BAD_REQUEST + return type('HttpResponse', (object,), {'status': status})(), message + return CheckUri class CredentialsLibTest(unittest2.TestCase): - def _GetServiceCreds(self, service_account_name=None, scopes=None): - scopes = scopes or ['scope1'] - kwargs = {} - if service_account_name is not None: - kwargs['service_account_name'] = service_account_name - service_account_name = service_account_name or 'default' - - def MockMetadataCalls(request): - request_url = request.get_full_url() - if request_url.endswith('scopes'): - return six.StringIO(''.join(scopes)) - elif request_url.endswith('service-accounts'): - return six.StringIO(service_account_name) - elif request_url.endswith( - '/service-accounts/%s/token' % service_account_name): - return six.StringIO('{"access_token": "token"}') - self.fail('Unexpected HTTP request to %s' % request_url) - - with mock.patch.object(credentials_lib, '_OpenNoProxy', - side_effect=MockMetadataCalls, - autospec=True) as opener_mock: - with mock.patch.object(util, 'DetectGce', autospec=True) as mock_detect: - mock_detect.return_value = True - validator = CreateUriValidator( - re.compile(r'.*/%s/.*' % service_account_name), - content='{"access_token": "token"}') - credentials = credentials_lib.GceAssertionCredentials(scopes, **kwargs) - self.assertIsNone(credentials._refresh(validator)) - self.assertEqual(3, opener_mock.call_count) - - def testGceServiceAccounts(self): - self._GetServiceCreds() - self._GetServiceCreds(service_account_name='my_service_account') + def _GetServiceCreds(self, service_account_name=None, scopes=None): + scopes = scopes or ['scope1'] + kwargs = {} + if service_account_name is not None: + kwargs['service_account_name'] = service_account_name + service_account_name = service_account_name or 'default' + + def MockMetadataCalls(request): + request_url = request.get_full_url() + if request_url.endswith('scopes'): + return six.StringIO(''.join(scopes)) + elif request_url.endswith('service-accounts'): + return six.StringIO(service_account_name) + elif request_url.endswith( + '/service-accounts/%s/token' % service_account_name): + return six.StringIO('{"access_token": "token"}') + self.fail('Unexpected HTTP request to %s' % request_url) + + with mock.patch.object(credentials_lib, '_OpenNoProxy', + side_effect=MockMetadataCalls, + autospec=True) as opener_mock: + with mock.patch.object(util, 'DetectGce', autospec=True) as mock_detect: + mock_detect.return_value = True + validator = CreateUriValidator( + re.compile(r'.*/%s/.*' % service_account_name), + content='{"access_token": "token"}') + credentials = credentials_lib.GceAssertionCredentials( + scopes, **kwargs) + self.assertIsNone(credentials._refresh(validator)) + self.assertEqual(3, opener_mock.call_count) + + def testGceServiceAccounts(self): + self._GetServiceCreds() + self._GetServiceCreds(service_account_name='my_service_account') class TestGetRunFlowFlags(unittest2.TestCase): - def setUp(self): - self._flags_actual = credentials_lib.FLAGS + def setUp(self): + self._flags_actual = credentials_lib.FLAGS - def tearDown(self): - credentials_lib.FLAGS = self._flags_actual + def tearDown(self): + credentials_lib.FLAGS = self._flags_actual - def test_with_gflags(self): - HOST = object() - PORT = object() + def test_with_gflags(self): + HOST = object() + PORT = object() - class MockFlags(object): - auth_host_name = HOST - auth_host_port = PORT - auth_local_webserver = False + class MockFlags(object): + auth_host_name = HOST + auth_host_port = PORT + auth_local_webserver = False - credentials_lib.FLAGS = MockFlags - flags = credentials_lib._GetRunFlowFlags() - self.assertEqual(flags.auth_host_name, HOST) - self.assertEqual(flags.auth_host_port, PORT) - self.assertEqual(flags.logging_level, 'ERROR') - self.assertEqual(flags.noauth_local_webserver, True) + credentials_lib.FLAGS = MockFlags + flags = credentials_lib._GetRunFlowFlags() + self.assertEqual(flags.auth_host_name, HOST) + self.assertEqual(flags.auth_host_port, PORT) + self.assertEqual(flags.logging_level, 'ERROR') + self.assertEqual(flags.noauth_local_webserver, True) - def test_without_gflags(self): - credentials_lib.FLAGS = None - flags = credentials_lib._GetRunFlowFlags() - self.assertEqual(flags.auth_host_name, 'localhost') - self.assertEqual(flags.auth_host_port, [8080, 8090]) - self.assertEqual(flags.logging_level, 'ERROR') - self.assertEqual(flags.noauth_local_webserver, False) + def test_without_gflags(self): + credentials_lib.FLAGS = None + flags = credentials_lib._GetRunFlowFlags() + self.assertEqual(flags.auth_host_name, 'localhost') + self.assertEqual(flags.auth_host_port, [8080, 8090]) + self.assertEqual(flags.logging_level, 'ERROR') + self.assertEqual(flags.noauth_local_webserver, False) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index abd2e0b..8020b3a 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -43,461 +43,467 @@ _FIELD_TYPE_CODECS = {} def MapUnrecognizedFields(field_name): - """Register field_name as a container for unrecognized fields in message.""" - def Register(cls): - _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name - return cls - return Register + """Register field_name as a container for unrecognized fields in message.""" + def Register(cls): + _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name + return cls + return Register def RegisterCustomMessageCodec(encoder, decoder): - """Register a custom encoder/decoder for this message class.""" - def Register(cls): - _CUSTOM_MESSAGE_CODECS[cls] = _Codec(encoder=encoder, decoder=decoder) - return cls - return Register + """Register a custom encoder/decoder for this message class.""" + def Register(cls): + _CUSTOM_MESSAGE_CODECS[cls] = _Codec(encoder=encoder, decoder=decoder) + return cls + return Register def RegisterCustomFieldCodec(encoder, decoder): - """Register a custom encoder/decoder for this field.""" - def Register(field): - _CUSTOM_FIELD_CODECS[field] = _Codec(encoder=encoder, decoder=decoder) - return field - return Register + """Register a custom encoder/decoder for this field.""" + def Register(field): + _CUSTOM_FIELD_CODECS[field] = _Codec(encoder=encoder, decoder=decoder) + return field + return Register def RegisterFieldTypeCodec(encoder, decoder): - """Register a custom encoder/decoder for all fields of this type.""" - def Register(field_type): - _FIELD_TYPE_CODECS[field_type] = _Codec(encoder=encoder, decoder=decoder) - return field_type - return Register + """Register a custom encoder/decoder for all fields of this type.""" + def Register(field_type): + _FIELD_TYPE_CODECS[field_type] = _Codec( + encoder=encoder, decoder=decoder) + return field_type + return Register # TODO(craigcitro): Delete this function with the switch to proto2. def CopyProtoMessage(message): - codec = protojson.ProtoJson() - return codec.decode_message(type(message), codec.encode_message(message)) + codec = protojson.ProtoJson() + return codec.decode_message(type(message), codec.encode_message(message)) def MessageToJson(message, include_fields=None): - """Convert the given message to JSON.""" - result = _ProtoJsonApiTools.Get().encode_message(message) - return _IncludeFields(result, message, include_fields) + """Convert the given message to JSON.""" + result = _ProtoJsonApiTools.Get().encode_message(message) + return _IncludeFields(result, message, include_fields) def JsonToMessage(message_type, message): - """Convert the given JSON to a message of type message_type.""" - return _ProtoJsonApiTools.Get().decode_message(message_type, message) + """Convert the given JSON to a message of type message_type.""" + return _ProtoJsonApiTools.Get().decode_message(message_type, message) # TODO(craigcitro): Do this directly, instead of via JSON. def DictToMessage(d, message_type): - """Convert the given dictionary to a message of type message_type.""" - return JsonToMessage(message_type, json.dumps(d)) + """Convert the given dictionary to a message of type message_type.""" + return JsonToMessage(message_type, json.dumps(d)) def MessageToDict(message): - """Convert the given message to a dictionary.""" - return json.loads(MessageToJson(message)) + """Convert the given message to a dictionary.""" + return json.loads(MessageToJson(message)) def PyValueToMessage(message_type, value): - """Convert the given python value to a message of type message_type.""" - return JsonToMessage(message_type, json.dumps(value)) + """Convert the given python value to a message of type message_type.""" + return JsonToMessage(message_type, json.dumps(value)) def MessageToPyValue(message): - """Convert the given message to a python value.""" - return json.loads(MessageToJson(message)) + """Convert the given message to a python value.""" + return json.loads(MessageToJson(message)) def MessageToRepr(msg, multiline=False, **kwargs): - """Return a repr-style string for a protorpc message. - - protorpc.Message.__repr__ does not return anything that could be considered - python code. Adding this function lets us print a protorpc message in such - a way that it could be pasted into code later, and used to compare against - other things. - - Args: - msg: protorpc.Message, the message to be repr'd. - multiline: bool, True if the returned string should have each field - assignment on its own line. - **kwargs: {str:str}, Additional flags for how to format the string. - - Known **kwargs: - shortstrings: bool, True if all string values should be truncated at - 100 characters, since when mocking the contents typically don't matter - except for IDs, and IDs are usually less than 100 characters. - no_modules: bool, True if the long module name should not be printed with - each type. - - Returns: - str, A string of valid python (assuming the right imports have been made) - that recreates the message passed into this function. - """ - - # TODO(user): craigcitro suggests a pretty-printer from apitools/gen. - - indent = kwargs.get('indent', 0) - - def IndentKwargs(kwargs): - kwargs = dict(kwargs) - kwargs['indent'] = kwargs.get('indent', 0) + 4 - return kwargs - - if isinstance(msg, list): - s = '[' - for item in msg: - if multiline: - s += '\n' + ' '*(indent + 4) - s += MessageToRepr( - item, multiline=multiline, **IndentKwargs(kwargs)) + ',' - if multiline: - s += '\n' + ' '*indent - s += ']' - return s - - if isinstance(msg, messages.Message): - s = type(msg).__name__ + '(' - if not kwargs.get('no_modules'): - s = msg.__module__ + '.' + s - names = sorted([field.name for field in msg.all_fields()]) - for name in names: - field = msg.field_by_name(name) - if multiline: - s += '\n' + ' '*(indent + 4) - value = getattr(msg, field.name) - s += field.name + '=' + MessageToRepr( - value, multiline=multiline, **IndentKwargs(kwargs)) + ',' - if multiline: - s += '\n'+' '*indent - s += ')' - return s - - if isinstance(msg, six.string_types): - if kwargs.get('shortstrings') and len(msg) > 100: - msg = msg[:100] - - if isinstance(msg, datetime.datetime): - - class SpecialTZInfo(datetime.tzinfo): - - def __init__(self, offset): - super(SpecialTZInfo, self).__init__() - self.offset = offset - - def __repr__(self): - s = 'TimeZoneOffset(' + repr(self.offset) + ')' + """Return a repr-style string for a protorpc message. + + protorpc.Message.__repr__ does not return anything that could be considered + python code. Adding this function lets us print a protorpc message in such + a way that it could be pasted into code later, and used to compare against + other things. + + Args: + msg: protorpc.Message, the message to be repr'd. + multiline: bool, True if the returned string should have each field + assignment on its own line. + **kwargs: {str:str}, Additional flags for how to format the string. + + Known **kwargs: + shortstrings: bool, True if all string values should be truncated at + 100 characters, since when mocking the contents typically don't matter + except for IDs, and IDs are usually less than 100 characters. + no_modules: bool, True if the long module name should not be printed with + each type. + + Returns: + str, A string of valid python (assuming the right imports have been made) + that recreates the message passed into this function. + """ + + # TODO(user): craigcitro suggests a pretty-printer from apitools/gen. + + indent = kwargs.get('indent', 0) + + def IndentKwargs(kwargs): + kwargs = dict(kwargs) + kwargs['indent'] = kwargs.get('indent', 0) + 4 + return kwargs + + if isinstance(msg, list): + s = '[' + for item in msg: + if multiline: + s += '\n' + ' ' * (indent + 4) + s += MessageToRepr( + item, multiline=multiline, **IndentKwargs(kwargs)) + ',' + if multiline: + s += '\n' + ' ' * indent + s += ']' + return s + + if isinstance(msg, messages.Message): + s = type(msg).__name__ + '(' if not kwargs.get('no_modules'): - s = 'protorpc.util.' + s + s = msg.__module__ + '.' + s + names = sorted([field.name for field in msg.all_fields()]) + for name in names: + field = msg.field_by_name(name) + if multiline: + s += '\n' + ' ' * (indent + 4) + value = getattr(msg, field.name) + s += field.name + '=' + MessageToRepr( + value, multiline=multiline, **IndentKwargs(kwargs)) + ',' + if multiline: + s += '\n' + ' ' * indent + s += ')' return s - msg = datetime.datetime( - msg.year, msg.month, msg.day, msg.hour, msg.minute, msg.second, - msg.microsecond, SpecialTZInfo(msg.tzinfo.utcoffset(0))) + if isinstance(msg, six.string_types): + if kwargs.get('shortstrings') and len(msg) > 100: + msg = msg[:100] - return repr(msg) + if isinstance(msg, datetime.datetime): + class SpecialTZInfo(datetime.tzinfo): -def _GetField(message, field_path): - for field in field_path: - if field not in dir(message): - raise KeyError('no field "%s"' % field) - message = getattr(message, field) - return message + def __init__(self, offset): + super(SpecialTZInfo, self).__init__() + self.offset = offset + def __repr__(self): + s = 'TimeZoneOffset(' + repr(self.offset) + ')' + if not kwargs.get('no_modules'): + s = 'protorpc.util.' + s + return s -def _SetField(dictblob, field_path, value): - for field in field_path[:-1]: - dictblob[field] = {} - dictblob = dictblob[field] - dictblob[field_path[-1]] = value + msg = datetime.datetime( + msg.year, msg.month, msg.day, msg.hour, msg.minute, msg.second, + msg.microsecond, SpecialTZInfo(msg.tzinfo.utcoffset(0))) + return repr(msg) -def _IncludeFields(encoded_message, message, include_fields): - """Add the requested fields to the encoded message.""" - if include_fields is None: - return encoded_message - result = json.loads(encoded_message) - for field_name in include_fields: - try: - value = _GetField(message, field_name.split('.')) - nullvalue = None - if isinstance(value, list): - nullvalue = [] - except KeyError: - raise exceptions.InvalidDataError( - 'No field named %s in message of type %s' % ( - field_name, type(message))) - _SetField(result, field_name.split('.'), nullvalue) - return json.dumps(result) +def _GetField(message, field_path): + for field in field_path: + if field not in dir(message): + raise KeyError('no field "%s"' % field) + message = getattr(message, field) + return message -def _GetFieldCodecs(field, attr): - result = [ - getattr(_CUSTOM_FIELD_CODECS.get(field), attr, None), - getattr(_FIELD_TYPE_CODECS.get(type(field)), attr, None), - ] - return [x for x in result if x is not None] +def _SetField(dictblob, field_path, value): + for field in field_path[:-1]: + dictblob[field] = {} + dictblob = dictblob[field] + dictblob[field_path[-1]] = value -class _ProtoJsonApiTools(protojson.ProtoJson): - """JSON encoder used by apitools clients.""" - _INSTANCE = None - - @classmethod - def Get(cls): - if cls._INSTANCE is None: - cls._INSTANCE = cls() - return cls._INSTANCE - - def decode_message(self, message_type, encoded_message): - if message_type in _CUSTOM_MESSAGE_CODECS: - return _CUSTOM_MESSAGE_CODECS[message_type].decoder(encoded_message) - # We turn off the default logging in protorpc. We may want to - # remove this later. - old_level = logging.getLogger().level - logging.getLogger().setLevel(logging.ERROR) - result = _DecodeCustomFieldNames(message_type, encoded_message) - result = super(_ProtoJsonApiTools, self).decode_message( - message_type, result) - logging.getLogger().setLevel(old_level) - result = _ProcessUnknownEnums(result, encoded_message) - result = _ProcessUnknownMessages(result, encoded_message) - return _DecodeUnknownFields(result, encoded_message) - - def decode_field(self, field, value): - """Decode the given JSON value. - Args: - field: a messages.Field for the field we're decoding. - value: a python value we'd like to decode. +def _IncludeFields(encoded_message, message, include_fields): + """Add the requested fields to the encoded message.""" + if include_fields is None: + return encoded_message + result = json.loads(encoded_message) + for field_name in include_fields: + try: + value = _GetField(message, field_name.split('.')) + nullvalue = None + if isinstance(value, list): + nullvalue = [] + except KeyError: + raise exceptions.InvalidDataError( + 'No field named %s in message of type %s' % ( + field_name, type(message))) + _SetField(result, field_name.split('.'), nullvalue) + return json.dumps(result) - Returns: - A value suitable for assignment to field. - """ - for decoder in _GetFieldCodecs(field, 'decoder'): - result = decoder(field, value) - value = result.value - if result.complete: - return value - if isinstance(field, messages.MessageField): - field_value = self.decode_message(field.message_type, json.dumps(value)) - elif isinstance(field, messages.EnumField): - value = GetCustomJsonEnumMapping(field.type, json_name=value) or value - try: - field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) - except messages.DecodeError: - if not isinstance(value, six.string_types): - raise - field_value = None - else: - field_value = super(_ProtoJsonApiTools, self).decode_field(field, value) - return field_value - def encode_message(self, message): - if isinstance(message, messages.FieldList): - return '[%s]' % (', '.join(self.encode_message(x) for x in message)) - if type(message) in _CUSTOM_MESSAGE_CODECS: - return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) - message = _EncodeUnknownFields(message) - result = super(_ProtoJsonApiTools, self).encode_message(message) - return _EncodeCustomFieldNames(message, result) +def _GetFieldCodecs(field, attr): + result = [ + getattr(_CUSTOM_FIELD_CODECS.get(field), attr, None), + getattr(_FIELD_TYPE_CODECS.get(type(field)), attr, None), + ] + return [x for x in result if x is not None] - def encode_field(self, field, value): - """Encode the given value as JSON. - Args: - field: a messages.Field for the field we're encoding. - value: a value for field. +class _ProtoJsonApiTools(protojson.ProtoJson): - Returns: - A python value suitable for json.dumps. - """ - for encoder in _GetFieldCodecs(field, 'encoder'): - result = encoder(field, value) - value = result.value - if result.complete: - return value - if isinstance(field, messages.EnumField): - if field.repeated: - remapped_value = [GetCustomJsonEnumMapping( - field.type, python_name=e.name) or e.name for e in value] - else: - remapped_value = GetCustomJsonEnumMapping( - field.type, python_name=value.name) - if remapped_value: - return remapped_value - if (isinstance(field, messages.MessageField) and - not isinstance(field, message_types.DateTimeField)): - value = json.loads(self.encode_message(value)) - return super(_ProtoJsonApiTools, self).encode_field(field, value) + """JSON encoder used by apitools clients.""" + _INSTANCE = None + + @classmethod + def Get(cls): + if cls._INSTANCE is None: + cls._INSTANCE = cls() + return cls._INSTANCE + + def decode_message(self, message_type, encoded_message): + if message_type in _CUSTOM_MESSAGE_CODECS: + return _CUSTOM_MESSAGE_CODECS[message_type].decoder(encoded_message) + # We turn off the default logging in protorpc. We may want to + # remove this later. + old_level = logging.getLogger().level + logging.getLogger().setLevel(logging.ERROR) + result = _DecodeCustomFieldNames(message_type, encoded_message) + result = super(_ProtoJsonApiTools, self).decode_message( + message_type, result) + logging.getLogger().setLevel(old_level) + result = _ProcessUnknownEnums(result, encoded_message) + result = _ProcessUnknownMessages(result, encoded_message) + return _DecodeUnknownFields(result, encoded_message) + + def decode_field(self, field, value): + """Decode the given JSON value. + + Args: + field: a messages.Field for the field we're decoding. + value: a python value we'd like to decode. + + Returns: + A value suitable for assignment to field. + """ + for decoder in _GetFieldCodecs(field, 'decoder'): + result = decoder(field, value) + value = result.value + if result.complete: + return value + if isinstance(field, messages.MessageField): + field_value = self.decode_message( + field.message_type, json.dumps(value)) + elif isinstance(field, messages.EnumField): + value = GetCustomJsonEnumMapping( + field.type, json_name=value) or value + try: + field_value = super( + _ProtoJsonApiTools, self).decode_field(field, value) + except messages.DecodeError: + if not isinstance(value, six.string_types): + raise + field_value = None + else: + field_value = super( + _ProtoJsonApiTools, self).decode_field(field, value) + return field_value + + def encode_message(self, message): + if isinstance(message, messages.FieldList): + return '[%s]' % (', '.join(self.encode_message(x) for x in message)) + if type(message) in _CUSTOM_MESSAGE_CODECS: + return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) + message = _EncodeUnknownFields(message) + result = super(_ProtoJsonApiTools, self).encode_message(message) + return _EncodeCustomFieldNames(message, result) + + def encode_field(self, field, value): + """Encode the given value as JSON. + + Args: + field: a messages.Field for the field we're encoding. + value: a value for field. + + Returns: + A python value suitable for json.dumps. + """ + for encoder in _GetFieldCodecs(field, 'encoder'): + result = encoder(field, value) + value = result.value + if result.complete: + return value + if isinstance(field, messages.EnumField): + if field.repeated: + remapped_value = [GetCustomJsonEnumMapping( + field.type, python_name=e.name) or e.name for e in value] + else: + remapped_value = GetCustomJsonEnumMapping( + field.type, python_name=value.name) + if remapped_value: + return remapped_value + if (isinstance(field, messages.MessageField) and + not isinstance(field, message_types.DateTimeField)): + value = json.loads(self.encode_message(value)) + return super(_ProtoJsonApiTools, self).encode_field(field, value) # TODO(craigcitro): Fold this and _IncludeFields in as codecs. def _DecodeUnknownFields(message, encoded_message): - """Rewrite unknown fields in message into message.destination.""" - destination = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) - if destination is None: + """Rewrite unknown fields in message into message.destination.""" + destination = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) + if destination is None: + return message + pair_field = message.field_by_name(destination) + if not isinstance(pair_field, messages.MessageField): + raise exceptions.InvalidDataFromServerError( + 'Unrecognized fields must be mapped to a compound ' + 'message type.') + pair_type = pair_field.message_type + # TODO(craigcitro): Add more error checking around the pair + # type being exactly what we suspect (field names, etc). + if isinstance(pair_type.value, messages.MessageField): + new_values = _DecodeUnknownMessages( + message, json.loads(encoded_message), pair_type) + else: + new_values = _DecodeUnrecognizedFields(message, pair_type) + setattr(message, destination, new_values) + # We could probably get away with not setting this, but + # why not clear it? + setattr(message, '_Message__unrecognized_fields', {}) return message - pair_field = message.field_by_name(destination) - if not isinstance(pair_field, messages.MessageField): - raise exceptions.InvalidDataFromServerError( - 'Unrecognized fields must be mapped to a compound ' - 'message type.') - pair_type = pair_field.message_type - # TODO(craigcitro): Add more error checking around the pair - # type being exactly what we suspect (field names, etc). - if isinstance(pair_type.value, messages.MessageField): - new_values = _DecodeUnknownMessages( - message, json.loads(encoded_message), pair_type) - else: - new_values = _DecodeUnrecognizedFields(message, pair_type) - setattr(message, destination, new_values) - # We could probably get away with not setting this, but - # why not clear it? - setattr(message, '_Message__unrecognized_fields', {}) - return message def _DecodeUnknownMessages(message, encoded_message, pair_type): - """Process unknown fields in encoded_message of a message type.""" - field_type = pair_type.value.type - new_values = [] - all_field_names = [x.name for x in message.all_fields()] - for name, value_dict in encoded_message.items(): - if name in all_field_names: - continue - value = PyValueToMessage(field_type, value_dict) - new_pair = pair_type(key=name, value=value) - new_values.append(new_pair) - return new_values + """Process unknown fields in encoded_message of a message type.""" + field_type = pair_type.value.type + new_values = [] + all_field_names = [x.name for x in message.all_fields()] + for name, value_dict in encoded_message.items(): + if name in all_field_names: + continue + value = PyValueToMessage(field_type, value_dict) + new_pair = pair_type(key=name, value=value) + new_values.append(new_pair) + return new_values def _DecodeUnrecognizedFields(message, pair_type): - """Process unrecognized fields in message.""" - new_values = [] - for unknown_field in message.all_unrecognized_fields(): - # TODO(craigcitro): Consider validating the variant if - # the assignment below doesn't take care of it. It may - # also be necessary to check it in the case that the - # type has multiple encodings. - value, _ = message.get_unrecognized_field_info(unknown_field) - value_type = pair_type.field_by_name('value') - if isinstance(value_type, messages.MessageField): - decoded_value = DictToMessage(value, pair_type.value.message_type) - else: - decoded_value = value - new_pair = pair_type(key=str(unknown_field), value=decoded_value) - new_values.append(new_pair) - return new_values + """Process unrecognized fields in message.""" + new_values = [] + for unknown_field in message.all_unrecognized_fields(): + # TODO(craigcitro): Consider validating the variant if + # the assignment below doesn't take care of it. It may + # also be necessary to check it in the case that the + # type has multiple encodings. + value, _ = message.get_unrecognized_field_info(unknown_field) + value_type = pair_type.field_by_name('value') + if isinstance(value_type, messages.MessageField): + decoded_value = DictToMessage(value, pair_type.value.message_type) + else: + decoded_value = value + new_pair = pair_type(key=str(unknown_field), value=decoded_value) + new_values.append(new_pair) + return new_values def _EncodeUnknownFields(message): - """Remap unknown fields in message out of message.source.""" - source = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) - if source is None: - return message - result = CopyProtoMessage(message) - pairs_field = message.field_by_name(source) - if not isinstance(pairs_field, messages.MessageField): - raise exceptions.InvalidUserInputError( - 'Invalid pairs field %s' % pairs_field) - pairs_type = pairs_field.message_type - value_variant = pairs_type.field_by_name('value').variant - pairs = getattr(message, source) - for pair in pairs: - if value_variant == messages.Variant.MESSAGE: - encoded_value = MessageToDict(pair.value) - else: - encoded_value = pair.value - result.set_unrecognized_field(pair.key, encoded_value, value_variant) - setattr(result, source, []) - return result + """Remap unknown fields in message out of message.source.""" + source = _UNRECOGNIZED_FIELD_MAPPINGS.get(type(message)) + if source is None: + return message + result = CopyProtoMessage(message) + pairs_field = message.field_by_name(source) + if not isinstance(pairs_field, messages.MessageField): + raise exceptions.InvalidUserInputError( + 'Invalid pairs field %s' % pairs_field) + pairs_type = pairs_field.message_type + value_variant = pairs_type.field_by_name('value').variant + pairs = getattr(message, source) + for pair in pairs: + if value_variant == messages.Variant.MESSAGE: + encoded_value = MessageToDict(pair.value) + else: + encoded_value = pair.value + result.set_unrecognized_field(pair.key, encoded_value, value_variant) + setattr(result, source, []) + return result def _SafeEncodeBytes(field, value): - """Encode the bytes in value as urlsafe base64.""" - try: - if field.repeated: - result = [base64.urlsafe_b64encode(byte) for byte in value] - else: - result = base64.urlsafe_b64encode(value) - complete = True - except TypeError: - result = value - complete = False - return CodecResult(value=result, complete=complete) + """Encode the bytes in value as urlsafe base64.""" + try: + if field.repeated: + result = [base64.urlsafe_b64encode(byte) for byte in value] + else: + result = base64.urlsafe_b64encode(value) + complete = True + except TypeError: + result = value + complete = False + return CodecResult(value=result, complete=complete) def _SafeDecodeBytes(unused_field, value): - """Decode the urlsafe base64 value into bytes.""" - try: - result = base64.urlsafe_b64decode(str(value)) - complete = True - except TypeError: - result = value - complete = False - return CodecResult(value=result, complete=complete) + """Decode the urlsafe base64 value into bytes.""" + try: + result = base64.urlsafe_b64decode(str(value)) + complete = True + except TypeError: + result = value + complete = False + return CodecResult(value=result, complete=complete) def _ProcessUnknownEnums(message, encoded_message): - """Add unknown enum values from encoded_message as unknown fields. - - ProtoRPC diverges from the usual protocol buffer behavior here and - doesn't allow unknown fields. Throwing on unknown fields makes it - impossible to let servers add new enum values and stay compatible - with older clients, which isn't reasonable for us. We simply store - unrecognized enum values as unknown fields, and all is well. - - Args: - message: Proto message we've decoded thus far. - encoded_message: JSON string we're decoding. - - Returns: - message, with any unknown enums stored as unrecognized fields. - """ - if not encoded_message: + """Add unknown enum values from encoded_message as unknown fields. + + ProtoRPC diverges from the usual protocol buffer behavior here and + doesn't allow unknown fields. Throwing on unknown fields makes it + impossible to let servers add new enum values and stay compatible + with older clients, which isn't reasonable for us. We simply store + unrecognized enum values as unknown fields, and all is well. + + Args: + message: Proto message we've decoded thus far. + encoded_message: JSON string we're decoding. + + Returns: + message, with any unknown enums stored as unrecognized fields. + """ + if not encoded_message: + return message + decoded_message = json.loads(encoded_message) + for field in message.all_fields(): + if (isinstance(field, messages.EnumField) and + field.name in decoded_message and + message.get_assigned_value(field.name) is None): + message.set_unrecognized_field(field.name, decoded_message[field.name], + messages.Variant.ENUM) return message - decoded_message = json.loads(encoded_message) - for field in message.all_fields(): - if (isinstance(field, messages.EnumField) and - field.name in decoded_message and - message.get_assigned_value(field.name) is None): - message.set_unrecognized_field(field.name, decoded_message[field.name], - messages.Variant.ENUM) - return message def _ProcessUnknownMessages(message, encoded_message): - """Store any remaining unknown fields as strings. - - ProtoRPC currently ignores unknown values for which no type can be - determined (and logs a "No variant found" message). For the purposes - of reserializing, this is quite harmful (since it throws away - information). Here we simply add those as unknown fields of type - string (so that they can easily be reserialized). - - Args: - message: Proto message we've decoded thus far. - encoded_message: JSON string we're decoding. - - Returns: - message, with any remaining unrecognized fields saved. - """ - if not encoded_message: + """Store any remaining unknown fields as strings. + + ProtoRPC currently ignores unknown values for which no type can be + determined (and logs a "No variant found" message). For the purposes + of reserializing, this is quite harmful (since it throws away + information). Here we simply add those as unknown fields of type + string (so that they can easily be reserialized). + + Args: + message: Proto message we've decoded thus far. + encoded_message: JSON string we're decoding. + + Returns: + message, with any remaining unrecognized fields saved. + """ + if not encoded_message: + return message + decoded_message = json.loads(encoded_message) + message_fields = [x.name for x in message.all_fields()] + list( + message.all_unrecognized_fields()) + missing_fields = [x for x in decoded_message.keys() + if x not in message_fields] + for field_name in missing_fields: + message.set_unrecognized_field(field_name, decoded_message[field_name], + messages.Variant.STRING) return message - decoded_message = json.loads(encoded_message) - message_fields = [x.name for x in message.all_fields()] + list( - message.all_unrecognized_fields()) - missing_fields = [x for x in decoded_message.keys() - if x not in message_fields] - for field_name in missing_fields: - message.set_unrecognized_field(field_name, decoded_message[field_name], - messages.Variant.STRING) - return message RegisterFieldTypeCodec(_SafeEncodeBytes, _SafeDecodeBytes)(messages.BytesField) @@ -510,127 +516,127 @@ _JSON_FIELD_MAPPINGS = {} def AddCustomJsonEnumMapping(enum_type, python_name, json_name): - """Add a custom wire encoding for a given enum value. - - This is primarily used in generated code, to handle enum values - which happen to be Python keywords. - - Args: - enum_type: (messages.Enum) An enum type - python_name: (string) Python name for this value. - json_name: (string) JSON name to be used on the wire. - """ - if not issubclass(enum_type, messages.Enum): - raise exceptions.TypecheckError( - 'Cannot set JSON enum mapping for non-enum "%s"' % enum_type) - enum_name = enum_type.definition_name() - if python_name not in enum_type.names(): - raise exceptions.InvalidDataError( - 'Enum value %s not a value for type %s' % (python_name, enum_type)) - field_mappings = _JSON_ENUM_MAPPINGS.setdefault(enum_name, {}) - _CheckForExistingMappings('enum', enum_type, python_name, json_name) - field_mappings[python_name] = json_name + """Add a custom wire encoding for a given enum value. + + This is primarily used in generated code, to handle enum values + which happen to be Python keywords. + + Args: + enum_type: (messages.Enum) An enum type + python_name: (string) Python name for this value. + json_name: (string) JSON name to be used on the wire. + """ + if not issubclass(enum_type, messages.Enum): + raise exceptions.TypecheckError( + 'Cannot set JSON enum mapping for non-enum "%s"' % enum_type) + enum_name = enum_type.definition_name() + if python_name not in enum_type.names(): + raise exceptions.InvalidDataError( + 'Enum value %s not a value for type %s' % (python_name, enum_type)) + field_mappings = _JSON_ENUM_MAPPINGS.setdefault(enum_name, {}) + _CheckForExistingMappings('enum', enum_type, python_name, json_name) + field_mappings[python_name] = json_name def AddCustomJsonFieldMapping(message_type, python_name, json_name): - """Add a custom wire encoding for a given message field. - - This is primarily used in generated code, to handle enum values - which happen to be Python keywords. - - Args: - message_type: (messages.Message) A message type - python_name: (string) Python name for this value. - json_name: (string) JSON name to be used on the wire. - """ - if not issubclass(message_type, messages.Message): - raise exceptions.TypecheckError( - 'Cannot set JSON field mapping for non-message "%s"' % message_type) - message_name = message_type.definition_name() - try: - _ = message_type.field_by_name(python_name) - except KeyError: - raise exceptions.InvalidDataError( - 'Field %s not recognized for type %s' % (python_name, message_type)) - field_mappings = _JSON_FIELD_MAPPINGS.setdefault(message_name, {}) - _CheckForExistingMappings('field', message_type, python_name, json_name) - field_mappings[python_name] = json_name + """Add a custom wire encoding for a given message field. + + This is primarily used in generated code, to handle enum values + which happen to be Python keywords. + + Args: + message_type: (messages.Message) A message type + python_name: (string) Python name for this value. + json_name: (string) JSON name to be used on the wire. + """ + if not issubclass(message_type, messages.Message): + raise exceptions.TypecheckError( + 'Cannot set JSON field mapping for non-message "%s"' % message_type) + message_name = message_type.definition_name() + try: + _ = message_type.field_by_name(python_name) + except KeyError: + raise exceptions.InvalidDataError( + 'Field %s not recognized for type %s' % (python_name, message_type)) + field_mappings = _JSON_FIELD_MAPPINGS.setdefault(message_name, {}) + _CheckForExistingMappings('field', message_type, python_name, json_name) + field_mappings[python_name] = json_name def GetCustomJsonEnumMapping(enum_type, python_name=None, json_name=None): - """Return the appropriate remapping for the given enum, or None.""" - return _FetchRemapping(enum_type.definition_name(), 'enum', - python_name=python_name, json_name=json_name, - mappings=_JSON_ENUM_MAPPINGS) + """Return the appropriate remapping for the given enum, or None.""" + return _FetchRemapping(enum_type.definition_name(), 'enum', + python_name=python_name, json_name=json_name, + mappings=_JSON_ENUM_MAPPINGS) def GetCustomJsonFieldMapping(message_type, python_name=None, json_name=None): - """Return the appropriate remapping for the given field, or None.""" - return _FetchRemapping(message_type.definition_name(), 'field', - python_name=python_name, json_name=json_name, - mappings=_JSON_FIELD_MAPPINGS) + """Return the appropriate remapping for the given field, or None.""" + return _FetchRemapping(message_type.definition_name(), 'field', + python_name=python_name, json_name=json_name, + mappings=_JSON_FIELD_MAPPINGS) def _FetchRemapping(type_name, mapping_type, python_name=None, json_name=None, mappings=None): - """Common code for fetching a key or value from a remapping dict.""" - if python_name and json_name: - raise exceptions.InvalidDataError( - 'Cannot specify both python_name and json_name for %s remapping' % ( - mapping_type,)) - if not (python_name or json_name): - raise exceptions.InvalidDataError( - 'Must specify either python_name or json_name for %s remapping' % ( - mapping_type,)) - field_remappings = mappings.get(type_name, {}) - if field_remappings: - if python_name: - return field_remappings.get(python_name) - elif json_name: - if json_name in list(field_remappings.values()): - return [k for k in field_remappings - if field_remappings[k] == json_name][0] - return None + """Common code for fetching a key or value from a remapping dict.""" + if python_name and json_name: + raise exceptions.InvalidDataError( + 'Cannot specify both python_name and json_name for %s remapping' % ( + mapping_type,)) + if not (python_name or json_name): + raise exceptions.InvalidDataError( + 'Must specify either python_name or json_name for %s remapping' % ( + mapping_type,)) + field_remappings = mappings.get(type_name, {}) + if field_remappings: + if python_name: + return field_remappings.get(python_name) + elif json_name: + if json_name in list(field_remappings.values()): + return [k for k in field_remappings + if field_remappings[k] == json_name][0] + return None def _CheckForExistingMappings(mapping_type, message_type, python_name, json_name): - """Validate that no mappings exist for the given values.""" - if mapping_type == 'field': - getter = GetCustomJsonFieldMapping - elif mapping_type == 'enum': - getter = GetCustomJsonEnumMapping - remapping = getter(message_type, python_name=python_name) - if remapping is not None: - raise exceptions.InvalidDataError( - 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( - mapping_type, python_name, remapping)) - remapping = getter(message_type, json_name=json_name) - if remapping is not None: - raise exceptions.InvalidDataError( - 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( - mapping_type, json_name, remapping)) + """Validate that no mappings exist for the given values.""" + if mapping_type == 'field': + getter = GetCustomJsonFieldMapping + elif mapping_type == 'enum': + getter = GetCustomJsonEnumMapping + remapping = getter(message_type, python_name=python_name) + if remapping is not None: + raise exceptions.InvalidDataError( + 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( + mapping_type, python_name, remapping)) + remapping = getter(message_type, json_name=json_name) + if remapping is not None: + raise exceptions.InvalidDataError( + 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( + mapping_type, json_name, remapping)) def _EncodeCustomFieldNames(message, encoded_value): - message_name = type(message).definition_name() - field_remappings = list(_JSON_FIELD_MAPPINGS.get(message_name, {}).items()) - if field_remappings: - decoded_value = json.loads(encoded_value) - for python_name, json_name in field_remappings: - if python_name in encoded_value: - decoded_value[json_name] = decoded_value.pop(python_name) - encoded_value = json.dumps(decoded_value) - return encoded_value + message_name = type(message).definition_name() + field_remappings = list(_JSON_FIELD_MAPPINGS.get(message_name, {}).items()) + if field_remappings: + decoded_value = json.loads(encoded_value) + for python_name, json_name in field_remappings: + if python_name in encoded_value: + decoded_value[json_name] = decoded_value.pop(python_name) + encoded_value = json.dumps(decoded_value) + return encoded_value def _DecodeCustomFieldNames(message_type, encoded_message): - message_name = message_type.definition_name() - field_remappings = _JSON_FIELD_MAPPINGS.get(message_name, {}) - if field_remappings: - decoded_message = json.loads(encoded_message) - for python_name, json_name in list(field_remappings.items()): - if json_name in decoded_message: - decoded_message[python_name] = decoded_message.pop(json_name) - encoded_message = json.dumps(decoded_message) - return encoded_message + message_name = message_type.definition_name() + field_remappings = _JSON_FIELD_MAPPINGS.get(message_name, {}) + if field_remappings: + decoded_message = json.loads(encoded_message) + for python_name, json_name in list(field_remappings.items()): + if json_name in decoded_message: + decoded_message[python_name] = decoded_message.pop(json_name) + encoded_message = json.dumps(decoded_message) + return encoded_message diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 81b02c0..91b7565 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -15,77 +15,77 @@ from apitools.base.py import exceptions class SimpleMessage(messages.Message): - field = messages.StringField(1) - repfield = messages.StringField(2, repeated=True) + field = messages.StringField(1) + repfield = messages.StringField(2, repeated=True) class BytesMessage(messages.Message): - field = messages.BytesField(1) - repfield = messages.BytesField(2, repeated=True) + field = messages.BytesField(1) + repfield = messages.BytesField(2, repeated=True) class TimeMessage(messages.Message): - timefield = message_types.DateTimeField(3) + timefield = message_types.DateTimeField(3) @encoding.MapUnrecognizedFields('additional_properties') class AdditionalPropertiesMessage(messages.Message): - class AdditionalProperty(messages.Message): - key = messages.StringField(1) - value = messages.StringField(2) + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.StringField(2) - additional_properties = messages.MessageField( - AdditionalProperty, 1, repeated=True) + additional_properties = messages.MessageField( + AdditionalProperty, 1, repeated=True) class CompoundPropertyType(messages.Message): - index = messages.IntegerField(1) - name = messages.StringField(2) + index = messages.IntegerField(1) + name = messages.StringField(2) class MessageWithEnum(messages.Message): - class ThisEnum(messages.Enum): - VALUE_ONE = 1 - VALUE_TWO = 2 + class ThisEnum(messages.Enum): + VALUE_ONE = 1 + VALUE_TWO = 2 - field_one = messages.EnumField(ThisEnum, 1) - field_two = messages.EnumField(ThisEnum, 2, default=ThisEnum.VALUE_TWO) - ignored_field = messages.EnumField(ThisEnum, 3) + field_one = messages.EnumField(ThisEnum, 1) + field_two = messages.EnumField(ThisEnum, 2, default=ThisEnum.VALUE_TWO) + ignored_field = messages.EnumField(ThisEnum, 3) @encoding.MapUnrecognizedFields('additional_properties') class AdditionalMessagePropertiesMessage(messages.Message): - class AdditionalProperty(messages.Message): - key = messages.StringField(1) - value = messages.MessageField(CompoundPropertyType, 2) + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.MessageField(CompoundPropertyType, 2) - additional_properties = messages.MessageField( - 'AdditionalProperty', 1, repeated=True) + additional_properties = messages.MessageField( + 'AdditionalProperty', 1, repeated=True) class HasNestedMessage(messages.Message): - nested = messages.MessageField(AdditionalPropertiesMessage, 1) - nested_list = messages.StringField(2, repeated=True) + nested = messages.MessageField(AdditionalPropertiesMessage, 1) + nested_list = messages.StringField(2, repeated=True) class ExtraNestedMessage(messages.Message): - nested = messages.MessageField(HasNestedMessage, 1) + nested = messages.MessageField(HasNestedMessage, 1) class MessageWithRemappings(messages.Message): - class SomeEnum(messages.Enum): - enum_value = 1 - second_value = 2 + class SomeEnum(messages.Enum): + enum_value = 1 + second_value = 2 - enum_field = messages.EnumField(SomeEnum, 1) - double_encoding = messages.EnumField(SomeEnum, 2) - another_field = messages.StringField(3) - repeated_enum = messages.EnumField(SomeEnum, 4, repeated=True) - repeated_field = messages.StringField(5, repeated=True) + enum_field = messages.EnumField(SomeEnum, 1) + double_encoding = messages.EnumField(SomeEnum, 2) + another_field = messages.StringField(3) + repeated_enum = messages.EnumField(SomeEnum, 4, repeated=True) + repeated_field = messages.StringField(5, repeated=True) encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, @@ -100,252 +100,255 @@ encoding.AddCustomJsonFieldMapping(MessageWithRemappings, class EncodingTest(unittest2.TestCase): - def testCopyProtoMessage(self): - msg = SimpleMessage(field='abc') - new_msg = encoding.CopyProtoMessage(msg) - self.assertEqual(msg.field, new_msg.field) - msg.field = 'def' - self.assertNotEqual(msg.field, new_msg.field) - - def testBytesEncoding(self): - b64_str = 'AAc+' - b64_msg = '{"field": "%s"}' % b64_str - urlsafe_b64_str = 'AAc-' - urlsafe_b64_msg = '{"field": "%s"}' % urlsafe_b64_str - data = base64.b64decode(b64_str) - msg = BytesMessage(field=data) - self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, urlsafe_b64_msg)) - self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, b64_msg)) - self.assertEqual(urlsafe_b64_msg, encoding.MessageToJson(msg)) - - enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % {'b': urlsafe_b64_str} - rep_msg = BytesMessage(repfield=[data, data]) - self.assertEqual(rep_msg, encoding.JsonToMessage(BytesMessage, enc_rep_msg)) - self.assertEqual(enc_rep_msg, encoding.MessageToJson(rep_msg)) - - def testIncludeFields(self): - msg = SimpleMessage() - self.assertEqual('{}', encoding.MessageToJson(msg)) - self.assertEqual( - '{"field": null}', - encoding.MessageToJson(msg, include_fields=['field'])) - self.assertEqual( - '{"repfield": []}', - encoding.MessageToJson(msg, include_fields=['repfield'])) - - def testNestedIncludeFields(self): - msg = HasNestedMessage( - nested=AdditionalPropertiesMessage( - additional_properties=[])) - self.assertEqual( - '{"nested": null}', - encoding.MessageToJson(msg, include_fields=['nested'])) - self.assertEqual( - '{"nested": {"additional_properties": []}}', - encoding.MessageToJson( - msg, include_fields=['nested.additional_properties'])) - msg = ExtraNestedMessage(nested=msg) - self.assertEqual( - '{"nested": {"nested": null}}', - encoding.MessageToJson(msg, include_fields=['nested.nested'])) - self.assertEqual( - '{"nested": {"nested_list": []}}', - encoding.MessageToJson(msg, include_fields=['nested.nested_list'])) - self.assertEqual( - '{"nested": {"nested": {"additional_properties": []}}}', - encoding.MessageToJson( - msg, include_fields=['nested.nested.additional_properties'])) - - def testAdditionalPropertyMapping(self): - msg = AdditionalPropertiesMessage() - msg.additional_properties = [ - AdditionalPropertiesMessage.AdditionalProperty( - key='key_one', value='value_one'), - AdditionalPropertiesMessage.AdditionalProperty( - key='key_two', value='value_two'), - ] - - encoded_msg = encoding.MessageToJson(msg) - self.assertEqual( - {'key_one': 'value_one', 'key_two': 'value_two'}, - json.loads(encoded_msg)) - - new_msg = encoding.JsonToMessage(type(msg), encoded_msg) - self.assertEqual( - set(('key_one', 'key_two')), - set([x.key for x in new_msg.additional_properties])) - self.assertIsNot(msg, new_msg) - - new_msg.additional_properties.pop() - self.assertEqual(1, len(new_msg.additional_properties)) - self.assertEqual(2, len(msg.additional_properties)) - - def testAdditionalMessageProperties(self): - json_msg = '{"input": {"index": 0, "name": "output"}}' - result = encoding.JsonToMessage( - AdditionalMessagePropertiesMessage, json_msg) - self.assertEqual(1, len(result.additional_properties)) - self.assertEqual(0, result.additional_properties[0].value.index) - - def testNestedFieldMapping(self): - nested_msg = AdditionalPropertiesMessage() - nested_msg.additional_properties = [ - AdditionalPropertiesMessage.AdditionalProperty( - key='key_one', value='value_one'), - AdditionalPropertiesMessage.AdditionalProperty( - key='key_two', value='value_two'), - ] - msg = HasNestedMessage(nested=nested_msg) - - encoded_msg = encoding.MessageToJson(msg) - self.assertEqual( - {'nested': {'key_one': 'value_one', 'key_two': 'value_two'}}, - json.loads(encoded_msg)) - - new_msg = encoding.JsonToMessage(type(msg), encoded_msg) - self.assertEqual( - set(('key_one', 'key_two')), - set([x.key for x in new_msg.nested.additional_properties])) - - new_msg.nested.additional_properties.pop() - self.assertEqual(1, len(new_msg.nested.additional_properties)) - self.assertEqual(2, len(msg.nested.additional_properties)) - - def testValidEnums(self): - message_json = '{"field_one": "VALUE_ONE"}' - message = encoding.JsonToMessage(MessageWithEnum, message_json) - self.assertEqual(MessageWithEnum.ThisEnum.VALUE_ONE, message.field_one) - self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) - self.assertEqual(json.loads(message_json), - json.loads(encoding.MessageToJson(message))) - - def testIgnoredEnums(self): - json_with_typo = '{"field_one": "VALUE_OEN"}' - message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) - self.assertEqual(None, message.field_one) - self.assertEqual(('VALUE_OEN', messages.Variant.ENUM), - message.get_unrecognized_field_info('field_one')) - self.assertEqual(json.loads(json_with_typo), - json.loads(encoding.MessageToJson(message))) - - empty_json = '' - message = encoding.JsonToMessage(MessageWithEnum, empty_json) - self.assertEqual(None, message.field_one) - - def testIgnoredEnumsWithDefaults(self): - json_with_typo = '{"field_two": "VALUE_OEN"}' - message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) - self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) - self.assertEqual(json.loads(json_with_typo), - json.loads(encoding.MessageToJson(message))) - - def testUnknownNestedRoundtrip(self): - json_message = '{"field": "abc", "submessage": {"a": 1, "b": "foo"}}' - message = encoding.JsonToMessage(SimpleMessage, json_message) - self.assertEqual(json.loads(json_message), - json.loads(encoding.MessageToJson(message))) - - def testJsonDatetime(self): - msg = TimeMessage(timefield=datetime.datetime( - 2014, 7, 2, 23, 33, 25, 541000, - tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) - self.assertEqual( - '{"timefield": "2014-07-02T23:33:25.541000+00:00"}', - encoding.MessageToJson(msg)) - - def testEnumRemapping(self): - msg = MessageWithRemappings( - enum_field=MessageWithRemappings.SomeEnum.enum_value) - json_message = encoding.MessageToJson(msg) - self.assertEqual('{"enum_field": "wire_name"}', json_message) - self.assertEqual( - msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - - def testRepeatedEnumRemapping(self): - msg = MessageWithRemappings( - repeated_enum=[ - MessageWithRemappings.SomeEnum.enum_value, - MessageWithRemappings.SomeEnum.second_value, - ]) - json_message = encoding.MessageToJson(msg) - self.assertEqual('{"repeated_enum": ["wire_name", "second_value"]}', - json_message) - self.assertEqual( - msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - - def testFieldRemapping(self): - msg = MessageWithRemappings(another_field='abc') - json_message = encoding.MessageToJson(msg) - self.assertEqual('{"anotherField": "abc"}', json_message) - self.assertEqual( - msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - - def testRepeatedFieldRemapping(self): - msg = MessageWithRemappings(repeated_field=['abc', 'def']) - json_message = encoding.MessageToJson(msg) - self.assertEqual('{"repeatedField": ["abc", "def"]}', json_message) - self.assertEqual( - msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - - def testMultipleRemapping(self): - msg = MessageWithRemappings( - double_encoding=MessageWithRemappings.SomeEnum.enum_value) - json_message = encoding.MessageToJson(msg) - self.assertEqual('{"doubleEncoding": "wire_name"}', json_message) - self.assertEqual( - msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - - def testNoRepeatedRemapping(self): - self.assertRaises( - exceptions.InvalidDataError, - encoding.AddCustomJsonFieldMapping, - MessageWithRemappings, 'double_encoding', 'something_else') - self.assertRaises( - exceptions.InvalidDataError, - encoding.AddCustomJsonFieldMapping, - MessageWithRemappings, 'enum_field', 'anotherField') - self.assertRaises( - exceptions.InvalidDataError, - encoding.AddCustomJsonEnumMapping, - MessageWithRemappings.SomeEnum, 'enum_value', 'another_name') - self.assertRaises( - exceptions.InvalidDataError, - encoding.AddCustomJsonEnumMapping, - MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') - - def testMessageToRepr(self): - # pylint:disable=bad-whitespace, Using the same string returned by - # MessageToRepr, with the module names fixed. - msg = SimpleMessage(field='field',repfield=['field','field',],) - self.assertEqual( - encoding.MessageToRepr(msg), - r"%s.SimpleMessage(field='field',repfield=['field','field',],)" % ( - __name__,)) - self.assertEqual( - encoding.MessageToRepr(msg, no_modules=True), - r"SimpleMessage(field='field',repfield=['field','field',],)") - - def testMessageToReprWithTime(self): - msg = TimeMessage(timefield=datetime.datetime( - 2014, 7, 2, 23, 33, 25, 541000, - tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) - self.assertEqual( - encoding.MessageToRepr(msg, multiline=True), - # pylint:disable=line-too-long, Too much effort to make MessageToRepr - # wrap lines properly. - """\ + def testCopyProtoMessage(self): + msg = SimpleMessage(field='abc') + new_msg = encoding.CopyProtoMessage(msg) + self.assertEqual(msg.field, new_msg.field) + msg.field = 'def' + self.assertNotEqual(msg.field, new_msg.field) + + def testBytesEncoding(self): + b64_str = 'AAc+' + b64_msg = '{"field": "%s"}' % b64_str + urlsafe_b64_str = 'AAc-' + urlsafe_b64_msg = '{"field": "%s"}' % urlsafe_b64_str + data = base64.b64decode(b64_str) + msg = BytesMessage(field=data) + self.assertEqual( + msg, encoding.JsonToMessage(BytesMessage, urlsafe_b64_msg)) + self.assertEqual(msg, encoding.JsonToMessage(BytesMessage, b64_msg)) + self.assertEqual(urlsafe_b64_msg, encoding.MessageToJson(msg)) + + enc_rep_msg = '{"repfield": ["%(b)s", "%(b)s"]}' % { + 'b': urlsafe_b64_str} + rep_msg = BytesMessage(repfield=[data, data]) + self.assertEqual( + rep_msg, encoding.JsonToMessage(BytesMessage, enc_rep_msg)) + self.assertEqual(enc_rep_msg, encoding.MessageToJson(rep_msg)) + + def testIncludeFields(self): + msg = SimpleMessage() + self.assertEqual('{}', encoding.MessageToJson(msg)) + self.assertEqual( + '{"field": null}', + encoding.MessageToJson(msg, include_fields=['field'])) + self.assertEqual( + '{"repfield": []}', + encoding.MessageToJson(msg, include_fields=['repfield'])) + + def testNestedIncludeFields(self): + msg = HasNestedMessage( + nested=AdditionalPropertiesMessage( + additional_properties=[])) + self.assertEqual( + '{"nested": null}', + encoding.MessageToJson(msg, include_fields=['nested'])) + self.assertEqual( + '{"nested": {"additional_properties": []}}', + encoding.MessageToJson( + msg, include_fields=['nested.additional_properties'])) + msg = ExtraNestedMessage(nested=msg) + self.assertEqual( + '{"nested": {"nested": null}}', + encoding.MessageToJson(msg, include_fields=['nested.nested'])) + self.assertEqual( + '{"nested": {"nested_list": []}}', + encoding.MessageToJson(msg, include_fields=['nested.nested_list'])) + self.assertEqual( + '{"nested": {"nested": {"additional_properties": []}}}', + encoding.MessageToJson( + msg, include_fields=['nested.nested.additional_properties'])) + + def testAdditionalPropertyMapping(self): + msg = AdditionalPropertiesMessage() + msg.additional_properties = [ + AdditionalPropertiesMessage.AdditionalProperty( + key='key_one', value='value_one'), + AdditionalPropertiesMessage.AdditionalProperty( + key='key_two', value='value_two'), + ] + + encoded_msg = encoding.MessageToJson(msg) + self.assertEqual( + {'key_one': 'value_one', 'key_two': 'value_two'}, + json.loads(encoded_msg)) + + new_msg = encoding.JsonToMessage(type(msg), encoded_msg) + self.assertEqual( + set(('key_one', 'key_two')), + set([x.key for x in new_msg.additional_properties])) + self.assertIsNot(msg, new_msg) + + new_msg.additional_properties.pop() + self.assertEqual(1, len(new_msg.additional_properties)) + self.assertEqual(2, len(msg.additional_properties)) + + def testAdditionalMessageProperties(self): + json_msg = '{"input": {"index": 0, "name": "output"}}' + result = encoding.JsonToMessage( + AdditionalMessagePropertiesMessage, json_msg) + self.assertEqual(1, len(result.additional_properties)) + self.assertEqual(0, result.additional_properties[0].value.index) + + def testNestedFieldMapping(self): + nested_msg = AdditionalPropertiesMessage() + nested_msg.additional_properties = [ + AdditionalPropertiesMessage.AdditionalProperty( + key='key_one', value='value_one'), + AdditionalPropertiesMessage.AdditionalProperty( + key='key_two', value='value_two'), + ] + msg = HasNestedMessage(nested=nested_msg) + + encoded_msg = encoding.MessageToJson(msg) + self.assertEqual( + {'nested': {'key_one': 'value_one', 'key_two': 'value_two'}}, + json.loads(encoded_msg)) + + new_msg = encoding.JsonToMessage(type(msg), encoded_msg) + self.assertEqual( + set(('key_one', 'key_two')), + set([x.key for x in new_msg.nested.additional_properties])) + + new_msg.nested.additional_properties.pop() + self.assertEqual(1, len(new_msg.nested.additional_properties)) + self.assertEqual(2, len(msg.nested.additional_properties)) + + def testValidEnums(self): + message_json = '{"field_one": "VALUE_ONE"}' + message = encoding.JsonToMessage(MessageWithEnum, message_json) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_ONE, message.field_one) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) + self.assertEqual(json.loads(message_json), + json.loads(encoding.MessageToJson(message))) + + def testIgnoredEnums(self): + json_with_typo = '{"field_one": "VALUE_OEN"}' + message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) + self.assertEqual(None, message.field_one) + self.assertEqual(('VALUE_OEN', messages.Variant.ENUM), + message.get_unrecognized_field_info('field_one')) + self.assertEqual(json.loads(json_with_typo), + json.loads(encoding.MessageToJson(message))) + + empty_json = '' + message = encoding.JsonToMessage(MessageWithEnum, empty_json) + self.assertEqual(None, message.field_one) + + def testIgnoredEnumsWithDefaults(self): + json_with_typo = '{"field_two": "VALUE_OEN"}' + message = encoding.JsonToMessage(MessageWithEnum, json_with_typo) + self.assertEqual(MessageWithEnum.ThisEnum.VALUE_TWO, message.field_two) + self.assertEqual(json.loads(json_with_typo), + json.loads(encoding.MessageToJson(message))) + + def testUnknownNestedRoundtrip(self): + json_message = '{"field": "abc", "submessage": {"a": 1, "b": "foo"}}' + message = encoding.JsonToMessage(SimpleMessage, json_message) + self.assertEqual(json.loads(json_message), + json.loads(encoding.MessageToJson(message))) + + def testJsonDatetime(self): + msg = TimeMessage(timefield=datetime.datetime( + 2014, 7, 2, 23, 33, 25, 541000, + tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) + self.assertEqual( + '{"timefield": "2014-07-02T23:33:25.541000+00:00"}', + encoding.MessageToJson(msg)) + + def testEnumRemapping(self): + msg = MessageWithRemappings( + enum_field=MessageWithRemappings.SomeEnum.enum_value) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"enum_field": "wire_name"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testRepeatedEnumRemapping(self): + msg = MessageWithRemappings( + repeated_enum=[ + MessageWithRemappings.SomeEnum.enum_value, + MessageWithRemappings.SomeEnum.second_value, + ]) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"repeated_enum": ["wire_name", "second_value"]}', + json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testFieldRemapping(self): + msg = MessageWithRemappings(another_field='abc') + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"anotherField": "abc"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testRepeatedFieldRemapping(self): + msg = MessageWithRemappings(repeated_field=['abc', 'def']) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"repeatedField": ["abc", "def"]}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testMultipleRemapping(self): + msg = MessageWithRemappings( + double_encoding=MessageWithRemappings.SomeEnum.enum_value) + json_message = encoding.MessageToJson(msg) + self.assertEqual('{"doubleEncoding": "wire_name"}', json_message) + self.assertEqual( + msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) + + def testNoRepeatedRemapping(self): + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonFieldMapping, + MessageWithRemappings, 'double_encoding', 'something_else') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonFieldMapping, + MessageWithRemappings, 'enum_field', 'anotherField') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonEnumMapping, + MessageWithRemappings.SomeEnum, 'enum_value', 'another_name') + self.assertRaises( + exceptions.InvalidDataError, + encoding.AddCustomJsonEnumMapping, + MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') + + def testMessageToRepr(self): + # pylint:disable=bad-whitespace, Using the same string returned by + # MessageToRepr, with the module names fixed. + msg = SimpleMessage(field='field', repfield=['field', 'field', ],) + self.assertEqual( + encoding.MessageToRepr(msg), + r"%s.SimpleMessage(field='field',repfield=['field','field',],)" % ( + __name__,)) + self.assertEqual( + encoding.MessageToRepr(msg, no_modules=True), + r"SimpleMessage(field='field',repfield=['field','field',],)") + + def testMessageToReprWithTime(self): + msg = TimeMessage(timefield=datetime.datetime( + 2014, 7, 2, 23, 33, 25, 541000, + tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) + self.assertEqual( + encoding.MessageToRepr(msg, multiline=True), + # pylint:disable=line-too-long, Too much effort to make MessageToRepr + # wrap lines properly. + """\ %s.TimeMessage( timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=protorpc.util.TimeZoneOffset(datetime.timedelta(0))), )""" % __name__) - self.assertEqual( - encoding.MessageToRepr(msg, multiline=True, no_modules=True), - # pylint:disable=line-too-long, Too much effort to make MessageToRepr - # wrap lines properly. - """\ + self.assertEqual( + encoding.MessageToRepr(msg, multiline=True, no_modules=True), + # pylint:disable=line-too-long, Too much effort to make MessageToRepr + # wrap lines properly. + """\ TimeMessage( timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=TimeZoneOffset(datetime.timedelta(0))), )""") if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index 9af1f37..1d73619 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -3,123 +3,146 @@ class Error(Exception): - """Base class for all exceptions.""" + + """Base class for all exceptions.""" class TypecheckError(Error, TypeError): - """An object of an incorrect type is provided.""" + + """An object of an incorrect type is provided.""" class NotFoundError(Error): - """A specified resource could not be found.""" + + """A specified resource could not be found.""" class UserError(Error): - """Base class for errors related to user input.""" + + """Base class for errors related to user input.""" class InvalidDataError(Error): - """Base class for any invalid data error.""" + + """Base class for any invalid data error.""" class CommunicationError(Error): - """Any communication error talking to an API server.""" + + """Any communication error talking to an API server.""" class HttpError(CommunicationError): - """Error making a request. Soon to be HttpError.""" - def __init__(self, response, content, url): - super(HttpError, self).__init__() - self.response = response - self.content = content - self.url = url + """Error making a request. Soon to be HttpError.""" - def __str__(self): - content = self.content.decode('ascii', 'replace') - return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( - self.url, self.response, content) + def __init__(self, response, content, url): + super(HttpError, self).__init__() + self.response = response + self.content = content + self.url = url - @property - def status_code(self): - # TODO(craigcitro): Turn this into something better than a - # KeyError if there is no status. - return int(self.response['status']) + def __str__(self): + content = self.content.decode('ascii', 'replace') + return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( + self.url, self.response, content) - @classmethod - def FromResponse(cls, http_response): - return cls(http_response.info, http_response.content, - http_response.request_url) + @property + def status_code(self): + # TODO(craigcitro): Turn this into something better than a + # KeyError if there is no status. + return int(self.response['status']) + + @classmethod + def FromResponse(cls, http_response): + return cls(http_response.info, http_response.content, + http_response.request_url) class InvalidUserInputError(InvalidDataError): - """User-provided input is invalid.""" + + """User-provided input is invalid.""" class InvalidDataFromServerError(InvalidDataError, CommunicationError): - """Data received from the server is malformed.""" + + """Data received from the server is malformed.""" class BatchError(Error): - """Error generated while constructing a batch request.""" + + """Error generated while constructing a batch request.""" class ConfigurationError(Error): - """Base class for configuration errors.""" + + """Base class for configuration errors.""" class GeneratedClientError(Error): - """The generated client configuration is invalid.""" + + """The generated client configuration is invalid.""" class ConfigurationValueError(UserError): - """Some part of the user-specified client configuration is invalid.""" + + """Some part of the user-specified client configuration is invalid.""" class ResourceUnavailableError(Error): - """User requested an unavailable resource.""" + + """User requested an unavailable resource.""" class CredentialsError(Error): - """Errors related to invalid credentials.""" + + """Errors related to invalid credentials.""" class TransferError(CommunicationError): - """Errors related to transfers.""" + + """Errors related to transfers.""" class TransferRetryError(TransferError): - """Retryable errors related to transfers.""" + + """Retryable errors related to transfers.""" class TransferInvalidError(TransferError): - """The given transfer is invalid.""" + + """The given transfer is invalid.""" class RequestError(CommunicationError): - """The request was not successful.""" + + """The request was not successful.""" class RetryAfterError(HttpError): - """The response contained a retry-after header.""" - def __init__(self, response, content, url, retry_after): - super(RetryAfterError, self).__init__(response, content, url) - self.retry_after = int(retry_after) + """The response contained a retry-after header.""" + + def __init__(self, response, content, url, retry_after): + super(RetryAfterError, self).__init__(response, content, url) + self.retry_after = int(retry_after) - @classmethod - def FromResponse(cls, http_response): - return cls(http_response.info, http_response.content, - http_response.request_url, http_response.retry_after) + @classmethod + def FromResponse(cls, http_response): + return cls(http_response.info, http_response.content, + http_response.request_url, http_response.retry_after) class BadStatusCodeError(HttpError): - """The request completed but returned a bad status code.""" + + """The request completed but returned a bad status code.""" class NotYetImplementedError(GeneratedClientError): - """This functionality is not yet implemented.""" + + """This functionality is not yet implemented.""" class StreamExhausted(Error): - """Attempted to read more bytes from a stream than were available.""" + + """Attempted to read more bytes from a stream than were available.""" diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 8b29c76..684afd4 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -36,60 +36,64 @@ DateTimeMessage = message_types.DateTimeMessage class DateField(messages.Field): - """Field definition for Date values.""" - # We insert our own metaclass here to avoid letting ProtoRPC - # register this as the default field type for strings. - # * since ProtoRPC does this via metaclasses, we don't have any - # choice but to use one ourselves - # * since a subclass's metaclass must inherit from its superclass's - # metaclass, we're forced to have this hard-to-read inheritance. - # - class __metaclass__(messages.Field.__metaclass__): # pylint: disable=invalid-name + """Field definition for Date values.""" - def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument - super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) + # We insert our own metaclass here to avoid letting ProtoRPC + # register this as the default field type for strings. + # * since ProtoRPC does this via metaclasses, we don't have any + # choice but to use one ourselves + # * since a subclass's metaclass must inherit from its superclass's + # metaclass, we're forced to have this hard-to-read inheritance. + # + class __metaclass__(messages.Field.__metaclass__): # pylint: disable=invalid-name - VARIANTS = frozenset([messages.Variant.STRING]) - DEFAULT_VARIANT = messages.Variant.STRING - type = datetime.date + def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument + super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) + + VARIANTS = frozenset([messages.Variant.STRING]) + DEFAULT_VARIANT = messages.Variant.STRING + type = datetime.date def _ValidateJsonValue(json_value): - entries = [(f, json_value.get_assigned_value(f.name)) - for f in json_value.all_fields()] - assigned_entries = [(f, value) for f, value in entries if value is not None] - if len(assigned_entries) != 1: - raise exceptions.InvalidDataError('Malformed JsonValue: %s' % json_value) + entries = [(f, json_value.get_assigned_value(f.name)) + for f in json_value.all_fields()] + assigned_entries = [(f, value) + for f, value in entries if value is not None] + if len(assigned_entries) != 1: + raise exceptions.InvalidDataError( + 'Malformed JsonValue: %s' % json_value) def _JsonValueToPythonValue(json_value): - """Convert the given JsonValue to a json string.""" - util.Typecheck(json_value, JsonValue) - _ValidateJsonValue(json_value) - if json_value.is_null: - return None - entries = [(f, json_value.get_assigned_value(f.name)) - for f in json_value.all_fields()] - assigned_entries = [(f, value) for f, value in entries if value is not None] - field, value = assigned_entries[0] - if not isinstance(field, messages.MessageField): - return value - elif field.message_type is JsonObject: - return _JsonObjectToPythonValue(value) - elif field.message_type is JsonArray: - return _JsonArrayToPythonValue(value) + """Convert the given JsonValue to a json string.""" + util.Typecheck(json_value, JsonValue) + _ValidateJsonValue(json_value) + if json_value.is_null: + return None + entries = [(f, json_value.get_assigned_value(f.name)) + for f in json_value.all_fields()] + assigned_entries = [(f, value) + for f, value in entries if value is not None] + field, value = assigned_entries[0] + if not isinstance(field, messages.MessageField): + return value + elif field.message_type is JsonObject: + return _JsonObjectToPythonValue(value) + elif field.message_type is JsonArray: + return _JsonArrayToPythonValue(value) def _JsonObjectToPythonValue(json_value): - util.Typecheck(json_value, JsonObject) - return dict([(prop.key, _JsonValueToPythonValue(prop.value)) for prop - in json_value.properties]) + util.Typecheck(json_value, JsonObject) + return dict([(prop.key, _JsonValueToPythonValue(prop.value)) for prop + in json_value.properties]) def _JsonArrayToPythonValue(json_value): - util.Typecheck(json_value, JsonArray) - return [_JsonValueToPythonValue(e) for e in json_value.entries] + util.Typecheck(json_value, JsonArray) + return [_JsonValueToPythonValue(e) for e in json_value.entries] _MAXINT64 = 2 << 63 - 1 @@ -97,81 +101,85 @@ _MININT64 = -(2 << 63) def _PythonValueToJsonValue(py_value): - """Convert the given python value to a JsonValue.""" - if py_value is None: - return JsonValue(is_null=True) - if isinstance(py_value, bool): - return JsonValue(boolean_value=py_value) - if isinstance(py_value, six.string_types): - return JsonValue(string_value=py_value) - if isinstance(py_value, numbers.Number): - if isinstance(py_value, six.integer_types): - if _MININT64 < py_value < _MAXINT64: - return JsonValue(integer_value=py_value) - return JsonValue(double_value=float(py_value)) - if isinstance(py_value, dict): - return JsonValue(object_value=_PythonValueToJsonObject(py_value)) - if isinstance(py_value, collections.Iterable): - return JsonValue(array_value=_PythonValueToJsonArray(py_value)) - raise exceptions.InvalidDataError( - 'Cannot convert "%s" to JsonValue' % py_value) + """Convert the given python value to a JsonValue.""" + if py_value is None: + return JsonValue(is_null=True) + if isinstance(py_value, bool): + return JsonValue(boolean_value=py_value) + if isinstance(py_value, six.string_types): + return JsonValue(string_value=py_value) + if isinstance(py_value, numbers.Number): + if isinstance(py_value, six.integer_types): + if _MININT64 < py_value < _MAXINT64: + return JsonValue(integer_value=py_value) + return JsonValue(double_value=float(py_value)) + if isinstance(py_value, dict): + return JsonValue(object_value=_PythonValueToJsonObject(py_value)) + if isinstance(py_value, collections.Iterable): + return JsonValue(array_value=_PythonValueToJsonArray(py_value)) + raise exceptions.InvalidDataError( + 'Cannot convert "%s" to JsonValue' % py_value) def _PythonValueToJsonObject(py_value): - util.Typecheck(py_value, dict) - return JsonObject( - properties=[ - JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) - for key, value in py_value.items()]) + util.Typecheck(py_value, dict) + return JsonObject( + properties=[ + JsonObject.Property(key=key, value=_PythonValueToJsonValue(value)) + for key, value in py_value.items()]) def _PythonValueToJsonArray(py_value): - return JsonArray(entries=list(map(_PythonValueToJsonValue, py_value))) + return JsonArray(entries=list(map(_PythonValueToJsonValue, py_value))) class JsonValue(messages.Message): - """Any valid JSON value.""" - # Is this JSON object `null`? - is_null = messages.BooleanField(1, default=False) - - # Exactly one of the following is provided if is_null is False; none - # should be provided if is_null is True. - boolean_value = messages.BooleanField(2) - string_value = messages.StringField(3) - # We keep two numeric fields to keep int64 round-trips exact. - double_value = messages.FloatField(4, variant=messages.Variant.DOUBLE) - integer_value = messages.IntegerField(5, variant=messages.Variant.INT64) - # Compound types - object_value = messages.MessageField('JsonObject', 6) - array_value = messages.MessageField('JsonArray', 7) + """Any valid JSON value.""" + # Is this JSON object `null`? + is_null = messages.BooleanField(1, default=False) -class JsonObject(messages.Message): - """A JSON object value. + # Exactly one of the following is provided if is_null is False; none + # should be provided if is_null is True. + boolean_value = messages.BooleanField(2) + string_value = messages.StringField(3) + # We keep two numeric fields to keep int64 round-trips exact. + double_value = messages.FloatField(4, variant=messages.Variant.DOUBLE) + integer_value = messages.IntegerField(5, variant=messages.Variant.INT64) + # Compound types + object_value = messages.MessageField('JsonObject', 6) + array_value = messages.MessageField('JsonArray', 7) - Messages: - Property: A property of a JsonObject. - Fields: - properties: A list of properties of a JsonObject. - """ +class JsonObject(messages.Message): + + """A JSON object value. - class Property(messages.Message): - """A property of a JSON object. + Messages: + Property: A property of a JsonObject. Fields: - key: Name of the property. - value: A JsonValue attribute. + properties: A list of properties of a JsonObject. """ - key = messages.StringField(1) - value = messages.MessageField(JsonValue, 2) - properties = messages.MessageField(Property, 1, repeated=True) + class Property(messages.Message): + + """A property of a JSON object. + + Fields: + key: Name of the property. + value: A JsonValue attribute. + """ + key = messages.StringField(1) + value = messages.MessageField(JsonValue, 2) + + properties = messages.MessageField(Property, 1, repeated=True) class JsonArray(messages.Message): - """A JSON array value.""" - entries = messages.MessageField(JsonValue, 1, repeated=True) + + """A JSON array value.""" + entries = messages.MessageField(JsonValue, 1, repeated=True) _JSON_PROTO_TO_PYTHON_MAP = { @@ -183,38 +191,38 @@ _JSON_PROTO_TYPES = tuple(_JSON_PROTO_TO_PYTHON_MAP.keys()) def _JsonProtoToPythonValue(json_proto): - util.Typecheck(json_proto, _JSON_PROTO_TYPES) - return _JSON_PROTO_TO_PYTHON_MAP[type(json_proto)](json_proto) + util.Typecheck(json_proto, _JSON_PROTO_TYPES) + return _JSON_PROTO_TO_PYTHON_MAP[type(json_proto)](json_proto) def _PythonValueToJsonProto(py_value): - if isinstance(py_value, dict): - return _PythonValueToJsonObject(py_value) - if (isinstance(py_value, collections.Iterable) and - not isinstance(py_value, six.string_types)): - return _PythonValueToJsonArray(py_value) - return _PythonValueToJsonValue(py_value) + if isinstance(py_value, dict): + return _PythonValueToJsonObject(py_value) + if (isinstance(py_value, collections.Iterable) and + not isinstance(py_value, six.string_types)): + return _PythonValueToJsonArray(py_value) + return _PythonValueToJsonValue(py_value) def _JsonProtoToJson(json_proto, unused_encoder=None): - return json.dumps(_JsonProtoToPythonValue(json_proto)) + return json.dumps(_JsonProtoToPythonValue(json_proto)) def _JsonToJsonProto(json_data, unused_decoder=None): - return _PythonValueToJsonProto(json.loads(json_data)) + return _PythonValueToJsonProto(json.loads(json_data)) def _JsonToJsonValue(json_data, unused_decoder=None): - result = _PythonValueToJsonProto(json.loads(json_data)) - if isinstance(result, JsonValue): - return result - elif isinstance(result, JsonObject): - return JsonValue(object_value=result) - elif isinstance(result, JsonArray): - return JsonValue(array_value=result) - else: - raise exceptions.InvalidDataError( - 'Malformed JsonValue: %s' % json_data) + result = _PythonValueToJsonProto(json.loads(json_data)) + if isinstance(result, JsonValue): + return result + elif isinstance(result, JsonObject): + return JsonValue(object_value=result) + elif isinstance(result, JsonArray): + return JsonValue(array_value=result) + else: + raise exceptions.InvalidDataError( + 'Malformed JsonValue: %s' % json_data) # pylint:disable=invalid-name @@ -230,14 +238,14 @@ encoding.RegisterCustomMessageCodec( def _EncodeDateTimeField(field, value): - result = protojson.ProtoJson().encode_field(field, value) - return encoding.CodecResult(value=result, complete=True) + result = protojson.ProtoJson().encode_field(field, value) + return encoding.CodecResult(value=result, complete=True) def _DecodeDateTimeField(unused_field, value): - result = protojson.ProtoJson().decode_field( - message_types.DateTimeField(1), value) - return encoding.CodecResult(value=result, complete=True) + result = protojson.ProtoJson().decode_field( + message_types.DateTimeField(1), value) + return encoding.CodecResult(value=result, complete=True) encoding.RegisterFieldTypeCodec(_EncodeDateTimeField, _DecodeDateTimeField)( @@ -245,40 +253,40 @@ encoding.RegisterFieldTypeCodec(_EncodeDateTimeField, _DecodeDateTimeField)( def _EncodeInt64Field(field, value): - """Handle the special case of int64 as a string.""" - capabilities = [ - messages.Variant.INT64, - messages.Variant.UINT64, - ] - if field.variant not in capabilities: - return encoding.CodecResult(value=value, complete=False) + """Handle the special case of int64 as a string.""" + capabilities = [ + messages.Variant.INT64, + messages.Variant.UINT64, + ] + if field.variant not in capabilities: + return encoding.CodecResult(value=value, complete=False) - if field.repeated: - result = [str(x) for x in value] - else: - result = str(value) - return encoding.CodecResult(value=result, complete=True) + if field.repeated: + result = [str(x) for x in value] + else: + result = str(value) + return encoding.CodecResult(value=result, complete=True) def _DecodeInt64Field(unused_field, value): - # Don't need to do anything special, they're decoded just fine - return encoding.CodecResult(value=value, complete=False) + # Don't need to do anything special, they're decoded just fine + return encoding.CodecResult(value=value, complete=False) encoding.RegisterFieldTypeCodec(_EncodeInt64Field, _DecodeInt64Field)( messages.IntegerField) def _EncodeDateField(field, value): - """Encoder for datetime.date objects.""" - if field.repeated: - result = [d.isoformat() for d in value] - else: - result = value.isoformat() - return encoding.CodecResult(value=result, complete=True) + """Encoder for datetime.date objects.""" + if field.repeated: + result = [d.isoformat() for d in value] + else: + result = value.isoformat() + return encoding.CodecResult(value=result, complete=True) def _DecodeDateField(unused_field, value): - date = datetime.datetime.strptime(value, '%Y-%m-%d').date() - return encoding.CodecResult(value=date, complete=True) + date = datetime.datetime.strptime(value, '%Y-%m-%d').date() + return encoding.CodecResult(value=date, complete=True) encoding.RegisterFieldTypeCodec(_EncodeDateField, _DecodeDateField)(DateField) diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 13cd439..f8ec125 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -15,161 +15,166 @@ from apitools.base.py import extra_types class ExtraTypesTest(unittest2.TestCase): - def assertRoundTrip(self, value): - if isinstance(value, extra_types._JSON_PROTO_TYPES): - self.assertEqual( - value, - extra_types._PythonValueToJsonProto( - extra_types._JsonProtoToPythonValue(value))) - else: - self.assertEqual( - value, - extra_types._JsonProtoToPythonValue( - extra_types._PythonValueToJsonProto(value))) - - def assertTranslations(self, py_value, json_proto): - self.assertEqual(py_value, extra_types._JsonProtoToPythonValue(json_proto)) - self.assertEqual(json_proto, extra_types._PythonValueToJsonProto(py_value)) - - def testInvalidProtos(self): - with self.assertRaises(exceptions.InvalidDataError): - extra_types._ValidateJsonValue(extra_types.JsonValue()) - with self.assertRaises(exceptions.InvalidDataError): - extra_types._ValidateJsonValue( - extra_types.JsonValue(is_null=True, string_value='a')) - with self.assertRaises(exceptions.InvalidDataError): - extra_types._ValidateJsonValue( - extra_types.JsonValue(integer_value=3, string_value='a')) - - def testNullEncoding(self): - self.assertTranslations(None, extra_types.JsonValue(is_null=True)) - - def testJsonNumberEncoding(self): - seventeen = extra_types.JsonValue(integer_value=17) - self.assertRoundTrip(17) - self.assertRoundTrip(seventeen) - self.assertTranslations(17, seventeen) - - json_pi = extra_types.JsonValue(double_value=math.pi) - self.assertRoundTrip(math.pi) - self.assertRoundTrip(json_pi) - self.assertTranslations(math.pi, json_pi) - - def testArrayEncoding(self): - array = [3, 'four', False] - json_array = extra_types.JsonArray(entries=[ - extra_types.JsonValue(integer_value=3), - extra_types.JsonValue(string_value='four'), - extra_types.JsonValue(boolean_value=False), - ]) - self.assertRoundTrip(array) - self.assertRoundTrip(json_array) - self.assertTranslations(array, json_array) - - def testArrayAsValue(self): - array_json = '[3, "four", false]' - array = [3, 'four', False] - value = encoding.JsonToMessage(extra_types.JsonValue, array_json) - self.assertTrue(isinstance(value, extra_types.JsonValue)) - self.assertEqual(array, encoding.MessageToPyValue(value)) - - def testObjectAsValue(self): - obj_json = '{"works": true}' - obj = {'works': True} - value = encoding.JsonToMessage(extra_types.JsonValue, obj_json) - self.assertTrue(isinstance(value, extra_types.JsonValue)) - self.assertEqual(obj, encoding.MessageToPyValue(value)) - - def testDictEncoding(self): - d = {'a': 6, 'b': 'eleventeen'} - json_d = extra_types.JsonObject(properties=[ - extra_types.JsonObject.Property( - key='a', value=extra_types.JsonValue(integer_value=6)), - extra_types.JsonObject.Property( - key='b', value=extra_types.JsonValue(string_value='eleventeen')), - ]) - self.assertRoundTrip(d) - # We don't know json_d will round-trip, because of randomness in - # python dictionary iteration ordering. We also need to force - # comparison as lists, since hashing protos isn't helpful. - translated_properties = extra_types._PythonValueToJsonProto(d).properties - for p in json_d.properties: - self.assertIn(p, translated_properties) - for p in translated_properties: - self.assertIn(p, json_d.properties) - - def testJsonObjectPropertyTranslation(self): - value = extra_types.JsonValue(string_value='abc') - obj = extra_types.JsonObject(properties=[ - extra_types.JsonObject.Property(key='attr_name', value=value)]) - json_value = '"abc"' - json_obj = '{"attr_name": "abc"}' - - self.assertRoundTrip(value) - self.assertRoundTrip(obj) - self.assertRoundTrip(json_value) - self.assertRoundTrip(json_obj) - - self.assertEqual(json_value, encoding.MessageToJson(value)) - self.assertEqual(json_obj, encoding.MessageToJson(obj)) - - def testDateField(self): - - class DateMsg(messages.Message): - start_date = extra_types.DateField(1) - all_dates = extra_types.DateField(2, repeated=True) - - msg = DateMsg( - start_date=datetime.date(1752, 9, 9), all_dates=[ - datetime.date(1979, 5, 6), - datetime.date(1980, 10, 24), - datetime.date(1981, 1, 19), + def assertRoundTrip(self, value): + if isinstance(value, extra_types._JSON_PROTO_TYPES): + self.assertEqual( + value, + extra_types._PythonValueToJsonProto( + extra_types._JsonProtoToPythonValue(value))) + else: + self.assertEqual( + value, + extra_types._JsonProtoToPythonValue( + extra_types._PythonValueToJsonProto(value))) + + def assertTranslations(self, py_value, json_proto): + self.assertEqual( + py_value, extra_types._JsonProtoToPythonValue(json_proto)) + self.assertEqual( + json_proto, extra_types._PythonValueToJsonProto(py_value)) + + def testInvalidProtos(self): + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue(extra_types.JsonValue()) + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue( + extra_types.JsonValue(is_null=True, string_value='a')) + with self.assertRaises(exceptions.InvalidDataError): + extra_types._ValidateJsonValue( + extra_types.JsonValue(integer_value=3, string_value='a')) + + def testNullEncoding(self): + self.assertTranslations(None, extra_types.JsonValue(is_null=True)) + + def testJsonNumberEncoding(self): + seventeen = extra_types.JsonValue(integer_value=17) + self.assertRoundTrip(17) + self.assertRoundTrip(seventeen) + self.assertTranslations(17, seventeen) + + json_pi = extra_types.JsonValue(double_value=math.pi) + self.assertRoundTrip(math.pi) + self.assertRoundTrip(json_pi) + self.assertTranslations(math.pi, json_pi) + + def testArrayEncoding(self): + array = [3, 'four', False] + json_array = extra_types.JsonArray(entries=[ + extra_types.JsonValue(integer_value=3), + extra_types.JsonValue(string_value='four'), + extra_types.JsonValue(boolean_value=False), + ]) + self.assertRoundTrip(array) + self.assertRoundTrip(json_array) + self.assertTranslations(array, json_array) + + def testArrayAsValue(self): + array_json = '[3, "four", false]' + array = [3, 'four', False] + value = encoding.JsonToMessage(extra_types.JsonValue, array_json) + self.assertTrue(isinstance(value, extra_types.JsonValue)) + self.assertEqual(array, encoding.MessageToPyValue(value)) + + def testObjectAsValue(self): + obj_json = '{"works": true}' + obj = {'works': True} + value = encoding.JsonToMessage(extra_types.JsonValue, obj_json) + self.assertTrue(isinstance(value, extra_types.JsonValue)) + self.assertEqual(obj, encoding.MessageToPyValue(value)) + + def testDictEncoding(self): + d = {'a': 6, 'b': 'eleventeen'} + json_d = extra_types.JsonObject(properties=[ + extra_types.JsonObject.Property( + key='a', value=extra_types.JsonValue(integer_value=6)), + extra_types.JsonObject.Property( + key='b', value=extra_types.JsonValue(string_value='eleventeen')), + ]) + self.assertRoundTrip(d) + # We don't know json_d will round-trip, because of randomness in + # python dictionary iteration ordering. We also need to force + # comparison as lists, since hashing protos isn't helpful. + translated_properties = extra_types._PythonValueToJsonProto( + d).properties + for p in json_d.properties: + self.assertIn(p, translated_properties) + for p in translated_properties: + self.assertIn(p, json_d.properties) + + def testJsonObjectPropertyTranslation(self): + value = extra_types.JsonValue(string_value='abc') + obj = extra_types.JsonObject(properties=[ + extra_types.JsonObject.Property(key='attr_name', value=value)]) + json_value = '"abc"' + json_obj = '{"attr_name": "abc"}' + + self.assertRoundTrip(value) + self.assertRoundTrip(obj) + self.assertRoundTrip(json_value) + self.assertRoundTrip(json_obj) + + self.assertEqual(json_value, encoding.MessageToJson(value)) + self.assertEqual(json_obj, encoding.MessageToJson(obj)) + + def testDateField(self): + + class DateMsg(messages.Message): + start_date = extra_types.DateField(1) + all_dates = extra_types.DateField(2, repeated=True) + + msg = DateMsg( + start_date=datetime.date(1752, 9, 9), all_dates=[ + datetime.date(1979, 5, 6), + datetime.date(1980, 10, 24), + datetime.date(1981, 1, 19), ]) - msg_dict = { - 'start_date': '1752-09-09', - 'all_dates': ['1979-05-06', '1980-10-24', '1981-01-19'], - } - self.assertEqual(msg_dict, json.loads(encoding.MessageToJson(msg))) - self.assertEqual(msg, encoding.JsonToMessage(DateMsg, json.dumps(msg_dict))) - - def testInt64(self): - # Testing roundtrip of type 'long' - - class DogeMsg(messages.Message): - such_string = messages.StringField(1) - wow = messages.IntegerField(2, variant=messages.Variant.INT64) - very_unsigned = messages.IntegerField(3, variant=messages.Variant.UINT64) - much_repeated = messages.IntegerField( - 4, variant=messages.Variant.INT64, repeated=True) - - def MtoJ(msg): - return encoding.MessageToJson(msg) - - def JtoM(class_type, json_str): - return encoding.JsonToMessage(class_type, json_str) - - def DoRoundtrip(class_type, json_msg=None, message=None, times=4): - if json_msg: - json_msg = MtoJ(JtoM(class_type, json_msg)) - if message: - message = JtoM(class_type, MtoJ(message)) - if times == 0: - result = json_msg if json_msg else message - return result - return DoRoundtrip(class_type=class_type, json_msg=json_msg, - message=message, times=times - 1) - - # Single - json_msg = ('{"such_string": "poot", "wow": "-1234",' - ' "very_unsigned": "999", "much_repeated": ["123", "456"]}') - out_json = MtoJ(JtoM(DogeMsg, json_msg)) - self.assertEqual(json.loads(out_json)['wow'], '-1234') - - # Repeated test case - msg = DogeMsg(such_string='wow', wow=-1234, - very_unsigned=800, much_repeated=[123, 456]) - self.assertEqual(msg, DoRoundtrip(DogeMsg, message=msg)) + msg_dict = { + 'start_date': '1752-09-09', + 'all_dates': ['1979-05-06', '1980-10-24', '1981-01-19'], + } + self.assertEqual(msg_dict, json.loads(encoding.MessageToJson(msg))) + self.assertEqual( + msg, encoding.JsonToMessage(DateMsg, json.dumps(msg_dict))) + + def testInt64(self): + # Testing roundtrip of type 'long' + + class DogeMsg(messages.Message): + such_string = messages.StringField(1) + wow = messages.IntegerField(2, variant=messages.Variant.INT64) + very_unsigned = messages.IntegerField( + 3, variant=messages.Variant.UINT64) + much_repeated = messages.IntegerField( + 4, variant=messages.Variant.INT64, repeated=True) + + def MtoJ(msg): + return encoding.MessageToJson(msg) + + def JtoM(class_type, json_str): + return encoding.JsonToMessage(class_type, json_str) + + def DoRoundtrip(class_type, json_msg=None, message=None, times=4): + if json_msg: + json_msg = MtoJ(JtoM(class_type, json_msg)) + if message: + message = JtoM(class_type, MtoJ(message)) + if times == 0: + result = json_msg if json_msg else message + return result + return DoRoundtrip(class_type=class_type, json_msg=json_msg, + message=message, times=times - 1) + + # Single + json_msg = ('{"such_string": "poot", "wow": "-1234",' + ' "very_unsigned": "999", "much_repeated": ["123", "456"]}') + out_json = MtoJ(JtoM(DogeMsg, json_msg)) + self.assertEqual(json.loads(out_json)['wow'], '-1234') + + # Repeated test case + msg = DogeMsg(such_string='wow', wow=-1234, + very_unsigned=800, much_repeated=[123, 456]) + self.assertEqual(msg, DoRoundtrip(DogeMsg, message=msg)) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 8267d0c..5ecf6d7 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -52,312 +52,319 @@ ExceptionRetryArgs = collections.namedtuple( @contextlib.contextmanager def _Httplib2Debuglevel(http_request, level, http=None): - """Temporarily change the value of httplib2.debuglevel, if necessary. - - If http_request has a `loggable_body` distinct from `body`, then we - need to prevent httplib2 from logging the full body. This sets - httplib2.debuglevel for the duration of the `with` block; however, - that alone won't change the value of existing HTTP connections. If - an httplib2.Http object is provided, we'll also change the level on - any cached connections attached to it. - - Args: - http_request: a Request we're logging. - level: (int) the debuglevel for logging. - http: (optional) an httplib2.Http whose connections we should - set the debuglevel on. - - Yields: - None. - """ - if http_request.loggable_body is None: + """Temporarily change the value of httplib2.debuglevel, if necessary. + + If http_request has a `loggable_body` distinct from `body`, then we + need to prevent httplib2 from logging the full body. This sets + httplib2.debuglevel for the duration of the `with` block; however, + that alone won't change the value of existing HTTP connections. If + an httplib2.Http object is provided, we'll also change the level on + any cached connections attached to it. + + Args: + http_request: a Request we're logging. + level: (int) the debuglevel for logging. + http: (optional) an httplib2.Http whose connections we should + set the debuglevel on. + + Yields: + None. + """ + if http_request.loggable_body is None: + yield + return + old_level = httplib2.debuglevel + http_levels = {} + httplib2.debuglevel = level + if http is not None: + for connection_key, connection in http.connections.items(): + # httplib2 stores two kinds of values in this dict, connection + # classes and instances. Since the connection types are all + # old-style classes, we can't easily distinguish by connection + # type -- so instead we use the key pattern. + if ':' not in connection_key: + continue + http_levels[connection_key] = connection.debuglevel + connection.set_debuglevel(level) yield - return - old_level = httplib2.debuglevel - http_levels = {} - httplib2.debuglevel = level - if http is not None: - for connection_key, connection in http.connections.items(): - # httplib2 stores two kinds of values in this dict, connection - # classes and instances. Since the connection types are all - # old-style classes, we can't easily distinguish by connection - # type -- so instead we use the key pattern. - if ':' not in connection_key: - continue - http_levels[connection_key] = connection.debuglevel - connection.set_debuglevel(level) - yield - httplib2.debuglevel = old_level - if http is not None: - for connection_key, old_level in http_levels.items(): - if connection_key in http.connections: - http.connections[connection_key].set_debuglevel(old_level) + httplib2.debuglevel = old_level + if http is not None: + for connection_key, old_level in http_levels.items(): + if connection_key in http.connections: + http.connections[connection_key].set_debuglevel(old_level) class Request(object): - """Class encapsulating the data for an HTTP request.""" - - def __init__(self, url='', http_method='GET', headers=None, body=''): - self.url = url - self.http_method = http_method - self.headers = headers or {} - self.__body = None - self.__loggable_body = None - self.body = body - - @property - def loggable_body(self): - return self.__loggable_body - - @loggable_body.setter - def loggable_body(self, value): - if self.body is None: - raise exceptions.RequestError( - 'Cannot set loggable body on request with no body') - self.__loggable_body = value - - @property - def body(self): - return self.__body - - @body.setter - def body(self, value): - """Sets the request body; handles logging and length measurement.""" - self.__body = value - if value is not None: - # Avoid calling len() which cannot exceed 4GiB in 32-bit python. - body_length = getattr(self.__body, 'length', None) or len(self.__body) - self.headers['content-length'] = str(body_length) - else: - self.headers.pop('content-length', None) - # This line ensures we don't try to print large requests. - if not isinstance(value, six.string_types): - self.loggable_body = '' + + """Class encapsulating the data for an HTTP request.""" + + def __init__(self, url='', http_method='GET', headers=None, body=''): + self.url = url + self.http_method = http_method + self.headers = headers or {} + self.__body = None + self.__loggable_body = None + self.body = body + + @property + def loggable_body(self): + return self.__loggable_body + + @loggable_body.setter + def loggable_body(self, value): + if self.body is None: + raise exceptions.RequestError( + 'Cannot set loggable body on request with no body') + self.__loggable_body = value + + @property + def body(self): + return self.__body + + @body.setter + def body(self, value): + """Sets the request body; handles logging and length measurement.""" + self.__body = value + if value is not None: + # Avoid calling len() which cannot exceed 4GiB in 32-bit python. + body_length = getattr( + self.__body, 'length', None) or len(self.__body) + self.headers['content-length'] = str(body_length) + else: + self.headers.pop('content-length', None) + # This line ensures we don't try to print large requests. + if not isinstance(value, six.string_types): + self.loggable_body = '' # Note: currently the order of fields here is important, since we want # to be able to pass in the result from httplib2.request. class Response(collections.namedtuple( - 'HttpResponse', ['info', 'content', 'request_url'])): - """Class encapsulating data for an HTTP response.""" - __slots__ = () - - def __len__(self): - return self.length - - @property - def length(self): - """Return the length of this response. - - We expose this as an attribute since using len() directly can fail - for responses larger than sys.maxint. - - Returns: - Response length (as int or long) - """ - def ProcessContentRange(content_range): - _, _, range_spec = content_range.partition(' ') - byte_range, _, _ = range_spec.partition('/') - start, _, end = byte_range.partition('-') - return int(end) - int(start) + 1 - - if '-content-encoding' in self.info and 'content-range' in self.info: - # httplib2 rewrites content-length in the case of a compressed - # transfer; we can't trust the content-length header in that - # case, but we *can* trust content-range, if it's present. - return ProcessContentRange(self.info['content-range']) - elif 'content-length' in self.info: - return int(self.info.get('content-length')) - elif 'content-range' in self.info: - return ProcessContentRange(self.info['content-range']) - return len(self.content) - - @property - def status_code(self): - return int(self.info['status']) - - @property - def retry_after(self): - if 'retry-after' in self.info: - return int(self.info['retry-after']) - - @property - def is_redirect(self): - return (self.status_code in _REDIRECT_STATUS_CODES and - 'location' in self.info) + 'HttpResponse', ['info', 'content', 'request_url'])): + + """Class encapsulating data for an HTTP response.""" + __slots__ = () + + def __len__(self): + return self.length + + @property + def length(self): + """Return the length of this response. + + We expose this as an attribute since using len() directly can fail + for responses larger than sys.maxint. + + Returns: + Response length (as int or long) + """ + def ProcessContentRange(content_range): + _, _, range_spec = content_range.partition(' ') + byte_range, _, _ = range_spec.partition('/') + start, _, end = byte_range.partition('-') + return int(end) - int(start) + 1 + + if '-content-encoding' in self.info and 'content-range' in self.info: + # httplib2 rewrites content-length in the case of a compressed + # transfer; we can't trust the content-length header in that + # case, but we *can* trust content-range, if it's present. + return ProcessContentRange(self.info['content-range']) + elif 'content-length' in self.info: + return int(self.info.get('content-length')) + elif 'content-range' in self.info: + return ProcessContentRange(self.info['content-range']) + return len(self.content) + + @property + def status_code(self): + return int(self.info['status']) + + @property + def retry_after(self): + if 'retry-after' in self.info: + return int(self.info['retry-after']) + + @property + def is_redirect(self): + return (self.status_code in _REDIRECT_STATUS_CODES and + 'location' in self.info) def CheckResponse(response): - if response is None: - # Caller shouldn't call us if the response is None, but handle anyway. - raise exceptions.RequestError('Request to url %s did not return a response.' - % response.request_url) - elif (response.status_code >= 500 or - response.status_code == TOO_MANY_REQUESTS): - raise exceptions.BadStatusCodeError.FromResponse(response) - elif response.status_code == http_client.UNAUTHORIZED: - # Sometimes we get a 401 after a connection break. - # TODO(craigcitro): this shouldn't be a retryable exception, but - # for now we retry. - raise exceptions.BadStatusCodeError.FromResponse(response) - elif response.retry_after: - raise exceptions.RetryAfterError.FromResponse(response) + if response is None: + # Caller shouldn't call us if the response is None, but handle anyway. + raise exceptions.RequestError('Request to url %s did not return a response.' + % response.request_url) + elif (response.status_code >= 500 or + response.status_code == TOO_MANY_REQUESTS): + raise exceptions.BadStatusCodeError.FromResponse(response) + elif response.status_code == http_client.UNAUTHORIZED: + # Sometimes we get a 401 after a connection break. + # TODO(craigcitro): this shouldn't be a retryable exception, but + # for now we retry. + raise exceptions.BadStatusCodeError.FromResponse(response) + elif response.retry_after: + raise exceptions.RetryAfterError.FromResponse(response) def RebuildHttpConnections(http): - """Rebuilds all http connections in the httplib2.Http instance. + """Rebuilds all http connections in the httplib2.Http instance. - httplib2 overloads the map in http.connections to contain two different - types of values: - { scheme string: connection class } and - { scheme + authority string : actual http connection } - Here we remove all of the entries for actual connections so that on the - next request httplib2 will rebuild them from the connection types. + httplib2 overloads the map in http.connections to contain two different + types of values: + { scheme string: connection class } and + { scheme + authority string : actual http connection } + Here we remove all of the entries for actual connections so that on the + next request httplib2 will rebuild them from the connection types. - Args: - http: An httplib2.Http instance. - """ - if getattr(http, 'connections', None): - for conn_key in list(http.connections.keys()): - if ':' in conn_key: - del http.connections[conn_key] + Args: + http: An httplib2.Http instance. + """ + if getattr(http, 'connections', None): + for conn_key in list(http.connections.keys()): + if ':' in conn_key: + del http.connections[conn_key] def RethrowExceptionHandler(*unused_args): - raise + raise def HandleExceptionsAndRebuildHttpConnections(retry_args): - """Exception handler for http failures. - - This catches known failures and rebuilds the underlying HTTP connections. - - Args: - retry_args: An ExceptionRetryArgs tuple. - """ - # If the server indicates how long to wait, use that value. Otherwise, - # calculate the wait time on our own. - retry_after = None - - # Transport failures - if isinstance(retry_args.exc, (http_client.BadStatusLine, - http_client.IncompleteRead, - http_client.ResponseNotReady)): - logging.debug('Caught HTTP error %s, retrying: %s', - type(retry_args.exc).__name__, retry_args.exc) - elif isinstance(retry_args.exc, socket.error): - logging.debug('Caught socket error, retrying: %s', retry_args.exc) - elif isinstance(retry_args.exc, socket.gaierror): - logging.debug('Caught socket address error, retrying: %s', retry_args.exc) - elif isinstance(retry_args.exc, socket.timeout): - logging.debug('Caught socket timeout error, retrying: %s', retry_args.exc) - elif isinstance(retry_args.exc, httplib2.ServerNotFoundError): - logging.debug('Caught server not found error, retrying: %s', retry_args.exc) - elif isinstance(retry_args.exc, ValueError): - # oauth2client tries to JSON-decode the response, which can result - # in a ValueError if the response was invalid. Until that is fixed in - # oauth2client, need to handle it here. - logging.debug('Response content was invalid (%s), retrying', - retry_args.exc) - elif isinstance(retry_args.exc, exceptions.RequestError): - logging.debug('Request returned no response, retrying') - # API-level failures - elif isinstance(retry_args.exc, exceptions.BadStatusCodeError): - logging.debug('Response returned status %s, retrying', - retry_args.exc.status_code) - elif isinstance(retry_args.exc, exceptions.RetryAfterError): - logging.debug('Response returned a retry-after header, retrying') - retry_after = retry_args.exc.retry_after - else: - raise - RebuildHttpConnections(retry_args.http) - logging.debug('Retrying request to url %s after exception %s', - retry_args.http_request.url, retry_args.exc) - time.sleep(retry_after or util.CalculateWaitForRetry(retry_args.num_retries)) + """Exception handler for http failures. + + This catches known failures and rebuilds the underlying HTTP connections. + + Args: + retry_args: An ExceptionRetryArgs tuple. + """ + # If the server indicates how long to wait, use that value. Otherwise, + # calculate the wait time on our own. + retry_after = None + + # Transport failures + if isinstance(retry_args.exc, (http_client.BadStatusLine, + http_client.IncompleteRead, + http_client.ResponseNotReady)): + logging.debug('Caught HTTP error %s, retrying: %s', + type(retry_args.exc).__name__, retry_args.exc) + elif isinstance(retry_args.exc, socket.error): + logging.debug('Caught socket error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, socket.gaierror): + logging.debug( + 'Caught socket address error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, socket.timeout): + logging.debug( + 'Caught socket timeout error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, httplib2.ServerNotFoundError): + logging.debug( + 'Caught server not found error, retrying: %s', retry_args.exc) + elif isinstance(retry_args.exc, ValueError): + # oauth2client tries to JSON-decode the response, which can result + # in a ValueError if the response was invalid. Until that is fixed in + # oauth2client, need to handle it here. + logging.debug('Response content was invalid (%s), retrying', + retry_args.exc) + elif isinstance(retry_args.exc, exceptions.RequestError): + logging.debug('Request returned no response, retrying') + # API-level failures + elif isinstance(retry_args.exc, exceptions.BadStatusCodeError): + logging.debug('Response returned status %s, retrying', + retry_args.exc.status_code) + elif isinstance(retry_args.exc, exceptions.RetryAfterError): + logging.debug('Response returned a retry-after header, retrying') + retry_after = retry_args.exc.retry_after + else: + raise + RebuildHttpConnections(retry_args.http) + logging.debug('Retrying request to url %s after exception %s', + retry_args.http_request.url, retry_args.exc) + time.sleep( + retry_after or util.CalculateWaitForRetry(retry_args.num_retries)) def MakeRequest(http, http_request, retries=7, redirections=5, retry_func=HandleExceptionsAndRebuildHttpConnections, check_response_func=CheckResponse): - """Send http_request via the given http, performing error/retry handling. - - Args: - http: An httplib2.Http instance, or a http multiplexer that delegates to - an underlying http, for example, HTTPMultiplexer. - http_request: A Request to send. - retries: (int, default 5) Number of retries to attempt on 5XX replies. - redirections: (int, default 5) Number of redirects to follow. - retry_func: Function to handle retries on exceptions. Arguments are - (Httplib2.Http, Request, Exception, int num_retries). - check_response_func: Function to validate the HTTP response. Arguments are - (Response, response content, url). - - Raises: - InvalidDataFromServerError: if there is no response after retries. - - Returns: - A Response object. - """ - retry = 0 - while True: - try: - return _MakeRequestNoRetry(http, http_request, redirections=redirections, - check_response_func=check_response_func) - # retry_func will consume the exception types it handles and raise. - # pylint: disable=broad-except - except Exception as e: - retry += 1 - if retry >= retries: - raise - else: - retry_func(ExceptionRetryArgs(http, http_request, e, retry)) + """Send http_request via the given http, performing error/retry handling. + + Args: + http: An httplib2.Http instance, or a http multiplexer that delegates to + an underlying http, for example, HTTPMultiplexer. + http_request: A Request to send. + retries: (int, default 5) Number of retries to attempt on 5XX replies. + redirections: (int, default 5) Number of redirects to follow. + retry_func: Function to handle retries on exceptions. Arguments are + (Httplib2.Http, Request, Exception, int num_retries). + check_response_func: Function to validate the HTTP response. Arguments are + (Response, response content, url). + + Raises: + InvalidDataFromServerError: if there is no response after retries. + + Returns: + A Response object. + """ + retry = 0 + while True: + try: + return _MakeRequestNoRetry(http, http_request, redirections=redirections, + check_response_func=check_response_func) + # retry_func will consume the exception types it handles and raise. + # pylint: disable=broad-except + except Exception as e: + retry += 1 + if retry >= retries: + raise + else: + retry_func(ExceptionRetryArgs(http, http_request, e, retry)) def _MakeRequestNoRetry(http, http_request, redirections=5, check_response_func=CheckResponse): - """Send http_request via the given http. - - This wrapper exists to handle translation between the plain httplib2 - request/response types and the Request and Response types above. - - Args: - http: An httplib2.Http instance, or a http multiplexer that delegates to - an underlying http, for example, HTTPMultiplexer. - http_request: A Request to send. - redirections: (int, default 5) Number of redirects to follow. - check_response_func: Function to validate the HTTP response. Arguments are - (Response, response content, url). - - Returns: - A Response object. - - Raises: - RequestError if no response could be parsed. - """ - connection_type = None - # Handle overrides for connection types. This is used if the caller - # wants control over the underlying connection for managing callbacks - # or hash digestion. - if getattr(http, 'connections', None): - url_scheme = parse.urlsplit(http_request.url).scheme - if url_scheme and url_scheme in http.connections: - connection_type = http.connections[url_scheme] - - # Custom printing only at debuglevel 4 - new_debuglevel = 4 if httplib2.debuglevel == 4 else 0 - with _Httplib2Debuglevel(http_request, new_debuglevel, http=http): - info, content = http.request( - str(http_request.url), method=str(http_request.http_method), - body=http_request.body, headers=http_request.headers, - redirections=redirections, connection_type=connection_type) - - if info is None: - raise exceptions.RequestError() - - response = Response(info, content, http_request.url) - check_response_func(response) - return response + """Send http_request via the given http. + + This wrapper exists to handle translation between the plain httplib2 + request/response types and the Request and Response types above. + + Args: + http: An httplib2.Http instance, or a http multiplexer that delegates to + an underlying http, for example, HTTPMultiplexer. + http_request: A Request to send. + redirections: (int, default 5) Number of redirects to follow. + check_response_func: Function to validate the HTTP response. Arguments are + (Response, response content, url). + + Returns: + A Response object. + + Raises: + RequestError if no response could be parsed. + """ + connection_type = None + # Handle overrides for connection types. This is used if the caller + # wants control over the underlying connection for managing callbacks + # or hash digestion. + if getattr(http, 'connections', None): + url_scheme = parse.urlsplit(http_request.url).scheme + if url_scheme and url_scheme in http.connections: + connection_type = http.connections[url_scheme] + + # Custom printing only at debuglevel 4 + new_debuglevel = 4 if httplib2.debuglevel == 4 else 0 + with _Httplib2Debuglevel(http_request, new_debuglevel, http=http): + info, content = http.request( + str(http_request.url), method=str(http_request.http_method), + body=http_request.body, headers=http_request.headers, + redirections=redirections, connection_type=connection_type) + + if info is None: + raise exceptions.RequestError() + + response = Response(info, content, http_request.url) + check_response_func(response) + return response def GetHttp(**kwds): - return httplib2.Http(**kwds) + return httplib2.Http(**kwds) diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index f99eea1..98a9194 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -6,23 +6,24 @@ from apitools.base.py import http_wrapper class RaisesExceptionOnLen(object): - """Supports length property but raises if __len__ is used.""" - def __len__(self): - raise Exception('len() called unnecessarily') + """Supports length property but raises if __len__ is used.""" - def length(self): - return 1 + def __len__(self): + raise Exception('len() called unnecessarily') + + def length(self): + return 1 class HttpWrapperTest(unittest2.TestCase): - def testRequestBodyUsesLengthProperty(self): - http_wrapper.Request(body=RaisesExceptionOnLen()) + def testRequestBodyUsesLengthProperty(self): + http_wrapper.Request(body=RaisesExceptionOnLen()) - def testRequestBodyWithLen(self): - http_wrapper.Request(body='burrito') + def testRequestBodyWithLen(self): + http_wrapper.Request(body='burrito') if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 8b3d936..7f5d56d 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -9,48 +9,48 @@ __all__ = [ def YieldFromList( - service, request, limit=None, batch_size=100, - method='List', field='items', predicate=None, - current_token_attribute='pageToken', - next_token_attribute='nextPageToken'): - """Make a series of List requests, keeping track of page tokens. + service, request, limit=None, batch_size=100, + method='List', field='items', predicate=None, + current_token_attribute='pageToken', + next_token_attribute='nextPageToken'): + """Make a series of List requests, keeping track of page tokens. - Args: - service: apitools_base.BaseApiService, A service with a .List() method. - request: protorpc.messages.Message, The request message corresponding to the - service's .List() method, with all the attributes populated except - the .maxResults and .pageToken attributes. - limit: int, The maximum number of records to yield. None if all available - records should be yielded. - batch_size: int, The number of items to retrieve per request. - method: str, The name of the method used to fetch resources. - field: str, The field in the response that will be a list of items. - predicate: lambda, A function that returns true for items to be yielded. - current_token_attribute: str, The name of the attribute in a request message - holding the page token for the page being requested. - next_token_attribute: str, The name of the attribute in a response message - holding the page token for the next page. + Args: + service: apitools_base.BaseApiService, A service with a .List() method. + request: protorpc.messages.Message, The request message corresponding to the + service's .List() method, with all the attributes populated except + the .maxResults and .pageToken attributes. + limit: int, The maximum number of records to yield. None if all available + records should be yielded. + batch_size: int, The number of items to retrieve per request. + method: str, The name of the method used to fetch resources. + field: str, The field in the response that will be a list of items. + predicate: lambda, A function that returns true for items to be yielded. + current_token_attribute: str, The name of the attribute in a request message + holding the page token for the page being requested. + next_token_attribute: str, The name of the attribute in a response message + holding the page token for the next page. - Yields: - protorpc.message.Message, The resources listed by the service. + Yields: + protorpc.message.Message, The resources listed by the service. - """ - request = copy.deepcopy(request) - request.maxResults = batch_size - request.pageToken = None - while limit is None or limit: - response = getattr(service, method)(request) - items = getattr(response, field) - if predicate: - items = list(filter(predicate, items)) - for item in items: - yield item - if limit is None: - continue - limit -= 1 - if not limit: - return - token = getattr(response, next_token_attribute) - if not token: - return - setattr(request, current_token_attribute, token) + """ + request = copy.deepcopy(request) + request.maxResults = batch_size + request.pageToken = None + while limit is None or limit: + response = getattr(service, method)(request) + items = getattr(response, field) + if predicate: + items = list(filter(predicate, items)) + for item in items: + yield item + if limit is None: + continue + limit -= 1 + if not limit: + return + token = getattr(response, next_token_attribute) + if not token: + return + setattr(request, current_token_attribute, token) diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py index 9901b81..7b1b1a1 100644 --- a/apitools/base/py/stream_slice.py +++ b/apitools/base/py/stream_slice.py @@ -5,57 +5,58 @@ from apitools.base.py import exceptions class StreamSlice(object): - """Provides a slice-like object for streams.""" - - def __init__(self, stream, max_bytes): - self.__stream = stream - self.__remaining_bytes = max_bytes - self.__max_bytes = max_bytes - - def __str__(self): - return 'Slice of stream %s with %s/%s bytes not yet read' % ( - self.__stream, self.__remaining_bytes, self.__max_bytes) - - def __len__(self): - return self.__max_bytes - - def __nonzero__(self): - # For 32-bit python2.x, len() cannot exceed a 32-bit number; avoid - # accidental len() calls from httplib in the form of "if this_object:". - return bool(self.__max_bytes) - - @property - def length(self): - # For 32-bit python2.x, len() cannot exceed a 32-bit number. - return self.__max_bytes - - def read(self, size=None): # pylint: disable=missing-docstring - """Read at most size bytes from this slice. - - Compared to other streams, there is one case where we may - unexpectedly raise an exception on read: if the underlying stream - is exhausted (i.e. returns no bytes on read), and the size of this - slice indicates we should still be able to read more bytes, we - raise exceptions.StreamExhausted. - - Args: - size: If provided, read no more than size bytes from the stream. - - Returns: - The bytes read from this slice. - - Raises: - exceptions.StreamExhausted - - """ - if size is not None: - read_size = min(size, self.__remaining_bytes) - else: - read_size = self.__remaining_bytes - data = self.__stream.read(read_size) - if read_size > 0 and not data: - raise exceptions.StreamExhausted( - 'Not enough bytes in stream; expected %d, exhausted after %d' % ( - self.__max_bytes, self.__max_bytes - self.__remaining_bytes)) - self.__remaining_bytes -= len(data) - return data + + """Provides a slice-like object for streams.""" + + def __init__(self, stream, max_bytes): + self.__stream = stream + self.__remaining_bytes = max_bytes + self.__max_bytes = max_bytes + + def __str__(self): + return 'Slice of stream %s with %s/%s bytes not yet read' % ( + self.__stream, self.__remaining_bytes, self.__max_bytes) + + def __len__(self): + return self.__max_bytes + + def __nonzero__(self): + # For 32-bit python2.x, len() cannot exceed a 32-bit number; avoid + # accidental len() calls from httplib in the form of "if this_object:". + return bool(self.__max_bytes) + + @property + def length(self): + # For 32-bit python2.x, len() cannot exceed a 32-bit number. + return self.__max_bytes + + def read(self, size=None): # pylint: disable=missing-docstring + """Read at most size bytes from this slice. + + Compared to other streams, there is one case where we may + unexpectedly raise an exception on read: if the underlying stream + is exhausted (i.e. returns no bytes on read), and the size of this + slice indicates we should still be able to read more bytes, we + raise exceptions.StreamExhausted. + + Args: + size: If provided, read no more than size bytes from the stream. + + Returns: + The bytes read from this slice. + + Raises: + exceptions.StreamExhausted + + """ + if size is not None: + read_size = min(size, self.__remaining_bytes) + else: + read_size = self.__remaining_bytes + data = self.__stream.read(read_size) + if read_size > 0 and not data: + raise exceptions.StreamExhausted( + 'Not enough bytes in stream; expected %d, exhausted after %d' % ( + self.__max_bytes, self.__max_bytes - self.__remaining_bytes)) + self.__remaining_bytes -= len(data) + return data diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py index e544952..de821e6 100644 --- a/apitools/base/py/stream_slice_test.py +++ b/apitools/base/py/stream_slice_test.py @@ -12,44 +12,44 @@ from apitools.base.py import stream_slice class StreamSliceTest(unittest2.TestCase): - def setUp(self): - self.stream = six.StringIO(string.ascii_letters) - self.value = self.stream.getvalue() - self.stream.seek(0) - - def testSimpleSlice(self): - ss = stream_slice.StreamSlice(self.stream, 10) - self.assertEqual('', ss.read(0)) - self.assertEqual(self.value[0:3], ss.read(3)) - self.assertIn('7/10', str(ss)) - self.assertEqual(self.value[3:10], ss.read()) - self.assertEqual('', ss.read()) - self.assertEqual('', ss.read(10)) - self.assertEqual(10, self.stream.tell()) - - def testEmptySlice(self): - ss = stream_slice.StreamSlice(self.stream, 0) - self.assertEqual('', ss.read(5)) - self.assertEqual('', ss.read()) - self.assertEqual(0, self.stream.tell()) - - def testOffsetStream(self): - self.stream.seek(26) - ss = stream_slice.StreamSlice(self.stream, 26) - self.assertEqual(self.value[26:36], ss.read(10)) - self.assertEqual(self.value[36:], ss.read()) - self.assertEqual('', ss.read()) - - def testTooShortStream(self): - ss = stream_slice.StreamSlice(self.stream, 1000) - self.assertEqual(self.value, ss.read()) - self.assertEqual('', ss.read(0)) - with self.assertRaises(exceptions.StreamExhausted) as e: - ss.read() - with self.assertRaises(exceptions.StreamExhausted) as e: - ss.read(10) - self.assertIn('exhausted after %d' % len(self.value), str(e.exception)) + def setUp(self): + self.stream = six.StringIO(string.ascii_letters) + self.value = self.stream.getvalue() + self.stream.seek(0) + + def testSimpleSlice(self): + ss = stream_slice.StreamSlice(self.stream, 10) + self.assertEqual('', ss.read(0)) + self.assertEqual(self.value[0:3], ss.read(3)) + self.assertIn('7/10', str(ss)) + self.assertEqual(self.value[3:10], ss.read()) + self.assertEqual('', ss.read()) + self.assertEqual('', ss.read(10)) + self.assertEqual(10, self.stream.tell()) + + def testEmptySlice(self): + ss = stream_slice.StreamSlice(self.stream, 0) + self.assertEqual('', ss.read(5)) + self.assertEqual('', ss.read()) + self.assertEqual(0, self.stream.tell()) + + def testOffsetStream(self): + self.stream.seek(26) + ss = stream_slice.StreamSlice(self.stream, 26) + self.assertEqual(self.value[26:36], ss.read(10)) + self.assertEqual(self.value[36:], ss.read()) + self.assertEqual('', ss.read()) + + def testTooShortStream(self): + ss = stream_slice.StreamSlice(self.stream, 1000) + self.assertEqual(self.value, ss.read()) + self.assertEqual('', ss.read(0)) + with self.assertRaises(exceptions.StreamExhausted) as e: + ss.read() + with self.assertRaises(exceptions.StreamExhausted) as e: + ss.read(10) + self.assertIn('exhausted after %d' % len(self.value), str(e.exception)) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index dc983b3..ba288b7 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -37,872 +37,882 @@ RESUMABLE_UPLOAD = 'resumable' def DownloadProgressPrinter(response, download): - """Print download progress based on response.""" - if 'content-range' in response.info: - print('Received %s' % response.info['content-range']) - else: - print('Received %d bytes' % response.length) + """Print download progress based on response.""" + if 'content-range' in response.info: + print('Received %s' % response.info['content-range']) + else: + print('Received %d bytes' % response.length) def DownloadCompletePrinter(response, download): - """Print information about a completed download.""" - print('Download complete') + """Print information about a completed download.""" + print('Download complete') def UploadProgressPrinter(response, upload): - """Print upload progress based on response.""" - print('Sent %s' % response.info['range']) + """Print upload progress based on response.""" + print('Sent %s' % response.info['range']) def UploadCompletePrinter(response, upload): - """Print information about a completed upload.""" - print('Upload complete') + """Print information about a completed upload.""" + print('Upload complete') class _Transfer(object): - """Generic bits common to Uploads and Downloads.""" - - def __init__(self, stream, close_stream=False, chunksize=None, - auto_transfer=True, http=None, num_retries=5): - self.__bytes_http = None - self.__close_stream = close_stream - self.__http = http - self.__stream = stream - self.__url = None - - self.__num_retries = 5 - # Let the @property do validation - self.num_retries = num_retries - - self.retry_func = http_wrapper.HandleExceptionsAndRebuildHttpConnections - self.auto_transfer = auto_transfer - self.chunksize = chunksize or 1048576 - - def __repr__(self): - return str(self) - - @property - def close_stream(self): - return self.__close_stream - - @property - def http(self): - return self.__http - - @property - def bytes_http(self): - return self.__bytes_http or self.http - - @bytes_http.setter - def bytes_http(self, value): - self.__bytes_http = value - - @property - def num_retries(self): - return self.__num_retries - - @num_retries.setter - def num_retries(self, value): - util.Typecheck(value, six.integer_types) - if value < 0: - raise exceptions.InvalidDataError( - 'Cannot have negative value for num_retries') - self.__num_retries = value - - @property - def stream(self): - return self.__stream - - @property - def url(self): - return self.__url - - def _Initialize(self, http, url): - """Initialize this download by setting self.http and self.url. - - We want the user to be able to override self.http by having set - the value in the constructor; in that case, we ignore the provided - http. - - Args: - http: An httplib2.Http instance or None. - url: The url for this transfer. - - Returns: - None. Initializes self. - """ - self.EnsureUninitialized() - if self.http is None: - self.__http = http or http_wrapper.GetHttp() - self.__url = url - - @property - def initialized(self): - return self.url is not None and self.http is not None - - @property - def _type_name(self): - return type(self).__name__ - - def EnsureInitialized(self): - if not self.initialized: - raise exceptions.TransferInvalidError( - 'Cannot use uninitialized %s', self._type_name) - def EnsureUninitialized(self): - if self.initialized: - raise exceptions.TransferInvalidError( - 'Cannot re-initialize %s', self._type_name) + """Generic bits common to Uploads and Downloads.""" + + def __init__(self, stream, close_stream=False, chunksize=None, + auto_transfer=True, http=None, num_retries=5): + self.__bytes_http = None + self.__close_stream = close_stream + self.__http = http + self.__stream = stream + self.__url = None + + self.__num_retries = 5 + # Let the @property do validation + self.num_retries = num_retries + + self.retry_func = http_wrapper.HandleExceptionsAndRebuildHttpConnections + self.auto_transfer = auto_transfer + self.chunksize = chunksize or 1048576 + + def __repr__(self): + return str(self) + + @property + def close_stream(self): + return self.__close_stream + + @property + def http(self): + return self.__http + + @property + def bytes_http(self): + return self.__bytes_http or self.http + + @bytes_http.setter + def bytes_http(self, value): + self.__bytes_http = value + + @property + def num_retries(self): + return self.__num_retries + + @num_retries.setter + def num_retries(self, value): + util.Typecheck(value, six.integer_types) + if value < 0: + raise exceptions.InvalidDataError( + 'Cannot have negative value for num_retries') + self.__num_retries = value + + @property + def stream(self): + return self.__stream + + @property + def url(self): + return self.__url + + def _Initialize(self, http, url): + """Initialize this download by setting self.http and self.url. + + We want the user to be able to override self.http by having set + the value in the constructor; in that case, we ignore the provided + http. + + Args: + http: An httplib2.Http instance or None. + url: The url for this transfer. + + Returns: + None. Initializes self. + """ + self.EnsureUninitialized() + if self.http is None: + self.__http = http or http_wrapper.GetHttp() + self.__url = url + + @property + def initialized(self): + return self.url is not None and self.http is not None + + @property + def _type_name(self): + return type(self).__name__ + + def EnsureInitialized(self): + if not self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot use uninitialized %s', self._type_name) + + def EnsureUninitialized(self): + if self.initialized: + raise exceptions.TransferInvalidError( + 'Cannot re-initialize %s', self._type_name) + + def __del__(self): + if self.__close_stream: + self.__stream.close() + + def _ExecuteCallback(self, callback, response): + # TODO(craigcitro): Push these into a queue. + if callback is not None: + threading.Thread(target=callback, args=(response, self)).start() - def __del__(self): - if self.__close_stream: - self.__stream.close() - def _ExecuteCallback(self, callback, response): - # TODO(craigcitro): Push these into a queue. - if callback is not None: - threading.Thread(target=callback, args=(response, self)).start() +class Download(_Transfer): + """Data for a single download. -class Download(_Transfer): - """Data for a single download. - - Public attributes: - chunksize: default chunksize to use for transfers. - """ - _ACCEPTABLE_STATUSES = set(( - http_client.OK, - http_client.NO_CONTENT, - http_client.PARTIAL_CONTENT, - http_client.REQUESTED_RANGE_NOT_SATISFIABLE, - )) - _REQUIRED_SERIALIZATION_KEYS = set(( - 'auto_transfer', 'progress', 'total_size', 'url')) - - def __init__(self, stream, progress_callback=None, finish_callback=None, - **kwds): - total_size = kwds.pop('total_size', None) - super(Download, self).__init__(stream, **kwds) - self.__initial_response = None - self.__progress = 0 - self.__total_size = total_size - self.__encoding = None - - self.progress_callback = progress_callback - self.finish_callback = finish_callback - - @property - def progress(self): - return self.__progress - - @property - def encoding(self): - return self.__encoding - - @classmethod - def FromFile(cls, filename, overwrite=False, auto_transfer=True, **kwds): - """Create a new download object from a filename.""" - path = os.path.expanduser(filename) - if os.path.exists(path) and not overwrite: - raise exceptions.InvalidUserInputError( - 'File %s exists and overwrite not specified' % path) - return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer, - **kwds) - - @classmethod - def FromStream(cls, stream, auto_transfer=True, total_size=None, **kwds): - """Create a new Download object from a stream.""" - return cls(stream, auto_transfer=auto_transfer, total_size=total_size, - **kwds) - - @classmethod - def FromData(cls, stream, json_data, http=None, auto_transfer=None, **kwds): - """Create a new Download object from a stream and serialized data.""" - info = json.loads(json_data) - missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) - if missing_keys: - raise exceptions.InvalidDataError( - 'Invalid serialization data, missing keys: %s' % ( - ', '.join(missing_keys))) - download = cls.FromStream(stream, **kwds) - if auto_transfer is not None: - download.auto_transfer = auto_transfer - else: - download.auto_transfer = info['auto_transfer'] - setattr(download, '_Download__progress', info['progress']) - setattr(download, '_Download__total_size', info['total_size']) - download._Initialize(http, info['url']) # pylint: disable=protected-access - return download - - @property - def serialization_data(self): - self.EnsureInitialized() - return { - 'auto_transfer': self.auto_transfer, - 'progress': self.progress, - 'total_size': self.total_size, - 'url': self.url, - } - - @property - def total_size(self): - return self.__total_size - - def __str__(self): - if not self.initialized: - return 'Download (uninitialized)' - else: - return 'Download with %d/%s bytes transferred from url %s' % ( - self.progress, self.total_size, self.url) - - def ConfigureRequest(self, http_request, url_builder): - url_builder.query_params['alt'] = 'media' - # TODO(craigcitro): We need to send range requests because by - # default httplib2 stores entire reponses in memory. Override - # httplib2's download method (as gsutil does) so that this is not - # necessary. - http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) - - def __SetTotal(self, info): - if 'content-range' in info: - _, _, total = info['content-range'].rpartition('/') - if total != '*': - self.__total_size = int(total) - # Note "total_size is None" means we don't know it; if no size - # info was returned on our initial range request, that means we - # have a 0-byte file. (That last statement has been verified - # empirically, but is not clearly documented anywhere.) - if self.total_size is None: - self.__total_size = 0 - - def InitializeDownload(self, http_request, http=None, client=None): - """Initialize this download by making a request. - - Args: - http_request: The HttpRequest to use to initialize this download. - http: The httplib2.Http instance for this request. - client: If provided, let this client process the final URL before - sending any additional requests. If client is provided and - http is not, client.http will be used instead. + Public attributes: + chunksize: default chunksize to use for transfers. """ - self.EnsureUninitialized() - if http is None and client is None: - raise exceptions.UserError('Must provide client or http.') - http = http or client.http - if client is not None: - http_request.url = client.FinalizeTransferUrl(http_request.url) - url = http_request.url - if self.auto_transfer: - response = http_wrapper.MakeRequest(self.bytes_http or http, http_request) - if response.status_code not in self._ACCEPTABLE_STATUSES: - raise exceptions.HttpError.FromResponse(response) - self.__initial_response = response - self.__SetTotal(response.info) - url = response.info.get('content-location', response.request_url) - if client is not None: - url = client.FinalizeTransferUrl(url) - self._Initialize(http, url) - # Unless the user has requested otherwise, we want to just - # go ahead and pump the bytes now. - if self.auto_transfer: - self.StreamInChunks() - - def __NormalizeStartEnd(self, start, end=None): - if end is not None: - if start < 0: - raise exceptions.TransferInvalidError( - 'Cannot have end index with negative start index') - elif start >= self.total_size: - raise exceptions.TransferInvalidError( - 'Cannot have start index greater than total size') - end = min(end, self.total_size - 1) - if end < start: - raise exceptions.TransferInvalidError( - 'Range requested with end[%s] < start[%s]' % (end, start)) - return start, end - else: - if start < 0: - start = max(0, start + self.total_size) - return start, self.total_size - - def __SetRangeHeader(self, request, start, end=None): - if start < 0: - request.headers['range'] = 'bytes=%d' % start - elif end is None: - request.headers['range'] = 'bytes=%d-' % start - else: - request.headers['range'] = 'bytes=%d-%d' % (start, end) - - def __GetChunk(self, start, end=None, additional_headers=None): - """Retrieve a chunk, and return the full response.""" - self.EnsureInitialized() - end_byte = end - if self.total_size and end: - end_byte = min(end, self.total_size) - request = http_wrapper.Request(url=self.url) - self.__SetRangeHeader(request, start, end=end_byte) - if additional_headers is not None: - request.headers.update(additional_headers) - return http_wrapper.MakeRequest( - self.bytes_http, request, retry_func=self.retry_func, - retries=self.num_retries) - - def __ProcessResponse(self, response): - """Process this response (by updating self and writing to self.stream).""" - if response.status_code not in self._ACCEPTABLE_STATUSES: - # We distinguish errors that mean we made a mistake in setting - # up the transfer versus something we should attempt again. - if response.status_code in (http_client.FORBIDDEN, http_client.NOT_FOUND): - raise exceptions.HttpError.FromResponse(response) - else: - raise exceptions.TransferRetryError(response.content) - if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): - self.stream.write(response.content) - self.__progress += response.length - if response.info and 'content-encoding' in response.info: - # TODO(craigcitro): Handle the case where this changes over a - # download. - self.__encoding = response.info['content-encoding'] - elif response.status_code == http_client.NO_CONTENT: - # It's important to write something to the stream for the case - # of a 0-byte download to a file, as otherwise python won't - # create the file. - self.stream.write('') - return response - - def GetRange(self, start, end=None, additional_headers=None): - """Retrieve a given byte range from this download, inclusive. - - Range must be of one of these three forms: - * 0 <= start, end = None: Fetch from start to the end of the file. - * 0 <= start <= end: Fetch the bytes from start to end. - * start < 0, end = None: Fetch the last -start bytes of the file. - - (These variations correspond to those described in the HTTP 1.1 - protocol for range headers in RFC 2616, sec. 14.35.1.) - - Args: - start: (int) Where to start fetching bytes. (See above.) - end: (int, optional) Where to stop fetching bytes. (See above.) - additional_headers: (bool, optional) Any additional headers to - pass with the request. - - Returns: - None. Streams bytes into self.stream. - """ - self.EnsureInitialized() - progress_end_normalized = False - if self.total_size is not None: - progress, end = self.__NormalizeStartEnd(start, end) - progress_end_normalized = True - else: - progress = start - while not progress_end_normalized or progress < end: - response = self.__GetChunk(progress, end=end, - additional_headers=additional_headers) - if not progress_end_normalized: - self.__SetTotal(response.info) - progress, end = self.__NormalizeStartEnd(start, end) - progress_end_normalized = True - response = self.__ProcessResponse(response) - progress += response.length - if not response: - raise exceptions.TransferRetryError( - 'Zero bytes unexpectedly returned in download response') - - def StreamInChunks(self, callback=None, finish_callback=None, - additional_headers=None): - """Stream the entire download.""" - callback = callback or self.progress_callback - finish_callback = finish_callback or self.finish_callback - - self.EnsureInitialized() - while True: - if self.__initial_response is not None: - response = self.__initial_response + _ACCEPTABLE_STATUSES = set(( + http_client.OK, + http_client.NO_CONTENT, + http_client.PARTIAL_CONTENT, + http_client.REQUESTED_RANGE_NOT_SATISFIABLE, + )) + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'progress', 'total_size', 'url')) + + def __init__(self, stream, progress_callback=None, finish_callback=None, + **kwds): + total_size = kwds.pop('total_size', None) + super(Download, self).__init__(stream, **kwds) self.__initial_response = None - else: - response = self.__GetChunk(self.progress, - additional_headers=additional_headers) - if self.total_size is None: - self.__SetTotal(response.info) - response = self.__ProcessResponse(response) - self._ExecuteCallback(callback, response) - if (response.status_code == http_client.OK or - self.progress >= self.total_size): - break - self._ExecuteCallback(finish_callback, response) + self.__progress = 0 + self.__total_size = total_size + self.__encoding = None + + self.progress_callback = progress_callback + self.finish_callback = finish_callback + + @property + def progress(self): + return self.__progress + + @property + def encoding(self): + return self.__encoding + + @classmethod + def FromFile(cls, filename, overwrite=False, auto_transfer=True, **kwds): + """Create a new download object from a filename.""" + path = os.path.expanduser(filename) + if os.path.exists(path) and not overwrite: + raise exceptions.InvalidUserInputError( + 'File %s exists and overwrite not specified' % path) + return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer, + **kwds) + + @classmethod + def FromStream(cls, stream, auto_transfer=True, total_size=None, **kwds): + """Create a new Download object from a stream.""" + return cls(stream, auto_transfer=auto_transfer, total_size=total_size, + **kwds) + + @classmethod + def FromData(cls, stream, json_data, http=None, auto_transfer=None, **kwds): + """Create a new Download object from a stream and serialized data.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) + download = cls.FromStream(stream, **kwds) + if auto_transfer is not None: + download.auto_transfer = auto_transfer + else: + download.auto_transfer = info['auto_transfer'] + setattr(download, '_Download__progress', info['progress']) + setattr(download, '_Download__total_size', info['total_size']) + download._Initialize( + http, info['url']) # pylint: disable=protected-access + return download + + @property + def serialization_data(self): + self.EnsureInitialized() + return { + 'auto_transfer': self.auto_transfer, + 'progress': self.progress, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def total_size(self): + return self.__total_size + + def __str__(self): + if not self.initialized: + return 'Download (uninitialized)' + else: + return 'Download with %d/%s bytes transferred from url %s' % ( + self.progress, self.total_size, self.url) + + def ConfigureRequest(self, http_request, url_builder): + url_builder.query_params['alt'] = 'media' + # TODO(craigcitro): We need to send range requests because by + # default httplib2 stores entire reponses in memory. Override + # httplib2's download method (as gsutil does) so that this is not + # necessary. + http_request.headers['Range'] = 'bytes=0-%d' % (self.chunksize - 1,) + + def __SetTotal(self, info): + if 'content-range' in info: + _, _, total = info['content-range'].rpartition('/') + if total != '*': + self.__total_size = int(total) + # Note "total_size is None" means we don't know it; if no size + # info was returned on our initial range request, that means we + # have a 0-byte file. (That last statement has been verified + # empirically, but is not clearly documented anywhere.) + if self.total_size is None: + self.__total_size = 0 + + def InitializeDownload(self, http_request, http=None, client=None): + """Initialize this download by making a request. + + Args: + http_request: The HttpRequest to use to initialize this download. + http: The httplib2.Http instance for this request. + client: If provided, let this client process the final URL before + sending any additional requests. If client is provided and + http is not, client.http will be used instead. + """ + self.EnsureUninitialized() + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + url = http_request.url + if self.auto_transfer: + response = http_wrapper.MakeRequest( + self.bytes_http or http, http_request) + if response.status_code not in self._ACCEPTABLE_STATUSES: + raise exceptions.HttpError.FromResponse(response) + self.__initial_response = response + self.__SetTotal(response.info) + url = response.info.get('content-location', response.request_url) + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + self.StreamInChunks() + + def __NormalizeStartEnd(self, start, end=None): + if end is not None: + if start < 0: + raise exceptions.TransferInvalidError( + 'Cannot have end index with negative start index') + elif start >= self.total_size: + raise exceptions.TransferInvalidError( + 'Cannot have start index greater than total size') + end = min(end, self.total_size - 1) + if end < start: + raise exceptions.TransferInvalidError( + 'Range requested with end[%s] < start[%s]' % (end, start)) + return start, end + else: + if start < 0: + start = max(0, start + self.total_size) + return start, self.total_size + + def __SetRangeHeader(self, request, start, end=None): + if start < 0: + request.headers['range'] = 'bytes=%d' % start + elif end is None: + request.headers['range'] = 'bytes=%d-' % start + else: + request.headers['range'] = 'bytes=%d-%d' % (start, end) + + def __GetChunk(self, start, end=None, additional_headers=None): + """Retrieve a chunk, and return the full response.""" + self.EnsureInitialized() + end_byte = end + if self.total_size and end: + end_byte = min(end, self.total_size) + request = http_wrapper.Request(url=self.url) + self.__SetRangeHeader(request, start, end=end_byte) + if additional_headers is not None: + request.headers.update(additional_headers) + return http_wrapper.MakeRequest( + self.bytes_http, request, retry_func=self.retry_func, + retries=self.num_retries) + + def __ProcessResponse(self, response): + """Process this response (by updating self and writing to self.stream).""" + if response.status_code not in self._ACCEPTABLE_STATUSES: + # We distinguish errors that mean we made a mistake in setting + # up the transfer versus something we should attempt again. + if response.status_code in (http_client.FORBIDDEN, http_client.NOT_FOUND): + raise exceptions.HttpError.FromResponse(response) + else: + raise exceptions.TransferRetryError(response.content) + if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): + self.stream.write(response.content) + self.__progress += response.length + if response.info and 'content-encoding' in response.info: + # TODO(craigcitro): Handle the case where this changes over a + # download. + self.__encoding = response.info['content-encoding'] + elif response.status_code == http_client.NO_CONTENT: + # It's important to write something to the stream for the case + # of a 0-byte download to a file, as otherwise python won't + # create the file. + self.stream.write('') + return response + + def GetRange(self, start, end=None, additional_headers=None): + """Retrieve a given byte range from this download, inclusive. + + Range must be of one of these three forms: + * 0 <= start, end = None: Fetch from start to the end of the file. + * 0 <= start <= end: Fetch the bytes from start to end. + * start < 0, end = None: Fetch the last -start bytes of the file. + + (These variations correspond to those described in the HTTP 1.1 + protocol for range headers in RFC 2616, sec. 14.35.1.) + + Args: + start: (int) Where to start fetching bytes. (See above.) + end: (int, optional) Where to stop fetching bytes. (See above.) + additional_headers: (bool, optional) Any additional headers to + pass with the request. + + Returns: + None. Streams bytes into self.stream. + """ + self.EnsureInitialized() + progress_end_normalized = False + if self.total_size is not None: + progress, end = self.__NormalizeStartEnd(start, end) + progress_end_normalized = True + else: + progress = start + while not progress_end_normalized or progress < end: + response = self.__GetChunk(progress, end=end, + additional_headers=additional_headers) + if not progress_end_normalized: + self.__SetTotal(response.info) + progress, end = self.__NormalizeStartEnd(start, end) + progress_end_normalized = True + response = self.__ProcessResponse(response) + progress += response.length + if not response: + raise exceptions.TransferRetryError( + 'Zero bytes unexpectedly returned in download response') + + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Stream the entire download.""" + callback = callback or self.progress_callback + finish_callback = finish_callback or self.finish_callback + + self.EnsureInitialized() + while True: + if self.__initial_response is not None: + response = self.__initial_response + self.__initial_response = None + else: + response = self.__GetChunk(self.progress, + additional_headers=additional_headers) + if self.total_size is None: + self.__SetTotal(response.info) + response = self.__ProcessResponse(response) + self._ExecuteCallback(callback, response) + if (response.status_code == http_client.OK or + self.progress >= self.total_size): + break + self._ExecuteCallback(finish_callback, response) class Upload(_Transfer): - """Data for a single Upload. - - Fields: - stream: The stream to upload. - mime_type: MIME type of the upload. - total_size: (optional) Total upload size for the stream. - close_stream: (default: False) Whether or not we should close the - stream when finished with the upload. - auto_transfer: (default: True) If True, stream all bytes as soon as - the upload is created. - """ - _REQUIRED_SERIALIZATION_KEYS = set(( - 'auto_transfer', 'mime_type', 'total_size', 'url')) - - def __init__(self, stream, mime_type, total_size=None, http=None, - close_stream=False, chunksize=None, auto_transfer=True, - progress_callback=None, finish_callback=None, - **kwds): - super(Upload, self).__init__( - stream, close_stream=close_stream, chunksize=chunksize, - auto_transfer=auto_transfer, http=http, **kwds) - self.__complete = False - self.__final_response = None - self.__mime_type = mime_type - self.__progress = 0 - self.__server_chunk_granularity = None - self.__strategy = None - - self.progress_callback = progress_callback - self.finish_callback = finish_callback - self.total_size = total_size - - @property - def progress(self): - return self.__progress - - @classmethod - def FromFile(cls, filename, mime_type=None, auto_transfer=True, **kwds): - """Create a new Upload object from a filename.""" - path = os.path.expanduser(filename) - if not os.path.exists(path): - raise exceptions.NotFoundError('Could not find file %s' % path) - if not mime_type: - mime_type, _ = mimetypes.guess_type(path) - if mime_type is None: - raise exceptions.InvalidUserInputError( - 'Could not guess mime type for %s' % path) - size = os.stat(path).st_size - return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, - auto_transfer=auto_transfer, **kwds) - - @classmethod - def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True, - **kwds): - """Create a new Upload object from a stream.""" - if mime_type is None: - raise exceptions.InvalidUserInputError( - 'No mime_type specified for stream') - return cls(stream, mime_type, total_size=total_size, close_stream=False, - auto_transfer=auto_transfer, **kwds) - - @classmethod - def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds): - """Create a new Upload of stream from serialized json_data using http.""" - info = json.loads(json_data) - missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) - if missing_keys: - raise exceptions.InvalidDataError( - 'Invalid serialization data, missing keys: %s' % ( - ', '.join(missing_keys))) - if 'total_size' in kwds: - raise exceptions.InvalidUserInputError( - 'Cannot override total_size on serialized Upload') - upload = cls.FromStream(stream, info['mime_type'], - total_size=info.get('total_size'), **kwds) - if isinstance(stream, io.IOBase) and not stream.seekable(): - raise exceptions.InvalidUserInputError( - 'Cannot restart resumable upload on non-seekable stream') - if auto_transfer is not None: - upload.auto_transfer = auto_transfer - else: - upload.auto_transfer = info['auto_transfer'] - upload.strategy = RESUMABLE_UPLOAD - upload._Initialize(http, info['url']) # pylint: disable=protected-access - upload.RefreshResumableUploadState() - upload.EnsureInitialized() - if upload.auto_transfer: - upload.StreamInChunks() - return upload - - @property - def serialization_data(self): - self.EnsureInitialized() - if self.strategy != RESUMABLE_UPLOAD: - raise exceptions.InvalidDataError( - 'Serialization only supported for resumable uploads') - return { - 'auto_transfer': self.auto_transfer, - 'mime_type': self.mime_type, - 'total_size': self.total_size, - 'url': self.url, - } - - @property - def complete(self): - return self.__complete - - @property - def mime_type(self): - return self.__mime_type - - def __str__(self): - if not self.initialized: - return 'Upload (uninitialized)' - else: - return 'Upload with %d/%s bytes transferred for url %s' % ( - self.progress, self.total_size or '???', self.url) - - @property - def strategy(self): - return self.__strategy - - @strategy.setter - def strategy(self, value): - if value not in (SIMPLE_UPLOAD, RESUMABLE_UPLOAD): - raise exceptions.UserError(( - 'Invalid value "%s" for upload strategy, must be one of ' - '"simple" or "resumable".') % value) - self.__strategy = value - - @property - def total_size(self): - return self.__total_size - - @total_size.setter - def total_size(self, value): - self.EnsureUninitialized() - self.__total_size = value - - def __SetDefaultUploadStrategy(self, upload_config, http_request): - """Determine and set the default upload strategy for this upload. - - We generally prefer simple or multipart, unless we're forced to - use resumable. This happens when any of (1) the upload is too - large, (2) the simple endpoint doesn't support multipart requests - and we have metadata, or (3) there is no simple upload endpoint. - - Args: - upload_config: Configuration for the upload endpoint. - http_request: The associated http request. - - Returns: - None. - """ - if self.strategy is not None: - return - strategy = SIMPLE_UPLOAD - if (self.total_size is not None and - self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): - strategy = RESUMABLE_UPLOAD - if http_request.body and not upload_config.simple_multipart: - strategy = RESUMABLE_UPLOAD - if not upload_config.simple_path: - strategy = RESUMABLE_UPLOAD - self.strategy = strategy - - def ConfigureRequest(self, upload_config, http_request, url_builder): - """Configure the request and url for this upload.""" - # Validate total_size vs. max_size - if (self.total_size and upload_config.max_size and - self.total_size > upload_config.max_size): - raise exceptions.InvalidUserInputError( - 'Upload too big: %s larger than max size %s' % ( - self.total_size, upload_config.max_size)) - # Validate mime type - if not util.AcceptableMimeType(upload_config.accept, self.mime_type): - raise exceptions.InvalidUserInputError( - 'MIME type %s does not match any accepted MIME ranges %s' % ( - self.mime_type, upload_config.accept)) - - self.__SetDefaultUploadStrategy(upload_config, http_request) - if self.strategy == SIMPLE_UPLOAD: - url_builder.relative_path = upload_config.simple_path - if http_request.body: - url_builder.query_params['uploadType'] = 'multipart' - self.__ConfigureMultipartRequest(http_request) - else: - url_builder.query_params['uploadType'] = 'media' - self.__ConfigureMediaRequest(http_request) - else: - url_builder.relative_path = upload_config.resumable_path - url_builder.query_params['uploadType'] = 'resumable' - self.__ConfigureResumableRequest(http_request) - - def __ConfigureMediaRequest(self, http_request): - """Configure http_request as a simple request for this upload.""" - http_request.headers['content-type'] = self.mime_type - http_request.body = self.stream.read() - http_request.loggable_body = '' - - def __ConfigureMultipartRequest(self, http_request): - """Configure http_request as a multipart request for this upload.""" - # This is a multipart/related upload. - msg_root = mime_multipart.MIMEMultipart('related') - # msg_root should not write out its own headers - setattr(msg_root, '_write_headers', lambda self: None) - - # attach the body as one part - msg = mime_nonmultipart.MIMENonMultipart( - *http_request.headers['content-type'].split('/')) - msg.set_payload(http_request.body) - msg_root.attach(msg) - - # attach the media as the second part - msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) - msg['Content-Transfer-Encoding'] = 'binary' - msg.set_payload(self.stream.read()) - msg_root.attach(msg) - - # NOTE: We encode the body, but can't use `email.message.Message.as_string` - # because it prepends `> ` to `From ` lines. - # NOTE: We must use six.StringIO() instead of io.StringIO() since the - # `email` library uses cStringIO in Py2 and io.StringIO in Py3. - fp = six.StringIO() - g = email_generator.Generator(fp, mangle_from_=False) - g.flatten(msg_root, unixfrom=False) - http_request.body = fp.getvalue() - - multipart_boundary = msg_root.get_boundary() - http_request.headers['content-type'] = ( - 'multipart/related; boundary=%r' % multipart_boundary) - - body_components = http_request.body.split(multipart_boundary) - headers, _, _ = body_components[-2].partition('\n\n') - body_components[-2] = '\n\n'.join([headers, '\n\n--']) - http_request.loggable_body = multipart_boundary.join(body_components) - - def __ConfigureResumableRequest(self, http_request): - http_request.headers['X-Upload-Content-Type'] = self.mime_type - if self.total_size is not None: - http_request.headers['X-Upload-Content-Length'] = str(self.total_size) - - def RefreshResumableUploadState(self): - """Talk to the server and refresh the state of this resumable upload. - - Returns: - Response if the upload is complete. - """ - if self.strategy != RESUMABLE_UPLOAD: - return - self.EnsureInitialized() - refresh_request = http_wrapper.Request( - url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) - refresh_response = http_wrapper.MakeRequest( - self.http, refresh_request, redirections=0, retries=self.num_retries) - range_header = self._GetRangeHeaderFromResponse(refresh_response) - if refresh_response.status_code in (http_client.OK, http_client.CREATED): - self.__complete = True - self.__progress = self.total_size - self.stream.seek(self.progress) - # If we're finished, the refresh response will contain the metadata - # originally requested. Cache it so it can be returned in StreamInChunks. - self.__final_response = refresh_response - elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: - if range_header is None: - self.__progress = 0 - else: - self.__progress = self.__GetLastByte(range_header) + 1 - self.stream.seek(self.progress) - else: - raise exceptions.HttpError.FromResponse(refresh_response) - - def _GetRangeHeaderFromResponse(self, response): - return response.info.get('Range', response.info.get('range')) - - def InitializeUpload(self, http_request, http=None, client=None): - """Initialize this upload from the given http_request.""" - if self.strategy is None: - raise exceptions.UserError( - 'No upload strategy set; did you call ConfigureRequest?') - if http is None and client is None: - raise exceptions.UserError('Must provide client or http.') - if self.strategy != RESUMABLE_UPLOAD: - return - http = http or client.http - if client is not None: - http_request.url = client.FinalizeTransferUrl(http_request.url) - self.EnsureUninitialized() - http_response = http_wrapper.MakeRequest(http, http_request, - retries=self.num_retries) - if http_response.status_code != http_client.OK: - raise exceptions.HttpError.FromResponse(http_response) - - self.__server_chunk_granularity = http_response.info.get( - 'X-Goog-Upload-Chunk-Granularity') - url = http_response.info['location'] - if client is not None: - url = client.FinalizeTransferUrl(url) - self._Initialize(http, url) - - # Unless the user has requested otherwise, we want to just - # go ahead and pump the bytes now. - if self.auto_transfer: - return self.StreamInChunks() - - def __GetLastByte(self, range_header): - _, _, end = range_header.partition('-') - # TODO(craigcitro): Validate start == 0? - return int(end) - - def __ValidateChunksize(self, chunksize=None): - if self.__server_chunk_granularity is None: - return - chunksize = chunksize or self.chunksize - if chunksize % self.__server_chunk_granularity: - raise exceptions.ConfigurationValueError( - 'Server requires chunksize to be a multiple of %d', - self.__server_chunk_granularity) - - def __StreamMedia(self, callback=None, finish_callback=None, - additional_headers=None, use_chunks=True): - """Helper function for StreamMedia / StreamInChunks.""" - if self.strategy != RESUMABLE_UPLOAD: - raise exceptions.InvalidUserInputError( - 'Cannot stream non-resumable upload') - callback = callback or self.progress_callback - finish_callback = finish_callback or self.finish_callback - # final_response is set if we resumed an already-completed upload. - response = self.__final_response - send_func = self.__SendChunk if use_chunks else self.__SendMediaBody - if use_chunks: - self.__ValidateChunksize(self.chunksize) - self.EnsureInitialized() - while not self.complete: - response = send_func(self.stream.tell(), - additional_headers=additional_headers) - if response.status_code in (http_client.OK, http_client.CREATED): - self.__complete = True - break - self.__progress = self.__GetLastByte(response.info['range']) - if self.progress + 1 != self.stream.tell(): - # TODO(craigcitro): Add a better way to recover here. - raise exceptions.CommunicationError( - 'Failed to transfer all bytes in chunk, upload paused at byte ' - '%d' % self.progress) - self._ExecuteCallback(callback, response) - if self.__complete and hasattr(self.stream, 'seek'): - current_pos = self.stream.tell() - self.stream.seek(0, os.SEEK_END) - end_pos = self.stream.tell() - self.stream.seek(current_pos) - if current_pos != end_pos: - raise exceptions.TransferInvalidError( - 'Upload complete with %s additional bytes left in stream' % - (int(end_pos) - int(current_pos))) - self._ExecuteCallback(finish_callback, response) - return response - - def StreamMedia(self, callback=None, finish_callback=None, - additional_headers=None): - """Send this resumable upload in a single request. - - Args: - callback: Progress callback function with inputs - (http_wrapper.Response, transfer.Upload) - finish_callback: Final callback function with inputs - (http_wrapper.Response, transfer.Upload) - additional_headers: Dict of headers to include with the upload - http_wrapper.Request. - - Returns: - http_wrapper.Response of final response. - """ - return self.__StreamMedia( - callback=callback, finish_callback=finish_callback, - additional_headers=additional_headers, use_chunks=False) - - def StreamInChunks(self, callback=None, finish_callback=None, - additional_headers=None): - """Send this (resumable) upload in chunks.""" - return self.__StreamMedia( - callback=callback, finish_callback=finish_callback, - additional_headers=additional_headers) - - def __SendMediaRequest(self, request, end): - """Helper function to make the request for SendMediaBody & SendChunk.""" - response = http_wrapper.MakeRequest( - self.bytes_http, request, retry_func=self.retry_func, - retries=self.num_retries) - if response.status_code not in (http_client.OK, http_client.CREATED, - http_wrapper.RESUME_INCOMPLETE): - # We want to reset our state to wherever the server left us - # before this failed request, and then raise. - self.RefreshResumableUploadState() - raise exceptions.HttpError.FromResponse(response) - if response.status_code == http_wrapper.RESUME_INCOMPLETE: - last_byte = self.__GetLastByte( - self._GetRangeHeaderFromResponse(response)) - if last_byte + 1 != end: - self.stream.seek(last_byte) - return response - - def __SendMediaBody(self, start, additional_headers=None): - """Send the entire media stream in a single request.""" - self.EnsureInitialized() - if self.total_size is None: - raise exceptions.TransferInvalidError( - 'Total size must be known for SendMediaBody') - body_stream = stream_slice.StreamSlice(self.stream, self.total_size - start) - - request = http_wrapper.Request(url=self.url, http_method='PUT', - body=body_stream) - request.headers['Content-Type'] = self.mime_type - if start == self.total_size: - # End of an upload with 0 bytes left to send; just finalize. - range_string = 'bytes */%s' % self.total_size - else: - range_string = 'bytes %s-%s/%s' % (start, self.total_size - 1, - self.total_size) - - request.headers['Content-Range'] = range_string - if additional_headers: - request.headers.update(additional_headers) - - return self.__SendMediaRequest(request, self.total_size) - - def __SendChunk(self, start, additional_headers=None): - """Send the specified chunk.""" - self.EnsureInitialized() - no_log_body = self.total_size is None - if self.total_size is None: - # For the streaming resumable case, we need to detect when we're at the - # end of the stream. - body_stream = buffered_stream.BufferedStream( - self.stream, start, self.chunksize) - end = body_stream.stream_end_position - if body_stream.stream_exhausted: - self.__total_size = end - # TODO: Here, change body_stream from a stream to a string object, - # which means reading a chunk into memory. This works around - # https://code.google.com/p/httplib2/issues/detail?id=176 which can - # cause httplib2 to skip bytes on 401's for file objects. - # Rework this solution to be more general. - body_stream = body_stream.read(self.chunksize) - else: - end = min(start + self.chunksize, self.total_size) - body_stream = stream_slice.StreamSlice(self.stream, end - start) - # TODO(craigcitro): Think about clearer errors on "no data in - # stream". - request = http_wrapper.Request(url=self.url, http_method='PUT', - body=body_stream) - request.headers['Content-Type'] = self.mime_type - if no_log_body: - # Disable logging of streaming body. - # TODO: Remove no_log_body and rework as part of a larger logs refactor. - request.loggable_body = '' - if self.total_size is None: - # Streaming resumable upload case, unknown total size. - range_string = 'bytes %s-%s/*' % (start, end - 1) - elif end == start: - # End of an upload with 0 bytes left to send; just finalize. - range_string = 'bytes */%s' % self.total_size - else: - # Normal resumable upload case with known sizes. - range_string = 'bytes %s-%s/%s' % (start, end - 1, self.total_size) - request.headers['Content-Range'] = range_string - if additional_headers: - request.headers.update(additional_headers) + """Data for a single Upload. + + Fields: + stream: The stream to upload. + mime_type: MIME type of the upload. + total_size: (optional) Total upload size for the stream. + close_stream: (default: False) Whether or not we should close the + stream when finished with the upload. + auto_transfer: (default: True) If True, stream all bytes as soon as + the upload is created. + """ + _REQUIRED_SERIALIZATION_KEYS = set(( + 'auto_transfer', 'mime_type', 'total_size', 'url')) - return self.__SendMediaRequest(request, end) + def __init__(self, stream, mime_type, total_size=None, http=None, + close_stream=False, chunksize=None, auto_transfer=True, + progress_callback=None, finish_callback=None, + **kwds): + super(Upload, self).__init__( + stream, close_stream=close_stream, chunksize=chunksize, + auto_transfer=auto_transfer, http=http, **kwds) + self.__complete = False + self.__final_response = None + self.__mime_type = mime_type + self.__progress = 0 + self.__server_chunk_granularity = None + self.__strategy = None + + self.progress_callback = progress_callback + self.finish_callback = finish_callback + self.total_size = total_size + + @property + def progress(self): + return self.__progress + + @classmethod + def FromFile(cls, filename, mime_type=None, auto_transfer=True, **kwds): + """Create a new Upload object from a filename.""" + path = os.path.expanduser(filename) + if not os.path.exists(path): + raise exceptions.NotFoundError('Could not find file %s' % path) + if not mime_type: + mime_type, _ = mimetypes.guess_type(path) + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'Could not guess mime type for %s' % path) + size = os.stat(path).st_size + return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, + auto_transfer=auto_transfer, **kwds) + + @classmethod + def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True, + **kwds): + """Create a new Upload object from a stream.""" + if mime_type is None: + raise exceptions.InvalidUserInputError( + 'No mime_type specified for stream') + return cls(stream, mime_type, total_size=total_size, close_stream=False, + auto_transfer=auto_transfer, **kwds) + + @classmethod + def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds): + """Create a new Upload of stream from serialized json_data using http.""" + info = json.loads(json_data) + missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) + if missing_keys: + raise exceptions.InvalidDataError( + 'Invalid serialization data, missing keys: %s' % ( + ', '.join(missing_keys))) + if 'total_size' in kwds: + raise exceptions.InvalidUserInputError( + 'Cannot override total_size on serialized Upload') + upload = cls.FromStream(stream, info['mime_type'], + total_size=info.get('total_size'), **kwds) + if isinstance(stream, io.IOBase) and not stream.seekable(): + raise exceptions.InvalidUserInputError( + 'Cannot restart resumable upload on non-seekable stream') + if auto_transfer is not None: + upload.auto_transfer = auto_transfer + else: + upload.auto_transfer = info['auto_transfer'] + upload.strategy = RESUMABLE_UPLOAD + upload._Initialize( + http, info['url']) # pylint: disable=protected-access + upload.RefreshResumableUploadState() + upload.EnsureInitialized() + if upload.auto_transfer: + upload.StreamInChunks() + return upload + + @property + def serialization_data(self): + self.EnsureInitialized() + if self.strategy != RESUMABLE_UPLOAD: + raise exceptions.InvalidDataError( + 'Serialization only supported for resumable uploads') + return { + 'auto_transfer': self.auto_transfer, + 'mime_type': self.mime_type, + 'total_size': self.total_size, + 'url': self.url, + } + + @property + def complete(self): + return self.__complete + + @property + def mime_type(self): + return self.__mime_type + + def __str__(self): + if not self.initialized: + return 'Upload (uninitialized)' + else: + return 'Upload with %d/%s bytes transferred for url %s' % ( + self.progress, self.total_size or '???', self.url) + + @property + def strategy(self): + return self.__strategy + + @strategy.setter + def strategy(self, value): + if value not in (SIMPLE_UPLOAD, RESUMABLE_UPLOAD): + raise exceptions.UserError(( + 'Invalid value "%s" for upload strategy, must be one of ' + '"simple" or "resumable".') % value) + self.__strategy = value + + @property + def total_size(self): + return self.__total_size + + @total_size.setter + def total_size(self, value): + self.EnsureUninitialized() + self.__total_size = value + + def __SetDefaultUploadStrategy(self, upload_config, http_request): + """Determine and set the default upload strategy for this upload. + + We generally prefer simple or multipart, unless we're forced to + use resumable. This happens when any of (1) the upload is too + large, (2) the simple endpoint doesn't support multipart requests + and we have metadata, or (3) there is no simple upload endpoint. + + Args: + upload_config: Configuration for the upload endpoint. + http_request: The associated http request. + + Returns: + None. + """ + if self.strategy is not None: + return + strategy = SIMPLE_UPLOAD + if (self.total_size is not None and + self.total_size > _RESUMABLE_UPLOAD_THRESHOLD): + strategy = RESUMABLE_UPLOAD + if http_request.body and not upload_config.simple_multipart: + strategy = RESUMABLE_UPLOAD + if not upload_config.simple_path: + strategy = RESUMABLE_UPLOAD + self.strategy = strategy + + def ConfigureRequest(self, upload_config, http_request, url_builder): + """Configure the request and url for this upload.""" + # Validate total_size vs. max_size + if (self.total_size and upload_config.max_size and + self.total_size > upload_config.max_size): + raise exceptions.InvalidUserInputError( + 'Upload too big: %s larger than max size %s' % ( + self.total_size, upload_config.max_size)) + # Validate mime type + if not util.AcceptableMimeType(upload_config.accept, self.mime_type): + raise exceptions.InvalidUserInputError( + 'MIME type %s does not match any accepted MIME ranges %s' % ( + self.mime_type, upload_config.accept)) + + self.__SetDefaultUploadStrategy(upload_config, http_request) + if self.strategy == SIMPLE_UPLOAD: + url_builder.relative_path = upload_config.simple_path + if http_request.body: + url_builder.query_params['uploadType'] = 'multipart' + self.__ConfigureMultipartRequest(http_request) + else: + url_builder.query_params['uploadType'] = 'media' + self.__ConfigureMediaRequest(http_request) + else: + url_builder.relative_path = upload_config.resumable_path + url_builder.query_params['uploadType'] = 'resumable' + self.__ConfigureResumableRequest(http_request) + + def __ConfigureMediaRequest(self, http_request): + """Configure http_request as a simple request for this upload.""" + http_request.headers['content-type'] = self.mime_type + http_request.body = self.stream.read() + http_request.loggable_body = '' + + def __ConfigureMultipartRequest(self, http_request): + """Configure http_request as a multipart request for this upload.""" + # This is a multipart/related upload. + msg_root = mime_multipart.MIMEMultipart('related') + # msg_root should not write out its own headers + setattr(msg_root, '_write_headers', lambda self: None) + + # attach the body as one part + msg = mime_nonmultipart.MIMENonMultipart( + *http_request.headers['content-type'].split('/')) + msg.set_payload(http_request.body) + msg_root.attach(msg) + + # attach the media as the second part + msg = mime_nonmultipart.MIMENonMultipart(*self.mime_type.split('/')) + msg['Content-Transfer-Encoding'] = 'binary' + msg.set_payload(self.stream.read()) + msg_root.attach(msg) + + # NOTE: We encode the body, but can't use `email.message.Message.as_string` + # because it prepends `> ` to `From ` lines. + # NOTE: We must use six.StringIO() instead of io.StringIO() since the + # `email` library uses cStringIO in Py2 and io.StringIO in Py3. + fp = six.StringIO() + g = email_generator.Generator(fp, mangle_from_=False) + g.flatten(msg_root, unixfrom=False) + http_request.body = fp.getvalue() + + multipart_boundary = msg_root.get_boundary() + http_request.headers['content-type'] = ( + 'multipart/related; boundary=%r' % multipart_boundary) + + body_components = http_request.body.split(multipart_boundary) + headers, _, _ = body_components[-2].partition('\n\n') + body_components[-2] = '\n\n'.join([headers, '\n\n--']) + http_request.loggable_body = multipart_boundary.join(body_components) + + def __ConfigureResumableRequest(self, http_request): + http_request.headers['X-Upload-Content-Type'] = self.mime_type + if self.total_size is not None: + http_request.headers[ + 'X-Upload-Content-Length'] = str(self.total_size) + + def RefreshResumableUploadState(self): + """Talk to the server and refresh the state of this resumable upload. + + Returns: + Response if the upload is complete. + """ + if self.strategy != RESUMABLE_UPLOAD: + return + self.EnsureInitialized() + refresh_request = http_wrapper.Request( + url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) + refresh_response = http_wrapper.MakeRequest( + self.http, refresh_request, redirections=0, retries=self.num_retries) + range_header = self._GetRangeHeaderFromResponse(refresh_response) + if refresh_response.status_code in (http_client.OK, http_client.CREATED): + self.__complete = True + self.__progress = self.total_size + self.stream.seek(self.progress) + # If we're finished, the refresh response will contain the metadata + # originally requested. Cache it so it can be returned in + # StreamInChunks. + self.__final_response = refresh_response + elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: + if range_header is None: + self.__progress = 0 + else: + self.__progress = self.__GetLastByte(range_header) + 1 + self.stream.seek(self.progress) + else: + raise exceptions.HttpError.FromResponse(refresh_response) + + def _GetRangeHeaderFromResponse(self, response): + return response.info.get('Range', response.info.get('range')) + + def InitializeUpload(self, http_request, http=None, client=None): + """Initialize this upload from the given http_request.""" + if self.strategy is None: + raise exceptions.UserError( + 'No upload strategy set; did you call ConfigureRequest?') + if http is None and client is None: + raise exceptions.UserError('Must provide client or http.') + if self.strategy != RESUMABLE_UPLOAD: + return + http = http or client.http + if client is not None: + http_request.url = client.FinalizeTransferUrl(http_request.url) + self.EnsureUninitialized() + http_response = http_wrapper.MakeRequest(http, http_request, + retries=self.num_retries) + if http_response.status_code != http_client.OK: + raise exceptions.HttpError.FromResponse(http_response) + + self.__server_chunk_granularity = http_response.info.get( + 'X-Goog-Upload-Chunk-Granularity') + url = http_response.info['location'] + if client is not None: + url = client.FinalizeTransferUrl(url) + self._Initialize(http, url) + + # Unless the user has requested otherwise, we want to just + # go ahead and pump the bytes now. + if self.auto_transfer: + return self.StreamInChunks() + + def __GetLastByte(self, range_header): + _, _, end = range_header.partition('-') + # TODO(craigcitro): Validate start == 0? + return int(end) + + def __ValidateChunksize(self, chunksize=None): + if self.__server_chunk_granularity is None: + return + chunksize = chunksize or self.chunksize + if chunksize % self.__server_chunk_granularity: + raise exceptions.ConfigurationValueError( + 'Server requires chunksize to be a multiple of %d', + self.__server_chunk_granularity) + + def __StreamMedia(self, callback=None, finish_callback=None, + additional_headers=None, use_chunks=True): + """Helper function for StreamMedia / StreamInChunks.""" + if self.strategy != RESUMABLE_UPLOAD: + raise exceptions.InvalidUserInputError( + 'Cannot stream non-resumable upload') + callback = callback or self.progress_callback + finish_callback = finish_callback or self.finish_callback + # final_response is set if we resumed an already-completed upload. + response = self.__final_response + send_func = self.__SendChunk if use_chunks else self.__SendMediaBody + if use_chunks: + self.__ValidateChunksize(self.chunksize) + self.EnsureInitialized() + while not self.complete: + response = send_func(self.stream.tell(), + additional_headers=additional_headers) + if response.status_code in (http_client.OK, http_client.CREATED): + self.__complete = True + break + self.__progress = self.__GetLastByte(response.info['range']) + if self.progress + 1 != self.stream.tell(): + # TODO(craigcitro): Add a better way to recover here. + raise exceptions.CommunicationError( + 'Failed to transfer all bytes in chunk, upload paused at byte ' + '%d' % self.progress) + self._ExecuteCallback(callback, response) + if self.__complete and hasattr(self.stream, 'seek'): + current_pos = self.stream.tell() + self.stream.seek(0, os.SEEK_END) + end_pos = self.stream.tell() + self.stream.seek(current_pos) + if current_pos != end_pos: + raise exceptions.TransferInvalidError( + 'Upload complete with %s additional bytes left in stream' % + (int(end_pos) - int(current_pos))) + self._ExecuteCallback(finish_callback, response) + return response + + def StreamMedia(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this resumable upload in a single request. + + Args: + callback: Progress callback function with inputs + (http_wrapper.Response, transfer.Upload) + finish_callback: Final callback function with inputs + (http_wrapper.Response, transfer.Upload) + additional_headers: Dict of headers to include with the upload + http_wrapper.Request. + + Returns: + http_wrapper.Response of final response. + """ + return self.__StreamMedia( + callback=callback, finish_callback=finish_callback, + additional_headers=additional_headers, use_chunks=False) + + def StreamInChunks(self, callback=None, finish_callback=None, + additional_headers=None): + """Send this (resumable) upload in chunks.""" + return self.__StreamMedia( + callback=callback, finish_callback=finish_callback, + additional_headers=additional_headers) + + def __SendMediaRequest(self, request, end): + """Helper function to make the request for SendMediaBody & SendChunk.""" + response = http_wrapper.MakeRequest( + self.bytes_http, request, retry_func=self.retry_func, + retries=self.num_retries) + if response.status_code not in (http_client.OK, http_client.CREATED, + http_wrapper.RESUME_INCOMPLETE): + # We want to reset our state to wherever the server left us + # before this failed request, and then raise. + self.RefreshResumableUploadState() + raise exceptions.HttpError.FromResponse(response) + if response.status_code == http_wrapper.RESUME_INCOMPLETE: + last_byte = self.__GetLastByte( + self._GetRangeHeaderFromResponse(response)) + if last_byte + 1 != end: + self.stream.seek(last_byte) + return response + + def __SendMediaBody(self, start, additional_headers=None): + """Send the entire media stream in a single request.""" + self.EnsureInitialized() + if self.total_size is None: + raise exceptions.TransferInvalidError( + 'Total size must be known for SendMediaBody') + body_stream = stream_slice.StreamSlice( + self.stream, self.total_size - start) + + request = http_wrapper.Request(url=self.url, http_method='PUT', + body=body_stream) + request.headers['Content-Type'] = self.mime_type + if start == self.total_size: + # End of an upload with 0 bytes left to send; just finalize. + range_string = 'bytes */%s' % self.total_size + else: + range_string = 'bytes %s-%s/%s' % (start, self.total_size - 1, + self.total_size) + + request.headers['Content-Range'] = range_string + if additional_headers: + request.headers.update(additional_headers) + + return self.__SendMediaRequest(request, self.total_size) + + def __SendChunk(self, start, additional_headers=None): + """Send the specified chunk.""" + self.EnsureInitialized() + no_log_body = self.total_size is None + if self.total_size is None: + # For the streaming resumable case, we need to detect when we're at the + # end of the stream. + body_stream = buffered_stream.BufferedStream( + self.stream, start, self.chunksize) + end = body_stream.stream_end_position + if body_stream.stream_exhausted: + self.__total_size = end + # TODO: Here, change body_stream from a stream to a string object, + # which means reading a chunk into memory. This works around + # https://code.google.com/p/httplib2/issues/detail?id=176 which can + # cause httplib2 to skip bytes on 401's for file objects. + # Rework this solution to be more general. + body_stream = body_stream.read(self.chunksize) + else: + end = min(start + self.chunksize, self.total_size) + body_stream = stream_slice.StreamSlice(self.stream, end - start) + # TODO(craigcitro): Think about clearer errors on "no data in + # stream". + request = http_wrapper.Request(url=self.url, http_method='PUT', + body=body_stream) + request.headers['Content-Type'] = self.mime_type + if no_log_body: + # Disable logging of streaming body. + # TODO: Remove no_log_body and rework as part of a larger logs + # refactor. + request.loggable_body = '' + if self.total_size is None: + # Streaming resumable upload case, unknown total size. + range_string = 'bytes %s-%s/*' % (start, end - 1) + elif end == start: + # End of an upload with 0 bytes left to send; just finalize. + range_string = 'bytes */%s' % self.total_size + else: + # Normal resumable upload case with known sizes. + range_string = 'bytes %s-%s/%s' % (start, end - 1, self.total_size) + + request.headers['Content-Range'] = range_string + if additional_headers: + request.headers.update(additional_headers) + + return self.__SendMediaRequest(request, end) diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 1e12de0..76e54d0 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -12,56 +12,56 @@ from apitools.base.py import transfer class TransferTest(unittest2.TestCase): - def testFromEncoding(self): - # Test a specific corner case in multipart encoding. + def testFromEncoding(self): + # Test a specific corner case in multipart encoding. - # Python's mime module by default encodes lines that start with - # "From " as ">From ", which we need to make sure we don't run afoul - # of when sending content that isn't intended to be so encoded. This - # test calls out that we get this right. We test for both the - # multipart and non-multipart case. - multipart_body = '{"body_field_one": 7}' - upload_contents = 'line one\nFrom \nline two' - upload_config = base_api.ApiUploadInfo( - accept=['*/*'], - max_size=None, - resumable_multipart=True, - resumable_path=u'/resumable/upload', - simple_multipart=True, - simple_path=u'/upload', - ) - url_builder = base_api._UrlBuilder('http://www.uploads.com') + # Python's mime module by default encodes lines that start with + # "From " as ">From ", which we need to make sure we don't run afoul + # of when sending content that isn't intended to be so encoded. This + # test calls out that we get this right. We test for both the + # multipart and non-multipart case. + multipart_body = '{"body_field_one": 7}' + upload_contents = 'line one\nFrom \nline two' + upload_config = base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload', + simple_multipart=True, + simple_path=u'/upload', + ) + url_builder = base_api._UrlBuilder('http://www.uploads.com') - # Test multipart: having a body argument in http_request forces - # multipart here. - upload = transfer.Upload.FromStream( - six.StringIO(upload_contents), - 'text/plain', - total_size=len(upload_contents)) - http_request = http_wrapper.Request( - 'http://www.uploads.com', - headers={'content-type': 'text/plain'}, - body=multipart_body) - upload.ConfigureRequest(upload_config, http_request, url_builder) - self.assertEqual(url_builder.query_params['uploadType'], 'multipart') - rewritten_upload_contents = '\n'.join( - http_request.body.split('--')[2].splitlines()[1:]) - self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) + # Test multipart: having a body argument in http_request forces + # multipart here. + upload = transfer.Upload.FromStream( + six.StringIO(upload_contents), + 'text/plain', + total_size=len(upload_contents)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}, + body=multipart_body) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual(url_builder.query_params['uploadType'], 'multipart') + rewritten_upload_contents = '\n'.join( + http_request.body.split('--')[2].splitlines()[1:]) + self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) - # Test non-multipart (aka media): no body argument means this is - # sent as media. - upload = transfer.Upload.FromStream( - six.StringIO(upload_contents), - 'text/plain', - total_size=len(upload_contents)) - http_request = http_wrapper.Request( - 'http://www.uploads.com', - headers={'content-type': 'text/plain'}) - upload.ConfigureRequest(upload_config, http_request, url_builder) - self.assertEqual(url_builder.query_params['uploadType'], 'media') - rewritten_upload_contents = http_request.body - self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) + # Test non-multipart (aka media): no body argument means this is + # sent as media. + upload = transfer.Upload.FromStream( + six.StringIO(upload_contents), + 'text/plain', + total_size=len(upload_contents)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual(url_builder.query_params['uploadType'], 'media') + rewritten_upload_contents = http_request.body + self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 5158bfa..7c6eaff 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -24,185 +24,187 @@ _RESERVED_URI_CHARS = r":/?#[]@!$&'()*+,;=" def DetectGae(): - """Determine whether or not we're running on GAE. + """Determine whether or not we're running on GAE. - This is based on: - https://developers.google.com/appengine/docs/python/#The_Environment + This is based on: + https://developers.google.com/appengine/docs/python/#The_Environment - Returns: - True iff we're running on GAE. - """ - server_software = os.environ.get('SERVER_SOFTWARE', '') - return (server_software.startswith('Development/') or - server_software.startswith('Google App Engine/')) + Returns: + True iff we're running on GAE. + """ + server_software = os.environ.get('SERVER_SOFTWARE', '') + return (server_software.startswith('Development/') or + server_software.startswith('Google App Engine/')) def DetectGce(): - """Determine whether or not we're running on GCE. + """Determine whether or not we're running on GCE. - This is based on: - https://cloud.google.com/compute/docs/metadata#runninggce + This is based on: + https://cloud.google.com/compute/docs/metadata#runninggce - Returns: - True iff we're running on a GCE instance. - """ - try: - o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open( - urllib_request.Request('http://metadata.google.internal')) - except urllib_error.URLError: - return False - return (o.getcode() == http_client.OK and - o.headers.get('metadata-flavor') == 'Google') + Returns: + True iff we're running on a GCE instance. + """ + try: + o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open( + urllib_request.Request('http://metadata.google.internal')) + except urllib_error.URLError: + return False + return (o.getcode() == http_client.OK and + o.headers.get('metadata-flavor') == 'Google') def NormalizeScopes(scope_spec): - """Normalize scope_spec to a set of strings.""" - if isinstance(scope_spec, six.string_types): - return set(scope_spec.split(' ')) - elif isinstance(scope_spec, collections.Iterable): - return set(scope_spec) - raise exceptions.TypecheckError( - 'NormalizeScopes expected string or iterable, found %s' % ( - type(scope_spec),)) + """Normalize scope_spec to a set of strings.""" + if isinstance(scope_spec, six.string_types): + return set(scope_spec.split(' ')) + elif isinstance(scope_spec, collections.Iterable): + return set(scope_spec) + raise exceptions.TypecheckError( + 'NormalizeScopes expected string or iterable, found %s' % ( + type(scope_spec),)) def Typecheck(arg, arg_type, msg=None): - if not isinstance(arg, arg_type): - if msg is None: - if isinstance(arg_type, tuple): - msg = 'Type of arg is "%s", not one of %r' % (type(arg), arg_type) - else: - msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) - raise exceptions.TypecheckError(msg) - return arg + if not isinstance(arg, arg_type): + if msg is None: + if isinstance(arg_type, tuple): + msg = 'Type of arg is "%s", not one of %r' % ( + type(arg), arg_type) + else: + msg = 'Type of arg is "%s", not "%s"' % (type(arg), arg_type) + raise exceptions.TypecheckError(msg) + return arg def ExpandRelativePath(method_config, params, relative_path=None): - """Determine the relative path for request.""" - path = relative_path or method_config.relative_path or '' - - for param in method_config.path_params: - param_template = '{%s}' % param - # For more details about "reserved word expansion", see: - # http://tools.ietf.org/html/rfc6570#section-3.2.2 - reserved_chars = '' - reserved_template = '{+%s}' % param - if reserved_template in path: - reserved_chars = _RESERVED_URI_CHARS - path = path.replace(reserved_template, param_template) - if param_template not in path: - raise exceptions.InvalidUserInputError( - 'Missing path parameter %s' % param) - try: - # TODO(craigcitro): Do we want to support some sophisticated - # mapping here? - value = params[param] - except KeyError: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - if value is None: - raise exceptions.InvalidUserInputError( - 'Request missing required parameter %s' % param) - try: - if not isinstance(value, six.string_types): - value = str(value) - path = path.replace(param_template, - urllib_parse.quote(value.encode('utf_8'), - reserved_chars)) - except TypeError as e: - raise exceptions.InvalidUserInputError( - 'Error setting required parameter %s to value %s: %s' % ( - param, value, e)) - return path + """Determine the relative path for request.""" + path = relative_path or method_config.relative_path or '' + + for param in method_config.path_params: + param_template = '{%s}' % param + # For more details about "reserved word expansion", see: + # http://tools.ietf.org/html/rfc6570#section-3.2.2 + reserved_chars = '' + reserved_template = '{+%s}' % param + if reserved_template in path: + reserved_chars = _RESERVED_URI_CHARS + path = path.replace(reserved_template, param_template) + if param_template not in path: + raise exceptions.InvalidUserInputError( + 'Missing path parameter %s' % param) + try: + # TODO(craigcitro): Do we want to support some sophisticated + # mapping here? + value = params[param] + except KeyError: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + if value is None: + raise exceptions.InvalidUserInputError( + 'Request missing required parameter %s' % param) + try: + if not isinstance(value, six.string_types): + value = str(value) + path = path.replace(param_template, + urllib_parse.quote(value.encode('utf_8'), + reserved_chars)) + except TypeError as e: + raise exceptions.InvalidUserInputError( + 'Error setting required parameter %s to value %s: %s' % ( + param, value, e)) + return path def CalculateWaitForRetry(retry_attempt, max_wait=60): - """Calculates amount of time to wait before a retry attempt. + """Calculates amount of time to wait before a retry attempt. - Wait time grows exponentially with the number of attempts. - A random amount of jitter is added to spread out retry attempts from different - clients. + Wait time grows exponentially with the number of attempts. + A random amount of jitter is added to spread out retry attempts from different + clients. - Args: - retry_attempt: Retry attempt counter. - max_wait: Upper bound for wait time. + Args: + retry_attempt: Retry attempt counter. + max_wait: Upper bound for wait time. - Returns: - Amount of time to wait before retrying request. - """ + Returns: + Amount of time to wait before retrying request. + """ - wait_time = 2 ** retry_attempt - # randrange requires a nonzero interval, so we want to drop it if - # the range is too small for jitter. - if retry_attempt: - max_jitter = (2 ** retry_attempt) / 2 - wait_time += random.randrange(-max_jitter, max_jitter) - return min(wait_time, max_wait) + wait_time = 2 ** retry_attempt + # randrange requires a nonzero interval, so we want to drop it if + # the range is too small for jitter. + if retry_attempt: + max_jitter = (2 ** retry_attempt) / 2 + wait_time += random.randrange(-max_jitter, max_jitter) + return min(wait_time, max_wait) def AcceptableMimeType(accept_patterns, mime_type): - """Return True iff mime_type is acceptable for one of accept_patterns. - - Note that this function assumes that all patterns in accept_patterns - will be simple types of the form "type/subtype", where one or both - of these can be "*". We do not support parameters (i.e. "; q=") in - patterns. - - Args: - accept_patterns: list of acceptable MIME types. - mime_type: the mime type we would like to match. - - Returns: - Whether or not mime_type matches (at least) one of these patterns. - """ - unsupported_patterns = [p for p in accept_patterns if ';' in p] - if unsupported_patterns: - raise exceptions.GeneratedClientError( - 'MIME patterns with parameter unsupported: "%s"' % ', '.join( - unsupported_patterns)) - def MimeTypeMatches(pattern, mime_type): - """Return True iff mime_type is acceptable for pattern.""" - # Some systems use a single '*' instead of '*/*'. - if pattern == '*': - pattern = '*/*' - return all(accept in ('*', provided) for accept, provided - in zip(pattern.split('/'), mime_type.split('/'))) - - return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) + """Return True iff mime_type is acceptable for one of accept_patterns. + + Note that this function assumes that all patterns in accept_patterns + will be simple types of the form "type/subtype", where one or both + of these can be "*". We do not support parameters (i.e. "; q=") in + patterns. + + Args: + accept_patterns: list of acceptable MIME types. + mime_type: the mime type we would like to match. + + Returns: + Whether or not mime_type matches (at least) one of these patterns. + """ + unsupported_patterns = [p for p in accept_patterns if ';' in p] + if unsupported_patterns: + raise exceptions.GeneratedClientError( + 'MIME patterns with parameter unsupported: "%s"' % ', '.join( + unsupported_patterns)) + + def MimeTypeMatches(pattern, mime_type): + """Return True iff mime_type is acceptable for pattern.""" + # Some systems use a single '*' instead of '*/*'. + if pattern == '*': + pattern = '*/*' + return all(accept in ('*', provided) for accept, provided + in zip(pattern.split('/'), mime_type.split('/'))) + + return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) def MapParamNames(params, request_type): - """Reverse parameter remappings for URL construction.""" - return [encoding.GetCustomJsonFieldMapping(request_type, json_name=p) or p - for p in params] + """Reverse parameter remappings for URL construction.""" + return [encoding.GetCustomJsonFieldMapping(request_type, json_name=p) or p + for p in params] def MapRequestParams(params, request_type): - """Perform any renames/remappings needed for URL construction. - - Currently, we have several ways to customize JSON encoding, in - particular of field names and enums. This works fine for JSON - bodies, but also needs to be applied for path and query parameters - in the URL. - - This function takes a dictionary from param names to values, and - performs any registered mappings. We also need the request type (to - look up the mappings). - - Args: - params: (dict) Map from param names to values - request_type: (protorpc.messages.Message) request type for this API call - - Returns: - A new dict of the same size, with all registered mappings applied. - """ - new_params = dict(params) - for param_name, value in params.items(): - field_remapping = encoding.GetCustomJsonFieldMapping( - request_type, python_name=param_name) - if field_remapping is not None: - new_params[field_remapping] = new_params.pop(param_name) - if isinstance(value, messages.Enum): - new_params[param_name] = encoding.GetCustomJsonEnumMapping( - type(value), python_name=str(value)) or str(value) - return new_params + """Perform any renames/remappings needed for URL construction. + + Currently, we have several ways to customize JSON encoding, in + particular of field names and enums. This works fine for JSON + bodies, but also needs to be applied for path and query parameters + in the URL. + + This function takes a dictionary from param names to values, and + performs any registered mappings. We also need the request type (to + look up the mappings). + + Args: + params: (dict) Map from param names to values + request_type: (protorpc.messages.Message) request type for this API call + + Returns: + A new dict of the same size, with all registered mappings applied. + """ + new_params = dict(params) + for param_name, value in params.items(): + field_remapping = encoding.GetCustomJsonFieldMapping( + request_type, python_name=param_name) + if field_remapping is not None: + new_params[field_remapping] = new_params.pop(param_name) + if isinstance(value, messages.Enum): + new_params[param_name] = encoding.GetCustomJsonEnumMapping( + type(value), python_name=str(value)) or str(value) + return new_params diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 025626c..c6c9e80 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -11,19 +11,19 @@ from apitools.base.py import util class MockedMethodConfig(object): - def __init__(self, relative_path, path_params): - self.relative_path = relative_path - self.path_params = path_params + def __init__(self, relative_path, path_params): + self.relative_path = relative_path + self.path_params = path_params class MessageWithRemappings(messages.Message): - class AnEnum(messages.Enum): - value_one = 1 - value_two = 2 + class AnEnum(messages.Enum): + value_one = 1 + value_two = 2 - str_field = messages.StringField(1) - enum_field = messages.EnumField('AnEnum', 2) + str_field = messages.StringField(1) + enum_field = messages.EnumField('AnEnum', 2) encoding.AddCustomJsonFieldMapping( @@ -34,157 +34,159 @@ encoding.AddCustomJsonEnumMapping( class UtilTest(unittest2.TestCase): - def testExpand(self): - method_config_xy = MockedMethodConfig(relative_path='{x}/y/{z}', - path_params=['x', 'z']) - self.assertEquals( - util.ExpandRelativePath(method_config_xy, {'x': '1', 'z': '2'}), - '1/y/2') - self.assertEquals( - util.ExpandRelativePath( - method_config_xy, - {'x': '1', 'z': '2'}, - relative_path='{x}/y/{z}/q'), - '1/y/2/q') - - def testReservedExpansion(self): - method_config_reserved = MockedMethodConfig(relative_path='{+x}/baz', - path_params=['x']) - self.assertEquals('foo/:bar:/baz', util.ExpandRelativePath( - method_config_reserved, {'x': 'foo/:bar:'})) - method_config_no_reserved = MockedMethodConfig(relative_path='{x}/baz', - path_params=['x']) - self.assertEquals('foo%2F%3Abar%3A/baz', util.ExpandRelativePath( - method_config_no_reserved, {'x': 'foo/:bar:'})) - - def testCalculateWaitForRetry(self): - self.assertTrue(util.CalculateWaitForRetry(1) in range(1, 4)) - self.assertTrue(util.CalculateWaitForRetry(2) in range(2, 7)) - self.assertTrue(util.CalculateWaitForRetry(3) in range(4, 13)) - self.assertTrue(util.CalculateWaitForRetry(4) in range(8, 25)) - - self.assertEquals(10, util.CalculateWaitForRetry(5, max_wait=10)) - - self.assertGreater(util.CalculateWaitForRetry(0), 0) - - def testTypecheck(self): - - class Class1(object): - pass - - class Class2(object): - pass - - class Class3(object): - pass - - instance_of_class1 = Class1() - - self.assertEquals( - instance_of_class1, util.Typecheck(instance_of_class1, Class1)) - - self.assertEquals( - instance_of_class1, - util.Typecheck(instance_of_class1, ((Class1, Class2), Class3))) - - self.assertEquals( - instance_of_class1, - util.Typecheck(instance_of_class1, (Class1, (Class2, Class3)))) - - self.assertEquals( - instance_of_class1, - util.Typecheck(instance_of_class1, Class1, 'message')) - - self.assertEquals( - instance_of_class1, - util.Typecheck( - instance_of_class1, ((Class1, Class2), Class3), 'message')) - - self.assertEquals( - instance_of_class1, - util.Typecheck( - instance_of_class1, (Class1, (Class2, Class3)), 'message')) - - try: - util.Typecheck(instance_of_class1, Class2) - self.fail('Type mismatch not detected when called with 2 arguments') - except exceptions.TypecheckError: - pass # expected - - try: - util.Typecheck(instance_of_class1, (Class2, Class3)) - self.fail( - 'Type mismatch not detected when called with 2 arguments including ' - 'type tuple') - except exceptions.TypecheckError: - pass # expected - - try: - util.Typecheck(instance_of_class1, Class2, 'message') - self.fail('Type mismatch not detected when called with 3 arguments') - except exceptions.TypecheckError: - pass # expected - - try: - util.Typecheck(instance_of_class1, (Class2, Class3), 'message') - self.fail( - 'Type mismatch not detected when called with 3 arguments including ' - 'type tuple') - except exceptions.TypecheckError: - pass # expected - - def testAcceptableMimeType(self): - valid_pairs = ( - ('*', 'text/plain'), - ('*/*', 'text/plain'), - ('text/*', 'text/plain'), - ('*/plain', 'text/plain'), - ('text/plain', 'text/plain'), - ) - - for accept, mime_type in valid_pairs: - self.assertTrue(util.AcceptableMimeType([accept], mime_type)) - - invalid_pairs = ( - ('text/*', 'application/json'), - ('text/plain', 'application/json'), - ) - - for accept, mime_type in invalid_pairs: - self.assertFalse(util.AcceptableMimeType([accept], mime_type)) - - self.assertTrue(util.AcceptableMimeType(['application/json', '*/*'], - 'text/plain')) - self.assertFalse(util.AcceptableMimeType(['application/json', 'img/*'], - 'text/plain')) - - def testUnsupportedMimeType(self): - self.assertRaises( - exceptions.GeneratedClientError, - util.AcceptableMimeType, ['text/html;q=0.9'], 'text/html') - - def testMapRequestParams(self): - params = { - 'str_field': 'foo', - 'enum_field': MessageWithRemappings.AnEnum.value_one, - } - remapped_params = { - 'path_field': 'foo', - 'enum_field': 'ONE', - } - self.assertEqual(remapped_params, - util.MapRequestParams(params, MessageWithRemappings)) - - params['enum_field'] = MessageWithRemappings.AnEnum.value_two - remapped_params['enum_field'] = 'value_two' - self.assertEqual(remapped_params, - util.MapRequestParams(params, MessageWithRemappings)) - - def testMapParamNames(self): - params = ['path_field', 'enum_field'] - remapped_params = ['str_field', 'enum_field'] - self.assertEqual(remapped_params, - util.MapParamNames(params, MessageWithRemappings)) + def testExpand(self): + method_config_xy = MockedMethodConfig(relative_path='{x}/y/{z}', + path_params=['x', 'z']) + self.assertEquals( + util.ExpandRelativePath(method_config_xy, {'x': '1', 'z': '2'}), + '1/y/2') + self.assertEquals( + util.ExpandRelativePath( + method_config_xy, + {'x': '1', 'z': '2'}, + relative_path='{x}/y/{z}/q'), + '1/y/2/q') + + def testReservedExpansion(self): + method_config_reserved = MockedMethodConfig(relative_path='{+x}/baz', + path_params=['x']) + self.assertEquals('foo/:bar:/baz', util.ExpandRelativePath( + method_config_reserved, {'x': 'foo/:bar:'})) + method_config_no_reserved = MockedMethodConfig(relative_path='{x}/baz', + path_params=['x']) + self.assertEquals('foo%2F%3Abar%3A/baz', util.ExpandRelativePath( + method_config_no_reserved, {'x': 'foo/:bar:'})) + + def testCalculateWaitForRetry(self): + self.assertTrue(util.CalculateWaitForRetry(1) in range(1, 4)) + self.assertTrue(util.CalculateWaitForRetry(2) in range(2, 7)) + self.assertTrue(util.CalculateWaitForRetry(3) in range(4, 13)) + self.assertTrue(util.CalculateWaitForRetry(4) in range(8, 25)) + + self.assertEquals(10, util.CalculateWaitForRetry(5, max_wait=10)) + + self.assertGreater(util.CalculateWaitForRetry(0), 0) + + def testTypecheck(self): + + class Class1(object): + pass + + class Class2(object): + pass + + class Class3(object): + pass + + instance_of_class1 = Class1() + + self.assertEquals( + instance_of_class1, util.Typecheck(instance_of_class1, Class1)) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, ((Class1, Class2), Class3))) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, (Class1, (Class2, Class3)))) + + self.assertEquals( + instance_of_class1, + util.Typecheck(instance_of_class1, Class1, 'message')) + + self.assertEquals( + instance_of_class1, + util.Typecheck( + instance_of_class1, ((Class1, Class2), Class3), 'message')) + + self.assertEquals( + instance_of_class1, + util.Typecheck( + instance_of_class1, (Class1, (Class2, Class3)), 'message')) + + try: + util.Typecheck(instance_of_class1, Class2) + self.fail( + 'Type mismatch not detected when called with 2 arguments') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, (Class2, Class3)) + self.fail( + 'Type mismatch not detected when called with 2 arguments including ' + 'type tuple') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, Class2, 'message') + self.fail( + 'Type mismatch not detected when called with 3 arguments') + except exceptions.TypecheckError: + pass # expected + + try: + util.Typecheck(instance_of_class1, (Class2, Class3), 'message') + self.fail( + 'Type mismatch not detected when called with 3 arguments including ' + 'type tuple') + except exceptions.TypecheckError: + pass # expected + + def testAcceptableMimeType(self): + valid_pairs = ( + ('*', 'text/plain'), + ('*/*', 'text/plain'), + ('text/*', 'text/plain'), + ('*/plain', 'text/plain'), + ('text/plain', 'text/plain'), + ) + + for accept, mime_type in valid_pairs: + self.assertTrue(util.AcceptableMimeType([accept], mime_type)) + + invalid_pairs = ( + ('text/*', 'application/json'), + ('text/plain', 'application/json'), + ) + + for accept, mime_type in invalid_pairs: + self.assertFalse(util.AcceptableMimeType([accept], mime_type)) + + self.assertTrue(util.AcceptableMimeType(['application/json', '*/*'], + 'text/plain')) + self.assertFalse(util.AcceptableMimeType(['application/json', 'img/*'], + 'text/plain')) + + def testUnsupportedMimeType(self): + self.assertRaises( + exceptions.GeneratedClientError, + util.AcceptableMimeType, ['text/html;q=0.9'], 'text/html') + + def testMapRequestParams(self): + params = { + 'str_field': 'foo', + 'enum_field': MessageWithRemappings.AnEnum.value_one, + } + remapped_params = { + 'path_field': 'foo', + 'enum_field': 'ONE', + } + self.assertEqual(remapped_params, + util.MapRequestParams(params, MessageWithRemappings)) + + params['enum_field'] = MessageWithRemappings.AnEnum.value_two + remapped_params['enum_field'] = 'value_two' + self.assertEqual(remapped_params, + util.MapRequestParams(params, MessageWithRemappings)) + + def testMapParamNames(self): + params = ['path_field', 'enum_field'] + remapped_params = ['str_field', 'enum_field'] + self.assertEqual(remapped_params, + util.MapParamNames(params, MessageWithRemappings)) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 0974316..8d78d55 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -22,58 +22,59 @@ _API_LIST = [ @contextlib.contextmanager def TempDir(): - original_dir = os.getcwd() - path = tempfile.mkdtemp() - try: - os.chdir(path) - yield path - finally: - os.chdir(original_dir) - shutil.rmtree(path) + original_dir = os.getcwd() + path = tempfile.mkdtemp() + try: + os.chdir(path) + yield path + finally: + os.chdir(original_dir) + shutil.rmtree(path) class ClientGenerationTest(unittest2.TestCase): - def setUp(self): - super(ClientGenerationTest, self).setUp() - self.gen_client_binary = 'gen_client' + def setUp(self): + super(ClientGenerationTest, self).setUp() + self.gen_client_binary = 'gen_client' - # TODO(craigcitro): Make apitools codegen support python 2.6. - # Maybe. - # - # unittest in 2.6 doesn't have skipIf. - @unittest2.skipUnless(sys.version_info[0] == 2 and - sys.version_info[1] == 7, - 'Only runs in Python 2.7') - def testGeneration(self): - for api in _API_LIST: - with TempDir(): - args = [ - self.gen_client_binary, - '--client_id=12345', - '--client_secret=67890', - '--discovery_url=%s' % api, - '--outdir=generated', - '--overwrite', - 'client', - ] - logging.info( - 'Testing API %s with command line: %s', api, ' '.join(args)) - retcode = subprocess.call(args) - if retcode == 128: - logging.error('Failed to fetch discovery doc, continuing.') - continue - self.assertEqual(0, retcode) + # TODO(craigcitro): Make apitools codegen support python 2.6. + # Maybe. + # + # unittest in 2.6 doesn't have skipIf. + @unittest2.skipUnless(sys.version_info[0] == 2 and + sys.version_info[1] == 7, + 'Only runs in Python 2.7') + def testGeneration(self): + for api in _API_LIST: + with TempDir(): + args = [ + self.gen_client_binary, + '--client_id=12345', + '--client_secret=67890', + '--discovery_url=%s' % api, + '--outdir=generated', + '--overwrite', + 'client', + ] + logging.info( + 'Testing API %s with command line: %s', api, ' '.join(args)) + retcode = subprocess.call(args) + if retcode == 128: + logging.error('Failed to fetch discovery doc, continuing.') + continue + self.assertEqual(0, retcode) - with tempfile.NamedTemporaryFile() as out: - cmdline_args = [ - os.path.join('generated', api.replace('.', '_') + '.py'), - 'help', - ] - retcode = subprocess.call(cmdline_args, stdout=out) - # appcommands returns 1 on help - self.assertEqual(1, retcode) + with tempfile.NamedTemporaryFile() as out: + cmdline_args = [ + os.path.join( + 'generated', api.replace('.', '_') + '.py'), + 'help', + ] + retcode = subprocess.call(cmdline_args, stdout=out) + # appcommands returns 1 on help + self.assertEqual(1, retcode) if __name__ == '__main__': - unittest2.main() + unittest2.main() diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 93c5d53..2710670 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -30,536 +30,557 @@ _VARIANT_TO_FLAG_TYPE_MAP = { class FlagInfo(messages.Message): - """Information about a flag and conversion to a message. - - Fields: - name: name of this flag. - type: type of the flag. - description: description of the flag. - default: default value for this flag. - enum_values: if this flag is an enum, the list of possible - values. - required: whether or not this flag is required. - fv: name of the flag_values object where this flag should - be registered. - conversion: template for type conversion. - special: (boolean, default: False) If True, this flag doesn't - correspond to an attribute on the request. - """ - name = messages.StringField(1) - type = messages.StringField(2) - description = messages.StringField(3) - default = messages.StringField(4) - enum_values = messages.StringField(5, repeated=True) - required = messages.BooleanField(6, default=False) - fv = messages.StringField(7) - conversion = messages.StringField(8) - special = messages.BooleanField(9, default=False) + + """Information about a flag and conversion to a message. + + Fields: + name: name of this flag. + type: type of the flag. + description: description of the flag. + default: default value for this flag. + enum_values: if this flag is an enum, the list of possible + values. + required: whether or not this flag is required. + fv: name of the flag_values object where this flag should + be registered. + conversion: template for type conversion. + special: (boolean, default: False) If True, this flag doesn't + correspond to an attribute on the request. + """ + name = messages.StringField(1) + type = messages.StringField(2) + description = messages.StringField(3) + default = messages.StringField(4) + enum_values = messages.StringField(5, repeated=True) + required = messages.BooleanField(6, default=False) + fv = messages.StringField(7) + conversion = messages.StringField(8) + special = messages.BooleanField(9, default=False) class ArgInfo(messages.Message): - """Information about a single positional command argument. - Fields: - name: argument name. - description: description of this argument. - conversion: template for type conversion. - """ - name = messages.StringField(1) - description = messages.StringField(2) - conversion = messages.StringField(3) + """Information about a single positional command argument. + + Fields: + name: argument name. + description: description of this argument. + conversion: template for type conversion. + """ + name = messages.StringField(1) + description = messages.StringField(2) + conversion = messages.StringField(3) class CommandInfo(messages.Message): - """Information about a single command. - - Fields: - name: name of this command. - class_name: name of the apitools_base.NewCmd class for this command. - description: description of this command. - flags: list of FlagInfo messages for the command-specific flags. - args: list of ArgInfo messages for the positional args. - request_type: name of the request type for this command. - client_method_path: path from the client object to the method - this command is wrapping. - """ - name = messages.StringField(1) - class_name = messages.StringField(2) - description = messages.StringField(3) - flags = messages.MessageField(FlagInfo, 4, repeated=True) - args = messages.MessageField(ArgInfo, 5, repeated=True) - request_type = messages.StringField(6) - client_method_path = messages.StringField(7) - has_upload = messages.BooleanField(8, default=False) - has_download = messages.BooleanField(9, default=False) + + """Information about a single command. + + Fields: + name: name of this command. + class_name: name of the apitools_base.NewCmd class for this command. + description: description of this command. + flags: list of FlagInfo messages for the command-specific flags. + args: list of ArgInfo messages for the positional args. + request_type: name of the request type for this command. + client_method_path: path from the client object to the method + this command is wrapping. + """ + name = messages.StringField(1) + class_name = messages.StringField(2) + description = messages.StringField(3) + flags = messages.MessageField(FlagInfo, 4, repeated=True) + args = messages.MessageField(ArgInfo, 5, repeated=True) + request_type = messages.StringField(6) + client_method_path = messages.StringField(7) + has_upload = messages.BooleanField(8, default=False) + has_download = messages.BooleanField(9, default=False) class CommandRegistry(object): - """Registry for CLI commands.""" - - def __init__(self, package, version, client_info, message_registry, - root_package, base_files_package, base_url, names): - self.__package = package - self.__version = version - self.__client_info = client_info - self.__names = names - self.__message_registry = message_registry - self.__root_package = root_package - self.__base_files_package = base_files_package - self.__base_url = base_url - self.__command_list = [] - self.__global_flags = [] - - def Validate(self): - self.__message_registry.Validate() - - def AddGlobalParameters(self, schema): - for field in schema.fields: - self.__global_flags.append(self.__FlagInfoFromField(field, schema)) - - def AddCommandForMethod(self, service_name, method_name, method_info, - request, _): - """Add the given method as a command.""" - command_name = self.__GetCommandName(method_info.method_id) - calling_path = '%s.%s' % (service_name, method_name) - request_type = self.__message_registry.LookupDescriptor(request) - description = method_info.description - if not description: - description = 'Call the %s method.' % method_info.method_id - field_map = dict((f.name, f) for f in request_type.fields) - args = [] - arg_names = [] - for field_name in method_info.ordered_params: - extended_field = field_map[field_name] - name = extended_field.name - args.append(ArgInfo( - name=name, - description=extended_field.description, - conversion=self.__GetConversion(extended_field, request_type), - )) - arg_names.append(name) - flags = [] - for extended_field in sorted(request_type.fields, key=lambda x: x.name): - field = extended_field.field_descriptor - if extended_field.name in arg_names: - continue - if self.__FieldIsRequired(field): - logging.warning( - 'Required field %s not in ordered_params for command %s', - extended_field.name, command_name) - flags.append(self.__FlagInfoFromField( - extended_field, request_type, fv='fv')) - if method_info.upload_config: - # TODO(craigcitro): Consider adding additional flags to allow - # determining the filename from the object metadata. - upload_flag_info = FlagInfo( - name='upload_filename', type='string', default='', - description='Filename to use for upload.', fv='fv', special=True) - flags.append(upload_flag_info) - mime_description = ( - 'MIME type to use for the upload. Only needed if ' - 'the extension on --upload_filename does not determine ' - 'the correct (or any) MIME type.') - mime_type_flag_info = FlagInfo( - name='upload_mime_type', type='string', default='', - description=mime_description, fv='fv', special=True) - flags.append(mime_type_flag_info) - if method_info.supports_download: - download_flag_info = FlagInfo( - name='download_filename', type='string', default='', - description='Filename to use for download.', fv='fv', special=True) - flags.append(download_flag_info) - overwrite_description = ( - 'If True, overwrite the existing file when downloading.') - overwrite_flag_info = FlagInfo( - name='overwrite', type='boolean', default='False', - description=overwrite_description, fv='fv', special=True) - flags.append(overwrite_flag_info) - command_info = CommandInfo( - name=command_name, - class_name=self.__names.ClassName(command_name), - description=description, - flags=flags, - args=args, - request_type=request_type.full_name, - client_method_path=calling_path, - has_upload=bool(method_info.upload_config), - has_download=bool(method_info.supports_download) - ) - self.__command_list.append(command_info) - - def __LookupMessage(self, message, field): - message_type = self.__message_registry.LookupDescriptor( - '%s.%s' % (message.name, field.type_name)) - if message_type is None: - message_type = self.__message_registry.LookupDescriptor(field.type_name) - return message_type - - def __GetCommandName(self, method_id): - command_name = method_id - prefix = '%s.' % self.__package - if command_name.startswith(prefix): - command_name = command_name[len(prefix):] - command_name = command_name.replace('.', '_') - return command_name - - def __GetConversion(self, extended_field, extended_message): - field = extended_field.field_descriptor - - type_name = '' - if field.variant in (messages.Variant.MESSAGE, messages.Variant.ENUM): - if field.type_name.startswith('protorpc.'): - type_name = field.type_name - else: - field_message = self.__LookupMessage(extended_message, field) - if field_message is None: - raise ValueError('Could not find type for field %s' % field.name) - type_name = 'messages.%s' % field_message.full_name - - template = '' - if field.variant in (messages.Variant.INT64, messages.Variant.UINT64): - template = 'int(%s)' - elif field.variant == messages.Variant.MESSAGE: - template = 'apitools_base.JsonToMessage(%s, %%s)' % type_name - elif field.variant == messages.Variant.ENUM: - template = '%s(%%s)' % type_name - elif field.variant == messages.Variant.STRING: - template = "%s.decode('utf8')" - - if self.__FieldIsRepeated(extended_field.field_descriptor): - if template: - template = '[%s for x in %%s]' % (template % 'x') - - return template - - def __FieldIsRequired(self, field): - return field.label == descriptor.FieldDescriptor.Label.REQUIRED - - def __FieldIsRepeated(self, field): - return field.label == descriptor.FieldDescriptor.Label.REPEATED - - def __FlagInfoFromField(self, extended_field, extended_message, fv=''): - field = extended_field.field_descriptor - flag_info = FlagInfo() - flag_info.name = str(field.name) - # TODO(craigcitro): We should key by variant. - flag_info.type = _VARIANT_TO_FLAG_TYPE_MAP[field.variant] - flag_info.description = extended_field.description - if field.default_value: - # TODO(craigcitro): Formatting? - flag_info.default = field.default_value - if flag_info.type == 'enum': - # TODO(craigcitro): Does protorpc do this for us? - enum_type = self.__LookupMessage(extended_message, field) - if enum_type is None: - raise ValueError('Cannot find enum type %s', field.type_name) - flag_info.enum_values = [x.name for x in enum_type.values] - # Note that this choice is completely arbitrary -- but we only - # push the value through if the user specifies it, so this - # doesn't hurt anything. - if flag_info.default is None: - flag_info.default = flag_info.enum_values[0] - if self.__FieldIsRequired(field): - flag_info.required = True - flag_info.fv = fv - flag_info.conversion = self.__GetConversion( - extended_field, extended_message) - return flag_info - - def __PrintFlagDeclarations(self, printer): - package = self.__client_info.package - function_name = '_Declare%sFlags' % (package[0].upper() + package[1:]) - printer() - printer() - printer('def %s():', function_name) - with printer.Indent(): - printer('"""Declare global flags in an idempotent way."""') - printer("if 'api_endpoint' in flags.FLAGS:") - with printer.Indent(): - printer('return') - printer('flags.DEFINE_string(') - with printer.Indent(' '): - printer("'api_endpoint',") - printer('%r,', self.__base_url) - printer("'URL of the API endpoint to use.',") - printer("short_name='%s_url')", self.__package) - printer('flags.DEFINE_string(') - with printer.Indent(' '): - printer("'history_file',") - printer('%r,', '~/.%s.%s.history' % (self.__package, self.__version)) - printer("'File with interactive shell history.')") - printer('flags.DEFINE_multistring(') - with printer.Indent(' '): - printer("'add_header', [],") - printer("'Additional http headers (as key=value strings). Can be '") - printer("'specified multiple times.')") - printer('flags.DEFINE_string(') - with printer.Indent(' '): - printer("'service_account_json_keyfile', '',") - printer("'Filename for a JSON service account key downloaded from '") - printer("'the Developer Console.')") - for flag_info in self.__global_flags: - self.__PrintFlag(printer, flag_info) - printer() - printer() - printer('FLAGS = flags.FLAGS') - printer('apitools_base_cli.DeclareBaseFlags()') - printer('%s()', function_name) - - def __PrintGetGlobalParams(self, printer): - printer('def GetGlobalParamsFromFlags():') - with printer.Indent(): - printer('"""Return a StandardQueryParameters based on flags."""') - printer('result = messages.StandardQueryParameters()') - - for flag_info in self.__global_flags: - rhs = 'FLAGS.%s' % flag_info.name - if flag_info.conversion: - rhs = flag_info.conversion % rhs - printer('if FLAGS[%r].present:', flag_info.name) - with printer.Indent(): - printer('result.%s = %s', flag_info.name, rhs) - printer('return result') - printer() - printer() - - def __PrintGetClient(self, printer): - printer('def GetClientFromFlags():') - with printer.Indent(): - printer('"""Return a client object, configured from flags."""') - printer('log_request = FLAGS.log_request or FLAGS.log_request_response') - printer('log_response = FLAGS.log_response or FLAGS.log_request_response') - printer('api_endpoint = apitools_base.NormalizeApiEndpoint(' - 'FLAGS.api_endpoint)') - printer("additional_http_headers = dict(x.split('=', 1) for x in " - "FLAGS.add_header)") - printer('credentials_args = {') - with printer.Indent(' '): - printer("'service_account_json_keyfile': os.path.expanduser(" - 'FLAGS.service_account_json_keyfile)') - printer('}') - printer('try:') - with printer.Indent(): - printer('client = client_lib.%s(', self.__client_info.client_class_name) - with printer.Indent(indent=' '): - printer('api_endpoint, log_request=log_request,') - printer('log_response=log_response,') - printer('credentials_args=credentials_args,') - printer('additional_http_headers=additional_http_headers)') - printer('except apitools_base.CredentialsError as e:') - with printer.Indent(): - printer("print 'Error creating credentials: %%s' %% e") - printer('sys.exit(1)') - printer('return client') - printer() - printer() - - def __PrintCommandDocstring(self, printer, command_info): - with printer.CommentContext(): - for line in textwrap.wrap('"""%s' % command_info.description, - printer.CalculateWidth()): - printer(line) - extended_descriptor.PrintIndentedDescriptions( - printer, command_info.args, 'Args') - extended_descriptor.PrintIndentedDescriptions( - printer, command_info.flags, 'Flags') - printer('"""') - - def __PrintFlag(self, printer, flag_info): - printer('flags.DEFINE_%s(', flag_info.type) - with printer.Indent(indent=' '): - printer('%r,', flag_info.name) - printer('%r,', flag_info.default) - if flag_info.type == 'enum': - printer('%r,', flag_info.enum_values) - - # TODO(craigcitro): Consider using 'drop_whitespace' elsewhere. - description_lines = textwrap.wrap( - flag_info.description, 75 - len(printer.indent), - drop_whitespace=False) - for line in description_lines[:-1]: - printer('%r', line) - last_line = description_lines[-1] if description_lines else '' - printer('%r%s', last_line, ',' if flag_info.fv else ')') - if flag_info.fv: - printer('flag_values=%s)', flag_info.fv) - if flag_info.required: - printer('flags.MarkFlagAsRequired(%r)', flag_info.name) - - def __PrintPyShell(self, printer): - printer('class PyShell(appcommands.Cmd):') - printer() - with printer.Indent(): - printer('def Run(self, _):') - with printer.Indent(): - printer('"""Run an interactive python shell with the client."""') - printer('client = GetClientFromFlags()') - printer('params = GetGlobalParamsFromFlags()') - printer('for field in params.all_fields():') - with printer.Indent(): - printer('value = params.get_assigned_value(field.name)') - printer('if value != field.default:') - with printer.Indent(): - printer('client.AddGlobalParam(field.name, value)') - printer('banner = """') - printer(' == %s interactive console ==' % ( - self.__client_info.package)) - printer(' client: a %s client' % self.__client_info.package) - printer(' apitools_base: base apitools module') - printer(' messages: the generated messages module') - printer('"""') - printer('local_vars = {') - with printer.Indent(indent=' '): - printer("'apitools_base': apitools_base,") - printer("'client': client,") - printer("'client_lib': client_lib,") - printer("'messages': messages,") - printer('}') - printer("if platform.system() == 'Linux':") - with printer.Indent(): - printer('console = apitools_base_cli.ConsoleWithReadline(') - with printer.Indent(indent=' '): - printer('local_vars, histfile=FLAGS.history_file)') - printer('else:') - with printer.Indent(): - printer('console = code.InteractiveConsole(local_vars)') - printer('try:') + + """Registry for CLI commands.""" + + def __init__(self, package, version, client_info, message_registry, + root_package, base_files_package, base_url, names): + self.__package = package + self.__version = version + self.__client_info = client_info + self.__names = names + self.__message_registry = message_registry + self.__root_package = root_package + self.__base_files_package = base_files_package + self.__base_url = base_url + self.__command_list = [] + self.__global_flags = [] + + def Validate(self): + self.__message_registry.Validate() + + def AddGlobalParameters(self, schema): + for field in schema.fields: + self.__global_flags.append(self.__FlagInfoFromField(field, schema)) + + def AddCommandForMethod(self, service_name, method_name, method_info, + request, _): + """Add the given method as a command.""" + command_name = self.__GetCommandName(method_info.method_id) + calling_path = '%s.%s' % (service_name, method_name) + request_type = self.__message_registry.LookupDescriptor(request) + description = method_info.description + if not description: + description = 'Call the %s method.' % method_info.method_id + field_map = dict((f.name, f) for f in request_type.fields) + args = [] + arg_names = [] + for field_name in method_info.ordered_params: + extended_field = field_map[field_name] + name = extended_field.name + args.append(ArgInfo( + name=name, + description=extended_field.description, + conversion=self.__GetConversion(extended_field, request_type), + )) + arg_names.append(name) + flags = [] + for extended_field in sorted(request_type.fields, key=lambda x: x.name): + field = extended_field.field_descriptor + if extended_field.name in arg_names: + continue + if self.__FieldIsRequired(field): + logging.warning( + 'Required field %s not in ordered_params for command %s', + extended_field.name, command_name) + flags.append(self.__FlagInfoFromField( + extended_field, request_type, fv='fv')) + if method_info.upload_config: + # TODO(craigcitro): Consider adding additional flags to allow + # determining the filename from the object metadata. + upload_flag_info = FlagInfo( + name='upload_filename', type='string', default='', + description='Filename to use for upload.', fv='fv', special=True) + flags.append(upload_flag_info) + mime_description = ( + 'MIME type to use for the upload. Only needed if ' + 'the extension on --upload_filename does not determine ' + 'the correct (or any) MIME type.') + mime_type_flag_info = FlagInfo( + name='upload_mime_type', type='string', default='', + description=mime_description, fv='fv', special=True) + flags.append(mime_type_flag_info) + if method_info.supports_download: + download_flag_info = FlagInfo( + name='download_filename', type='string', default='', + description='Filename to use for download.', fv='fv', special=True) + flags.append(download_flag_info) + overwrite_description = ( + 'If True, overwrite the existing file when downloading.') + overwrite_flag_info = FlagInfo( + name='overwrite', type='boolean', default='False', + description=overwrite_description, fv='fv', special=True) + flags.append(overwrite_flag_info) + command_info = CommandInfo( + name=command_name, + class_name=self.__names.ClassName(command_name), + description=description, + flags=flags, + args=args, + request_type=request_type.full_name, + client_method_path=calling_path, + has_upload=bool(method_info.upload_config), + has_download=bool(method_info.supports_download) + ) + self.__command_list.append(command_info) + + def __LookupMessage(self, message, field): + message_type = self.__message_registry.LookupDescriptor( + '%s.%s' % (message.name, field.type_name)) + if message_type is None: + message_type = self.__message_registry.LookupDescriptor( + field.type_name) + return message_type + + def __GetCommandName(self, method_id): + command_name = method_id + prefix = '%s.' % self.__package + if command_name.startswith(prefix): + command_name = command_name[len(prefix):] + command_name = command_name.replace('.', '_') + return command_name + + def __GetConversion(self, extended_field, extended_message): + field = extended_field.field_descriptor + + type_name = '' + if field.variant in (messages.Variant.MESSAGE, messages.Variant.ENUM): + if field.type_name.startswith('protorpc.'): + type_name = field.type_name + else: + field_message = self.__LookupMessage(extended_message, field) + if field_message is None: + raise ValueError( + 'Could not find type for field %s' % field.name) + type_name = 'messages.%s' % field_message.full_name + + template = '' + if field.variant in (messages.Variant.INT64, messages.Variant.UINT64): + template = 'int(%s)' + elif field.variant == messages.Variant.MESSAGE: + template = 'apitools_base.JsonToMessage(%s, %%s)' % type_name + elif field.variant == messages.Variant.ENUM: + template = '%s(%%s)' % type_name + elif field.variant == messages.Variant.STRING: + template = "%s.decode('utf8')" + + if self.__FieldIsRepeated(extended_field.field_descriptor): + if template: + template = '[%s for x in %%s]' % (template % 'x') + + return template + + def __FieldIsRequired(self, field): + return field.label == descriptor.FieldDescriptor.Label.REQUIRED + + def __FieldIsRepeated(self, field): + return field.label == descriptor.FieldDescriptor.Label.REPEATED + + def __FlagInfoFromField(self, extended_field, extended_message, fv=''): + field = extended_field.field_descriptor + flag_info = FlagInfo() + flag_info.name = str(field.name) + # TODO(craigcitro): We should key by variant. + flag_info.type = _VARIANT_TO_FLAG_TYPE_MAP[field.variant] + flag_info.description = extended_field.description + if field.default_value: + # TODO(craigcitro): Formatting? + flag_info.default = field.default_value + if flag_info.type == 'enum': + # TODO(craigcitro): Does protorpc do this for us? + enum_type = self.__LookupMessage(extended_message, field) + if enum_type is None: + raise ValueError('Cannot find enum type %s', field.type_name) + flag_info.enum_values = [x.name for x in enum_type.values] + # Note that this choice is completely arbitrary -- but we only + # push the value through if the user specifies it, so this + # doesn't hurt anything. + if flag_info.default is None: + flag_info.default = flag_info.enum_values[0] + if self.__FieldIsRequired(field): + flag_info.required = True + flag_info.fv = fv + flag_info.conversion = self.__GetConversion( + extended_field, extended_message) + return flag_info + + def __PrintFlagDeclarations(self, printer): + package = self.__client_info.package + function_name = '_Declare%sFlags' % (package[0].upper() + package[1:]) + printer() + printer() + printer('def %s():', function_name) with printer.Indent(): - printer('console.interact(banner)') - printer('except SystemExit as e:') + printer('"""Declare global flags in an idempotent way."""') + printer("if 'api_endpoint' in flags.FLAGS:") + with printer.Indent(): + printer('return') + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'api_endpoint',") + printer('%r,', self.__base_url) + printer("'URL of the API endpoint to use.',") + printer("short_name='%s_url')", self.__package) + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'history_file',") + printer('%r,', '~/.%s.%s.history' % + (self.__package, self.__version)) + printer("'File with interactive shell history.')") + printer('flags.DEFINE_multistring(') + with printer.Indent(' '): + printer("'add_header', [],") + printer( + "'Additional http headers (as key=value strings). Can be '") + printer("'specified multiple times.')") + printer('flags.DEFINE_string(') + with printer.Indent(' '): + printer("'service_account_json_keyfile', '',") + printer( + "'Filename for a JSON service account key downloaded from '") + printer("'the Developer Console.')") + for flag_info in self.__global_flags: + self.__PrintFlag(printer, flag_info) + printer() + printer() + printer('FLAGS = flags.FLAGS') + printer('apitools_base_cli.DeclareBaseFlags()') + printer('%s()', function_name) + + def __PrintGetGlobalParams(self, printer): + printer('def GetGlobalParamsFromFlags():') with printer.Indent(): - printer('return e.code') - printer() - printer() - - def WriteFile(self, printer): - """Write a simple CLI (currently just a stub).""" - printer('#!/usr/bin/env python') - printer('"""CLI for %s, version %s."""', self.__package, self.__version) - printer('# NOTE: This file is autogenerated and should not be edited by ' - 'hand.') - # TODO(craigcitro): Add a build stamp, along with some other - # information. - printer() - printer('import code') - printer('import os') - printer('import platform') - printer('import sys') - printer() - printer('import protorpc') - printer('from protorpc import message_types') - printer('from protorpc import messages') - printer() - appcommands_import = 'from google.apputils import appcommands' - printer(appcommands_import) - - flags_import = 'import gflags as flags' - printer(flags_import) - printer() - printer('import %s as apitools_base', self.__base_files_package) - printer('from %s import cli as apitools_base_cli', - self.__base_files_package) - import_prefix = '' - printer('%simport %s as client_lib', - import_prefix, self.__client_info.client_rule_name) - printer('%simport %s as messages', - import_prefix, self.__client_info.messages_rule_name) - self.__PrintFlagDeclarations(printer) - printer() - printer() - self.__PrintGetGlobalParams(printer) - self.__PrintGetClient(printer) - self.__PrintPyShell(printer) - self.__PrintCommands(printer) - printer('def main(_):') - with printer.Indent(): - printer("appcommands.AddCmd('pyshell', PyShell)") - for command_info in self.__command_list: - printer("appcommands.AddCmd('%s', %s)", - command_info.name, command_info.class_name) - printer() - printer('apitools_base_cli.SetupLogger()') - # TODO(craigcitro): Just call SetDefaultCommand as soon as - # another appcommands release happens and this exists - # externally. - printer("if hasattr(appcommands, 'SetDefaultCommand'):") - with printer.Indent(): - printer("appcommands.SetDefaultCommand('pyshell')") - printer() - printer() - printer('run_main = apitools_base_cli.run_main') - printer() - printer("if __name__ == '__main__':") - with printer.Indent(): - printer('appcommands.Run()') - - def __PrintCommands(self, printer): - """Print all commands in this registry using printer.""" - for command_info in self.__command_list: - arg_list = [arg_info.name for arg_info in command_info.args] - printer('class %s(apitools_base_cli.NewCmd):', command_info.class_name) - with printer.Indent(): - printer('"""Command wrapping %s."""', command_info.client_method_path) + printer('"""Return a StandardQueryParameters based on flags."""') + printer('result = messages.StandardQueryParameters()') + + for flag_info in self.__global_flags: + rhs = 'FLAGS.%s' % flag_info.name + if flag_info.conversion: + rhs = flag_info.conversion % rhs + printer('if FLAGS[%r].present:', flag_info.name) + with printer.Indent(): + printer('result.%s = %s', flag_info.name, rhs) + printer('return result') printer() - printer('usage = """%s%s%s"""', - command_info.name, - ' ' if arg_list else '', - ' '.join('<%s>' % argname for argname in arg_list)) printer() - printer('def __init__(self, name, fv):') + + def __PrintGetClient(self, printer): + printer('def GetClientFromFlags():') with printer.Indent(): - printer('super(%s, self).__init__(name, fv)', command_info.class_name) - for flag in command_info.flags: - self.__PrintFlag(printer, flag) + printer('"""Return a client object, configured from flags."""') + printer( + 'log_request = FLAGS.log_request or FLAGS.log_request_response') + printer( + 'log_response = FLAGS.log_response or FLAGS.log_request_response') + printer('api_endpoint = apitools_base.NormalizeApiEndpoint(' + 'FLAGS.api_endpoint)') + printer("additional_http_headers = dict(x.split('=', 1) for x in " + "FLAGS.add_header)") + printer('credentials_args = {') + with printer.Indent(' '): + printer("'service_account_json_keyfile': os.path.expanduser(" + 'FLAGS.service_account_json_keyfile)') + printer('}') + printer('try:') + with printer.Indent(): + printer( + 'client = client_lib.%s(', self.__client_info.client_class_name) + with printer.Indent(indent=' '): + printer('api_endpoint, log_request=log_request,') + printer('log_response=log_response,') + printer('credentials_args=credentials_args,') + printer('additional_http_headers=additional_http_headers)') + printer('except apitools_base.CredentialsError as e:') + with printer.Indent(): + printer("print 'Error creating credentials: %%s' %% e") + printer('sys.exit(1)') + printer('return client') + printer() + printer() + + def __PrintCommandDocstring(self, printer, command_info): + with printer.CommentContext(): + for line in textwrap.wrap('"""%s' % command_info.description, + printer.CalculateWidth()): + printer(line) + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.args, 'Args') + extended_descriptor.PrintIndentedDescriptions( + printer, command_info.flags, 'Flags') + printer('"""') + + def __PrintFlag(self, printer, flag_info): + printer('flags.DEFINE_%s(', flag_info.type) + with printer.Indent(indent=' '): + printer('%r,', flag_info.name) + printer('%r,', flag_info.default) + if flag_info.type == 'enum': + printer('%r,', flag_info.enum_values) + + # TODO(craigcitro): Consider using 'drop_whitespace' elsewhere. + description_lines = textwrap.wrap( + flag_info.description, 75 - len(printer.indent), + drop_whitespace=False) + for line in description_lines[:-1]: + printer('%r', line) + last_line = description_lines[-1] if description_lines else '' + printer('%r%s', last_line, ',' if flag_info.fv else ')') + if flag_info.fv: + printer('flag_values=%s)', flag_info.fv) + if flag_info.required: + printer('flags.MarkFlagAsRequired(%r)', flag_info.name) + + def __PrintPyShell(self, printer): + printer('class PyShell(appcommands.Cmd):') printer() - printer('def RunWithArgs(%s):', ', '.join(['self'] + arg_list)) with printer.Indent(): - self.__PrintCommandDocstring(printer, command_info) - printer('client = GetClientFromFlags()') - printer('global_params = GetGlobalParamsFromFlags()') - printer('request = messages.%s(', command_info.request_type) - with printer.Indent(indent=' '): - for arg in command_info.args: - rhs = arg.name - if arg.conversion: - rhs = arg.conversion % arg.name - printer('%s=%s,', arg.name, rhs) - printer(')') - for flag_info in command_info.flags: - if flag_info.special: - continue - rhs = 'FLAGS.%s' % flag_info.name - if flag_info.conversion: - rhs = flag_info.conversion % rhs - printer('if FLAGS[%r].present:', flag_info.name) + printer('def Run(self, _):') with printer.Indent(): - printer('request.%s = %s', flag_info.name, rhs) - call_args = ['request', 'global_params=global_params'] - if command_info.has_upload: - call_args.append('upload=upload') - printer('upload = None') - printer('if FLAGS.upload_filename:') + printer( + '"""Run an interactive python shell with the client."""') + printer('client = GetClientFromFlags()') + printer('params = GetGlobalParamsFromFlags()') + printer('for field in params.all_fields():') + with printer.Indent(): + printer('value = params.get_assigned_value(field.name)') + printer('if value != field.default:') + with printer.Indent(): + printer('client.AddGlobalParam(field.name, value)') + printer('banner = """') + printer(' == %s interactive console ==' % ( + self.__client_info.package)) + printer(' client: a %s client' % + self.__client_info.package) + printer(' apitools_base: base apitools module') + printer(' messages: the generated messages module') + printer('"""') + printer('local_vars = {') + with printer.Indent(indent=' '): + printer("'apitools_base': apitools_base,") + printer("'client': client,") + printer("'client_lib': client_lib,") + printer("'messages': messages,") + printer('}') + printer("if platform.system() == 'Linux':") + with printer.Indent(): + printer('console = apitools_base_cli.ConsoleWithReadline(') + with printer.Indent(indent=' '): + printer('local_vars, histfile=FLAGS.history_file)') + printer('else:') + with printer.Indent(): + printer('console = code.InteractiveConsole(local_vars)') + printer('try:') + with printer.Indent(): + printer('console.interact(banner)') + printer('except SystemExit as e:') + with printer.Indent(): + printer('return e.code') + printer() + printer() + + def WriteFile(self, printer): + """Write a simple CLI (currently just a stub).""" + printer('#!/usr/bin/env python') + printer('"""CLI for %s, version %s."""', + self.__package, self.__version) + printer('# NOTE: This file is autogenerated and should not be edited by ' + 'hand.') + # TODO(craigcitro): Add a build stamp, along with some other + # information. + printer() + printer('import code') + printer('import os') + printer('import platform') + printer('import sys') + printer() + printer('import protorpc') + printer('from protorpc import message_types') + printer('from protorpc import messages') + printer() + appcommands_import = 'from google.apputils import appcommands' + printer(appcommands_import) + + flags_import = 'import gflags as flags' + printer(flags_import) + printer() + printer('import %s as apitools_base', self.__base_files_package) + printer('from %s import cli as apitools_base_cli', + self.__base_files_package) + import_prefix = '' + printer('%simport %s as client_lib', + import_prefix, self.__client_info.client_rule_name) + printer('%simport %s as messages', + import_prefix, self.__client_info.messages_rule_name) + self.__PrintFlagDeclarations(printer) + printer() + printer() + self.__PrintGetGlobalParams(printer) + self.__PrintGetClient(printer) + self.__PrintPyShell(printer) + self.__PrintCommands(printer) + printer('def main(_):') + with printer.Indent(): + printer("appcommands.AddCmd('pyshell', PyShell)") + for command_info in self.__command_list: + printer("appcommands.AddCmd('%s', %s)", + command_info.name, command_info.class_name) + printer() + printer('apitools_base_cli.SetupLogger()') + # TODO(craigcitro): Just call SetDefaultCommand as soon as + # another appcommands release happens and this exists + # externally. + printer("if hasattr(appcommands, 'SetDefaultCommand'):") with printer.Indent(): - printer('upload = apitools_base.Upload.FromFile(') - printer(' FLAGS.upload_filename, FLAGS.upload_mime_type,') - printer(' progress_callback=' - 'apitools_base.UploadProgressPrinter,') - printer(' finish_callback=' - 'apitools_base.UploadCompletePrinter)') - if command_info.has_download: - call_args.append('download=download') - printer('download = None') - printer('if FLAGS.download_filename:') + printer("appcommands.SetDefaultCommand('pyshell')") + printer() + printer() + printer('run_main = apitools_base_cli.run_main') + printer() + printer("if __name__ == '__main__':") + with printer.Indent(): + printer('appcommands.Run()') + + def __PrintCommands(self, printer): + """Print all commands in this registry using printer.""" + for command_info in self.__command_list: + arg_list = [arg_info.name for arg_info in command_info.args] + printer( + 'class %s(apitools_base_cli.NewCmd):', command_info.class_name) with printer.Indent(): - printer('download = apitools_base.Download.FromFile(' - 'FLAGS.download_filename, overwrite=FLAGS.overwrite,') - printer(' progress_callback=' - 'apitools_base.DownloadProgressPrinter,') - printer(' finish_callback=' - 'apitools_base.DownloadCompletePrinter)') - printer('result = client.%s(', command_info.client_method_path) - with printer.Indent(indent=' '): - printer('%s)', ', '.join(call_args)) - printer('print apitools_base_cli.FormatOutput(result)') - printer() - printer() + printer( + '"""Command wrapping %s."""', command_info.client_method_path) + printer() + printer('usage = """%s%s%s"""', + command_info.name, + ' ' if arg_list else '', + ' '.join('<%s>' % argname for argname in arg_list)) + printer() + printer('def __init__(self, name, fv):') + with printer.Indent(): + printer( + 'super(%s, self).__init__(name, fv)', command_info.class_name) + for flag in command_info.flags: + self.__PrintFlag(printer, flag) + printer() + printer('def RunWithArgs(%s):', ', '.join(['self'] + arg_list)) + with printer.Indent(): + self.__PrintCommandDocstring(printer, command_info) + printer('client = GetClientFromFlags()') + printer('global_params = GetGlobalParamsFromFlags()') + printer( + 'request = messages.%s(', command_info.request_type) + with printer.Indent(indent=' '): + for arg in command_info.args: + rhs = arg.name + if arg.conversion: + rhs = arg.conversion % arg.name + printer('%s=%s,', arg.name, rhs) + printer(')') + for flag_info in command_info.flags: + if flag_info.special: + continue + rhs = 'FLAGS.%s' % flag_info.name + if flag_info.conversion: + rhs = flag_info.conversion % rhs + printer('if FLAGS[%r].present:', flag_info.name) + with printer.Indent(): + printer('request.%s = %s', flag_info.name, rhs) + call_args = ['request', 'global_params=global_params'] + if command_info.has_upload: + call_args.append('upload=upload') + printer('upload = None') + printer('if FLAGS.upload_filename:') + with printer.Indent(): + printer('upload = apitools_base.Upload.FromFile(') + printer( + ' FLAGS.upload_filename, FLAGS.upload_mime_type,') + printer(' progress_callback=' + 'apitools_base.UploadProgressPrinter,') + printer(' finish_callback=' + 'apitools_base.UploadCompletePrinter)') + if command_info.has_download: + call_args.append('download=download') + printer('download = None') + printer('if FLAGS.download_filename:') + with printer.Indent(): + printer('download = apitools_base.Download.FromFile(' + 'FLAGS.download_filename, overwrite=FLAGS.overwrite,') + printer(' progress_callback=' + 'apitools_base.DownloadProgressPrinter,') + printer(' finish_callback=' + 'apitools_base.DownloadCompletePrinter)') + printer( + 'result = client.%s(', command_info.client_method_path) + with printer.Indent(indent=' '): + printer('%s)', ', '.join(call_args)) + printer('print apitools_base_cli.FormatOutput(result)') + printer() + printer() diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index d10532b..51bac6d 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -25,420 +25,441 @@ import apitools.base.py as apitools_base class ExtendedEnumValueDescriptor(messages.Message): - """Enum value descriptor with additional fields. - Fields: - name: Name of enumeration value. - number: Number of enumeration value. - description: Description of this enum value. - """ - name = messages.StringField(1) - number = messages.IntegerField(2, variant=messages.Variant.INT32) + """Enum value descriptor with additional fields. - description = messages.StringField(100) + Fields: + name: Name of enumeration value. + number: Number of enumeration value. + description: Description of this enum value. + """ + name = messages.StringField(1) + number = messages.IntegerField(2, variant=messages.Variant.INT32) + + description = messages.StringField(100) class ExtendedEnumDescriptor(messages.Message): - """Enum class descriptor with additional fields. - Fields: - name: Name of Enum without any qualification. - values: Values defined by Enum class. - description: Description of this enum class. - full_name: Fully qualified name of this enum class. - enum_mappings: Mappings from python to JSON names for enum values. - """ + """Enum class descriptor with additional fields. + + Fields: + name: Name of Enum without any qualification. + values: Values defined by Enum class. + description: Description of this enum class. + full_name: Fully qualified name of this enum class. + enum_mappings: Mappings from python to JSON names for enum values. + """ - class JsonEnumMapping(messages.Message): - """Mapping from a python name to the wire name for an enum.""" - python_name = messages.StringField(1) - json_name = messages.StringField(2) + class JsonEnumMapping(messages.Message): - name = messages.StringField(1) - values = messages.MessageField(ExtendedEnumValueDescriptor, 2, repeated=True) + """Mapping from a python name to the wire name for an enum.""" + python_name = messages.StringField(1) + json_name = messages.StringField(2) - description = messages.StringField(100) - full_name = messages.StringField(101) - enum_mappings = messages.MessageField('JsonEnumMapping', 102, repeated=True) + name = messages.StringField(1) + values = messages.MessageField( + ExtendedEnumValueDescriptor, 2, repeated=True) + + description = messages.StringField(100) + full_name = messages.StringField(101) + enum_mappings = messages.MessageField( + 'JsonEnumMapping', 102, repeated=True) class ExtendedFieldDescriptor(messages.Message): - """Field descriptor with additional fields. - Fields: - field_descriptor: The underlying field descriptor. - name: The name of this field. - description: Description of this field. - """ - field_descriptor = messages.MessageField( - protorpc_descriptor.FieldDescriptor, 100) - # We duplicate the names for easier bookkeeping. - name = messages.StringField(101) - description = messages.StringField(102) + """Field descriptor with additional fields. + + Fields: + field_descriptor: The underlying field descriptor. + name: The name of this field. + description: Description of this field. + """ + field_descriptor = messages.MessageField( + protorpc_descriptor.FieldDescriptor, 100) + # We duplicate the names for easier bookkeeping. + name = messages.StringField(101) + description = messages.StringField(102) class ExtendedMessageDescriptor(messages.Message): - """Message descriptor with additional fields. - - Fields: - name: Name of Message without any qualification. - fields: Fields defined for message. - message_types: Nested Message classes defined on message. - enum_types: Nested Enum classes defined on message. - description: Description of this message. - full_name: Full qualified name of this message. - decorators: Decorators to include in the definition when printing. - Printed in the given order from top to bottom (so the last entry - is the innermost decorator). - alias_for: This type is just an alias for the named type. - field_mappings: Mappings from python to json field names. - """ - - class JsonFieldMapping(messages.Message): - """Mapping from a python name to the wire name for a field.""" - python_name = messages.StringField(1) - json_name = messages.StringField(2) - - name = messages.StringField(1) - fields = messages.MessageField(ExtendedFieldDescriptor, 2, repeated=True) - message_types = messages.MessageField( - 'extended_descriptor.ExtendedMessageDescriptor', 3, repeated=True) - enum_types = messages.MessageField(ExtendedEnumDescriptor, 4, repeated=True) - - description = messages.StringField(100) - full_name = messages.StringField(101) - decorators = messages.StringField(102, repeated=True) - alias_for = messages.StringField(103) - field_mappings = messages.MessageField('JsonFieldMapping', 104, repeated=True) + + """Message descriptor with additional fields. + + Fields: + name: Name of Message without any qualification. + fields: Fields defined for message. + message_types: Nested Message classes defined on message. + enum_types: Nested Enum classes defined on message. + description: Description of this message. + full_name: Full qualified name of this message. + decorators: Decorators to include in the definition when printing. + Printed in the given order from top to bottom (so the last entry + is the innermost decorator). + alias_for: This type is just an alias for the named type. + field_mappings: Mappings from python to json field names. + """ + + class JsonFieldMapping(messages.Message): + + """Mapping from a python name to the wire name for a field.""" + python_name = messages.StringField(1) + json_name = messages.StringField(2) + + name = messages.StringField(1) + fields = messages.MessageField(ExtendedFieldDescriptor, 2, repeated=True) + message_types = messages.MessageField( + 'extended_descriptor.ExtendedMessageDescriptor', 3, repeated=True) + enum_types = messages.MessageField( + ExtendedEnumDescriptor, 4, repeated=True) + + description = messages.StringField(100) + full_name = messages.StringField(101) + decorators = messages.StringField(102, repeated=True) + alias_for = messages.StringField(103) + field_mappings = messages.MessageField( + 'JsonFieldMapping', 104, repeated=True) class ExtendedFileDescriptor(messages.Message): - """File descriptor with additional fields. - Fields: - package: Fully qualified name of package that definitions belong to. - message_types: Message definitions contained in file. - enum_types: Enum definitions contained in file. - description: Description of this file. - additional_imports: Extra imports used in this package. - """ - package = messages.StringField(2) + """File descriptor with additional fields. + + Fields: + package: Fully qualified name of package that definitions belong to. + message_types: Message definitions contained in file. + enum_types: Enum definitions contained in file. + description: Description of this file. + additional_imports: Extra imports used in this package. + """ + package = messages.StringField(2) - message_types = messages.MessageField( - ExtendedMessageDescriptor, 4, repeated=True) - enum_types = messages.MessageField( - ExtendedEnumDescriptor, 5, repeated=True) + message_types = messages.MessageField( + ExtendedMessageDescriptor, 4, repeated=True) + enum_types = messages.MessageField( + ExtendedEnumDescriptor, 5, repeated=True) - description = messages.StringField(100) - additional_imports = messages.StringField(101, repeated=True) + description = messages.StringField(100) + additional_imports = messages.StringField(101, repeated=True) def _WriteFile(file_descriptor, package, version, proto_printer): - """Write the given extended file descriptor to the printer.""" - proto_printer.PrintPreamble(package, version, file_descriptor) - _PrintEnums(proto_printer, file_descriptor.enum_types) - _PrintMessages(proto_printer, file_descriptor.message_types) - custom_json_mappings = _FetchCustomMappings(file_descriptor.enum_types) - custom_json_mappings.extend( - _FetchCustomMappings(file_descriptor.message_types)) - for mapping in custom_json_mappings: - proto_printer.PrintCustomJsonMapping(mapping) + """Write the given extended file descriptor to the printer.""" + proto_printer.PrintPreamble(package, version, file_descriptor) + _PrintEnums(proto_printer, file_descriptor.enum_types) + _PrintMessages(proto_printer, file_descriptor.message_types) + custom_json_mappings = _FetchCustomMappings(file_descriptor.enum_types) + custom_json_mappings.extend( + _FetchCustomMappings(file_descriptor.message_types)) + for mapping in custom_json_mappings: + proto_printer.PrintCustomJsonMapping(mapping) def WriteMessagesFile(file_descriptor, package, version, printer): - """Write the given extended file descriptor to out as a message file.""" - _WriteFile(file_descriptor, package, version, - _Proto2Printer(printer)) + """Write the given extended file descriptor to out as a message file.""" + _WriteFile(file_descriptor, package, version, + _Proto2Printer(printer)) def WritePythonFile(file_descriptor, package, version, printer): - """Write the given extended file descriptor to out.""" - _WriteFile(file_descriptor, package, version, - _ProtoRpcPrinter(printer)) + """Write the given extended file descriptor to out.""" + _WriteFile(file_descriptor, package, version, + _ProtoRpcPrinter(printer)) def PrintIndentedDescriptions(printer, ls, name, prefix=''): - if ls: - with printer.Indent(indent=prefix): - with printer.CommentContext(): - width = printer.CalculateWidth() - len(prefix) - printer() - printer(name + ':') - for x in ls: - description = '%s: %s' % (x.name, x.description) - for line in textwrap.wrap(description, width, initial_indent=' ', - subsequent_indent=' '): - printer(line) + if ls: + with printer.Indent(indent=prefix): + with printer.CommentContext(): + width = printer.CalculateWidth() - len(prefix) + printer() + printer(name + ':') + for x in ls: + description = '%s: %s' % (x.name, x.description) + for line in textwrap.wrap(description, width, initial_indent=' ', + subsequent_indent=' '): + printer(line) def _FetchCustomMappings(descriptor_ls): - """Find and return all custom mappings for descriptors in descriptor_ls.""" - custom_mappings = [] - for descriptor in descriptor_ls: - if isinstance(descriptor, ExtendedEnumDescriptor): - custom_mappings.extend( - _FormatCustomJsonMapping('Enum', m, descriptor) - for m in descriptor.enum_mappings) - elif isinstance(descriptor, ExtendedMessageDescriptor): - custom_mappings.extend( - _FormatCustomJsonMapping('Field', m, descriptor) - for m in descriptor.field_mappings) - custom_mappings.extend(_FetchCustomMappings(descriptor.enum_types)) - custom_mappings.extend(_FetchCustomMappings(descriptor.message_types)) - return custom_mappings + """Find and return all custom mappings for descriptors in descriptor_ls.""" + custom_mappings = [] + for descriptor in descriptor_ls: + if isinstance(descriptor, ExtendedEnumDescriptor): + custom_mappings.extend( + _FormatCustomJsonMapping('Enum', m, descriptor) + for m in descriptor.enum_mappings) + elif isinstance(descriptor, ExtendedMessageDescriptor): + custom_mappings.extend( + _FormatCustomJsonMapping('Field', m, descriptor) + for m in descriptor.field_mappings) + custom_mappings.extend(_FetchCustomMappings(descriptor.enum_types)) + custom_mappings.extend( + _FetchCustomMappings(descriptor.message_types)) + return custom_mappings def _FormatCustomJsonMapping(mapping_type, mapping, descriptor): - return '\n'.join(( - 'encoding.AddCustomJson%sMapping(' % mapping_type, - " %s, '%s', '%s')" % (descriptor.full_name, mapping.python_name, - mapping.json_name) - )) + return '\n'.join(( + 'encoding.AddCustomJson%sMapping(' % mapping_type, + " %s, '%s', '%s')" % (descriptor.full_name, mapping.python_name, + mapping.json_name) + )) def _EmptyMessage(message_type): - return not any((message_type.enum_types, - message_type.message_types, - message_type.fields)) + return not any((message_type.enum_types, + message_type.message_types, + message_type.fields)) class ProtoPrinter(six.with_metaclass(abc.ABCMeta, object)): - """Interface for proto printers.""" - @abc.abstractmethod - def PrintPreamble(self, package, version, file_descriptor): - """Print the file docstring and import lines.""" + """Interface for proto printers.""" + + @abc.abstractmethod + def PrintPreamble(self, package, version, file_descriptor): + """Print the file docstring and import lines.""" - @abc.abstractmethod - def PrintEnum(self, enum_type): - """Print the given enum declaration.""" + @abc.abstractmethod + def PrintEnum(self, enum_type): + """Print the given enum declaration.""" - @abc.abstractmethod - def PrintMessage(self, message_type): - """Print the given message declaration.""" + @abc.abstractmethod + def PrintMessage(self, message_type): + """Print the given message declaration.""" class _Proto2Printer(ProtoPrinter): - """Printer for proto2 definitions.""" - - def __init__(self, printer): - self.__printer = printer - - def __PrintEnumCommentLines(self, enum_type): - description = enum_type.description or '%s enum type.' % enum_type.name - for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): - self.__printer('// %s', line) - PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values', - prefix='// ') - - def __PrintEnumValueCommentLines(self, enum_value): - if enum_value.description: - width = self.__printer.CalculateWidth() - 3 - for line in textwrap.wrap(enum_value.description, width): - self.__printer('// %s', line) - - def PrintEnum(self, enum_type): - self.__PrintEnumCommentLines(enum_type) - self.__printer('enum %s {', enum_type.name) - with self.__printer.Indent(): - enum_values = sorted(enum_type.values, key=operator.attrgetter('number')) - for enum_value in enum_values: + + """Printer for proto2 definitions.""" + + def __init__(self, printer): + self.__printer = printer + + def __PrintEnumCommentLines(self, enum_type): + description = enum_type.description or '%s enum type.' % enum_type.name + for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + self.__printer('// %s', line) + PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values', + prefix='// ') + + def __PrintEnumValueCommentLines(self, enum_value): + if enum_value.description: + width = self.__printer.CalculateWidth() - 3 + for line in textwrap.wrap(enum_value.description, width): + self.__printer('// %s', line) + + def PrintEnum(self, enum_type): + self.__PrintEnumCommentLines(enum_type) + self.__printer('enum %s {', enum_type.name) + with self.__printer.Indent(): + enum_values = sorted( + enum_type.values, key=operator.attrgetter('number')) + for enum_value in enum_values: + self.__printer() + self.__PrintEnumValueCommentLines(enum_value) + self.__printer('%s = %s;', enum_value.name, enum_value.number) + self.__printer('}') self.__printer() - self.__PrintEnumValueCommentLines(enum_value) - self.__printer('%s = %s;', enum_value.name, enum_value.number) - self.__printer('}') - self.__printer() - - def PrintPreamble(self, package, version, file_descriptor): - self.__printer('// Generated message classes for %s version %s.', - package, version) - self.__printer('// NOTE: This file is autogenerated and should not be ' - 'edited by hand.') - description_lines = textwrap.wrap(file_descriptor.description, 75) - if description_lines: - self.__printer('//') - for line in description_lines: - self.__printer('// %s', line) - self.__printer() - self.__printer('syntax = "proto2";') - self.__printer('package %s;', file_descriptor.package) - - def __PrintMessageCommentLines(self, message_type): - """Print the description of this message.""" - description = message_type.description or '%s message type.' % ( - message_type.name) - width = self.__printer.CalculateWidth() - 3 - for line in textwrap.wrap(description, width): - self.__printer('// %s', line) - PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums', - prefix='// ') - PrintIndentedDescriptions(self.__printer, message_type.message_types, - 'Messages', prefix='// ') - PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields', - prefix='// ') - - def __PrintFieldDescription(self, description): - for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): - self.__printer('// %s', line) - - def __PrintFields(self, fields): - for extended_field in fields: - field = extended_field.field_descriptor - field_type = messages.Field.lookup_field_type_by_variant(field.variant) - self.__printer() - self.__PrintFieldDescription(extended_field.description) - label = str(field.label).lower() - if field_type in (messages.EnumField, messages.MessageField): - proto_type = field.type_name - else: - proto_type = str(field.variant).lower() - default_statement = '' - if field.default_value: - if field_type in [messages.BytesField, messages.StringField]: - default_value = '"%s"' % field.default_value - elif field_type is messages.BooleanField: - default_value = str(field.default_value).lower() - else: - default_value = str(field.default_value) - - default_statement = ' [default = %s]' % default_value - self.__printer( - '%s %s %s = %d%s;', - label, proto_type, field.name, field.number, default_statement) - - def PrintMessage(self, message_type): - self.__printer() - self.__PrintMessageCommentLines(message_type) - if _EmptyMessage(message_type): - self.__printer('message %s {}', message_type.name) - return - self.__printer('message %s {', message_type.name) - with self.__printer.Indent(): - _PrintEnums(self, message_type.enum_types) - _PrintMessages(self, message_type.message_types) - self.__PrintFields(message_type.fields) - self.__printer('}') - - def PrintCustomJsonMapping(self, mapping_lines): - raise NotImplementedError('Custom JSON encoding not supported for proto2') + + def PrintPreamble(self, package, version, file_descriptor): + self.__printer('// Generated message classes for %s version %s.', + package, version) + self.__printer('// NOTE: This file is autogenerated and should not be ' + 'edited by hand.') + description_lines = textwrap.wrap(file_descriptor.description, 75) + if description_lines: + self.__printer('//') + for line in description_lines: + self.__printer('// %s', line) + self.__printer() + self.__printer('syntax = "proto2";') + self.__printer('package %s;', file_descriptor.package) + + def __PrintMessageCommentLines(self, message_type): + """Print the description of this message.""" + description = message_type.description or '%s message type.' % ( + message_type.name) + width = self.__printer.CalculateWidth() - 3 + for line in textwrap.wrap(description, width): + self.__printer('// %s', line) + PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums', + prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.message_types, + 'Messages', prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields', + prefix='// ') + + def __PrintFieldDescription(self, description): + for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + self.__printer('// %s', line) + + def __PrintFields(self, fields): + for extended_field in fields: + field = extended_field.field_descriptor + field_type = messages.Field.lookup_field_type_by_variant( + field.variant) + self.__printer() + self.__PrintFieldDescription(extended_field.description) + label = str(field.label).lower() + if field_type in (messages.EnumField, messages.MessageField): + proto_type = field.type_name + else: + proto_type = str(field.variant).lower() + default_statement = '' + if field.default_value: + if field_type in [messages.BytesField, messages.StringField]: + default_value = '"%s"' % field.default_value + elif field_type is messages.BooleanField: + default_value = str(field.default_value).lower() + else: + default_value = str(field.default_value) + + default_statement = ' [default = %s]' % default_value + self.__printer( + '%s %s %s = %d%s;', + label, proto_type, field.name, field.number, default_statement) + + def PrintMessage(self, message_type): + self.__printer() + self.__PrintMessageCommentLines(message_type) + if _EmptyMessage(message_type): + self.__printer('message %s {}', message_type.name) + return + self.__printer('message %s {', message_type.name) + with self.__printer.Indent(): + _PrintEnums(self, message_type.enum_types) + _PrintMessages(self, message_type.message_types) + self.__PrintFields(message_type.fields) + self.__printer('}') + + def PrintCustomJsonMapping(self, mapping_lines): + raise NotImplementedError( + 'Custom JSON encoding not supported for proto2') class _ProtoRpcPrinter(ProtoPrinter): - """Printer for ProtoRPC definitions.""" - - def __init__(self, printer): - self.__printer = printer - - def __PrintClassSeparator(self): - self.__printer() - if not self.__printer.indent: - self.__printer() - - def __PrintEnumDocstringLines(self, enum_type): - description = enum_type.description or '%s enum type.' % enum_type.name - for line in textwrap.wrap('"""%s' % description, - self.__printer.CalculateWidth()): - self.__printer(line) - PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values') - self.__printer('"""') - - def PrintEnum(self, enum_type): - self.__printer('class %s(messages.Enum):', enum_type.name) - with self.__printer.Indent(): - self.__PrintEnumDocstringLines(enum_type) - enum_values = sorted(enum_type.values, key=operator.attrgetter('number')) - for enum_value in enum_values: - self.__printer('%s = %s', enum_value.name, enum_value.number) - if not enum_type.values: - self.__printer('pass') - self.__PrintClassSeparator() - - def __PrintAdditionalImports(self, imports): - """Print additional imports needed for protorpc.""" - google_imports = [x for x in imports if 'google' in x] - other_imports = [x for x in imports if 'google' not in x] - if other_imports: - for import_ in sorted(other_imports): - self.__printer(import_) - self.__printer() - # Note: If we ever were going to add imports from this package, we'd - # need to sort those out and put them at the end. - if google_imports: - for import_ in sorted(google_imports): - self.__printer(import_) - self.__printer() - - def PrintPreamble(self, package, version, file_descriptor): - self.__printer('"""Generated message classes for %s version %s.', - package, version) - self.__printer() - for line in textwrap.wrap(file_descriptor.description, 78): - self.__printer(line) - self.__printer('"""') - self.__printer('# NOTE: This file is autogenerated and should not be ' - 'edited by hand.') - self.__printer() - self.__PrintAdditionalImports(file_descriptor.additional_imports) - self.__printer() - self.__printer("package = '%s'", file_descriptor.package) - self.__printer() - self.__printer() - - def __PrintMessageDocstringLines(self, message_type): - """Print the docstring for this message.""" - description = message_type.description or '%s message type.' % ( - message_type.name) - short_description = ( - _EmptyMessage(message_type) and - len(description) < (self.__printer.CalculateWidth() - 6)) - with self.__printer.CommentContext(): - if short_description: - # Note that we use explicit string interpolation here since - # we're in comment context. - self.__printer('"""%s"""' % description) - return - for line in textwrap.wrap('"""%s' % description, - self.__printer.CalculateWidth()): - self.__printer(line) - - PrintIndentedDescriptions(self.__printer, message_type.enum_types, - 'Enums') - PrintIndentedDescriptions( - self.__printer, message_type.message_types, 'Messages') - PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields') - self.__printer('"""') - self.__printer() - - def PrintMessage(self, message_type): - if message_type.alias_for: - self.__printer('%s = %s', message_type.name, message_type.alias_for) - self.__PrintClassSeparator() - return - for decorator in message_type.decorators: - self.__printer('@%s', decorator) - self.__printer('class %s(messages.Message):', message_type.name) - with self.__printer.Indent(): - self.__PrintMessageDocstringLines(message_type) - _PrintEnums(self, message_type.enum_types) - _PrintMessages(self, message_type.message_types) - _PrintFields(message_type.fields, self.__printer) - self.__PrintClassSeparator() - - def PrintCustomJsonMapping(self, mapping): - self.__printer(mapping) + + """Printer for ProtoRPC definitions.""" + + def __init__(self, printer): + self.__printer = printer + + def __PrintClassSeparator(self): + self.__printer() + if not self.__printer.indent: + self.__printer() + + def __PrintEnumDocstringLines(self, enum_type): + description = enum_type.description or '%s enum type.' % enum_type.name + for line in textwrap.wrap('"""%s' % description, + self.__printer.CalculateWidth()): + self.__printer(line) + PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values') + self.__printer('"""') + + def PrintEnum(self, enum_type): + self.__printer('class %s(messages.Enum):', enum_type.name) + with self.__printer.Indent(): + self.__PrintEnumDocstringLines(enum_type) + enum_values = sorted( + enum_type.values, key=operator.attrgetter('number')) + for enum_value in enum_values: + self.__printer('%s = %s', enum_value.name, enum_value.number) + if not enum_type.values: + self.__printer('pass') + self.__PrintClassSeparator() + + def __PrintAdditionalImports(self, imports): + """Print additional imports needed for protorpc.""" + google_imports = [x for x in imports if 'google' in x] + other_imports = [x for x in imports if 'google' not in x] + if other_imports: + for import_ in sorted(other_imports): + self.__printer(import_) + self.__printer() + # Note: If we ever were going to add imports from this package, we'd + # need to sort those out and put them at the end. + if google_imports: + for import_ in sorted(google_imports): + self.__printer(import_) + self.__printer() + + def PrintPreamble(self, package, version, file_descriptor): + self.__printer('"""Generated message classes for %s version %s.', + package, version) + self.__printer() + for line in textwrap.wrap(file_descriptor.description, 78): + self.__printer(line) + self.__printer('"""') + self.__printer('# NOTE: This file is autogenerated and should not be ' + 'edited by hand.') + self.__printer() + self.__PrintAdditionalImports(file_descriptor.additional_imports) + self.__printer() + self.__printer("package = '%s'", file_descriptor.package) + self.__printer() + self.__printer() + + def __PrintMessageDocstringLines(self, message_type): + """Print the docstring for this message.""" + description = message_type.description or '%s message type.' % ( + message_type.name) + short_description = ( + _EmptyMessage(message_type) and + len(description) < (self.__printer.CalculateWidth() - 6)) + with self.__printer.CommentContext(): + if short_description: + # Note that we use explicit string interpolation here since + # we're in comment context. + self.__printer('"""%s"""' % description) + return + for line in textwrap.wrap('"""%s' % description, + self.__printer.CalculateWidth()): + self.__printer(line) + + PrintIndentedDescriptions(self.__printer, message_type.enum_types, + 'Enums') + PrintIndentedDescriptions( + self.__printer, message_type.message_types, 'Messages') + PrintIndentedDescriptions( + self.__printer, message_type.fields, 'Fields') + self.__printer('"""') + self.__printer() + + def PrintMessage(self, message_type): + if message_type.alias_for: + self.__printer( + '%s = %s', message_type.name, message_type.alias_for) + self.__PrintClassSeparator() + return + for decorator in message_type.decorators: + self.__printer('@%s', decorator) + self.__printer('class %s(messages.Message):', message_type.name) + with self.__printer.Indent(): + self.__PrintMessageDocstringLines(message_type) + _PrintEnums(self, message_type.enum_types) + _PrintMessages(self, message_type.message_types) + _PrintFields(message_type.fields, self.__printer) + self.__PrintClassSeparator() + + def PrintCustomJsonMapping(self, mapping): + self.__printer(mapping) def _PrintEnums(proto_printer, enum_types): - """Print all enums to the given proto_printer.""" - enum_types = sorted(enum_types, key=operator.attrgetter('name')) - for enum_type in enum_types: - proto_printer.PrintEnum(enum_type) + """Print all enums to the given proto_printer.""" + enum_types = sorted(enum_types, key=operator.attrgetter('name')) + for enum_type in enum_types: + proto_printer.PrintEnum(enum_type) def _PrintMessages(proto_printer, message_list): - message_list = sorted(message_list, key=operator.attrgetter('name')) - for message_type in message_list: - proto_printer.PrintMessage(message_type) + message_list = sorted(message_list, key=operator.attrgetter('name')) + for message_type in message_list: + proto_printer.PrintMessage(message_type) _MESSAGE_FIELD_MAP = { @@ -447,60 +468,62 @@ _MESSAGE_FIELD_MAP = { def _PrintFields(fields, printer): - for extended_field in fields: - field = extended_field.field_descriptor - printed_field_info = { - 'name': field.name, - 'module': 'messages', - 'type_name': '', - 'type_format': '', - 'number': field.number, - 'label_format': '', - 'variant_format': '', - 'default_format': '', - } - - message_field = _MESSAGE_FIELD_MAP.get(field.type_name) - if message_field: - printed_field_info['module'] = 'message_types' - field_type = message_field - elif field.type_name == 'extra_types.DateField': - printed_field_info['module'] = 'extra_types' - field_type = apitools_base.DateField - else: - field_type = messages.Field.lookup_field_type_by_variant(field.variant) - - if field_type in (messages.EnumField, messages.MessageField): - printed_field_info['type_format'] = "'%s', " % field.type_name - - if field.label == protorpc_descriptor.FieldDescriptor.Label.REQUIRED: - printed_field_info['label_format'] = ', required=True' - elif field.label == protorpc_descriptor.FieldDescriptor.Label.REPEATED: - printed_field_info['label_format'] = ', repeated=True' - - if field_type.DEFAULT_VARIANT != field.variant: - printed_field_info['variant_format'] = ', variant=messages.Variant.%s' % ( - field.variant,) - - if field.default_value: - if field_type in [messages.BytesField, messages.StringField]: - default_value = repr(field.default_value) - elif field_type is messages.EnumField: - try: - default_value = str(int(field.default_value)) - except ValueError: - default_value = repr(field.default_value) - else: - default_value = field.default_value - - printed_field_info['default_format'] = ', default=%s' % (default_value,) - - printed_field_info['type_name'] = field_type.__name__ - args = ''.join('%%(%s)s' % field for field in ( - 'type_format', - 'number', - 'label_format', - 'variant_format', - 'default_format')) - format_str = '%%(name)s = %%(module)s.%%(type_name)s(%s)' % args - printer(format_str % printed_field_info) + for extended_field in fields: + field = extended_field.field_descriptor + printed_field_info = { + 'name': field.name, + 'module': 'messages', + 'type_name': '', + 'type_format': '', + 'number': field.number, + 'label_format': '', + 'variant_format': '', + 'default_format': '', + } + + message_field = _MESSAGE_FIELD_MAP.get(field.type_name) + if message_field: + printed_field_info['module'] = 'message_types' + field_type = message_field + elif field.type_name == 'extra_types.DateField': + printed_field_info['module'] = 'extra_types' + field_type = apitools_base.DateField + else: + field_type = messages.Field.lookup_field_type_by_variant( + field.variant) + + if field_type in (messages.EnumField, messages.MessageField): + printed_field_info['type_format'] = "'%s', " % field.type_name + + if field.label == protorpc_descriptor.FieldDescriptor.Label.REQUIRED: + printed_field_info['label_format'] = ', required=True' + elif field.label == protorpc_descriptor.FieldDescriptor.Label.REPEATED: + printed_field_info['label_format'] = ', repeated=True' + + if field_type.DEFAULT_VARIANT != field.variant: + printed_field_info['variant_format'] = ', variant=messages.Variant.%s' % ( + field.variant,) + + if field.default_value: + if field_type in [messages.BytesField, messages.StringField]: + default_value = repr(field.default_value) + elif field_type is messages.EnumField: + try: + default_value = str(int(field.default_value)) + except ValueError: + default_value = repr(field.default_value) + else: + default_value = field.default_value + + printed_field_info[ + 'default_format'] = ', default=%s' % (default_value,) + + printed_field_info['type_name'] = field_type.__name__ + args = ''.join('%%(%s)s' % field for field in ( + 'type_format', + 'number', + 'label_format', + 'variant_format', + 'default_format')) + format_str = '%%(name)s = %%(module)s.%%(type_name)s(%s)' % args + printer(format_str % printed_field_info) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 0473c9f..7b7b648 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -94,157 +94,161 @@ flags.RegisterValidator( def _CopyLocalFile(filename): - with contextlib.closing(open(filename, 'w')) as out: - src_data = pkgutil.get_data( - 'apitools.base.py', filename) - if src_data is None: - raise exceptions.GeneratedClientError('Could not find file %s' % filename) - out.write(src_data) + with contextlib.closing(open(filename, 'w')) as out: + src_data = pkgutil.get_data( + 'apitools.base.py', filename) + if src_data is None: + raise exceptions.GeneratedClientError( + 'Could not find file %s' % filename) + out.write(src_data) def _GetCodegenFromFlags(): - """Create a codegen object from flags.""" - if FLAGS.discovery_url: - try: - discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) - except exceptions.CommunicationError: - return None - else: - infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' - discovery_doc = json.load(open(infile)) - names = util.Names( - FLAGS.strip_prefix, - FLAGS.experimental_name_convention, - FLAGS.experimental_capitalize_enums) - - if FLAGS.client_json: - try: - with open(FLAGS.client_json) as client_json: - f = json.loads(client_json.read()) - web = f.get('web', {}) - client_id = web.get('client_id') - client_secret = web.get('client_secret') - except IOError: - raise exceptions.NotFoundError( - 'Failed to open client json file: %s' % FLAGS.client_json) - else: - client_id = FLAGS.client_id - client_secret = FLAGS.client_secret - - if not client_id: - logging.warning('No client ID supplied') - client_id = '' - - if not client_secret: - logging.warning('No client secret supplied') - client_secret = '' - - client_info = util.ClientInfo.Create( - discovery_doc, FLAGS.scope, client_id, client_secret, - FLAGS.user_agent, names, FLAGS.api_key) - outdir = os.path.expanduser(FLAGS.outdir) or client_info.default_directory - if os.path.exists(outdir) and not FLAGS.overwrite: - raise exceptions.ConfigurationValueError( - 'Output directory exists, pass --overwrite to replace ' - 'the existing files.') - - root_package = FLAGS.root_package or util.GetPackage(outdir) # pylint: disable=line-too-long - return gen_client_lib.DescriptorGenerator( - discovery_doc, client_info, names, root_package, outdir, - base_package=FLAGS.base_package, - generate_cli=FLAGS.generate_cli, - use_proto2=FLAGS.experimental_proto2_output, - unelidable_request_methods=FLAGS.unelidable_request_methods) + """Create a codegen object from flags.""" + if FLAGS.discovery_url: + try: + discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) + except exceptions.CommunicationError: + return None + else: + infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' + discovery_doc = json.load(open(infile)) + names = util.Names( + FLAGS.strip_prefix, + FLAGS.experimental_name_convention, + FLAGS.experimental_capitalize_enums) + + if FLAGS.client_json: + try: + with open(FLAGS.client_json) as client_json: + f = json.loads(client_json.read()) + web = f.get('web', {}) + client_id = web.get('client_id') + client_secret = web.get('client_secret') + except IOError: + raise exceptions.NotFoundError( + 'Failed to open client json file: %s' % FLAGS.client_json) + else: + client_id = FLAGS.client_id + client_secret = FLAGS.client_secret + + if not client_id: + logging.warning('No client ID supplied') + client_id = '' + + if not client_secret: + logging.warning('No client secret supplied') + client_secret = '' + + client_info = util.ClientInfo.Create( + discovery_doc, FLAGS.scope, client_id, client_secret, + FLAGS.user_agent, names, FLAGS.api_key) + outdir = os.path.expanduser(FLAGS.outdir) or client_info.default_directory + if os.path.exists(outdir) and not FLAGS.overwrite: + raise exceptions.ConfigurationValueError( + 'Output directory exists, pass --overwrite to replace ' + 'the existing files.') + + root_package = FLAGS.root_package or util.GetPackage( + outdir) # pylint: disable=line-too-long + return gen_client_lib.DescriptorGenerator( + discovery_doc, client_info, names, root_package, outdir, + base_package=FLAGS.base_package, + generate_cli=FLAGS.generate_cli, + use_proto2=FLAGS.experimental_proto2_output, + unelidable_request_methods=FLAGS.unelidable_request_methods) # TODO(craigcitro): Delete this if we don't need this functionality. def _WriteBaseFiles(codegen): - with util.Chdir(codegen.outdir): - _CopyLocalFile('app2.py') - _CopyLocalFile('base_api.py') - _CopyLocalFile('base_cli.py') - _CopyLocalFile('credentials_lib.py') - _CopyLocalFile('exceptions.py') + with util.Chdir(codegen.outdir): + _CopyLocalFile('app2.py') + _CopyLocalFile('base_api.py') + _CopyLocalFile('base_cli.py') + _CopyLocalFile('credentials_lib.py') + _CopyLocalFile('exceptions.py') def _WriteProtoFiles(codegen): - with util.Chdir(codegen.outdir): - with open(codegen.client_info.messages_proto_file_name, 'w') as out: - codegen.WriteMessagesProtoFile(out) - with open(codegen.client_info.services_proto_file_name, 'w') as out: - codegen.WriteServicesProtoFile(out) + with util.Chdir(codegen.outdir): + with open(codegen.client_info.messages_proto_file_name, 'w') as out: + codegen.WriteMessagesProtoFile(out) + with open(codegen.client_info.services_proto_file_name, 'w') as out: + codegen.WriteServicesProtoFile(out) def _WriteGeneratedFiles(codegen): - if codegen.use_proto2: - _WriteProtoFiles(codegen) - with util.Chdir(codegen.outdir): - with open(codegen.client_info.messages_file_name, 'w') as out: - codegen.WriteMessagesFile(out) - with open(codegen.client_info.client_file_name, 'w') as out: - codegen.WriteClientLibrary(out) - if FLAGS.generate_cli: - with open(codegen.client_info.cli_file_name, 'w') as out: - codegen.WriteCli(out) - os.chmod(codegen.client_info.cli_file_name, 0o755) + if codegen.use_proto2: + _WriteProtoFiles(codegen) + with util.Chdir(codegen.outdir): + with open(codegen.client_info.messages_file_name, 'w') as out: + codegen.WriteMessagesFile(out) + with open(codegen.client_info.client_file_name, 'w') as out: + codegen.WriteClientLibrary(out) + if FLAGS.generate_cli: + with open(codegen.client_info.cli_file_name, 'w') as out: + codegen.WriteCli(out) + os.chmod(codegen.client_info.cli_file_name, 0o755) def _WriteInit(codegen): - with util.Chdir(codegen.outdir): - with open('__init__.py', 'w') as out: - codegen.WriteInit(out) + with util.Chdir(codegen.outdir): + with open('__init__.py', 'w') as out: + codegen.WriteInit(out) class GenerateClient(appcommands.Cmd): - """Driver for client code generation.""" - def Run(self, _): - """Create a client library.""" - codegen = _GetCodegenFromFlags() - if codegen is None: - logging.error('Failed to create codegen, exiting.') - return 128 - _WriteGeneratedFiles(codegen) - _WriteInit(codegen) + """Driver for client code generation.""" + + def Run(self, _): + """Create a client library.""" + codegen = _GetCodegenFromFlags() + if codegen is None: + logging.error('Failed to create codegen, exiting.') + return 128 + _WriteGeneratedFiles(codegen) + _WriteInit(codegen) class GenerateProto(appcommands.Cmd): - """Generate just the two proto files for a given API.""" - def Run(self, _): - """Create proto definitions for an API.""" - codegen = _GetCodegenFromFlags() - _WriteProtoFiles(codegen) + """Generate just the two proto files for a given API.""" + + def Run(self, _): + """Create proto definitions for an API.""" + codegen = _GetCodegenFromFlags() + _WriteProtoFiles(codegen) # pylint:disable=invalid-name def run_main(): - """Function to be used as setuptools script entry point.""" - # Put the flags for this module somewhere the flags module will look - # for them. - - # pylint:disable=protected-access - new_name = flags._GetMainModule() - sys.modules[new_name] = sys.modules['__main__'] - for flag in FLAGS.FlagsByModuleDict().get(__name__, []): - FLAGS._RegisterFlagByModule(new_name, flag) - for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): - FLAGS._RegisterKeyFlagForModule(new_name, key_flag) - # pylint:enable=protected-access - - # Now set __main__ appropriately so that appcommands will be - # happy. - sys.modules['__main__'] = sys.modules[__name__] - appcommands.Run() - sys.modules['__main__'] = sys.modules.pop(new_name) + """Function to be used as setuptools script entry point.""" + # Put the flags for this module somewhere the flags module will look + # for them. + + # pylint:disable=protected-access + new_name = flags._GetMainModule() + sys.modules[new_name] = sys.modules['__main__'] + for flag in FLAGS.FlagsByModuleDict().get(__name__, []): + FLAGS._RegisterFlagByModule(new_name, flag) + for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): + FLAGS._RegisterKeyFlagForModule(new_name, key_flag) + # pylint:enable=protected-access + + # Now set __main__ appropriately so that appcommands will be + # happy. + sys.modules['__main__'] = sys.modules[__name__] + appcommands.Run() + sys.modules['__main__'] = sys.modules.pop(new_name) def main(_): - appcommands.AddCmd('client', GenerateClient) - appcommands.AddCmd('proto', GenerateProto) + appcommands.AddCmd('client', GenerateClient) + appcommands.AddCmd('proto', GenerateProto) if __name__ == '__main__': - appcommands.Run() + appcommands.Run() diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 732bf38..1c11866 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -16,158 +16,161 @@ from apitools.gen import util def _StandardQueryParametersSchema(discovery_doc): - """Sets up dict of standard query parameters.""" - standard_query_schema = { - 'id': 'StandardQueryParameters', - 'type': 'object', - 'description': 'Query parameters accepted by all methods.', - 'properties': discovery_doc.get('parameters', {}), - } - # We add an entry for the trace, since Discovery doesn't. - standard_query_schema['properties']['trace'] = { - 'type': 'string', - 'description': base_cli.TRACE_HELP, - 'location': 'query', - } - return standard_query_schema + """Sets up dict of standard query parameters.""" + standard_query_schema = { + 'id': 'StandardQueryParameters', + 'type': 'object', + 'description': 'Query parameters accepted by all methods.', + 'properties': discovery_doc.get('parameters', {}), + } + # We add an entry for the trace, since Discovery doesn't. + standard_query_schema['properties']['trace'] = { + 'type': 'string', + 'description': base_cli.TRACE_HELP, + 'location': 'query', + } + return standard_query_schema def _ComputePaths(package, version, discovery_doc): - full_path = urllib_parse.urljoin( - discovery_doc['rootUrl'], discovery_doc['servicePath']) - api_path_component = '/'.join((package, version, '')) - if api_path_component not in full_path: - return full_path, '' - prefix, _, suffix = full_path.rpartition(api_path_component) - return prefix + api_path_component, suffix + full_path = urllib_parse.urljoin( + discovery_doc['rootUrl'], discovery_doc['servicePath']) + api_path_component = '/'.join((package, version, '')) + if api_path_component not in full_path: + return full_path, '' + prefix, _, suffix = full_path.rpartition(api_path_component) + return prefix + api_path_component, suffix class DescriptorGenerator(object): - """Code generator for a given discovery document.""" - - def __init__(self, discovery_doc, client_info, names, root_package, outdir, - base_package, generate_cli=False, use_proto2=False, - unelidable_request_methods=None): - self.__discovery_doc = discovery_doc - self.__client_info = client_info - self.__outdir = outdir - self.__use_proto2 = use_proto2 - self.__description = util.CleanDescription( - self.__discovery_doc.get('description', '')) - self.__package = self.__client_info.package - self.__version = self.__client_info.version - self.__generate_cli = generate_cli - self.__root_package = root_package - self.__base_files_package = base_package - self.__base_files_target = ( - '//cloud/bigscience/apitools/base/py:apitools_base') - self.__names = names - self.__base_url, self.__base_path = _ComputePaths( - self.__package, self.__client_info.url_version, self.__discovery_doc) - - # Order is important here: we need the schemas before we can - # define the services. - self.__message_registry = message_registry.MessageRegistry( - self.__client_info, self.__names, self.__description, - self.__root_package, self.__base_files_package) - schemas = self.__discovery_doc.get('schemas', {}) - for schema_name, schema in schemas.items(): - self.__message_registry.AddDescriptorFromSchema(schema_name, schema) - - # We need to add one more message type for the global parameters. - standard_query_schema = _StandardQueryParametersSchema( - self.__discovery_doc) - self.__message_registry.AddDescriptorFromSchema( - standard_query_schema['id'], standard_query_schema) - - # Now that we know all the messages, we need to correct some - # fields from MessageFields to EnumFields. - self.__message_registry.FixupMessageFields() - - self.__command_registry = command_registry.CommandRegistry( - self.__package, self.__version, self.__client_info, - self.__message_registry, self.__root_package, self.__base_files_package, - self.__base_url, self.__names) - self.__command_registry.AddGlobalParameters( - self.__message_registry.LookupDescriptorOrDie( - 'StandardQueryParameters')) - - self.__services_registry = service_registry.ServiceRegistry( - self.__client_info, - self.__message_registry, - self.__command_registry, - self.__base_url, - self.__base_path, - self.__names, - self.__root_package, - self.__base_files_package, - unelidable_request_methods or []) - services = self.__discovery_doc.get('resources', {}) - for service_name, methods in sorted(services.items()): - self.__services_registry.AddServiceFromResource(service_name, methods) - # We might also have top-level methods. - api_methods = self.__discovery_doc.get('methods', []) - if api_methods: - self.__services_registry.AddServiceFromResource( - 'api', {'methods': api_methods}) - self.__client_info = self.__client_info._replace( # pylint:disable=protected-access - scopes=self.__services_registry.scopes) - - @property - def client_info(self): - return self.__client_info - - @property - def discovery_doc(self): - return self.__discovery_doc - - @property - def names(self): - return self.__names - - @property - def outdir(self): - return self.__outdir - - @property - def use_proto2(self): - return self.__use_proto2 - - def _GetPrinter(self, out): - printer = util.SimplePrettyPrinter(out) - return printer - - def WriteInit(self, out): - """Write a simple __init__.py for the generated client.""" - printer = self._GetPrinter(out) - printer('"""Common imports for generated %s client library."""', - self.__client_info.package) - printer('# pylint:disable=wildcard-import') - printer() - printer('import pkgutil') - printer() - printer('from %s import *', self.__base_files_package) - if self.__generate_cli: - printer('from %s.%s import *', - self.__root_package, self.__client_info.cli_rule_name) - printer('from %s.%s import *', - self.__root_package, self.__client_info.client_rule_name) - printer('from %s.%s import *', - self.__root_package, self.__client_info.messages_rule_name) - printer() - printer('__path__ = pkgutil.extend_path(__path__, __name__)') - - def WriteMessagesFile(self, out): - self.__message_registry.WriteFile(self._GetPrinter(out)) - - def WriteMessagesProtoFile(self, out): - self.__message_registry.WriteProtoFile(self._GetPrinter(out)) - - def WriteServicesProtoFile(self, out): - self.__services_registry.WriteProtoFile(self._GetPrinter(out)) - - def WriteClientLibrary(self, out): - self.__services_registry.WriteFile(self._GetPrinter(out)) - - def WriteCli(self, out): - self.__command_registry.WriteFile(self._GetPrinter(out)) + + """Code generator for a given discovery document.""" + + def __init__(self, discovery_doc, client_info, names, root_package, outdir, + base_package, generate_cli=False, use_proto2=False, + unelidable_request_methods=None): + self.__discovery_doc = discovery_doc + self.__client_info = client_info + self.__outdir = outdir + self.__use_proto2 = use_proto2 + self.__description = util.CleanDescription( + self.__discovery_doc.get('description', '')) + self.__package = self.__client_info.package + self.__version = self.__client_info.version + self.__generate_cli = generate_cli + self.__root_package = root_package + self.__base_files_package = base_package + self.__base_files_target = ( + '//cloud/bigscience/apitools/base/py:apitools_base') + self.__names = names + self.__base_url, self.__base_path = _ComputePaths( + self.__package, self.__client_info.url_version, self.__discovery_doc) + + # Order is important here: we need the schemas before we can + # define the services. + self.__message_registry = message_registry.MessageRegistry( + self.__client_info, self.__names, self.__description, + self.__root_package, self.__base_files_package) + schemas = self.__discovery_doc.get('schemas', {}) + for schema_name, schema in schemas.items(): + self.__message_registry.AddDescriptorFromSchema( + schema_name, schema) + + # We need to add one more message type for the global parameters. + standard_query_schema = _StandardQueryParametersSchema( + self.__discovery_doc) + self.__message_registry.AddDescriptorFromSchema( + standard_query_schema['id'], standard_query_schema) + + # Now that we know all the messages, we need to correct some + # fields from MessageFields to EnumFields. + self.__message_registry.FixupMessageFields() + + self.__command_registry = command_registry.CommandRegistry( + self.__package, self.__version, self.__client_info, + self.__message_registry, self.__root_package, self.__base_files_package, + self.__base_url, self.__names) + self.__command_registry.AddGlobalParameters( + self.__message_registry.LookupDescriptorOrDie( + 'StandardQueryParameters')) + + self.__services_registry = service_registry.ServiceRegistry( + self.__client_info, + self.__message_registry, + self.__command_registry, + self.__base_url, + self.__base_path, + self.__names, + self.__root_package, + self.__base_files_package, + unelidable_request_methods or []) + services = self.__discovery_doc.get('resources', {}) + for service_name, methods in sorted(services.items()): + self.__services_registry.AddServiceFromResource( + service_name, methods) + # We might also have top-level methods. + api_methods = self.__discovery_doc.get('methods', []) + if api_methods: + self.__services_registry.AddServiceFromResource( + 'api', {'methods': api_methods}) + self.__client_info = self.__client_info._replace( # pylint:disable=protected-access + scopes=self.__services_registry.scopes) + + @property + def client_info(self): + return self.__client_info + + @property + def discovery_doc(self): + return self.__discovery_doc + + @property + def names(self): + return self.__names + + @property + def outdir(self): + return self.__outdir + + @property + def use_proto2(self): + return self.__use_proto2 + + def _GetPrinter(self, out): + printer = util.SimplePrettyPrinter(out) + return printer + + def WriteInit(self, out): + """Write a simple __init__.py for the generated client.""" + printer = self._GetPrinter(out) + printer('"""Common imports for generated %s client library."""', + self.__client_info.package) + printer('# pylint:disable=wildcard-import') + printer() + printer('import pkgutil') + printer() + printer('from %s import *', self.__base_files_package) + if self.__generate_cli: + printer('from %s.%s import *', + self.__root_package, self.__client_info.cli_rule_name) + printer('from %s.%s import *', + self.__root_package, self.__client_info.client_rule_name) + printer('from %s.%s import *', + self.__root_package, self.__client_info.messages_rule_name) + printer() + printer('__path__ = pkgutil.extend_path(__path__, __name__)') + + def WriteMessagesFile(self, out): + self.__message_registry.WriteFile(self._GetPrinter(out)) + + def WriteMessagesProtoFile(self, out): + self.__message_registry.WriteProtoFile(self._GetPrinter(out)) + + def WriteServicesProtoFile(self, out): + self.__services_registry.WriteProtoFile(self._GetPrinter(out)) + + def WriteClientLibrary(self, out): + self.__services_registry.WriteFile(self._GetPrinter(out)) + + def WriteCli(self, out): + self.__command_registry.WriteFile(self._GetPrinter(out)) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 76c5911..b367c0d 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -16,425 +16,434 @@ TypeInfo = collections.namedtuple('TypeInfo', ('type_name', 'variant')) class MessageRegistry(object): - """Registry for message types. - - This closely mirrors a messages.FileDescriptor, but adds additional - attributes (such as message and field descriptions) and some extra - code for validation and cycle detection. - """ - - # Type information from these two maps comes from here: - # https://developers.google.com/discovery/v1/type-format - PRIMITIVE_TYPE_INFO_MAP = { - 'string': TypeInfo(type_name='string', - variant=messages.StringField.DEFAULT_VARIANT), - 'integer': TypeInfo(type_name='integer', - variant=messages.IntegerField.DEFAULT_VARIANT), - 'boolean': TypeInfo(type_name='boolean', - variant=messages.BooleanField.DEFAULT_VARIANT), - 'number': TypeInfo(type_name='number', - variant=messages.FloatField.DEFAULT_VARIANT), - 'any': TypeInfo(type_name='extra_types.JsonValue', - variant=messages.Variant.MESSAGE), - } - - PRIMITIVE_FORMAT_MAP = { - 'int32': TypeInfo(type_name='integer', - variant=messages.Variant.INT32), - 'uint32': TypeInfo(type_name='integer', - variant=messages.Variant.UINT32), - 'int64': TypeInfo(type_name='string', - variant=messages.Variant.INT64), - 'uint64': TypeInfo(type_name='string', - variant=messages.Variant.UINT64), - 'double': TypeInfo(type_name='number', - variant=messages.Variant.DOUBLE), - 'float': TypeInfo(type_name='number', - variant=messages.Variant.FLOAT), - 'byte': TypeInfo(type_name='byte', - variant=messages.BytesField.DEFAULT_VARIANT), - 'date': TypeInfo(type_name='extra_types.DateField', - variant=messages.Variant.STRING), - 'date-time': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', - variant=messages.Variant.MESSAGE), - } - - def __init__(self, client_info, names, description, - root_package_dir, base_files_package): - self.__names = names - self.__client_info = client_info - self.__package = client_info.package - self.__description = util.CleanDescription(description) - self.__root_package_dir = root_package_dir - self.__base_files_package = base_files_package - self.__file_descriptor = extended_descriptor.ExtendedFileDescriptor( - package=self.__package, description=self.__description) - # Add required imports - self.__file_descriptor.additional_imports = [ - 'from protorpc import messages', - ] - # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. - self.__message_registry = collections.OrderedDict() - # A set of types that we're currently adding (for cycle detection). - self.__nascent_types = set() - # A set of types for which we've seen a reference but no - # definition; if this set is nonempty, validation fails. - self.__unknown_types = set() - # Used for tracking paths during message creation - self.__current_path = [] - # Where to register created messages - self.__current_env = self.__file_descriptor - # TODO(craigcitro): Add a `Finalize` method. - - @property - def file_descriptor(self): - self.Validate() - return self.__file_descriptor - - def WriteProtoFile(self, printer): - """Write the messages file to out as proto.""" - self.Validate() - extended_descriptor.WriteMessagesFile( - self.__file_descriptor, self.__package, self.__client_info.version, - printer) - - def WriteFile(self, printer): - """Write the messages file to out.""" - self.Validate() - extended_descriptor.WritePythonFile( - self.__file_descriptor, self.__package, self.__client_info.version, - printer) - - def Validate(self): - mysteries = self.__nascent_types or self.__unknown_types - if mysteries: - raise ValueError('Malformed MessageRegistry: %s' % mysteries) - - def __ComputeFullName(self, name): - return '.'.join(map(six.text_type, self.__current_path[:] + [name])) - - def __AddImport(self, new_import): - if new_import not in self.__file_descriptor.additional_imports: - self.__file_descriptor.additional_imports.append(new_import) - - def __DeclareDescriptor(self, name): - self.__nascent_types.add(self.__ComputeFullName(name)) - - def __RegisterDescriptor(self, new_descriptor): - """Register the given descriptor in this registry.""" - if not isinstance(new_descriptor, ( - extended_descriptor.ExtendedMessageDescriptor, - extended_descriptor.ExtendedEnumDescriptor)): - raise ValueError('Cannot add descriptor of type %s' % ( - type(new_descriptor),)) - full_name = self.__ComputeFullName(new_descriptor.name) - if full_name in self.__message_registry: - raise ValueError('Attempt to re-register descriptor %s' % full_name) - if full_name not in self.__nascent_types: - raise ValueError('Directly adding types is not supported') - new_descriptor.full_name = full_name - self.__message_registry[full_name] = new_descriptor - if isinstance(new_descriptor, - extended_descriptor.ExtendedMessageDescriptor): - self.__current_env.message_types.append(new_descriptor) - elif isinstance(new_descriptor, extended_descriptor.ExtendedEnumDescriptor): - self.__current_env.enum_types.append(new_descriptor) - self.__unknown_types.discard(full_name) - self.__nascent_types.remove(full_name) - - def LookupDescriptor(self, name): - return self.__GetDescriptorByName(name) - - def LookupDescriptorOrDie(self, name): - message_descriptor = self.LookupDescriptor(name) - if message_descriptor is None: - raise ValueError('No message descriptor named "%s"', name) - return message_descriptor - - def __GetDescriptor(self, name): - return self.__GetDescriptorByName(self.__ComputeFullName(name)) - - def __GetDescriptorByName(self, name): - if name in self.__message_registry: - return self.__message_registry[name] - if name in self.__nascent_types: - raise ValueError( - 'Cannot retrieve type currently being created: %s' % name) - return None - - @contextlib.contextmanager - def __DescriptorEnv(self, message_descriptor): - # TODO(craigcitro): Typecheck? - previous_env = self.__current_env - self.__current_path.append(message_descriptor.name) - self.__current_env = message_descriptor - yield - self.__current_path.pop() - self.__current_env = previous_env - - def AddEnumDescriptor(self, name, description, - enum_values, enum_descriptions): - """Add a new EnumDescriptor named name with the given enum values.""" - message = extended_descriptor.ExtendedEnumDescriptor() - message.name = self.__names.ClassName(name) - message.description = util.CleanDescription(description) - self.__DeclareDescriptor(message.name) - for index, (enum_name, enum_description) in enumerate( - zip(enum_values, enum_descriptions)): - enum_value = extended_descriptor.ExtendedEnumValueDescriptor() - enum_value.name = self.__names.NormalizeEnumName(enum_name) - if enum_value.name != enum_name: - message.enum_mappings.append( - extended_descriptor.ExtendedEnumDescriptor.JsonEnumMapping( - python_name=enum_value.name, json_name=enum_name)) - self.__AddImport('from %s import encoding' % self.__base_files_package) - enum_value.number = index - enum_value.description = util.CleanDescription( - enum_description or '') - message.values.append(enum_value) - self.__RegisterDescriptor(message) - - def __DeclareMessageAlias(self, schema, alias_for): - """Declare schema as an alias for alias_for.""" - # TODO(craigcitro): This is a hack. Remove it. - message = extended_descriptor.ExtendedMessageDescriptor() - message.name = self.__names.ClassName(schema['id']) - message.alias_for = alias_for - self.__DeclareDescriptor(message.name) - self.__AddImport('from %s import extra_types' % self.__base_files_package) - self.__RegisterDescriptor(message) - - def __AddAdditionalProperties(self, message, schema, properties): - """Add an additionalProperties field to message.""" - additional_properties_info = schema['additionalProperties'] - entries_type_name = self.__AddAdditionalPropertyType( - message.name, additional_properties_info) - description = util.CleanDescription( - additional_properties_info.get('description')) - if description is None: - description = 'Additional properties of type %s' % message.name - attrs = { - 'items': { - '$ref': entries_type_name, - }, - 'description': description, - 'type': 'array', + + """Registry for message types. + + This closely mirrors a messages.FileDescriptor, but adds additional + attributes (such as message and field descriptions) and some extra + code for validation and cycle detection. + """ + + # Type information from these two maps comes from here: + # https://developers.google.com/discovery/v1/type-format + PRIMITIVE_TYPE_INFO_MAP = { + 'string': TypeInfo(type_name='string', + variant=messages.StringField.DEFAULT_VARIANT), + 'integer': TypeInfo(type_name='integer', + variant=messages.IntegerField.DEFAULT_VARIANT), + 'boolean': TypeInfo(type_name='boolean', + variant=messages.BooleanField.DEFAULT_VARIANT), + 'number': TypeInfo(type_name='number', + variant=messages.FloatField.DEFAULT_VARIANT), + 'any': TypeInfo(type_name='extra_types.JsonValue', + variant=messages.Variant.MESSAGE), } - field_name = 'additionalProperties' - message.fields.append(self.__FieldDescriptorFromProperties( - field_name, len(properties) + 1, attrs)) - self.__AddImport('from %s import encoding' % self.__base_files_package) - message.decorators.append( - 'encoding.MapUnrecognizedFields(%r)' % field_name) - - def AddDescriptorFromSchema(self, schema_name, schema): - """Add a new MessageDescriptor named schema_name based on schema.""" - # TODO(craigcitro): Is schema_name redundant? - if self.__GetDescriptor(schema_name): - return - if schema.get('enum'): - self.__DeclareEnum(schema_name, schema) - return - if schema.get('type') == 'any': - self.__DeclareMessageAlias(schema, 'extra_types.JsonValue') - return - if schema.get('type') != 'object': - raise ValueError( - 'Cannot create message descriptors for type %s', schema.get('type')) - message = extended_descriptor.ExtendedMessageDescriptor() - message.name = self.__names.ClassName(schema['id']) - message.description = util.CleanDescription(schema.get( - 'description', 'A %s object.' % message.name)) - self.__DeclareDescriptor(message.name) - with self.__DescriptorEnv(message): - properties = schema.get('properties', {}) - for index, (name, attrs) in enumerate(sorted(properties.items())): - field = self.__FieldDescriptorFromProperties(name, index + 1, attrs) - message.fields.append(field) - if field.name != name: - message.field_mappings.append( - extended_descriptor.ExtendedMessageDescriptor.JsonFieldMapping( - python_name=field.name, json_name=name)) - self.__AddImport( - 'from %s import encoding' % self.__base_files_package) - if 'additionalProperties' in schema: - self.__AddAdditionalProperties(message, schema, properties) - self.__RegisterDescriptor(message) - - def __AddAdditionalPropertyType(self, name, property_schema): - """Add a new nested AdditionalProperty message.""" - new_type_name = 'AdditionalProperty' - property_schema = dict(property_schema) - # We drop the description here on purpose, so the resulting - # messages are less repetitive. - property_schema.pop('description', None) - description = 'An additional property for a %s object.' % name - schema = { - 'id': new_type_name, - 'type': 'object', - 'description': description, - 'properties': { - 'key': { - 'type': 'string', - 'description': 'Name of the additional property.', - }, - 'value': property_schema, - }, + + PRIMITIVE_FORMAT_MAP = { + 'int32': TypeInfo(type_name='integer', + variant=messages.Variant.INT32), + 'uint32': TypeInfo(type_name='integer', + variant=messages.Variant.UINT32), + 'int64': TypeInfo(type_name='string', + variant=messages.Variant.INT64), + 'uint64': TypeInfo(type_name='string', + variant=messages.Variant.UINT64), + 'double': TypeInfo(type_name='number', + variant=messages.Variant.DOUBLE), + 'float': TypeInfo(type_name='number', + variant=messages.Variant.FLOAT), + 'byte': TypeInfo(type_name='byte', + variant=messages.BytesField.DEFAULT_VARIANT), + 'date': TypeInfo(type_name='extra_types.DateField', + variant=messages.Variant.STRING), + 'date-time': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', + variant=messages.Variant.MESSAGE), } - self.AddDescriptorFromSchema(new_type_name, schema) - return new_type_name - - def __AddEntryType(self, entry_type_name, entry_schema, parent_name): - """Add a type for a list entry.""" - entry_schema.pop('description', None) - description = 'Single entry in a %s.' % parent_name - schema = { - 'id': entry_type_name, - 'type': 'object', - 'description': description, - 'properties': { - 'entry': { - 'type': 'array', - 'items': entry_schema, + + def __init__(self, client_info, names, description, + root_package_dir, base_files_package): + self.__names = names + self.__client_info = client_info + self.__package = client_info.package + self.__description = util.CleanDescription(description) + self.__root_package_dir = root_package_dir + self.__base_files_package = base_files_package + self.__file_descriptor = extended_descriptor.ExtendedFileDescriptor( + package=self.__package, description=self.__description) + # Add required imports + self.__file_descriptor.additional_imports = [ + 'from protorpc import messages', + ] + # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. + self.__message_registry = collections.OrderedDict() + # A set of types that we're currently adding (for cycle detection). + self.__nascent_types = set() + # A set of types for which we've seen a reference but no + # definition; if this set is nonempty, validation fails. + self.__unknown_types = set() + # Used for tracking paths during message creation + self.__current_path = [] + # Where to register created messages + self.__current_env = self.__file_descriptor + # TODO(craigcitro): Add a `Finalize` method. + + @property + def file_descriptor(self): + self.Validate() + return self.__file_descriptor + + def WriteProtoFile(self, printer): + """Write the messages file to out as proto.""" + self.Validate() + extended_descriptor.WriteMessagesFile( + self.__file_descriptor, self.__package, self.__client_info.version, + printer) + + def WriteFile(self, printer): + """Write the messages file to out.""" + self.Validate() + extended_descriptor.WritePythonFile( + self.__file_descriptor, self.__package, self.__client_info.version, + printer) + + def Validate(self): + mysteries = self.__nascent_types or self.__unknown_types + if mysteries: + raise ValueError('Malformed MessageRegistry: %s' % mysteries) + + def __ComputeFullName(self, name): + return '.'.join(map(six.text_type, self.__current_path[:] + [name])) + + def __AddImport(self, new_import): + if new_import not in self.__file_descriptor.additional_imports: + self.__file_descriptor.additional_imports.append(new_import) + + def __DeclareDescriptor(self, name): + self.__nascent_types.add(self.__ComputeFullName(name)) + + def __RegisterDescriptor(self, new_descriptor): + """Register the given descriptor in this registry.""" + if not isinstance(new_descriptor, ( + extended_descriptor.ExtendedMessageDescriptor, + extended_descriptor.ExtendedEnumDescriptor)): + raise ValueError('Cannot add descriptor of type %s' % ( + type(new_descriptor),)) + full_name = self.__ComputeFullName(new_descriptor.name) + if full_name in self.__message_registry: + raise ValueError( + 'Attempt to re-register descriptor %s' % full_name) + if full_name not in self.__nascent_types: + raise ValueError('Directly adding types is not supported') + new_descriptor.full_name = full_name + self.__message_registry[full_name] = new_descriptor + if isinstance(new_descriptor, + extended_descriptor.ExtendedMessageDescriptor): + self.__current_env.message_types.append(new_descriptor) + elif isinstance(new_descriptor, extended_descriptor.ExtendedEnumDescriptor): + self.__current_env.enum_types.append(new_descriptor) + self.__unknown_types.discard(full_name) + self.__nascent_types.remove(full_name) + + def LookupDescriptor(self, name): + return self.__GetDescriptorByName(name) + + def LookupDescriptorOrDie(self, name): + message_descriptor = self.LookupDescriptor(name) + if message_descriptor is None: + raise ValueError('No message descriptor named "%s"', name) + return message_descriptor + + def __GetDescriptor(self, name): + return self.__GetDescriptorByName(self.__ComputeFullName(name)) + + def __GetDescriptorByName(self, name): + if name in self.__message_registry: + return self.__message_registry[name] + if name in self.__nascent_types: + raise ValueError( + 'Cannot retrieve type currently being created: %s' % name) + return None + + @contextlib.contextmanager + def __DescriptorEnv(self, message_descriptor): + # TODO(craigcitro): Typecheck? + previous_env = self.__current_env + self.__current_path.append(message_descriptor.name) + self.__current_env = message_descriptor + yield + self.__current_path.pop() + self.__current_env = previous_env + + def AddEnumDescriptor(self, name, description, + enum_values, enum_descriptions): + """Add a new EnumDescriptor named name with the given enum values.""" + message = extended_descriptor.ExtendedEnumDescriptor() + message.name = self.__names.ClassName(name) + message.description = util.CleanDescription(description) + self.__DeclareDescriptor(message.name) + for index, (enum_name, enum_description) in enumerate( + zip(enum_values, enum_descriptions)): + enum_value = extended_descriptor.ExtendedEnumValueDescriptor() + enum_value.name = self.__names.NormalizeEnumName(enum_name) + if enum_value.name != enum_name: + message.enum_mappings.append( + extended_descriptor.ExtendedEnumDescriptor.JsonEnumMapping( + python_name=enum_value.name, json_name=enum_name)) + self.__AddImport('from %s import encoding' % + self.__base_files_package) + enum_value.number = index + enum_value.description = util.CleanDescription( + enum_description or '') + message.values.append(enum_value) + self.__RegisterDescriptor(message) + + def __DeclareMessageAlias(self, schema, alias_for): + """Declare schema as an alias for alias_for.""" + # TODO(craigcitro): This is a hack. Remove it. + message = extended_descriptor.ExtendedMessageDescriptor() + message.name = self.__names.ClassName(schema['id']) + message.alias_for = alias_for + self.__DeclareDescriptor(message.name) + self.__AddImport('from %s import extra_types' % + self.__base_files_package) + self.__RegisterDescriptor(message) + + def __AddAdditionalProperties(self, message, schema, properties): + """Add an additionalProperties field to message.""" + additional_properties_info = schema['additionalProperties'] + entries_type_name = self.__AddAdditionalPropertyType( + message.name, additional_properties_info) + description = util.CleanDescription( + additional_properties_info.get('description')) + if description is None: + description = 'Additional properties of type %s' % message.name + attrs = { + 'items': { + '$ref': entries_type_name, }, - }, - } - self.AddDescriptorFromSchema(entry_type_name, schema) - return entry_type_name - - def __FieldDescriptorFromProperties(self, name, index, attrs): - """Create a field descriptor for these attrs.""" - field = descriptor.FieldDescriptor() - field.name = self.__names.CleanName(name) - field.number = index - field.label = self.__ComputeLabel(attrs) - new_type_name_hint = self.__names.ClassName( - '%sValue' % self.__names.ClassName(name)) - type_info = self.__GetTypeInfo(attrs, new_type_name_hint) - field.type_name = type_info.type_name - field.variant = type_info.variant - if 'default' in attrs: - # TODO(craigcitro): Correctly handle non-primitive default values. - default = attrs['default'] - if field.type_name != 'string' and field.variant != messages.Variant.ENUM: - default = str(json.loads(default)) - if field.variant == messages.Variant.ENUM: - default = self.__names.NormalizeEnumName(default) - field.default_value = default - extended_field = extended_descriptor.ExtendedFieldDescriptor() - extended_field.name = field.name - extended_field.description = util.CleanDescription( - attrs.get('description', 'A %s attribute.' % field.type_name)) - extended_field.field_descriptor = field - return extended_field - - @staticmethod - def __ComputeLabel(attrs): - if attrs.get('required', False): - return descriptor.FieldDescriptor.Label.REQUIRED - elif attrs.get('type') == 'array': - return descriptor.FieldDescriptor.Label.REPEATED - elif attrs.get('repeated'): - return descriptor.FieldDescriptor.Label.REPEATED - return descriptor.FieldDescriptor.Label.OPTIONAL - - def __DeclareEnum(self, enum_name, attrs): - description = util.CleanDescription(attrs.get('description', '')) - enum_values = attrs['enum'] - enum_descriptions = attrs.get('enumDescriptions', [''] * len(enum_values)) - self.AddEnumDescriptor(enum_name, description, - enum_values, enum_descriptions) - self.__AddIfUnknown(enum_name) - return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) - - def __AddIfUnknown(self, type_name): - type_name = self.__names.ClassName(type_name) - full_type_name = self.__ComputeFullName(type_name) - if (full_type_name not in self.__message_registry.keys() and - type_name not in self.__message_registry.keys()): - self.__unknown_types.add(type_name) - - def __GetTypeInfo(self, attrs, name_hint): - """Return a TypeInfo object for attrs, creating one if needed.""" - - type_ref = self.__names.ClassName(attrs.get('$ref')) - type_name = attrs.get('type') - if not (type_ref or type_name): - raise ValueError('No type found for %s' % attrs) - - if type_ref: - self.__AddIfUnknown(type_ref) - # We don't actually know this is a message -- it might be an - # enum. However, we can't check that until we've created all the - # types, so we come back and fix this up later. - return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) - - if 'enum' in attrs: - enum_name = '%sValuesEnum' % name_hint - return self.__DeclareEnum(enum_name, attrs) - - if 'format' in attrs: - type_info = self.PRIMITIVE_FORMAT_MAP.get(attrs['format']) - if type_info is None: - # If we don't recognize the format, the spec says we fall back - # to just using the type name. + 'description': description, + 'type': 'array', + } + field_name = 'additionalProperties' + message.fields.append(self.__FieldDescriptorFromProperties( + field_name, len(properties) + 1, attrs)) + self.__AddImport('from %s import encoding' % self.__base_files_package) + message.decorators.append( + 'encoding.MapUnrecognizedFields(%r)' % field_name) + + def AddDescriptorFromSchema(self, schema_name, schema): + """Add a new MessageDescriptor named schema_name based on schema.""" + # TODO(craigcitro): Is schema_name redundant? + if self.__GetDescriptor(schema_name): + return + if schema.get('enum'): + self.__DeclareEnum(schema_name, schema) + return + if schema.get('type') == 'any': + self.__DeclareMessageAlias(schema, 'extra_types.JsonValue') + return + if schema.get('type') != 'object': + raise ValueError( + 'Cannot create message descriptors for type %s', schema.get('type')) + message = extended_descriptor.ExtendedMessageDescriptor() + message.name = self.__names.ClassName(schema['id']) + message.description = util.CleanDescription(schema.get( + 'description', 'A %s object.' % message.name)) + self.__DeclareDescriptor(message.name) + with self.__DescriptorEnv(message): + properties = schema.get('properties', {}) + for index, (name, attrs) in enumerate(sorted(properties.items())): + field = self.__FieldDescriptorFromProperties( + name, index + 1, attrs) + message.fields.append(field) + if field.name != name: + message.field_mappings.append( + extended_descriptor.ExtendedMessageDescriptor.JsonFieldMapping( + python_name=field.name, json_name=name)) + self.__AddImport( + 'from %s import encoding' % self.__base_files_package) + if 'additionalProperties' in schema: + self.__AddAdditionalProperties(message, schema, properties) + self.__RegisterDescriptor(message) + + def __AddAdditionalPropertyType(self, name, property_schema): + """Add a new nested AdditionalProperty message.""" + new_type_name = 'AdditionalProperty' + property_schema = dict(property_schema) + # We drop the description here on purpose, so the resulting + # messages are less repetitive. + property_schema.pop('description', None) + description = 'An additional property for a %s object.' % name + schema = { + 'id': new_type_name, + 'type': 'object', + 'description': description, + 'properties': { + 'key': { + 'type': 'string', + 'description': 'Name of the additional property.', + }, + 'value': property_schema, + }, + } + self.AddDescriptorFromSchema(new_type_name, schema) + return new_type_name + + def __AddEntryType(self, entry_type_name, entry_schema, parent_name): + """Add a type for a list entry.""" + entry_schema.pop('description', None) + description = 'Single entry in a %s.' % parent_name + schema = { + 'id': entry_type_name, + 'type': 'object', + 'description': description, + 'properties': { + 'entry': { + 'type': 'array', + 'items': entry_schema, + }, + }, + } + self.AddDescriptorFromSchema(entry_type_name, schema) + return entry_type_name + + def __FieldDescriptorFromProperties(self, name, index, attrs): + """Create a field descriptor for these attrs.""" + field = descriptor.FieldDescriptor() + field.name = self.__names.CleanName(name) + field.number = index + field.label = self.__ComputeLabel(attrs) + new_type_name_hint = self.__names.ClassName( + '%sValue' % self.__names.ClassName(name)) + type_info = self.__GetTypeInfo(attrs, new_type_name_hint) + field.type_name = type_info.type_name + field.variant = type_info.variant + if 'default' in attrs: + # TODO(craigcitro): Correctly handle non-primitive default values. + default = attrs['default'] + if field.type_name != 'string' and field.variant != messages.Variant.ENUM: + default = str(json.loads(default)) + if field.variant == messages.Variant.ENUM: + default = self.__names.NormalizeEnumName(default) + field.default_value = default + extended_field = extended_descriptor.ExtendedFieldDescriptor() + extended_field.name = field.name + extended_field.description = util.CleanDescription( + attrs.get('description', 'A %s attribute.' % field.type_name)) + extended_field.field_descriptor = field + return extended_field + + @staticmethod + def __ComputeLabel(attrs): + if attrs.get('required', False): + return descriptor.FieldDescriptor.Label.REQUIRED + elif attrs.get('type') == 'array': + return descriptor.FieldDescriptor.Label.REPEATED + elif attrs.get('repeated'): + return descriptor.FieldDescriptor.Label.REPEATED + return descriptor.FieldDescriptor.Label.OPTIONAL + + def __DeclareEnum(self, enum_name, attrs): + description = util.CleanDescription(attrs.get('description', '')) + enum_values = attrs['enum'] + enum_descriptions = attrs.get( + 'enumDescriptions', [''] * len(enum_values)) + self.AddEnumDescriptor(enum_name, description, + enum_values, enum_descriptions) + self.__AddIfUnknown(enum_name) + return TypeInfo(type_name=enum_name, variant=messages.Variant.ENUM) + + def __AddIfUnknown(self, type_name): + type_name = self.__names.ClassName(type_name) + full_type_name = self.__ComputeFullName(type_name) + if (full_type_name not in self.__message_registry.keys() and + type_name not in self.__message_registry.keys()): + self.__unknown_types.add(type_name) + + def __GetTypeInfo(self, attrs, name_hint): + """Return a TypeInfo object for attrs, creating one if needed.""" + + type_ref = self.__names.ClassName(attrs.get('$ref')) + type_name = attrs.get('type') + if not (type_ref or type_name): + raise ValueError('No type found for %s' % attrs) + + if type_ref: + self.__AddIfUnknown(type_ref) + # We don't actually know this is a message -- it might be an + # enum. However, we can't check that until we've created all the + # types, so we come back and fix this up later. + return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) + + if 'enum' in attrs: + enum_name = '%sValuesEnum' % name_hint + return self.__DeclareEnum(enum_name, attrs) + + if 'format' in attrs: + type_info = self.PRIMITIVE_FORMAT_MAP.get(attrs['format']) + if type_info is None: + # If we don't recognize the format, the spec says we fall back + # to just using the type name. + if type_name in self.PRIMITIVE_TYPE_INFO_MAP: + return self.PRIMITIVE_TYPE_INFO_MAP[type_name] + raise ValueError('Unknown type/format "%s"/"%s"' % ( + attrs['format'], type_name)) + if (type_info.type_name.startswith('protorpc.message_types.') or + type_info.type_name.startswith('message_types.')): + self.__AddImport('from protorpc import message_types') + if type_info.type_name.startswith('extra_types.'): + self.__AddImport( + 'from %s import extra_types' % self.__base_files_package) + return type_info + if type_name in self.PRIMITIVE_TYPE_INFO_MAP: - return self.PRIMITIVE_TYPE_INFO_MAP[type_name] - raise ValueError('Unknown type/format "%s"/"%s"' % ( - attrs['format'], type_name)) - if (type_info.type_name.startswith('protorpc.message_types.') or - type_info.type_name.startswith('message_types.')): - self.__AddImport('from protorpc import message_types') - if type_info.type_name.startswith('extra_types.'): - self.__AddImport( - 'from %s import extra_types' % self.__base_files_package) - return type_info - - if type_name in self.PRIMITIVE_TYPE_INFO_MAP: - type_info = self.PRIMITIVE_TYPE_INFO_MAP[type_name] - return type_info - - if type_name == 'array': - items = attrs.get('items') - if not items: - raise ValueError('Array type with no item type: %s' % attrs) - entry_name_hint = self.__names.ClassName( - items.get('title') or '%sListEntry' % name_hint) - entry_label = self.__ComputeLabel(items) - if entry_label == descriptor.FieldDescriptor.Label.REPEATED: - parent_name = self.__names.ClassName(items.get('title') or name_hint) - entry_type_name = self.__AddEntryType( - entry_name_hint, items.get('items'), parent_name) - return TypeInfo( - type_name=entry_type_name, variant=messages.Variant.MESSAGE) - else: - return self.__GetTypeInfo(items, entry_name_hint) - elif type_name == 'any': - self.__AddImport('from %s import extra_types' % self.__base_files_package) - return self.PRIMITIVE_TYPE_INFO_MAP['any'] - elif type_name == 'object': - # TODO(craigcitro): Think of a better way to come up with names. - if not name_hint: - raise ValueError('Cannot create subtype without some name hint') - schema = dict(attrs) - schema['id'] = name_hint - self.AddDescriptorFromSchema(name_hint, schema) - self.__AddIfUnknown(name_hint) - return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) - - raise ValueError('Unknown type: %s' % type_name) - - def FixupMessageFields(self): - for message_type in self.file_descriptor.message_types: - self._FixupMessage(message_type) - - def _FixupMessage(self, message_type): - with self.__DescriptorEnv(message_type): - for field in message_type.fields: - if field.field_descriptor.variant == messages.Variant.MESSAGE: - field_type_name = field.field_descriptor.type_name - field_type = self.LookupDescriptor(field_type_name) - if isinstance(field_type, extended_descriptor.ExtendedEnumDescriptor): - field.field_descriptor.variant = messages.Variant.ENUM - for submessage_type in message_type.message_types: - self._FixupMessage(submessage_type) + type_info = self.PRIMITIVE_TYPE_INFO_MAP[type_name] + return type_info + + if type_name == 'array': + items = attrs.get('items') + if not items: + raise ValueError('Array type with no item type: %s' % attrs) + entry_name_hint = self.__names.ClassName( + items.get('title') or '%sListEntry' % name_hint) + entry_label = self.__ComputeLabel(items) + if entry_label == descriptor.FieldDescriptor.Label.REPEATED: + parent_name = self.__names.ClassName( + items.get('title') or name_hint) + entry_type_name = self.__AddEntryType( + entry_name_hint, items.get('items'), parent_name) + return TypeInfo( + type_name=entry_type_name, variant=messages.Variant.MESSAGE) + else: + return self.__GetTypeInfo(items, entry_name_hint) + elif type_name == 'any': + self.__AddImport('from %s import extra_types' % + self.__base_files_package) + return self.PRIMITIVE_TYPE_INFO_MAP['any'] + elif type_name == 'object': + # TODO(craigcitro): Think of a better way to come up with names. + if not name_hint: + raise ValueError( + 'Cannot create subtype without some name hint') + schema = dict(attrs) + schema['id'] = name_hint + self.AddDescriptorFromSchema(name_hint, schema) + self.__AddIfUnknown(name_hint) + return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) + + raise ValueError('Unknown type: %s' % type_name) + + def FixupMessageFields(self): + for message_type in self.file_descriptor.message_types: + self._FixupMessage(message_type) + + def _FixupMessage(self, message_type): + with self.__DescriptorEnv(message_type): + for field in message_type.fields: + if field.field_descriptor.variant == messages.Variant.MESSAGE: + field_type_name = field.field_descriptor.type_name + field_type = self.LookupDescriptor(field_type_name) + if isinstance(field_type, extended_descriptor.ExtendedEnumDescriptor): + field.field_descriptor.variant = messages.Variant.ENUM + for submessage_type in message_type.message_types: + self._FixupMessage(submessage_type) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 63185c8..5dd2315 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -16,419 +16,437 @@ _MIME_PATTERN_RE = re.compile(r'(?i)[a-z0-9_*-]+/[a-z0-9_*-]+') class ServiceRegistry(object): - """Registry for service types.""" - - def __init__(self, client_info, message_registry, command_registry, - base_url, base_path, names, - root_package_dir, base_files_package, - unelidable_request_methods): - self.__client_info = client_info - self.__package = client_info.package - self.__names = names - self.__service_method_info_map = collections.OrderedDict() - self.__message_registry = message_registry - self.__command_registry = command_registry - self.__base_url = base_url - self.__base_path = base_path - self.__root_package_dir = root_package_dir - self.__base_files_package = base_files_package - self.__unelidable_request_methods = unelidable_request_methods - self.__all_scopes = set(self.__client_info.scopes) - - def Validate(self): - self.__message_registry.Validate() - - @property - def scopes(self): - return sorted(list(self.__all_scopes)) - - def __GetServiceClassName(self, service_name): - return self.__names.ClassName( - '%sService' % self.__names.ClassName(service_name)) - - def __PrintDocstring(self, printer, method_info, method_name, name): - """Print a docstring for a service method.""" - if method_info.description: - description = util.CleanDescription(method_info.description) - first_line, newline, remaining = method_info.description.partition( - '\n') - if not first_line.endswith('.'): - first_line = '%s.' % first_line - description = '%s%s%s' % (first_line, newline, remaining) - else: - description = '%s method for the %s service.' % (method_name, name) - with printer.CommentContext(): - printer('"""%s' % description) - printer() - printer('Args:') - printer(' request: (%s) input message', method_info.request_type_name) - printer(' global_params: (StandardQueryParameters, default: None) ' - 'global arguments') - if method_info.upload_config: - printer(' upload: (Upload, default: None) If present, upload') - printer(' this stream with the request.') - if method_info.supports_download: - printer(' download: (Download, default: None) If present, download') - printer(' data from the request via this stream.') - printer('Returns:') - printer(' (%s) The response message.', method_info.response_type_name) - printer('"""') - - def __WriteSingleService( - self, printer, name, method_info_map, client_class_name): - printer() - class_name = self.__GetServiceClassName(name) - printer('class %s(base_api.BaseApiService):', class_name) - with printer.Indent(): - printer('"""Service class for the %s resource."""', name) - printer() - printer('_NAME = %s', repr(name)) - - # Print the configs for the methods first. - printer() - printer('def __init__(self, client):') - with printer.Indent(): - printer('super(%s.%s, self).__init__(client)', - client_class_name, class_name) - printer('self._method_configs = {') - with printer.Indent(indent=' '): - for method_name, method_info in method_info_map.items(): - printer("'%s': base_api.ApiMethodInfo(", method_name) - with printer.Indent(indent=' '): - attrs = sorted(x.name for x in method_info.all_fields()) - for attr in attrs: - if attr in ('upload_config', 'description'): - continue - printer('%s=%r,', attr, getattr(method_info, attr)) - printer('),') - printer('}') - printer() - printer('self._upload_configs = {') - with printer.Indent(indent=' '): - for method_name, method_info in method_info_map.items(): - upload_config = method_info.upload_config - if upload_config is not None: - printer("'%s': base_api.ApiUploadInfo(", method_name) - with printer.Indent(indent=' '): - attrs = sorted(x.name for x in upload_config.all_fields()) - for attr in attrs: - printer('%s=%r,', attr, getattr(upload_config, attr)) - printer('),') - printer('}') - - # Now write each method in turn. - for method_name, method_info in method_info_map.items(): + + """Registry for service types.""" + + def __init__(self, client_info, message_registry, command_registry, + base_url, base_path, names, + root_package_dir, base_files_package, + unelidable_request_methods): + self.__client_info = client_info + self.__package = client_info.package + self.__names = names + self.__service_method_info_map = collections.OrderedDict() + self.__message_registry = message_registry + self.__command_registry = command_registry + self.__base_url = base_url + self.__base_path = base_path + self.__root_package_dir = root_package_dir + self.__base_files_package = base_files_package + self.__unelidable_request_methods = unelidable_request_methods + self.__all_scopes = set(self.__client_info.scopes) + + def Validate(self): + self.__message_registry.Validate() + + @property + def scopes(self): + return sorted(list(self.__all_scopes)) + + def __GetServiceClassName(self, service_name): + return self.__names.ClassName( + '%sService' % self.__names.ClassName(service_name)) + + def __PrintDocstring(self, printer, method_info, method_name, name): + """Print a docstring for a service method.""" + if method_info.description: + description = util.CleanDescription(method_info.description) + first_line, newline, remaining = method_info.description.partition( + '\n') + if not first_line.endswith('.'): + first_line = '%s.' % first_line + description = '%s%s%s' % (first_line, newline, remaining) + else: + description = '%s method for the %s service.' % (method_name, name) + with printer.CommentContext(): + printer('"""%s' % description) printer() - params = ['self', 'request', 'global_params=None'] + printer('Args:') + printer(' request: (%s) input message', method_info.request_type_name) + printer(' global_params: (StandardQueryParameters, default: None) ' + 'global arguments') if method_info.upload_config: - params.append('upload=None') + printer(' upload: (Upload, default: None) If present, upload') + printer(' this stream with the request.') if method_info.supports_download: - params.append('download=None') - printer('def %s(%s):', method_name, ', '.join(params)) + printer( + ' download: (Download, default: None) If present, download') + printer(' data from the request via this stream.') + printer('Returns:') + printer(' (%s) The response message.', method_info.response_type_name) + printer('"""') + + def __WriteSingleService( + self, printer, name, method_info_map, client_class_name): + printer() + class_name = self.__GetServiceClassName(name) + printer('class %s(base_api.BaseApiService):', class_name) + with printer.Indent(): + printer('"""Service class for the %s resource."""', name) + printer() + printer('_NAME = %s', repr(name)) + + # Print the configs for the methods first. + printer() + printer('def __init__(self, client):') + with printer.Indent(): + printer('super(%s.%s, self).__init__(client)', + client_class_name, class_name) + printer('self._method_configs = {') + with printer.Indent(indent=' '): + for method_name, method_info in method_info_map.items(): + printer("'%s': base_api.ApiMethodInfo(", method_name) + with printer.Indent(indent=' '): + attrs = sorted( + x.name for x in method_info.all_fields()) + for attr in attrs: + if attr in ('upload_config', 'description'): + continue + printer( + '%s=%r,', attr, getattr(method_info, attr)) + printer('),') + printer('}') + printer() + printer('self._upload_configs = {') + with printer.Indent(indent=' '): + for method_name, method_info in method_info_map.items(): + upload_config = method_info.upload_config + if upload_config is not None: + printer( + "'%s': base_api.ApiUploadInfo(", method_name) + with printer.Indent(indent=' '): + attrs = sorted( + x.name for x in upload_config.all_fields()) + for attr in attrs: + printer( + '%s=%r,', attr, getattr(upload_config, attr)) + printer('),') + printer('}') + + # Now write each method in turn. + for method_name, method_info in method_info_map.items(): + printer() + params = ['self', 'request', 'global_params=None'] + if method_info.upload_config: + params.append('upload=None') + if method_info.supports_download: + params.append('download=None') + printer('def %s(%s):', method_name, ', '.join(params)) + with printer.Indent(): + self.__PrintDocstring( + printer, method_info, method_name, name) + printer("config = self.GetMethodConfig('%s')", method_name) + upload_config = method_info.upload_config + if upload_config is not None: + printer( + "upload_config = self.GetUploadConfig('%s')", method_name) + arg_lines = [ + 'config, request, global_params=global_params'] + if method_info.upload_config: + arg_lines.append( + 'upload=upload, upload_config=upload_config') + if method_info.supports_download: + arg_lines.append('download=download') + printer('return self._RunMethod(') + with printer.Indent(indent=' '): + for line in arg_lines[:-1]: + printer('%s,', line) + printer('%s)', arg_lines[-1]) + + def __WriteProtoServiceDeclaration(self, printer, name, method_info_map): + """Write a single service declaration to a proto file.""" + printer() + printer('service %s {', self.__GetServiceClassName(name)) + with printer.Indent(): + for method_name, method_info in method_info_map.items(): + for line in textwrap.wrap(method_info.description, + printer.CalculateWidth() - 3): + printer('// %s', line) + printer('rpc %s (%s) returns (%s);', + method_name, + method_info.request_type_name, + method_info.response_type_name) + printer('}') + + def WriteProtoFile(self, printer): + """Write the services in this registry to out as proto.""" + self.Validate() + client_info = self.__client_info + printer('// Generated services for %s version %s.', + client_info.package, client_info.version) + printer() + printer('syntax = "proto2";') + printer('package %s;', self.__package) + printer('import "%s";', client_info.messages_proto_file_name) + printer() + for name, method_info_map in self.__service_method_info_map.items(): + self.__WriteProtoServiceDeclaration(printer, name, method_info_map) + + def WriteFile(self, printer): + """Write the services in this registry to out.""" + self.Validate() + client_info = self.__client_info + printer('"""Generated client library for %s version %s."""', + client_info.package, client_info.version) + printer('# NOTE: This file is autogenerated and should not be edited by ' + 'hand.') + printer('from %s import base_api', self.__base_files_package) + import_prefix = '' + printer('%simport %s as messages', import_prefix, + client_info.messages_rule_name) + printer() + printer() + printer('class %s(base_api.BaseApiClient):', + client_info.client_class_name) with printer.Indent(): - self.__PrintDocstring(printer, method_info, method_name, name) - printer("config = self.GetMethodConfig('%s')", method_name) - upload_config = method_info.upload_config - if upload_config is not None: - printer("upload_config = self.GetUploadConfig('%s')", method_name) - arg_lines = ['config, request, global_params=global_params'] - if method_info.upload_config: - arg_lines.append('upload=upload, upload_config=upload_config') - if method_info.supports_download: - arg_lines.append('download=download') - printer('return self._RunMethod(') - with printer.Indent(indent=' '): - for line in arg_lines[:-1]: - printer('%s,', line) - printer('%s)', arg_lines[-1]) - - def __WriteProtoServiceDeclaration(self, printer, name, method_info_map): - """Write a single service declaration to a proto file.""" - printer() - printer('service %s {', self.__GetServiceClassName(name)) - with printer.Indent(): - for method_name, method_info in method_info_map.items(): - for line in textwrap.wrap(method_info.description, - printer.CalculateWidth() - 3): - printer('// %s', line) - printer('rpc %s (%s) returns (%s);', - method_name, - method_info.request_type_name, - method_info.response_type_name) - printer('}') - - def WriteProtoFile(self, printer): - """Write the services in this registry to out as proto.""" - self.Validate() - client_info = self.__client_info - printer('// Generated services for %s version %s.', - client_info.package, client_info.version) - printer() - printer('syntax = "proto2";') - printer('package %s;', self.__package) - printer('import "%s";', client_info.messages_proto_file_name) - printer() - for name, method_info_map in self.__service_method_info_map.items(): - self.__WriteProtoServiceDeclaration(printer, name, method_info_map) - - def WriteFile(self, printer): - """Write the services in this registry to out.""" - self.Validate() - client_info = self.__client_info - printer('"""Generated client library for %s version %s."""', - client_info.package, client_info.version) - printer('# NOTE: This file is autogenerated and should not be edited by ' - 'hand.') - printer('from %s import base_api', self.__base_files_package) - import_prefix = '' - printer('%simport %s as messages', import_prefix, - client_info.messages_rule_name) - printer() - printer() - printer('class %s(base_api.BaseApiClient):', client_info.client_class_name) - with printer.Indent(): - printer('"""Generated client library for service %s version %s."""', - client_info.package, client_info.version) - printer() - printer('MESSAGES_MODULE = messages') - printer() - client_info_items = client_info._asdict().items() # pylint:disable=protected-access - for attr, val in client_info_items: - if attr == 'scopes' and not val: - val = ['https://www.googleapis.com/auth/userinfo.email'] - printer('_%s = %r' % (attr.upper(), val)) - printer() - printer("def __init__(self, url='', credentials=None,") - with printer.Indent(indent=' '): - printer('get_credentials=True, http=None, model=None,') - printer('log_request=False, log_response=False,') - printer('credentials_args=None, default_global_params=None,') - printer('additional_http_headers=None):') - with printer.Indent(): - printer('"""Create a new %s handle."""', client_info.package) - printer('url = url or %r', self.__base_url) - printer('super(%s, self).__init__(', client_info.client_class_name) - printer(' url, credentials=credentials,') - printer(' get_credentials=get_credentials, http=http, model=model,') - printer(' log_request=log_request, log_response=log_response,') - printer(' credentials_args=credentials_args,') - printer(' default_global_params=default_global_params,') - printer(' additional_http_headers=additional_http_headers)') - for name in self.__service_method_info_map.keys(): - printer('self.%s = self.%s(self)', - name, self.__GetServiceClassName(name)) - for name, method_info_map in self.__service_method_info_map.items(): - self.__WriteSingleService( - printer, name, method_info_map, client_info.client_class_name) - - def __RegisterService(self, service_name, method_info_map): - if service_name in self.__service_method_info_map: - raise ValueError('Attempt to re-register descriptor %s' % service_name) - self.__service_method_info_map[service_name] = method_info_map - - def __CreateRequestType(self, method_description, body_type=None): - """Create a request type for this method.""" - schema = {} - schema['id'] = self.__names.ClassName('%sRequest' % ( - self.__names.ClassName(method_description['id'], separator='.'),)) - schema['type'] = 'object' - schema['properties'] = collections.OrderedDict() - if 'parameterOrder' not in method_description: - ordered_parameters = list(method_description.get('parameters', [])) - else: - ordered_parameters = method_description['parameterOrder'][:] - for k in method_description['parameters']: - if k not in ordered_parameters: - ordered_parameters.append(k) - for parameter_name in ordered_parameters: - field_name = self.__names.CleanName(parameter_name) - field = dict(method_description['parameters'][parameter_name]) - if 'type' not in field: - raise ValueError('No type found in parameter %s' % field) - schema['properties'][field_name] = field - if body_type is not None: - body_field_name = self.__GetRequestField(method_description, body_type) - if body_field_name in schema['properties']: - raise ValueError('Failed to normalize request resource name') - if 'description' not in body_type: - body_type['description'] = ( - 'A %s resource to be passed as the request body.' % ( - self.__GetRequestType(body_type),)) - schema['properties'][body_field_name] = body_type - self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) - return schema['id'] - - def __CreateVoidResponseType(self, method_description): - """Create an empty response type.""" - schema = {} - method_name = self.__names.ClassName( - method_description['id'], separator='.') - schema['id'] = self.__names.ClassName('%sResponse' % method_name) - schema['type'] = 'object' - schema['description'] = 'An empty %s response.' % method_name - self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) - return schema['id'] - - def __NeedRequestType(self, method_description, request_type): - """Determine if this method needs a new request type created.""" - if not request_type: - return True - if method_description.get('id', '') in self.__unelidable_request_methods: - return True - message = self.__message_registry.LookupDescriptorOrDie(request_type) - if message is None: - return True - field_names = [x.name for x in message.fields] - parameters = method_description.get('parameters', {}) - for param_name, param_info in parameters.items(): - if (param_info.get('location') != 'path' or - self.__names.CleanName(param_name) not in field_names): - break - else: - return False - return True - - def __MaxSizeToInt(self, max_size): - """Convert max_size to an int.""" - size_groups = re.match(r'(?P\d+)(?P.B)?$', max_size) - if size_groups is None: - raise ValueError('Could not parse maxSize') - size, unit = size_groups.group('size', 'unit') - shift = 0 - if unit is not None: - unit_dict = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} - shift = unit_dict.get(unit.upper()) - if shift is None: - raise ValueError('Unknown unit %s' % unit) - return int(size) * (1 << shift) - - def __ComputeUploadConfig(self, media_upload_config, method_id): - """Fill out the upload config for this method.""" - config = base_api.ApiUploadInfo() - if 'maxSize' in media_upload_config: - config.max_size = self.__MaxSizeToInt( - media_upload_config['maxSize']) - if 'accept' not in media_upload_config: - logging.warn( - 'No accept types found for upload configuration in ' - 'method %s, using */*', method_id) - config.accept.extend([ - str(a) for a in media_upload_config.get('accept', '*/*')]) - - for accept_pattern in config.accept: - if not _MIME_PATTERN_RE.match(accept_pattern): - logging.warn('Unexpected MIME type: %s', accept_pattern) - protocols = media_upload_config.get('protocols', {}) - for protocol in ('simple', 'resumable'): - media = protocols.get(protocol, {}) - for attr in ('multipart', 'path'): - if attr in media: - setattr(config, '%s_%s' % (protocol, attr), media[attr]) - return config - - def __ComputeMethodInfo(self, method_description, request, response, - request_field): - """Compute the base_api.ApiMethodInfo for this method.""" - relative_path = self.__names.NormalizeRelativePath( - ''.join((self.__base_path, method_description['path']))) - method_id = method_description['id'] - ordered_params = [] - for param_name in method_description.get('parameterOrder', []): - if method_description['parameters'][param_name].get('required', False): - ordered_params.append(param_name) - method_info = base_api.ApiMethodInfo( - relative_path=relative_path, - method_id=method_id, - http_method=method_description['httpMethod'], - description=util.CleanDescription( - method_description.get('description', '')), - query_params=[], - path_params=[], - ordered_params=ordered_params, - request_type_name=self.__names.ClassName(request), - response_type_name=self.__names.ClassName(response), - request_field=request_field, - ) - if method_description.get('supportsMediaUpload', False): - method_info.upload_config = self.__ComputeUploadConfig( - method_description.get('mediaUpload'), method_id) - method_info.supports_download = method_description.get( - 'supportsMediaDownload', False) - self.__all_scopes.update(method_description.get('scopes', ())) - for param, desc in method_description.get('parameters', {}).items(): - param = self.__names.CleanName(param) - location = desc['location'] - if location == 'query': - method_info.query_params.append(param) - elif location == 'path': - method_info.path_params.append(param) - else: - raise ValueError('Unknown parameter location %s for parameter %s' % ( - location, param)) - method_info.path_params.sort() - method_info.query_params.sort() - return method_info - - def __BodyFieldName(self, body_type): - if body_type is None: - return '' - return self.__names.FieldName(body_type['$ref']) - - def __GetRequestType(self, body_type): - return self.__names.ClassName(body_type.get('$ref')) - - def __GetRequestField(self, method_description, body_type): - """Determine the request field for this method.""" - body_field_name = self.__BodyFieldName(body_type) - if body_field_name in method_description.get('parameters', {}): - body_field_name = self.__names.FieldName( - '%s_resource' % body_field_name) - # It's exceedingly unlikely that we'd get two name collisions, which - # means it's bound to happen at some point. - while body_field_name in method_description.get('parameters', {}): - body_field_name = self.__names.FieldName( - '%s_body' % body_field_name) - return body_field_name - - def AddServiceFromResource(self, service_name, methods): - """Add a new service named service_name with the given methods.""" - method_descriptions = methods.get('methods', {}) - method_info_map = collections.OrderedDict() - items = sorted(method_descriptions.items()) - for method_name, method_description in items: - method_name = self.__names.MethodName(method_name) - - # NOTE: According to the discovery document, if the request or - # response is present, it will simply contain a `$ref`. - body_type = method_description.get('request') - if body_type is None: - request_type = None - else: - request_type = self.__GetRequestType(body_type) - if self.__NeedRequestType(method_description, request_type): - request = self.__CreateRequestType( - method_description, body_type=body_type) - request_field = self.__GetRequestField( - method_description, body_type) - else: - request = request_type - request_field = base_api.REQUEST_IS_BODY - - if 'response' in method_description: - response = method_description['response']['$ref'] - else: - response = self.__CreateVoidResponseType(method_description) - - method_info_map[method_name] = self.__ComputeMethodInfo( - method_description, request, response, request_field) - self.__command_registry.AddCommandForMethod( - service_name, method_name, method_info_map[method_name], - request, response) - - nested_services = methods.get('resources', {}) - services = sorted(nested_services.items()) - for subservice_name, submethods in services: - new_service_name = '%s_%s' % (service_name, subservice_name) - self.AddServiceFromResource(new_service_name, submethods) - - self.__RegisterService(service_name, method_info_map) + printer('"""Generated client library for service %s version %s."""', + client_info.package, client_info.version) + printer() + printer('MESSAGES_MODULE = messages') + printer() + client_info_items = client_info._asdict( + ).items() # pylint:disable=protected-access + for attr, val in client_info_items: + if attr == 'scopes' and not val: + val = ['https://www.googleapis.com/auth/userinfo.email'] + printer('_%s = %r' % (attr.upper(), val)) + printer() + printer("def __init__(self, url='', credentials=None,") + with printer.Indent(indent=' '): + printer('get_credentials=True, http=None, model=None,') + printer('log_request=False, log_response=False,') + printer('credentials_args=None, default_global_params=None,') + printer('additional_http_headers=None):') + with printer.Indent(): + printer('"""Create a new %s handle."""', client_info.package) + printer('url = url or %r', self.__base_url) + printer( + 'super(%s, self).__init__(', client_info.client_class_name) + printer(' url, credentials=credentials,') + printer( + ' get_credentials=get_credentials, http=http, model=model,') + printer( + ' log_request=log_request, log_response=log_response,') + printer(' credentials_args=credentials_args,') + printer(' default_global_params=default_global_params,') + printer(' additional_http_headers=additional_http_headers)') + for name in self.__service_method_info_map.keys(): + printer('self.%s = self.%s(self)', + name, self.__GetServiceClassName(name)) + for name, method_info_map in self.__service_method_info_map.items(): + self.__WriteSingleService( + printer, name, method_info_map, client_info.client_class_name) + + def __RegisterService(self, service_name, method_info_map): + if service_name in self.__service_method_info_map: + raise ValueError( + 'Attempt to re-register descriptor %s' % service_name) + self.__service_method_info_map[service_name] = method_info_map + + def __CreateRequestType(self, method_description, body_type=None): + """Create a request type for this method.""" + schema = {} + schema['id'] = self.__names.ClassName('%sRequest' % ( + self.__names.ClassName(method_description['id'], separator='.'),)) + schema['type'] = 'object' + schema['properties'] = collections.OrderedDict() + if 'parameterOrder' not in method_description: + ordered_parameters = list(method_description.get('parameters', [])) + else: + ordered_parameters = method_description['parameterOrder'][:] + for k in method_description['parameters']: + if k not in ordered_parameters: + ordered_parameters.append(k) + for parameter_name in ordered_parameters: + field_name = self.__names.CleanName(parameter_name) + field = dict(method_description['parameters'][parameter_name]) + if 'type' not in field: + raise ValueError('No type found in parameter %s' % field) + schema['properties'][field_name] = field + if body_type is not None: + body_field_name = self.__GetRequestField( + method_description, body_type) + if body_field_name in schema['properties']: + raise ValueError('Failed to normalize request resource name') + if 'description' not in body_type: + body_type['description'] = ( + 'A %s resource to be passed as the request body.' % ( + self.__GetRequestType(body_type),)) + schema['properties'][body_field_name] = body_type + self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) + return schema['id'] + + def __CreateVoidResponseType(self, method_description): + """Create an empty response type.""" + schema = {} + method_name = self.__names.ClassName( + method_description['id'], separator='.') + schema['id'] = self.__names.ClassName('%sResponse' % method_name) + schema['type'] = 'object' + schema['description'] = 'An empty %s response.' % method_name + self.__message_registry.AddDescriptorFromSchema(schema['id'], schema) + return schema['id'] + + def __NeedRequestType(self, method_description, request_type): + """Determine if this method needs a new request type created.""" + if not request_type: + return True + if method_description.get('id', '') in self.__unelidable_request_methods: + return True + message = self.__message_registry.LookupDescriptorOrDie(request_type) + if message is None: + return True + field_names = [x.name for x in message.fields] + parameters = method_description.get('parameters', {}) + for param_name, param_info in parameters.items(): + if (param_info.get('location') != 'path' or + self.__names.CleanName(param_name) not in field_names): + break + else: + return False + return True + + def __MaxSizeToInt(self, max_size): + """Convert max_size to an int.""" + size_groups = re.match(r'(?P\d+)(?P.B)?$', max_size) + if size_groups is None: + raise ValueError('Could not parse maxSize') + size, unit = size_groups.group('size', 'unit') + shift = 0 + if unit is not None: + unit_dict = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40} + shift = unit_dict.get(unit.upper()) + if shift is None: + raise ValueError('Unknown unit %s' % unit) + return int(size) * (1 << shift) + + def __ComputeUploadConfig(self, media_upload_config, method_id): + """Fill out the upload config for this method.""" + config = base_api.ApiUploadInfo() + if 'maxSize' in media_upload_config: + config.max_size = self.__MaxSizeToInt( + media_upload_config['maxSize']) + if 'accept' not in media_upload_config: + logging.warn( + 'No accept types found for upload configuration in ' + 'method %s, using */*', method_id) + config.accept.extend([ + str(a) for a in media_upload_config.get('accept', '*/*')]) + + for accept_pattern in config.accept: + if not _MIME_PATTERN_RE.match(accept_pattern): + logging.warn('Unexpected MIME type: %s', accept_pattern) + protocols = media_upload_config.get('protocols', {}) + for protocol in ('simple', 'resumable'): + media = protocols.get(protocol, {}) + for attr in ('multipart', 'path'): + if attr in media: + setattr(config, '%s_%s' % (protocol, attr), media[attr]) + return config + + def __ComputeMethodInfo(self, method_description, request, response, + request_field): + """Compute the base_api.ApiMethodInfo for this method.""" + relative_path = self.__names.NormalizeRelativePath( + ''.join((self.__base_path, method_description['path']))) + method_id = method_description['id'] + ordered_params = [] + for param_name in method_description.get('parameterOrder', []): + if method_description['parameters'][param_name].get('required', False): + ordered_params.append(param_name) + method_info = base_api.ApiMethodInfo( + relative_path=relative_path, + method_id=method_id, + http_method=method_description['httpMethod'], + description=util.CleanDescription( + method_description.get('description', '')), + query_params=[], + path_params=[], + ordered_params=ordered_params, + request_type_name=self.__names.ClassName(request), + response_type_name=self.__names.ClassName(response), + request_field=request_field, + ) + if method_description.get('supportsMediaUpload', False): + method_info.upload_config = self.__ComputeUploadConfig( + method_description.get('mediaUpload'), method_id) + method_info.supports_download = method_description.get( + 'supportsMediaDownload', False) + self.__all_scopes.update(method_description.get('scopes', ())) + for param, desc in method_description.get('parameters', {}).items(): + param = self.__names.CleanName(param) + location = desc['location'] + if location == 'query': + method_info.query_params.append(param) + elif location == 'path': + method_info.path_params.append(param) + else: + raise ValueError('Unknown parameter location %s for parameter %s' % ( + location, param)) + method_info.path_params.sort() + method_info.query_params.sort() + return method_info + + def __BodyFieldName(self, body_type): + if body_type is None: + return '' + return self.__names.FieldName(body_type['$ref']) + + def __GetRequestType(self, body_type): + return self.__names.ClassName(body_type.get('$ref')) + + def __GetRequestField(self, method_description, body_type): + """Determine the request field for this method.""" + body_field_name = self.__BodyFieldName(body_type) + if body_field_name in method_description.get('parameters', {}): + body_field_name = self.__names.FieldName( + '%s_resource' % body_field_name) + # It's exceedingly unlikely that we'd get two name collisions, which + # means it's bound to happen at some point. + while body_field_name in method_description.get('parameters', {}): + body_field_name = self.__names.FieldName( + '%s_body' % body_field_name) + return body_field_name + + def AddServiceFromResource(self, service_name, methods): + """Add a new service named service_name with the given methods.""" + method_descriptions = methods.get('methods', {}) + method_info_map = collections.OrderedDict() + items = sorted(method_descriptions.items()) + for method_name, method_description in items: + method_name = self.__names.MethodName(method_name) + + # NOTE: According to the discovery document, if the request or + # response is present, it will simply contain a `$ref`. + body_type = method_description.get('request') + if body_type is None: + request_type = None + else: + request_type = self.__GetRequestType(body_type) + if self.__NeedRequestType(method_description, request_type): + request = self.__CreateRequestType( + method_description, body_type=body_type) + request_field = self.__GetRequestField( + method_description, body_type) + else: + request = request_type + request_field = base_api.REQUEST_IS_BODY + + if 'response' in method_description: + response = method_description['response']['$ref'] + else: + response = self.__CreateVoidResponseType(method_description) + + method_info_map[method_name] = self.__ComputeMethodInfo( + method_description, request, response, request_field) + self.__command_registry.AddCommandForMethod( + service_name, method_name, method_info_map[method_name], + request, response) + + nested_services = methods.get('resources', {}) + services = sorted(nested_services.items()) + for subservice_name, submethods in services: + new_service_name = '%s_%s' % (service_name, subservice_name) + self.AddServiceFromResource(new_service_name, submethods) + + self.__RegisterService(service_name, method_info_map) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index c6f4afb..b3e6c1c 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -14,289 +14,292 @@ import urllib2 import six - class Error(Exception): - """Base error for apitools generation.""" + + """Base error for apitools generation.""" class CommunicationError(Error): - """Error in network communication.""" + + """Error in network communication.""" def _SortLengthFirst(a, b): - return -cmp(len(a), len(b)) or cmp(a, b) + return -cmp(len(a), len(b)) or cmp(a, b) class Names(object): - """Utility class for cleaning and normalizing names in a fixed style.""" - DEFAULT_NAME_CONVENTION = 'LOWER_CAMEL' - NAME_CONVENTIONS = ['LOWER_CAMEL', 'LOWER_WITH_UNDER', 'NONE'] - - def __init__(self, strip_prefixes, - name_convention=None, - capitalize_enums=False): - self.__strip_prefixes = sorted(strip_prefixes, cmp=_SortLengthFirst) - self.__name_convention = name_convention or self.DEFAULT_NAME_CONVENTION - self.__capitalize_enums = capitalize_enums - - @staticmethod - def __FromCamel(name, separator='_'): - name = re.sub(r'([a-z0-9])([A-Z])', r'\1%s\2' % separator, name) - return name.lower() - - @staticmethod - def __ToCamel(name, separator='_'): - # TODO(craigcitro): Consider what to do about leading or trailing - # underscores (such as `_refValue` in discovery). - return ''.join(s[0:1].upper() + s[1:] for s in name.split(separator)) - - @staticmethod - def __ToLowerCamel(name, separator='_'): - name = Names.__ToCamel(name, separator=separator) - return name[0].lower() + name[1:] - - def __StripName(self, name): - """Strip strip_prefix entries from name.""" - if not name: - return name - for prefix in self.__strip_prefixes: - if name.startswith(prefix): - return name[len(prefix):] - return name - - @staticmethod - def CleanName(name): - """Perform generic name cleaning.""" - name = re.sub('[^_A-Za-z0-9]', '_', name) - if name[0].isdigit(): - name = '_%s' % name - while keyword.iskeyword(name): - name = '%s_' % name - # If we end up with __ as a prefix, we'll run afoul of python - # field renaming, so we manually correct for it. - if name.startswith('__'): - name = 'f%s' % name - return name - - @staticmethod - def NormalizeRelativePath(path): - """Normalize camelCase entries in path.""" - path_components = path.split('/') - normalized_components = [] - for component in path_components: - if re.match(r'{[A-Za-z0-9_]+}$', component): - normalized_components.append( - '{%s}' % Names.CleanName(component[1:-1])) - else: - normalized_components.append(component) - return '/'.join(normalized_components) - - def NormalizeEnumName(self, enum_name): - if self.__capitalize_enums: - enum_name = enum_name.upper() - return self.CleanName(enum_name) - - def ClassName(self, name, separator='_'): - """Generate a valid class name from name.""" - # TODO(craigcitro): Get rid of this case here and in MethodName. - if name is None: - return name - # TODO(craigcitro): This is a hack to handle the case of specific - # protorpc class names; clean this up. - if name.startswith('protorpc.') or name.startswith('message_types.'): - return name - name = self.__StripName(name) - name = self.__ToCamel(name, separator=separator) - return self.CleanName(name) - - def MethodName(self, name, separator='_'): - """Generate a valid method name from name.""" - if name is None: - return None - name = Names.__ToCamel(name, separator=separator) - return Names.CleanName(name) - - def FieldName(self, name): - """Generate a valid field name from name.""" - # TODO(craigcitro): We shouldn't need to strip this name, but some - # of the service names here are excessive. Fix the API and then - # remove this. - name = self.__StripName(name) - if self.__name_convention == 'LOWER_CAMEL': - name = Names.__ToLowerCamel(name) - elif self.__name_convention == 'LOWER_WITH_UNDER': - name = Names.__FromCamel(name) - return Names.CleanName(name) + + """Utility class for cleaning and normalizing names in a fixed style.""" + DEFAULT_NAME_CONVENTION = 'LOWER_CAMEL' + NAME_CONVENTIONS = ['LOWER_CAMEL', 'LOWER_WITH_UNDER', 'NONE'] + + def __init__(self, strip_prefixes, + name_convention=None, + capitalize_enums=False): + self.__strip_prefixes = sorted(strip_prefixes, cmp=_SortLengthFirst) + self.__name_convention = name_convention or self.DEFAULT_NAME_CONVENTION + self.__capitalize_enums = capitalize_enums + + @staticmethod + def __FromCamel(name, separator='_'): + name = re.sub(r'([a-z0-9])([A-Z])', r'\1%s\2' % separator, name) + return name.lower() + + @staticmethod + def __ToCamel(name, separator='_'): + # TODO(craigcitro): Consider what to do about leading or trailing + # underscores (such as `_refValue` in discovery). + return ''.join(s[0:1].upper() + s[1:] for s in name.split(separator)) + + @staticmethod + def __ToLowerCamel(name, separator='_'): + name = Names.__ToCamel(name, separator=separator) + return name[0].lower() + name[1:] + + def __StripName(self, name): + """Strip strip_prefix entries from name.""" + if not name: + return name + for prefix in self.__strip_prefixes: + if name.startswith(prefix): + return name[len(prefix):] + return name + + @staticmethod + def CleanName(name): + """Perform generic name cleaning.""" + name = re.sub('[^_A-Za-z0-9]', '_', name) + if name[0].isdigit(): + name = '_%s' % name + while keyword.iskeyword(name): + name = '%s_' % name + # If we end up with __ as a prefix, we'll run afoul of python + # field renaming, so we manually correct for it. + if name.startswith('__'): + name = 'f%s' % name + return name + + @staticmethod + def NormalizeRelativePath(path): + """Normalize camelCase entries in path.""" + path_components = path.split('/') + normalized_components = [] + for component in path_components: + if re.match(r'{[A-Za-z0-9_]+}$', component): + normalized_components.append( + '{%s}' % Names.CleanName(component[1:-1])) + else: + normalized_components.append(component) + return '/'.join(normalized_components) + + def NormalizeEnumName(self, enum_name): + if self.__capitalize_enums: + enum_name = enum_name.upper() + return self.CleanName(enum_name) + + def ClassName(self, name, separator='_'): + """Generate a valid class name from name.""" + # TODO(craigcitro): Get rid of this case here and in MethodName. + if name is None: + return name + # TODO(craigcitro): This is a hack to handle the case of specific + # protorpc class names; clean this up. + if name.startswith('protorpc.') or name.startswith('message_types.'): + return name + name = self.__StripName(name) + name = self.__ToCamel(name, separator=separator) + return self.CleanName(name) + + def MethodName(self, name, separator='_'): + """Generate a valid method name from name.""" + if name is None: + return None + name = Names.__ToCamel(name, separator=separator) + return Names.CleanName(name) + + def FieldName(self, name): + """Generate a valid field name from name.""" + # TODO(craigcitro): We shouldn't need to strip this name, but some + # of the service names here are excessive. Fix the API and then + # remove this. + name = self.__StripName(name) + if self.__name_convention == 'LOWER_CAMEL': + name = Names.__ToLowerCamel(name) + elif self.__name_convention == 'LOWER_WITH_UNDER': + name = Names.__FromCamel(name) + return Names.CleanName(name) @contextlib.contextmanager def Chdir(dirname, create=True): - if not os.path.exists(dirname): - if not create: - raise OSError('Cannot find directory %s' % dirname) - else: - os.mkdir(dirname) - previous_directory = os.getcwd() - os.chdir(dirname) - yield - os.chdir(previous_directory) + if not os.path.exists(dirname): + if not create: + raise OSError('Cannot find directory %s' % dirname) + else: + os.mkdir(dirname) + previous_directory = os.getcwd() + os.chdir(dirname) + yield + os.chdir(previous_directory) def NormalizeVersion(version): - # Currently, '.' is the only character that might cause us trouble. - return version.replace('.', '_') + # Currently, '.' is the only character that might cause us trouble. + return version.replace('.', '_') class ClientInfo(collections.namedtuple('ClientInfo', ( - 'package', 'scopes', 'version', 'client_id', 'client_secret', - 'user_agent', 'client_class_name', 'url_version', 'api_key'))): - """Container for client-related info and names.""" - - @classmethod - def Create(cls, discovery_doc, - scope_ls, client_id, client_secret, user_agent, names, api_key): - """Create a new ClientInfo object from a discovery document.""" - scopes = set( - discovery_doc.get('auth', {}).get('oauth2', {}).get('scopes', {})) - scopes.update(scope_ls) - client_info = { - 'package': discovery_doc['name'], - 'version': NormalizeVersion(discovery_doc['version']), - 'url_version': discovery_doc['version'], - 'scopes': sorted(list(scopes)), - 'client_id': client_id, - 'client_secret': client_secret, - 'user_agent': user_agent, - 'api_key': api_key, - } - client_class_name = ''.join( - map(names.ClassName, (client_info['package'], client_info['version']))) - client_info['client_class_name'] = client_class_name - return cls(**client_info) - - @property - def default_directory(self): - return self.package - - @property - def cli_rule_name(self): - return '%s_%s' % (self.package, self.version) - - @property - def cli_file_name(self): - return '%s.py' % self.cli_rule_name - - @property - def client_rule_name(self): - return '%s_%s_client' % (self.package, self.version) - - @property - def client_file_name(self): - return '%s.py' % self.client_rule_name - - @property - def messages_rule_name(self): - return '%s_%s_messages' % (self.package, self.version) - - @property - def services_rule_name(self): - return '%s_%s_services' % (self.package, self.version) - - @property - def messages_file_name(self): - return '%s.py' % self.messages_rule_name - - @property - def messages_proto_file_name(self): - return '%s.proto' % self.messages_rule_name - - @property - def services_proto_file_name(self): - return '%s.proto' % self.services_rule_name + 'package', 'scopes', 'version', 'client_id', 'client_secret', + 'user_agent', 'client_class_name', 'url_version', 'api_key'))): + + """Container for client-related info and names.""" + + @classmethod + def Create(cls, discovery_doc, + scope_ls, client_id, client_secret, user_agent, names, api_key): + """Create a new ClientInfo object from a discovery document.""" + scopes = set( + discovery_doc.get('auth', {}).get('oauth2', {}).get('scopes', {})) + scopes.update(scope_ls) + client_info = { + 'package': discovery_doc['name'], + 'version': NormalizeVersion(discovery_doc['version']), + 'url_version': discovery_doc['version'], + 'scopes': sorted(list(scopes)), + 'client_id': client_id, + 'client_secret': client_secret, + 'user_agent': user_agent, + 'api_key': api_key, + } + client_class_name = ''.join( + map(names.ClassName, (client_info['package'], client_info['version']))) + client_info['client_class_name'] = client_class_name + return cls(**client_info) + + @property + def default_directory(self): + return self.package + + @property + def cli_rule_name(self): + return '%s_%s' % (self.package, self.version) + + @property + def cli_file_name(self): + return '%s.py' % self.cli_rule_name + + @property + def client_rule_name(self): + return '%s_%s_client' % (self.package, self.version) + + @property + def client_file_name(self): + return '%s.py' % self.client_rule_name + + @property + def messages_rule_name(self): + return '%s_%s_messages' % (self.package, self.version) + + @property + def services_rule_name(self): + return '%s_%s_services' % (self.package, self.version) + + @property + def messages_file_name(self): + return '%s.py' % self.messages_rule_name + + @property + def messages_proto_file_name(self): + return '%s.proto' % self.messages_rule_name + + @property + def services_proto_file_name(self): + return '%s.proto' % self.services_rule_name def GetPackage(path): - path_components = path.split(os.path.sep) - return '.'.join(path_components) + path_components = path.split(os.path.sep) + return '.'.join(path_components) def CleanDescription(description): - """Return a version of description safe for printing in a docstring.""" - if not isinstance(description, six.string_types): - return description - return description.replace('"""', '" " "') + """Return a version of description safe for printing in a docstring.""" + if not isinstance(description, six.string_types): + return description + return description.replace('"""', '" " "') class SimplePrettyPrinter(object): - """Simple pretty-printer that supports an indent contextmanager.""" - - def __init__(self, out): - self.__out = out - self.__indent = '' - self.__skip = False - self.__comment_context = False - - @property - def indent(self): - return self.__indent - - def CalculateWidth(self, max_width=78): - return max_width - len(self.indent) - - @contextlib.contextmanager - def Indent(self, indent=' '): - previous_indent = self.__indent - self.__indent = '%s%s' % (previous_indent, indent) - yield - self.__indent = previous_indent - - @contextlib.contextmanager - def CommentContext(self): - """Print without any argument formatting.""" - old_context = self.__comment_context - self.__comment_context = True - yield - self.__comment_context = old_context - - def __call__(self, *args): - if self.__comment_context and args[1:]: - raise Error('Cannot do string interpolation in comment context') - if args and args[0]: - if not self.__comment_context: - line = (args[0] % args[1:]).rstrip() - else: - line = args[0].rstrip() - line = line.encode('ascii', 'backslashreplace') - print('%s%s' % (self.__indent, line), file=self.__out) - else: - print('', file=self.__out) + """Simple pretty-printer that supports an indent contextmanager.""" + + def __init__(self, out): + self.__out = out + self.__indent = '' + self.__skip = False + self.__comment_context = False + + @property + def indent(self): + return self.__indent + + def CalculateWidth(self, max_width=78): + return max_width - len(self.indent) + + @contextlib.contextmanager + def Indent(self, indent=' '): + previous_indent = self.__indent + self.__indent = '%s%s' % (previous_indent, indent) + yield + self.__indent = previous_indent + + @contextlib.contextmanager + def CommentContext(self): + """Print without any argument formatting.""" + old_context = self.__comment_context + self.__comment_context = True + yield + self.__comment_context = old_context + + def __call__(self, *args): + if self.__comment_context and args[1:]: + raise Error('Cannot do string interpolation in comment context') + if args and args[0]: + if not self.__comment_context: + line = (args[0] % args[1:]).rstrip() + else: + line = args[0].rstrip() + line = line.encode('ascii', 'backslashreplace') + print('%s%s' % (self.__indent, line), file=self.__out) + else: + print('', file=self.__out) def NormalizeDiscoveryUrl(discovery_url): - """Expands a few abbreviations into full discovery urls.""" - if discovery_url.startswith('http'): - return discovery_url - elif '.' not in discovery_url: - raise ValueError('Unrecognized value "%s" for discovery url') - api_name, _, api_version = discovery_url.partition('.') - return 'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % ( - api_name, api_version) + """Expands a few abbreviations into full discovery urls.""" + if discovery_url.startswith('http'): + return discovery_url + elif '.' not in discovery_url: + raise ValueError('Unrecognized value "%s" for discovery url') + api_name, _, api_version = discovery_url.partition('.') + return 'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % ( + api_name, api_version) def FetchDiscoveryDoc(discovery_url, retries=5): - """Fetch the discovery document at the given url.""" - discovery_url = NormalizeDiscoveryUrl(discovery_url) - discovery_doc = None - last_exception = None - for _ in range(retries): - try: - discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) - break - except (urllib2.HTTPError, urllib2.URLError) as last_exception: - logging.warning('Attempting to fetch discovery doc again after "%s"', - last_exception) - if discovery_doc is None: - raise CommunicationError('Could not find discovery doc at url "%s": %s' % ( - discovery_url, last_exception)) - return discovery_doc + """Fetch the discovery document at the given url.""" + discovery_url = NormalizeDiscoveryUrl(discovery_url) + discovery_doc = None + last_exception = None + for _ in range(retries): + try: + discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) + break + except (urllib2.HTTPError, urllib2.URLError) as last_exception: + logging.warning('Attempting to fetch discovery doc again after "%s"', + last_exception) + if discovery_doc is None: + raise CommunicationError('Could not find discovery doc at url "%s": %s' % ( + discovery_url, last_exception)) + return discovery_doc diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index e9ce36a..4044d80 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -14,162 +14,165 @@ import apitools.base.py as apitools_base import storage _CLIENT = None + + def _GetClient(): - global _CLIENT - if _CLIENT is None: - _CLIENT = storage.StorageV1() - return _CLIENT + global _CLIENT + if _CLIENT is None: + _CLIENT = storage.StorageV1() + return _CLIENT + class DownloadsTest(unittest.TestCase): - _DEFAULT_BUCKET = 'apitools' - _TESTDATA_PREFIX = 'testdata' - - def setUp(self): - self.__client = _GetClient() - self.__ResetDownload() - - def __ResetDownload(self, auto_transfer=False): - self.__buffer = io.StringIO() - self.__download = storage.Download.FromStream( - self.__buffer, auto_transfer=auto_transfer) - - def __GetTestdataFileContents(self, filename): - file_contents = open('testdata/%s' % filename).read() - self.assertIsNotNone( - file_contents, msg=('Could not read file %s' % filename)) - return file_contents - - @classmethod - def __GetRequest(cls, filename): - object_name = os.path.join(cls._TESTDATA_PREFIX, filename) - return storage.StorageObjectsGetRequest( - bucket=cls._DEFAULT_BUCKET, object=object_name) - - def __GetFile(self, request): - response = self.__client.objects.Get(request, download=self.__download) - self.assertIsNone(response, msg=( - 'Unexpected nonempty response for file download: %s' % response)) - - def __GetAndStream(self, request): - self.__GetFile(request) - self.__download.StreamInChunks() - - def testZeroBytes(self): - request = self.__GetRequest('zero_byte_file') - self.__GetAndStream(request) - self.assertEqual(0, self.__buffer.tell()) - - def testObjectDoesNotExist(self): - with self.assertRaises(apitools_base.HttpError): - self.__GetFile(self.__GetRequest('nonexistent_file')) - - def testAutoTransfer(self): - self.__ResetDownload(auto_transfer=True) - self.__GetFile(self.__GetRequest('fifteen_byte_file')) - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - self.assertEqual(15, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents, self.__buffer.read()) - - def testFilenameWithSpaces(self): - self.__ResetDownload(auto_transfer=True) - self.__GetFile(self.__GetRequest('filename with spaces')) - # NOTE(craigcitro): We add _ here to make this play nice with blaze. - file_contents = self.__GetTestdataFileContents('filename_with_spaces') - self.assertEqual(15, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents, self.__buffer.read()) - - def testGetRange(self): - # TODO(craigcitro): Test about a thousand more corner cases. - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - self.__GetFile(self.__GetRequest('fifteen_byte_file')) - self.__download.GetRange(5, 10) - self.assertEqual(6, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents[5:11], self.__buffer.read()) - - def testGetRangeWithNegativeStart(self): - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - self.__GetFile(self.__GetRequest('fifteen_byte_file')) - self.__download.GetRange(-3) - self.assertEqual(3, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents[-3:], self.__buffer.read()) - - def testGetRangeWithPositiveStart(self): - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - self.__GetFile(self.__GetRequest('fifteen_byte_file')) - self.__download.GetRange(2) - self.assertEqual(13, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents[2:15], self.__buffer.read()) - - def testSmallChunksizes(self): - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - request = self.__GetRequest('fifteen_byte_file') - for chunksize in (2, 3, 15, 100): - self.__ResetDownload() - self.__download.chunksize = chunksize - self.__GetAndStream(request) - self.assertEqual(15, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents, self.__buffer.read(15)) - - def testLargeFileChunksizes(self): - request = self.__GetRequest('thirty_meg_file') - for chunksize in (1048576, 40 * 1048576): - self.__ResetDownload() - self.__download.chunksize = chunksize - self.__GetAndStream(request) - self.__buffer.seek(0) - - def testAutoGzipObject(self): - # TODO(craigcitro): Move this to a new object once we have a more - # permanent one, see: http://b/12250275 - request = storage.StorageObjectsGetRequest( - bucket='ottenl-gzip', object='50K.txt') - # First, try without auto-transfer. - self.__GetFile(request) - self.assertEqual(0, self.__buffer.tell()) - self.__download.StreamInChunks() - self.assertEqual(50000, self.__buffer.tell()) - # Next, try with auto-transfer. - self.__ResetDownload(auto_transfer=True) - self.__GetFile(request) - self.assertEqual(50000, self.__buffer.tell()) - - def testSmallGzipObject(self): - request = self.__GetRequest('zero-gzipd.html') - self.__GetFile(request) - self.assertEqual(0, self.__buffer.tell()) - additional_headers = {'accept-encoding': 'gzip, deflate'} - self.__download.StreamInChunks(additional_headers=additional_headers) - self.assertEqual(0, self.__buffer.tell()) - - def testSerializedDownload(self): - - def _ProgressCallback(unused_response, download_object): - print 'Progress %s' % download_object.progress - - file_contents = self.__GetTestdataFileContents('fifteen_byte_file') - object_name = os.path.join(self._TESTDATA_PREFIX, 'fifteen_byte_file') - request = storage.StorageObjectsGetRequest( - bucket=self._DEFAULT_BUCKET, object=object_name) - response = self.__client.objects.Get(request) - self.__buffer = io.StringIO() - download_data = json.dumps({ - 'auto_transfer': False, - 'progress': 0, - 'total_size': response.size, - 'url': response.mediaLink, - }) - self.__download = storage.Download.FromData(self.__buffer, download_data, - http=self.__client.http) - self.__download.StreamInChunks(callback=_ProgressCallback) - self.assertEqual(15, self.__buffer.tell()) - self.__buffer.seek(0) - self.assertEqual(file_contents, self.__buffer.read(15)) + _DEFAULT_BUCKET = 'apitools' + _TESTDATA_PREFIX = 'testdata' + + def setUp(self): + self.__client = _GetClient() + self.__ResetDownload() + + def __ResetDownload(self, auto_transfer=False): + self.__buffer = io.StringIO() + self.__download = storage.Download.FromStream( + self.__buffer, auto_transfer=auto_transfer) + + def __GetTestdataFileContents(self, filename): + file_contents = open('testdata/%s' % filename).read() + self.assertIsNotNone( + file_contents, msg=('Could not read file %s' % filename)) + return file_contents + + @classmethod + def __GetRequest(cls, filename): + object_name = os.path.join(cls._TESTDATA_PREFIX, filename) + return storage.StorageObjectsGetRequest( + bucket=cls._DEFAULT_BUCKET, object=object_name) + + def __GetFile(self, request): + response = self.__client.objects.Get(request, download=self.__download) + self.assertIsNone(response, msg=( + 'Unexpected nonempty response for file download: %s' % response)) + + def __GetAndStream(self, request): + self.__GetFile(request) + self.__download.StreamInChunks() + + def testZeroBytes(self): + request = self.__GetRequest('zero_byte_file') + self.__GetAndStream(request) + self.assertEqual(0, self.__buffer.tell()) + + def testObjectDoesNotExist(self): + with self.assertRaises(apitools_base.HttpError): + self.__GetFile(self.__GetRequest('nonexistent_file')) + + def testAutoTransfer(self): + self.__ResetDownload(auto_transfer=True) + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read()) + + def testFilenameWithSpaces(self): + self.__ResetDownload(auto_transfer=True) + self.__GetFile(self.__GetRequest('filename with spaces')) + # NOTE(craigcitro): We add _ here to make this play nice with blaze. + file_contents = self.__GetTestdataFileContents('filename_with_spaces') + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read()) + + def testGetRange(self): + # TODO(craigcitro): Test about a thousand more corner cases. + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(5, 10) + self.assertEqual(6, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[5:11], self.__buffer.read()) + + def testGetRangeWithNegativeStart(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(-3) + self.assertEqual(3, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[-3:], self.__buffer.read()) + + def testGetRangeWithPositiveStart(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + self.__GetFile(self.__GetRequest('fifteen_byte_file')) + self.__download.GetRange(2) + self.assertEqual(13, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents[2:15], self.__buffer.read()) + + def testSmallChunksizes(self): + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + request = self.__GetRequest('fifteen_byte_file') + for chunksize in (2, 3, 15, 100): + self.__ResetDownload() + self.__download.chunksize = chunksize + self.__GetAndStream(request) + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read(15)) + + def testLargeFileChunksizes(self): + request = self.__GetRequest('thirty_meg_file') + for chunksize in (1048576, 40 * 1048576): + self.__ResetDownload() + self.__download.chunksize = chunksize + self.__GetAndStream(request) + self.__buffer.seek(0) + + def testAutoGzipObject(self): + # TODO(craigcitro): Move this to a new object once we have a more + # permanent one, see: http://b/12250275 + request = storage.StorageObjectsGetRequest( + bucket='ottenl-gzip', object='50K.txt') + # First, try without auto-transfer. + self.__GetFile(request) + self.assertEqual(0, self.__buffer.tell()) + self.__download.StreamInChunks() + self.assertEqual(50000, self.__buffer.tell()) + # Next, try with auto-transfer. + self.__ResetDownload(auto_transfer=True) + self.__GetFile(request) + self.assertEqual(50000, self.__buffer.tell()) + + def testSmallGzipObject(self): + request = self.__GetRequest('zero-gzipd.html') + self.__GetFile(request) + self.assertEqual(0, self.__buffer.tell()) + additional_headers = {'accept-encoding': 'gzip, deflate'} + self.__download.StreamInChunks(additional_headers=additional_headers) + self.assertEqual(0, self.__buffer.tell()) + + def testSerializedDownload(self): + + def _ProgressCallback(unused_response, download_object): + print 'Progress %s' % download_object.progress + + file_contents = self.__GetTestdataFileContents('fifteen_byte_file') + object_name = os.path.join(self._TESTDATA_PREFIX, 'fifteen_byte_file') + request = storage.StorageObjectsGetRequest( + bucket=self._DEFAULT_BUCKET, object=object_name) + response = self.__client.objects.Get(request) + self.__buffer = io.StringIO() + download_data = json.dumps({ + 'auto_transfer': False, + 'progress': 0, + 'total_size': response.size, + 'url': response.mediaLink, + }) + self.__download = storage.Download.FromData(self.__buffer, download_data, + http=self.__client.http) + self.__download.StreamInChunks(callback=_ProgressCallback) + self.assertEqual(15, self.__buffer.tell()) + self.__buffer.seek(0) + self.assertEqual(file_contents, self.__buffer.read(15)) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index 9c9dcf8..0d25a30 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -15,128 +15,131 @@ import apitools.base.py as apitools_base import storage _CLIENT = None + + def _GetClient(): - global _CLIENT - if _CLIENT is None: - _CLIENT = storage.StorageV1() - return _CLIENT + global _CLIENT + if _CLIENT is None: + _CLIENT = storage.StorageV1() + return _CLIENT + class UploadsTest(unittest.TestCase): - _DEFAULT_BUCKET = 'apitools' - _TESTDATA_PREFIX = 'uploads' - - def setUp(self): - self.__client = _GetClient() - self.__files = [] - self.__content = '' - self.__buffer = None - self.__upload = None - - def tearDown(self): - self.__DeleteFiles() - - def __ResetUpload(self, size, auto_transfer=True): - self.__content = ''.join( - random.choice(string.ascii_letters) for _ in range(size)) - self.__buffer = io.StringIO(self.__content) - self.__upload = storage.Upload.FromStream( - self.__buffer, 'text/plain', auto_transfer=auto_transfer) - - def __DeleteFiles(self): - for filename in self.__files: - self.__DeleteFile(filename) - - def __DeleteFile(self, filename): - object_name = os.path.join(self._TESTDATA_PREFIX, filename) - req = storage.StorageObjectsDeleteRequest( - bucket=self._DEFAULT_BUCKET, object=object_name) - self.__client.objects.Delete(req) - - def __InsertRequest(self, filename): - object_name = os.path.join(self._TESTDATA_PREFIX, filename) - return storage.StorageObjectsInsertRequest( - name=object_name, bucket=self._DEFAULT_BUCKET) - - def __GetRequest(self, filename): - object_name = os.path.join(self._TESTDATA_PREFIX, filename) - return storage.StorageObjectsGetRequest( - object=object_name, bucket=self._DEFAULT_BUCKET) - - def __InsertFile(self, filename, request=None): - if request is None: - request = self.__InsertRequest(filename) - response = self.__client.objects.Insert(request, upload=self.__upload) - self.assertIsNotNone(response) - self.__files.append(filename) - return response - - def testZeroBytes(self): - filename = 'zero_byte_file' - self.__ResetUpload(0) - response = self.__InsertFile(filename) - self.assertEqual(0, response.size) - - def testSimpleUpload(self): - filename = 'fifteen_byte_file' - self.__ResetUpload(15) - response = self.__InsertFile(filename) - self.assertEqual(15, response.size) - - def testMultipartUpload(self): - filename = 'fifteen_byte_file' - self.__ResetUpload(15) - request = self.__InsertRequest(filename) - request.object = storage.Object(contentLanguage='en') - response = self.__InsertFile(filename, request=request) - self.assertEqual(15, response.size) - self.assertEqual('en', response.contentLanguage) - - def testAutoUpload(self): - filename = 'ten_meg_file' - size = 10 << 20 - self.__ResetUpload(size) - request = self.__InsertRequest(filename) - response = self.__InsertFile(filename, request=request) - self.assertEqual(size, response.size) - - def testBreakAndResumeUpload(self): - filename = ('ten_meg_file_' + - ''.join(random.sample(string.ascii_letters, 5))) - size = 10 << 20 - self.__ResetUpload(size, auto_transfer=False) - self.__upload.strategy = 'resumable' - self.__upload.total_size = size - # Start the upload - request = self.__InsertRequest(filename) - initial_response = self.__client.objects.Insert( - request, upload=self.__upload) - self.assertIsNotNone(initial_response) - self.assertEqual(0, self.__buffer.tell()) - # Pretend the process died, and resume with a new attempt at the - # same upload. - upload_data = json.dumps(self.__upload.serialization_data) - second_upload_attempt = apitools_base.Upload.FromData( - self.__buffer, upload_data, self.__upload.http) - second_upload_attempt._Upload__SendChunk(0) - self.assertEqual(second_upload_attempt.chunksize, self.__buffer.tell()) - # Simulate a third try, and stream from there. - final_upload_attempt = apitools_base.Upload.FromData( - self.__buffer, upload_data, self.__upload.http) - final_upload_attempt.StreamInChunks() - self.assertEqual(size, self.__buffer.tell()) - # Verify the upload - object_info = self.__client.objects.Get(self.__GetRequest(filename)) - self.assertEqual(size, object_info.size) - # Confirm that a new attempt successfully does nothing. - completed_upload_attempt = apitools_base.Upload.FromData( - self.__buffer, upload_data, self.__upload.http) - self.assertTrue(completed_upload_attempt.complete) - completed_upload_attempt.StreamInChunks() - # Verify the upload didn't pick up extra bytes. - object_info = self.__client.objects.Get(self.__GetRequest(filename)) - self.assertEqual(size, object_info.size) - # TODO(craigcitro): Add tests for callbacks (especially around - # finish callback). + _DEFAULT_BUCKET = 'apitools' + _TESTDATA_PREFIX = 'uploads' + + def setUp(self): + self.__client = _GetClient() + self.__files = [] + self.__content = '' + self.__buffer = None + self.__upload = None + + def tearDown(self): + self.__DeleteFiles() + + def __ResetUpload(self, size, auto_transfer=True): + self.__content = ''.join( + random.choice(string.ascii_letters) for _ in range(size)) + self.__buffer = io.StringIO(self.__content) + self.__upload = storage.Upload.FromStream( + self.__buffer, 'text/plain', auto_transfer=auto_transfer) + + def __DeleteFiles(self): + for filename in self.__files: + self.__DeleteFile(filename) + + def __DeleteFile(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + req = storage.StorageObjectsDeleteRequest( + bucket=self._DEFAULT_BUCKET, object=object_name) + self.__client.objects.Delete(req) + + def __InsertRequest(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + return storage.StorageObjectsInsertRequest( + name=object_name, bucket=self._DEFAULT_BUCKET) + + def __GetRequest(self, filename): + object_name = os.path.join(self._TESTDATA_PREFIX, filename) + return storage.StorageObjectsGetRequest( + object=object_name, bucket=self._DEFAULT_BUCKET) + + def __InsertFile(self, filename, request=None): + if request is None: + request = self.__InsertRequest(filename) + response = self.__client.objects.Insert(request, upload=self.__upload) + self.assertIsNotNone(response) + self.__files.append(filename) + return response + + def testZeroBytes(self): + filename = 'zero_byte_file' + self.__ResetUpload(0) + response = self.__InsertFile(filename) + self.assertEqual(0, response.size) + + def testSimpleUpload(self): + filename = 'fifteen_byte_file' + self.__ResetUpload(15) + response = self.__InsertFile(filename) + self.assertEqual(15, response.size) + + def testMultipartUpload(self): + filename = 'fifteen_byte_file' + self.__ResetUpload(15) + request = self.__InsertRequest(filename) + request.object = storage.Object(contentLanguage='en') + response = self.__InsertFile(filename, request=request) + self.assertEqual(15, response.size) + self.assertEqual('en', response.contentLanguage) + + def testAutoUpload(self): + filename = 'ten_meg_file' + size = 10 << 20 + self.__ResetUpload(size) + request = self.__InsertRequest(filename) + response = self.__InsertFile(filename, request=request) + self.assertEqual(size, response.size) + + def testBreakAndResumeUpload(self): + filename = ('ten_meg_file_' + + ''.join(random.sample(string.ascii_letters, 5))) + size = 10 << 20 + self.__ResetUpload(size, auto_transfer=False) + self.__upload.strategy = 'resumable' + self.__upload.total_size = size + # Start the upload + request = self.__InsertRequest(filename) + initial_response = self.__client.objects.Insert( + request, upload=self.__upload) + self.assertIsNotNone(initial_response) + self.assertEqual(0, self.__buffer.tell()) + # Pretend the process died, and resume with a new attempt at the + # same upload. + upload_data = json.dumps(self.__upload.serialization_data) + second_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + second_upload_attempt._Upload__SendChunk(0) + self.assertEqual(second_upload_attempt.chunksize, self.__buffer.tell()) + # Simulate a third try, and stream from there. + final_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + final_upload_attempt.StreamInChunks() + self.assertEqual(size, self.__buffer.tell()) + # Verify the upload + object_info = self.__client.objects.Get(self.__GetRequest(filename)) + self.assertEqual(size, object_info.size) + # Confirm that a new attempt successfully does nothing. + completed_upload_attempt = apitools_base.Upload.FromData( + self.__buffer, upload_data, self.__upload.http) + self.assertTrue(completed_upload_attempt.complete) + completed_upload_attempt.StreamInChunks() + # Verify the upload didn't pick up extra bytes. + object_info = self.__client.objects.Get(self.__GetRequest(filename)) + self.assertEqual(size, object_info.size) + # TODO(craigcitro): Add tests for callbacks (especially around + # finish callback). if __name__ == '__main__': - unittest.main() + unittest.main() -- GitLab From 1d74abfe833c39c8974818fc0d74b8896336c488 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 30 Apr 2015 14:00:18 -0700 Subject: [PATCH 098/295] Finish autopep8. --- apitools/base/py/app2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 629a476..e4f439b 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -62,7 +62,7 @@ class NewCmd(appcommands.Cmd): self._max_args = len(self._argspec.args or ()) self._min_args = self._max_args - len(self._argspec.defaults or ()) if self._star_args: - self._max_args = sys.maxint + self._max_args = sys.maxsize self._debug_mode = FLAGS.debug_mode self.surface_in_shell = True -- GitLab From 8d3fef6f8aa295f9fbf07a1c83639ddb63f9a0b9 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 30 Apr 2015 15:40:28 -0700 Subject: [PATCH 099/295] Fix long lines with autopep8. --- apitools/base/py/base_api.py | 12 ++++++++---- apitools/base/py/credentials_lib.py | 6 ++++-- apitools/base/py/encoding.py | 6 ++++-- apitools/base/py/extra_types.py | 3 ++- apitools/base/py/transfer.py | 12 ++++++++---- apitools/base/py/util.py | 3 ++- apitools/gen/command_registry.py | 3 ++- apitools/gen/extended_descriptor.py | 6 ++++-- apitools/gen/message_registry.py | 9 ++++++--- apitools/gen/service_registry.py | 9 ++++++--- 10 files changed, 46 insertions(+), 23 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index c6d7f9d..2580040 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -150,11 +150,13 @@ class _UrlBuilder(object): base_url = urllib.parse.urlunsplit(( urlparts.scheme, urlparts.netloc, '', None, None)) relative_path = urlparts.path or '' - return cls(base_url, relative_path=relative_path, query_params=query_params) + return cls( + base_url, relative_path=relative_path, query_params=query_params) @property def base_url(self): - return urllib.parse.urlunsplit((self.__scheme, self.__netloc, '', '', '')) + return urllib.parse.urlunsplit( + (self.__scheme, self.__netloc, '', '', '')) @base_url.setter def base_url(self, value): @@ -371,7 +373,8 @@ class BaseApiClient(object): # TODO(craigcitro): Decide where these two functions should live. def SerializeMessage(self, message): - return encoding.MessageToJson(message, include_fields=self.__include_fields) + return encoding.MessageToJson( + message, include_fields=self.__include_fields) def DeserializeMessage(self, response_type, data): """Deserialize the given data as method_config.response_type.""" @@ -476,7 +479,8 @@ class BaseApiService(object): query_info[k] = v.isoformat() return query_info - def __ConstructRelativePath(self, method_config, request, relative_path=None): + def __ConstructRelativePath( + self, method_config, request, relative_path=None): """Determine the relative path for request.""" python_param_names = util.MapParamNames( method_config.path_params, type(request)) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 61a13a5..bcef08e 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -67,7 +67,8 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, if service_account_json_keyfile: with open(service_account_json_keyfile) as keyfile: service_account_info = json.load(keyfile) - if service_account_info.get('type') != oauth2client.client.SERVICE_ACCOUNT: + if service_account_info.get( + 'type') != oauth2client.client.SERVICE_ACCOUNT: raise exceptions.CredentialsError( 'Invalid service account credentials: %s' % ( service_account_json_keyfile,)) @@ -285,7 +286,8 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): except urllib.error.URLError as e: raise exceptions.CommunicationError( 'Could not reach metadata service: %s' % e.reason) - return util.NormalizeScopes(scope.strip() for scope in response.readlines()) + return util.NormalizeScopes(scope.strip() + for scope in response.readlines()) def _refresh(self, do_request): """Refresh self.access_token. diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 8020b3a..9fbc320 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -257,7 +257,8 @@ class _ProtoJsonApiTools(protojson.ProtoJson): def decode_message(self, message_type, encoded_message): if message_type in _CUSTOM_MESSAGE_CODECS: - return _CUSTOM_MESSAGE_CODECS[message_type].decoder(encoded_message) + return _CUSTOM_MESSAGE_CODECS[ + message_type].decoder(encoded_message) # We turn off the default logging in protorpc. We may want to # remove this later. old_level = logging.getLogger().level @@ -305,7 +306,8 @@ class _ProtoJsonApiTools(protojson.ProtoJson): def encode_message(self, message): if isinstance(message, messages.FieldList): - return '[%s]' % (', '.join(self.encode_message(x) for x in message)) + return '[%s]' % (', '.join(self.encode_message(x) + for x in message)) if type(message) in _CUSTOM_MESSAGE_CODECS: return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) message = _EncodeUnknownFields(message) diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 684afd4..03d1182 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -46,7 +46,8 @@ class DateField(messages.Field): # * since a subclass's metaclass must inherit from its superclass's # metaclass, we're forced to have this hard-to-read inheritance. # - class __metaclass__(messages.Field.__metaclass__): # pylint: disable=invalid-name + class __metaclass__( + messages.Field.__metaclass__): # pylint: disable=invalid-name def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index ba288b7..44023ea 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -218,7 +218,8 @@ class Download(_Transfer): **kwds) @classmethod - def FromData(cls, stream, json_data, http=None, auto_transfer=None, **kwds): + def FromData( + cls, stream, json_data, http=None, auto_transfer=None, **kwds): """Create a new Download object from a stream and serialized data.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) @@ -356,11 +357,13 @@ class Download(_Transfer): if response.status_code not in self._ACCEPTABLE_STATUSES: # We distinguish errors that mean we made a mistake in setting # up the transfer versus something we should attempt again. - if response.status_code in (http_client.FORBIDDEN, http_client.NOT_FOUND): + if response.status_code in ( + http_client.FORBIDDEN, http_client.NOT_FOUND): raise exceptions.HttpError.FromResponse(response) else: raise exceptions.TransferRetryError(response.content) - if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): + if response.status_code in ( + http_client.OK, http_client.PARTIAL_CONTENT): self.stream.write(response.content) self.__progress += response.length if response.info and 'content-encoding' in response.info: @@ -698,7 +701,8 @@ class Upload(_Transfer): refresh_response = http_wrapper.MakeRequest( self.http, refresh_request, redirections=0, retries=self.num_retries) range_header = self._GetRangeHeaderFromResponse(refresh_response) - if refresh_response.status_code in (http_client.OK, http_client.CREATED): + if refresh_response.status_code in ( + http_client.OK, http_client.CREATED): self.__complete = True self.__progress = self.total_size self.stream.seek(self.progress) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 7c6eaff..ad35768 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -170,7 +170,8 @@ def AcceptableMimeType(accept_patterns, mime_type): return all(accept in ('*', provided) for accept, provided in zip(pattern.split('/'), mime_type.split('/'))) - return any(MimeTypeMatches(pattern, mime_type) for pattern in accept_patterns) + return any(MimeTypeMatches(pattern, mime_type) + for pattern in accept_patterns) def MapParamNames(params, request_type): diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 2710670..664331c 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -143,7 +143,8 @@ class CommandRegistry(object): )) arg_names.append(name) flags = [] - for extended_field in sorted(request_type.fields, key=lambda x: x.name): + for extended_field in sorted( + request_type.fields, key=lambda x: x.name): field = extended_field.field_descriptor if extended_field.name in arg_names: continue diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 51bac6d..d90533d 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -240,7 +240,8 @@ class _Proto2Printer(ProtoPrinter): def __PrintEnumCommentLines(self, enum_type): description = enum_type.description or '%s enum type.' % enum_type.name - for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + for line in textwrap.wrap( + description, self.__printer.CalculateWidth() - 3): self.__printer('// %s', line) PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values', prefix='// ') @@ -293,7 +294,8 @@ class _Proto2Printer(ProtoPrinter): prefix='// ') def __PrintFieldDescription(self, description): - for line in textwrap.wrap(description, self.__printer.CalculateWidth() - 3): + for line in textwrap.wrap( + description, self.__printer.CalculateWidth() - 3): self.__printer('// %s', line) def __PrintFields(self, fields): diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index b367c0d..ab3d292 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -373,7 +373,8 @@ class MessageRegistry(object): # We don't actually know this is a message -- it might be an # enum. However, we can't check that until we've created all the # types, so we come back and fix this up later. - return TypeInfo(type_name=type_ref, variant=messages.Variant.MESSAGE) + return TypeInfo( + type_name=type_ref, variant=messages.Variant.MESSAGE) if 'enum' in attrs: enum_name = '%sValuesEnum' % name_hint @@ -429,7 +430,8 @@ class MessageRegistry(object): schema['id'] = name_hint self.AddDescriptorFromSchema(name_hint, schema) self.__AddIfUnknown(name_hint) - return TypeInfo(type_name=name_hint, variant=messages.Variant.MESSAGE) + return TypeInfo( + type_name=name_hint, variant=messages.Variant.MESSAGE) raise ValueError('Unknown type: %s' % type_name) @@ -443,7 +445,8 @@ class MessageRegistry(object): if field.field_descriptor.variant == messages.Variant.MESSAGE: field_type_name = field.field_descriptor.type_name field_type = self.LookupDescriptor(field_type_name) - if isinstance(field_type, extended_descriptor.ExtendedEnumDescriptor): + if isinstance( + field_type, extended_descriptor.ExtendedEnumDescriptor): field.field_descriptor.variant = messages.Variant.ENUM for submessage_type in message_type.message_types: self._FixupMessage(submessage_type) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 5dd2315..b9afaf6 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -233,7 +233,8 @@ class ServiceRegistry(object): for name in self.__service_method_info_map.keys(): printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) - for name, method_info_map in self.__service_method_info_map.items(): + for name, method_info_map in self.__service_method_info_map.items( + ): self.__WriteSingleService( printer, name, method_info_map, client_info.client_class_name) @@ -291,7 +292,8 @@ class ServiceRegistry(object): """Determine if this method needs a new request type created.""" if not request_type: return True - if method_description.get('id', '') in self.__unelidable_request_methods: + if method_description.get( + 'id', '') in self.__unelidable_request_methods: return True message = self.__message_registry.LookupDescriptorOrDie(request_type) if message is None: @@ -352,7 +354,8 @@ class ServiceRegistry(object): method_id = method_description['id'] ordered_params = [] for param_name in method_description.get('parameterOrder', []): - if method_description['parameters'][param_name].get('required', False): + if method_description['parameters'][ + param_name].get('required', False): ordered_params.append(param_name) method_info = base_api.ApiMethodInfo( relative_path=relative_path, -- GitLab From 4572ff0bcc26850eb66ce9f6960b95a54c826491 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 1 May 2015 09:14:07 -0700 Subject: [PATCH 100/295] Lint cleanup. This adds a linting rule to tox, and cleans up all lint errors in apitools. All of these were pretty trivial (though sadly not automatable). --- .gitignore | 3 + apitools/base/py/app2.py | 14 +- apitools/base/py/base_api.py | 21 +- apitools/base/py/base_api_test.py | 8 +- apitools/base/py/base_cli.py | 4 +- apitools/base/py/batch.py | 65 +++-- apitools/base/py/cli.py | 1 + apitools/base/py/credentials_lib.py | 70 +++-- apitools/base/py/credentials_lib_test.py | 3 +- apitools/base/py/encoding.py | 29 +- apitools/base/py/encoding_test.py | 25 +- apitools/base/py/extra_types.py | 7 +- apitools/base/py/extra_types_test.py | 7 +- apitools/base/py/http_wrapper.py | 22 +- apitools/base/py/list_pager.py | 16 +- apitools/base/py/stream_slice.py | 6 +- apitools/base/py/transfer.py | 77 ++--- apitools/base/py/util.py | 7 +- apitools/base/py/util_test.py | 26 +- apitools/gen/client_generation_test.py | 4 +- apitools/gen/command_registry.py | 53 ++-- apitools/gen/extended_descriptor.py | 26 +- apitools/gen/gen_client_lib.py | 10 +- apitools/gen/message_registry.py | 25 +- apitools/gen/service_registry.py | 43 +-- apitools/gen/util.py | 18 +- default.pylintrc | 345 +++++++++++++++++++++++ run_pylint.py | 226 +++++++++++++++ samples/storage_sample/downloads_test.py | 7 +- samples/storage_sample/uploads_test.py | 2 +- tox.ini | 15 + 31 files changed, 906 insertions(+), 279 deletions(-) create mode 100644 default.pylintrc create mode 100644 run_pylint.py diff --git a/.gitignore b/.gitignore index 2254491..b51fd87 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ distribute-* .coverage coverage.xml nosetests.xml + +# Make sure a generated file isn't accidentally committed. +reduced.pylintrc diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index e4f439b..531a977 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -99,12 +99,13 @@ class NewCmd(appcommands.Cmd): # that we can per-command flags in the REPL. args = argv[1:] fail = None + fail_template = '%s positional args, found %d, expected at %s %d' if len(args) < self._min_args: - fail = 'Not enough positional args; found %d, expected at least %d' % ( - len(args), self._min_args) + fail = fail_template % ('Not enough', len(args), + 'least', self._min_args) if len(args) > self._max_args: - fail = 'Too many positional args; found %d, expected at most %d' % ( - len(args), self._max_args) + fail = fail_template % ('Too many', len(args), + 'most', self._max_args) if fail: print(fail) if self.usage: @@ -207,7 +208,7 @@ class CommandLoop(cmd.Cmd): def _set_prompt(self): self.prompt = self._default_prompt - def do_EOF(self, *unused_args): + def do_EOF(self, *unused_args): # pylint: disable=invalid-name """Terminate the running command loop. This function raises an exception to avoid the need to do @@ -227,6 +228,7 @@ class CommandLoop(cmd.Cmd): def postloop(self): print('Goodbye.') + # pylint: disable=arguments-differ def completedefault(self, unused_text, line, unused_begidx, unused_endidx): if not line: return [] @@ -240,6 +242,7 @@ class CommandLoop(cmd.Cmd): print(usage) print('%s%s' % (self.prompt, line), end=' ') return [] + # pylint: enable=arguments-differ def emptyline(self): print('Available commands:', end=' ') @@ -326,7 +329,6 @@ class CommandLoop(cmd.Cmd): def postcmd(self, stop, line): return bool(stop) or line == 'EOF' -# pylint: enable=g-bad-name class Repl(NewCmd): diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 2580040..d10314b 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -197,9 +197,9 @@ class BaseApiClient(object): _USER_AGENT = '' def __init__(self, url, credentials=None, get_credentials=True, http=None, - model=None, log_request=False, log_response=False, num_retries=5, - credentials_args=None, default_global_params=None, - additional_http_headers=None): + model=None, log_request=False, log_response=False, + num_retries=5, credentials_args=None, + default_global_params=None, additional_http_headers=None): _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) if default_global_params is not None: util.Typecheck(default_global_params, self.params_type) @@ -464,8 +464,8 @@ class BaseApiService(object): for param in global_param_names) # Next, add the query params. query_param_names = util.MapParamNames(query_params, type(request)) - query_info.update( - (param, getattr(request, param, None)) for param in query_param_names) + query_info.update((param, getattr(request, param, None)) + for param in query_param_names) query_info = dict((k, v) for k, v in query_info.items() if v is not None) query_info = self.__EncodePrettyPrint(query_info) @@ -479,8 +479,8 @@ class BaseApiService(object): query_info[k] = v.isoformat() return query_info - def __ConstructRelativePath( - self, method_config, request, relative_path=None): + def __ConstructRelativePath(self, method_config, request, + relative_path=None): """Determine the relative path for request.""" python_param_names = util.MapParamNames( method_config.path_params, type(request)) @@ -516,8 +516,8 @@ class BaseApiService(object): if self.__client.response_type_model == 'json': return http_response.content else: - response_type = _LoadClass( - method_config.response_type_name, self.__client.MESSAGES_MODULE) + response_type = _LoadClass(method_config.response_type_name, + self.__client.MESSAGES_MODULE) return self.__client.DeserializeMessage( response_type, http_response.content) @@ -598,7 +598,8 @@ class BaseApiService(object): 'Cannot yet use both upload and download at once') http_request = self.PrepareHttpRequest( - method_config, request, global_params, upload, upload_config, download) + method_config, request, global_params, upload, upload_config, + download) # TODO(craigcitro): Make num_retries customizable on Transfer # objects, and pass in self.__client.num_retries when initializing diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 8bd665f..8e10366 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -100,8 +100,9 @@ class BaseApiTest(unittest2.TestCase): self.assertEqual(response_message, service.ProcessHttpResponse( method_config, http_response)) with service.client.JsonResponseModel(): - self.assertEqual(http_response.content, service.ProcessHttpResponse( - method_config, http_response)) + self.assertEqual( + http_response.content, + service.ProcessHttpResponse(method_config, http_response)) def testAdditionalHeaders(self): additional_headers = {'Request-Is-Awesome': '1'} @@ -168,7 +169,8 @@ class BaseApiTest(unittest2.TestCase): request_type_name='MessageWithRemappings', path_params=['remapped_field', 'enum_field']) request = MessageWithRemappings( - str_field='gonna', enum_field=MessageWithRemappings.AnEnum.value_one) + str_field='gonna', + enum_field=MessageWithRemappings.AnEnum.value_one) service = FakeService() expected_url = service.client.url + 'parameters/gonna/remap/ONE%2FTWO' http_request = service.PrepareHttpRequest(method_config, request) diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index 70872b8..3fe3b28 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -38,7 +38,7 @@ _OUTPUT_FORMATTER_MAP = { def DeclareBaseFlags(): """Declare base flags for all CLIs.""" # TODO(craigcitro): FlagValidators? - global _BASE_FLAGS_DECLARED + global _BASE_FLAGS_DECLARED # pylint: disable=global-statement if _BASE_FLAGS_DECLARED: return flags.DEFINE_boolean( @@ -120,7 +120,7 @@ class ConsoleWithReadline(code.InteractiveConsole): atexit.register(lambda: readline.write_history_file(histfile)) -def run_main(): +def run_main(): # pylint: disable=invalid-name """Function to be used as setuptools script entry point. Appcommands assumes that it always runs as __main__, but launching diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 1fbf1bd..9477531 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -46,8 +46,8 @@ class BatchApiRequest(object): """Holds request and response information for each request. - ApiCalls are ultimately exposed to the client once the HTTP batch request - has been completed. + ApiCalls are ultimately exposed to the client once the HTTP + batch request has been completed. Attributes: http_request: A client-supplied http_wrapper.Request to be @@ -56,6 +56,7 @@ class BatchApiRequest(object): response to the user request, or None if an error occurred. exception: An apiclient.errors.HttpError object if an error occurred, or None. + """ def __init__(self, request, retryable_codes, service, method_config): @@ -63,9 +64,11 @@ class BatchApiRequest(object): Args: request: An http_wrapper.Request object. - retryable_codes: A list of integer HTTP codes that can be retried. + retryable_codes: A list of integer HTTP codes that can + be retried. service: A service inheriting from base_api.BaseApiService. method_config: Method config for the desired API request. + """ self.__retryable_codes = list( set(retryable_codes + [http_client.UNAUTHORIZED])) @@ -97,8 +100,8 @@ class BatchApiRequest(object): @property def terminal_state(self): - return (self.__http_response and ( - self.__http_response.status_code not in self.__retryable_codes)) + response_code = getattr(self.__http_response, 'status_code', 0) + return response_code not in self.__retryable_codes def HandleResponse(self, http_response, exception): """Handles an incoming http response to the request in http_request. @@ -108,7 +111,9 @@ class BatchApiRequest(object): Args: http_response: Deserialized http_wrapper.Response object. - exception: apiclient.errors.HttpError object if an error occurred. + exception: apiclient.errors.HttpError object if an error + occurred. + """ self.__http_response = http_response self.__exception = exception @@ -134,12 +139,14 @@ class BatchApiRequest(object): service: A class inheriting base_api.BaseApiService. method: A string indicated desired method from the service. See the example in the class docstring. - request: An input message appropriate for the specified service.method. + request: An input message appropriate for the specified + service.method. global_params: Optional additional parameters to pass into method.PrepareHttpRequest. Returns: None + """ # Retrieve the configs for the desired method and service. method_config = service.GetMethodConfig(method) @@ -160,7 +167,8 @@ class BatchApiRequest(object): Args: http: httplib2.Http object for use in the request. - sleep_between_polls: Integer number of seconds to sleep between polls. + sleep_between_polls: Integer number of seconds to sleep between + polls. max_retries: Max retries. Any requests that have not succeeded by this number of retries simply report the last response or exception, whatever it happened to be. @@ -175,8 +183,8 @@ class BatchApiRequest(object): if attempt: time.sleep(sleep_between_polls) - # Create a batch_http_request object and populate it with incomplete - # requests. + # Create a batch_http_request object and populate it with + # incomplete requests. batch_http_request = BatchHttpRequest(batch_url=self.batch_url) for request in requests: batch_http_request.Add( @@ -187,9 +195,9 @@ class BatchApiRequest(object): requests = [request for request in self.api_requests if not request.terminal_state] - if (any(request.authorization_failed for request in requests) - and hasattr(http.request, 'credentials')): - http.request.credentials.refresh(http) + if hasattr(http.request, 'credentials'): + if any(request.authorization_failed for request in requests): + http.request.credentials.refresh(http) if not requests: break @@ -207,10 +215,11 @@ class BatchHttpRequest(object): Args: batch_url: URL to send batch requests to. callback: A callback to be called for each response, of the - form callback(response, exception). The first parameter is - the deserialized Response object. The second is an - apiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no error occurred. + form callback(response, exception). The first parameter is + the deserialized Response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no error + occurred. """ # Endpoint to which these requests are sent. self.__batch_url = batch_url @@ -235,9 +244,10 @@ class BatchHttpRequest(object): request_id: String identifier for a individual request. Returns: - A Content-ID header with the id_ encoded into it. A UUID is prepended to - the value because Content-ID headers are supposed to be universally - unique. + A Content-ID header with the id_ encoded into it. A UUID is + prepended to the value because Content-ID headers are + supposed to be universally unique. + """ return '<%s+%s>' % (self.__base_id, urllib_parse.quote(request_id)) @@ -354,16 +364,17 @@ class BatchHttpRequest(object): Args: request: A http_wrapper.Request to add to the batch. callback: A callback to be called for this response, of the - form callback(response, exception). The first parameter is the - deserialized response object. The second is an - apiclient.errors.HttpError exception object if an HTTP error - occurred while processing the request, or None if no errors occurred. + form callback(response, exception). The first parameter is the + deserialized response object. The second is an + apiclient.errors.HttpError exception object if an HTTP error + occurred while processing the request, or None if no errors + occurred. Returns: None """ - self.__request_response_handlers[self._NewId()] = RequestResponseAndHandler( - request, None, callback) + handler = RequestResponseAndHandler(request, None, callback) + self.__request_response_handlers[self._NewId()] = handler def _Execute(self, http): """Serialize batch request, send to server, process response. @@ -417,7 +428,7 @@ class BatchHttpRequest(object): # Disable protected access because namedtuple._replace(...) # is not actually meant to be protected. self.__request_response_handlers[request_id] = ( - self.__request_response_handlers[request_id]._replace( # pylint: disable=protected-access + self.__request_response_handlers[request_id]._replace( response=response)) def Execute(self, http): diff --git a/apitools/base/py/cli.py b/apitools/base/py/cli.py index b24470b..6f7aa32 100644 --- a/apitools/base/py/cli.py +++ b/apitools/base/py/cli.py @@ -8,6 +8,7 @@ cause pain. """ # pylint:disable=wildcard-import +# pylint:disable=unused-wildcard-import from apitools.base.py.app2 import * from apitools.base.py.base_cli import * diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index bcef08e..6651aba 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -18,14 +18,12 @@ from oauth2client import tools # for gflags declarations from six.moves import http_client from six.moves import urllib -import logging - from apitools.base.py import exceptions from apitools.base.py import util try: - import gflags as flags - FLAGS = flags.FLAGS + import gflags + FLAGS = gflags.FLAGS except ImportError: FLAGS = None @@ -47,7 +45,8 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, credentials_filename=None, service_account_name=None, service_account_keyfile=None, service_account_json_keyfile=None, - api_key=None, client=None): + api_key=None, # pylint: disable=unused-argument + client=None): # pylint: disable=unused-argument """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) if ((service_account_name and not service_account_keyfile) or @@ -67,18 +66,20 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, if service_account_json_keyfile: with open(service_account_json_keyfile) as keyfile: service_account_info = json.load(keyfile) - if service_account_info.get( - 'type') != oauth2client.client.SERVICE_ACCOUNT: + account_type = service_account_info.get('type') + if account_type != oauth2client.client.SERVICE_ACCOUNT: raise exceptions.CredentialsError( 'Invalid service account credentials: %s' % ( service_account_json_keyfile,)) - credentials = oauth2client.service_account._ServiceAccountCredentials( # pylint: disable=protected-access + # pylint: disable=protected-access + credentials = oauth2client.service_account._ServiceAccountCredentials( service_account_id=service_account_info['client_id'], service_account_email=service_account_info['client_email'], private_key_id=service_account_info['private_key_id'], private_key_pkcs8_text=service_account_info['private_key'], scopes=scopes, **service_account_kwargs) + # pylint: enable=protected-access return credentials if service_account_name is not None: credentials = ServiceAccountCredentialsFromFile( @@ -147,16 +148,18 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """Initializes the credentials instance. Args: - scopes: The scopes to get. If None, whatever scopes that are available - to the instance are used. - service_account_name: The service account to retrieve the scopes from. + scopes: The scopes to get. If None, whatever scopes that are + available to the instance are used. + service_account_name: The service account to retrieve the scopes + from. **kwds: Additional keyword args. + """ # If there is a connectivity issue with the metadata server, - # detection calls may fail even if we've already successfully identified - # these scopes in the same execution. However, the available scopes don't - # change once an instance is created, so there is no reason to perform - # more than one query. + # detection calls may fail even if we've already successfully + # identified these scopes in the same execution. However, the + # available scopes don't change once an instance is created, + # so there is no reason to perform more than one query. # # TODO(craigcitro): Move this into oauth2client. self.__service_account_name = service_account_name @@ -193,7 +196,8 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """ creds = { # Credentials metadata dict. 'scopes': sorted(list(scopes)) if scopes else None, - 'svc_acct_name': self.__service_account_name} + 'svc_acct_name': self.__service_account_name, + } if _EnsureFileExists(cache_filename): locked_file = oauth2client.locked_file.LockedFile( cache_filename, 'r+b', 'rb') @@ -203,10 +207,9 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): if cached_creds_str: # Cached credentials metadata dict. cached_creds = json.loads(cached_creds_str) - if (creds['svc_acct_name'] == cached_creds['svc_acct_name'] and - (creds['scopes'] is None or - creds['scopes'] == cached_creds['scopes'])): - scopes = cached_creds['scopes'] + if creds['svc_acct_name'] == cached_creds['svc_acct_name']: + if creds['scopes'] in (None, cached_creds['scopes']): + scopes = cached_creds['scopes'] finally: locked_file.unlock_and_close() return scopes @@ -232,8 +235,9 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'svc_acct_name': self.__service_account_name} locked_file.file_handle().write( json.dumps(creds, encoding='ascii')) - # If it's not locked, the locking process will write the same - # data to the file, so just continue. + # If it's not locked, the locking process will + # write the same data to the file, so just + # continue. finally: locked_file.unlock_and_close() @@ -243,8 +247,8 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'GCE credentials requested outside a GCE instance') if not self.GetServiceAccount(self.__service_account_name): raise exceptions.ResourceUnavailableError( - 'GCE credentials requested but service account %s does not exist.' % - self.__service_account_name) + 'GCE credentials requested but service account ' + '%s does not exist.' % self.__service_account_name) if scopes: scope_ls = util.NormalizeScopes(scopes) instance_scopes = self.GetInstanceScopes() @@ -292,12 +296,13 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): def _refresh(self, do_request): """Refresh self.access_token. - This function replaces AppAssertionCredentials._refresh, which does not use - the credential store and is therefore poorly suited for multi-threaded - scenarios. + This function replaces AppAssertionCredentials._refresh, which + does not use the credential store and is therefore poorly + suited for multi-threaded scenarios. Args: do_request: A function matching httplib2.Http.request's signature. + """ # pylint: disable=protected-access oauth2client.client.OAuth2Credentials._refresh(self, do_request) @@ -329,9 +334,9 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): self.access_token = credential_info['access_token'] if 'expires_in' in credential_info: - self.token_expiry = ( - datetime.timedelta(seconds=int(credential_info['expires_in'])) + - datetime.datetime.utcnow()) + expires_in = int(credential_info['expires_in']) + self.token_expiry = datetime.timedelta( + seconds=expires_in + datetime.datetime.utcnow()) else: self.token_expiry = None self.invalid = False @@ -351,6 +356,11 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): credentials.invalid = data['invalid'] return credentials + @property + def serialization_data(self): + raise NotImplementedError( + 'Cannot serialize credentials for GCE service accounts.') + # TODO(craigcitro): Currently, we can't even *load* # `oauth2client.appengine` without being on appengine, because of how diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index af099a3..12f71c3 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -49,7 +49,8 @@ class CredentialsLibTest(unittest2.TestCase): with mock.patch.object(credentials_lib, '_OpenNoProxy', side_effect=MockMetadataCalls, autospec=True) as opener_mock: - with mock.patch.object(util, 'DetectGce', autospec=True) as mock_detect: + with mock.patch.object(util, 'DetectGce', + autospec=True) as mock_detect: mock_detect.return_value = True validator = CreateUriValidator( re.compile(r'.*/%s/.*' % service_account_name), diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 9fbc320..c35e4cf 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -43,7 +43,7 @@ _FIELD_TYPE_CODECS = {} def MapUnrecognizedFields(field_name): - """Register field_name as a container for unrecognized fields in message.""" + """Register field_name as a container for unrecognized fields.""" def Register(cls): _UNRECOGNIZED_FIELD_MAPPINGS[cls] = field_name return cls @@ -128,15 +128,17 @@ def MessageToRepr(msg, multiline=False, **kwargs): **kwargs: {str:str}, Additional flags for how to format the string. Known **kwargs: - shortstrings: bool, True if all string values should be truncated at - 100 characters, since when mocking the contents typically don't matter - except for IDs, and IDs are usually less than 100 characters. + shortstrings: bool, True if all string values should be + truncated at 100 characters, since when mocking the contents + typically don't matter except for IDs, and IDs are usually + less than 100 characters. no_modules: bool, True if the long module name should not be printed with each type. Returns: str, A string of valid python (assuming the right imports have been made) that recreates the message passed into this function. + """ # TODO(user): craigcitro suggests a pretty-printer from apitools/gen. @@ -307,8 +309,9 @@ class _ProtoJsonApiTools(protojson.ProtoJson): def encode_message(self, message): if isinstance(message, messages.FieldList): return '[%s]' % (', '.join(self.encode_message(x) - for x in message)) - if type(message) in _CUSTOM_MESSAGE_CODECS: + for x in message)) + message_type = type(message) + if message_type in _CUSTOM_MESSAGE_CODECS: return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) message = _EncodeUnknownFields(message) result = super(_ProtoJsonApiTools, self).encode_message(message) @@ -474,8 +477,8 @@ def _ProcessUnknownEnums(message, encoded_message): if (isinstance(field, messages.EnumField) and field.name in decoded_message and message.get_assigned_value(field.name) is None): - message.set_unrecognized_field(field.name, decoded_message[field.name], - messages.Variant.ENUM) + message.set_unrecognized_field( + field.name, decoded_message[field.name], messages.Variant.ENUM) return message @@ -553,13 +556,15 @@ def AddCustomJsonFieldMapping(message_type, python_name, json_name): """ if not issubclass(message_type, messages.Message): raise exceptions.TypecheckError( - 'Cannot set JSON field mapping for non-message "%s"' % message_type) + 'Cannot set JSON field mapping for ' + 'non-message "%s"' % message_type) message_name = message_type.definition_name() try: _ = message_type.field_by_name(python_name) except KeyError: raise exceptions.InvalidDataError( - 'Field %s not recognized for type %s' % (python_name, message_type)) + 'Field %s not recognized for type %s' % ( + python_name, message_type)) field_mappings = _JSON_FIELD_MAPPINGS.setdefault(message_name, {}) _CheckForExistingMappings('field', message_type, python_name, json_name) field_mappings[python_name] = json_name @@ -584,8 +589,8 @@ def _FetchRemapping(type_name, mapping_type, python_name=None, json_name=None, """Common code for fetching a key or value from a remapping dict.""" if python_name and json_name: raise exceptions.InvalidDataError( - 'Cannot specify both python_name and json_name for %s remapping' % ( - mapping_type,)) + 'Cannot specify both python_name and json_name ' + 'for %s remapping' % mapping_type) if not (python_name or json_name): raise exceptions.InvalidDataError( 'Must specify either python_name or json_name for %s remapping' % ( diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 91b7565..36861a7 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -317,9 +317,11 @@ class EncodingTest(unittest2.TestCase): MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') def testMessageToRepr(self): - # pylint:disable=bad-whitespace, Using the same string returned by - # MessageToRepr, with the module names fixed. + # Using the same string returned by MessageToRepr, with the + # module names fixed. + # pylint: disable=bad-whitespace msg = SimpleMessage(field='field', repfield=['field', 'field', ],) + # pylint: enable=bad-whitespace self.assertEqual( encoding.MessageToRepr(msg), r"%s.SimpleMessage(field='field',repfield=['field','field',],)" % ( @@ -334,20 +336,15 @@ class EncodingTest(unittest2.TestCase): tzinfo=util.TimeZoneOffset(datetime.timedelta(0)))) self.assertEqual( encoding.MessageToRepr(msg, multiline=True), - # pylint:disable=line-too-long, Too much effort to make MessageToRepr - # wrap lines properly. - """\ -%s.TimeMessage( - timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=protorpc.util.TimeZoneOffset(datetime.timedelta(0))), -)""" % __name__) + ('%s.TimeMessage(\n ' + 'timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, ' + 'tzinfo=protorpc.util.TimeZoneOffset(' + 'datetime.timedelta(0))),\n)') % __name__) self.assertEqual( encoding.MessageToRepr(msg, multiline=True, no_modules=True), - # pylint:disable=line-too-long, Too much effort to make MessageToRepr - # wrap lines properly. - """\ -TimeMessage( - timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, tzinfo=TimeZoneOffset(datetime.timedelta(0))), -)""") + 'TimeMessage(\n ' + 'timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, ' + 'tzinfo=TimeZoneOffset(datetime.timedelta(0))),\n)') if __name__ == '__main__': diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 03d1182..68fc716 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -46,11 +46,12 @@ class DateField(messages.Field): # * since a subclass's metaclass must inherit from its superclass's # metaclass, we're forced to have this hard-to-read inheritance. # - class __metaclass__( - messages.Field.__metaclass__): # pylint: disable=invalid-name + # pylint: disable=invalid-name + class __metaclass__(messages.Field.__metaclass__): - def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument + def __init__(cls, name, bases, dct): super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) + # pylint: enable=invalid-name VARIANTS = frozenset([messages.Variant.STRING]) DEFAULT_VARIANT = messages.Variant.STRING diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index f8ec125..4505d70 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -88,7 +88,8 @@ class ExtraTypesTest(unittest2.TestCase): extra_types.JsonObject.Property( key='a', value=extra_types.JsonValue(integer_value=6)), extra_types.JsonObject.Property( - key='b', value=extra_types.JsonValue(string_value='eleventeen')), + key='b', + value=extra_types.JsonValue(string_value='eleventeen')), ]) self.assertRoundTrip(d) # We don't know json_d will round-trip, because of randomness in @@ -165,8 +166,8 @@ class ExtraTypesTest(unittest2.TestCase): message=message, times=times - 1) # Single - json_msg = ('{"such_string": "poot", "wow": "-1234",' - ' "very_unsigned": "999", "much_repeated": ["123", "456"]}') + json_msg = ('{"such_string": "poot", "wow": "-1234", ' + '"very_unsigned": "999", "much_repeated": ["123", "456"]}') out_json = MtoJ(JtoM(DogeMsg, json_msg)) self.assertEqual(json.loads(out_json)['wow'], '-1234') diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 5ecf6d7..e5fd2fe 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -193,8 +193,9 @@ class Response(collections.namedtuple( def CheckResponse(response): if response is None: # Caller shouldn't call us if the response is None, but handle anyway. - raise exceptions.RequestError('Request to url %s did not return a response.' - % response.request_url) + raise exceptions.RequestError( + 'Request to url %s did not return a response.' % + response.request_url) elif (response.status_code >= 500 or response.status_code == TOO_MANY_REQUESTS): raise exceptions.BadStatusCodeError.FromResponse(response) @@ -295,21 +296,23 @@ def MakeRequest(http, http_request, retries=7, redirections=5, retries: (int, default 5) Number of retries to attempt on 5XX replies. redirections: (int, default 5) Number of redirects to follow. retry_func: Function to handle retries on exceptions. Arguments are - (Httplib2.Http, Request, Exception, int num_retries). - check_response_func: Function to validate the HTTP response. Arguments are - (Response, response content, url). + (Httplib2.Http, Request, Exception, int num_retries). + check_response_func: Function to validate the HTTP response. + Arguments are (Response, response content, url). Raises: InvalidDataFromServerError: if there is no response after retries. Returns: A Response object. + """ retry = 0 while True: try: - return _MakeRequestNoRetry(http, http_request, redirections=redirections, - check_response_func=check_response_func) + return _MakeRequestNoRetry( + http, http_request, redirections=redirections, + check_response_func=check_response_func) # retry_func will consume the exception types it handles and raise. # pylint: disable=broad-except except Exception as e: @@ -332,14 +335,15 @@ def _MakeRequestNoRetry(http, http_request, redirections=5, an underlying http, for example, HTTPMultiplexer. http_request: A Request to send. redirections: (int, default 5) Number of redirects to follow. - check_response_func: Function to validate the HTTP response. Arguments are - (Response, response content, url). + check_response_func: Function to validate the HTTP response. + Arguments are (Response, response content, url). Returns: A Response object. Raises: RequestError if no response could be parsed. + """ connection_type = None # Handle overrides for connection types. This is used if the caller diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 7f5d56d..84d60aa 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -17,19 +17,21 @@ def YieldFromList( Args: service: apitools_base.BaseApiService, A service with a .List() method. - request: protorpc.messages.Message, The request message corresponding to the - service's .List() method, with all the attributes populated except - the .maxResults and .pageToken attributes. + request: protorpc.messages.Message, The request message + corresponding to the service's .List() method, with all the + attributes populated except the .maxResults and .pageToken + attributes. limit: int, The maximum number of records to yield. None if all available records should be yielded. batch_size: int, The number of items to retrieve per request. method: str, The name of the method used to fetch resources. field: str, The field in the response that will be a list of items. predicate: lambda, A function that returns true for items to be yielded. - current_token_attribute: str, The name of the attribute in a request message - holding the page token for the page being requested. - next_token_attribute: str, The name of the attribute in a response message - holding the page token for the next page. + current_token_attribute: str, The name of the attribute in a + request message holding the page token for the page being + requested. + next_token_attribute: str, The name of the attribute in a + response message holding the page token for the next page. Yields: protorpc.message.Message, The resources listed by the service. diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py index 7b1b1a1..bd43daf 100644 --- a/apitools/base/py/stream_slice.py +++ b/apitools/base/py/stream_slice.py @@ -56,7 +56,9 @@ class StreamSlice(object): data = self.__stream.read(read_size) if read_size > 0 and not data: raise exceptions.StreamExhausted( - 'Not enough bytes in stream; expected %d, exhausted after %d' % ( - self.__max_bytes, self.__max_bytes - self.__remaining_bytes)) + 'Not enough bytes in stream; expected %d, exhausted ' + 'after %d' % ( + self.__max_bytes, + self.__max_bytes - self.__remaining_bytes)) self.__remaining_bytes -= len(data) return data diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 44023ea..bcdf794 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -36,7 +36,7 @@ SIMPLE_UPLOAD = 'simple' RESUMABLE_UPLOAD = 'resumable' -def DownloadProgressPrinter(response, download): +def DownloadProgressPrinter(response, unused_download): """Print download progress based on response.""" if 'content-range' in response.info: print('Received %s' % response.info['content-range']) @@ -44,17 +44,17 @@ def DownloadProgressPrinter(response, download): print('Received %d bytes' % response.length) -def DownloadCompletePrinter(response, download): +def DownloadCompletePrinter(unused_response, unused_download): """Print information about a completed download.""" print('Download complete') -def UploadProgressPrinter(response, upload): +def UploadProgressPrinter(response, unused_upload): """Print upload progress based on response.""" print('Sent %s' % response.info['range']) -def UploadCompletePrinter(response, upload): +def UploadCompletePrinter(unused_response, unused_upload): """Print information about a completed upload.""" print('Upload complete') @@ -75,7 +75,8 @@ class _Transfer(object): # Let the @property do validation self.num_retries = num_retries - self.retry_func = http_wrapper.HandleExceptionsAndRebuildHttpConnections + self.retry_func = ( + http_wrapper.HandleExceptionsAndRebuildHttpConnections) self.auto_transfer = auto_transfer self.chunksize = chunksize or 1048576 @@ -208,8 +209,8 @@ class Download(_Transfer): if os.path.exists(path) and not overwrite: raise exceptions.InvalidUserInputError( 'File %s exists and overwrite not specified' % path) - return cls(open(path, 'wb'), close_stream=True, auto_transfer=auto_transfer, - **kwds) + return cls(open(path, 'wb'), close_stream=True, + auto_transfer=auto_transfer, **kwds) @classmethod def FromStream(cls, stream, auto_transfer=True, total_size=None, **kwds): @@ -218,8 +219,8 @@ class Download(_Transfer): **kwds) @classmethod - def FromData( - cls, stream, json_data, http=None, auto_transfer=None, **kwds): + def FromData(cls, stream, json_data, http=None, auto_transfer=None, + **kwds): """Create a new Download object from a stream and serialized data.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) @@ -234,8 +235,8 @@ class Download(_Transfer): download.auto_transfer = info['auto_transfer'] setattr(download, '_Download__progress', info['progress']) setattr(download, '_Download__total_size', info['total_size']) - download._Initialize( - http, info['url']) # pylint: disable=protected-access + download._Initialize( # pylint: disable=protected-access + http, info['url']) return download @property @@ -353,17 +354,17 @@ class Download(_Transfer): retries=self.num_retries) def __ProcessResponse(self, response): - """Process this response (by updating self and writing to self.stream).""" + """Process response (by updating self and writing to self.stream).""" if response.status_code not in self._ACCEPTABLE_STATUSES: # We distinguish errors that mean we made a mistake in setting # up the transfer versus something we should attempt again. - if response.status_code in ( - http_client.FORBIDDEN, http_client.NOT_FOUND): + if response.status_code in (http_client.FORBIDDEN, + http_client.NOT_FOUND): raise exceptions.HttpError.FromResponse(response) else: raise exceptions.TransferRetryError(response.content) - if response.status_code in ( - http_client.OK, http_client.PARTIAL_CONTENT): + if response.status_code in (http_client.OK, + http_client.PARTIAL_CONTENT): self.stream.write(response.content) self.__progress += response.length if response.info and 'content-encoding' in response.info: @@ -429,8 +430,8 @@ class Download(_Transfer): response = self.__initial_response self.__initial_response = None else: - response = self.__GetChunk(self.progress, - additional_headers=additional_headers) + response = self.__GetChunk( + self.progress, additional_headers=additional_headers) if self.total_size is None: self.__SetTotal(response.info) response = self.__ProcessResponse(response) @@ -470,6 +471,7 @@ class Upload(_Transfer): self.__progress = 0 self.__server_chunk_granularity = None self.__strategy = None + self.__total_size = None self.progress_callback = progress_callback self.finish_callback = finish_callback @@ -491,8 +493,8 @@ class Upload(_Transfer): raise exceptions.InvalidUserInputError( 'Could not guess mime type for %s' % path) size = os.stat(path).st_size - return cls(open(path, 'rb'), mime_type, total_size=size, close_stream=True, - auto_transfer=auto_transfer, **kwds) + return cls(open(path, 'rb'), mime_type, total_size=size, + close_stream=True, auto_transfer=auto_transfer, **kwds) @classmethod def FromStream(cls, stream, mime_type, total_size=None, auto_transfer=True, @@ -501,12 +503,12 @@ class Upload(_Transfer): if mime_type is None: raise exceptions.InvalidUserInputError( 'No mime_type specified for stream') - return cls(stream, mime_type, total_size=total_size, close_stream=False, - auto_transfer=auto_transfer, **kwds) + return cls(stream, mime_type, total_size=total_size, + close_stream=False, auto_transfer=auto_transfer, **kwds) @classmethod def FromData(cls, stream, json_data, http, auto_transfer=None, **kwds): - """Create a new Upload of stream from serialized json_data using http.""" + """Create a new Upload of stream from serialized json_data and http.""" info = json.loads(json_data) missing_keys = cls._REQUIRED_SERIALIZATION_KEYS - set(info.keys()) if missing_keys: @@ -526,8 +528,8 @@ class Upload(_Transfer): else: upload.auto_transfer = info['auto_transfer'] upload.strategy = RESUMABLE_UPLOAD - upload._Initialize( - http, info['url']) # pylint: disable=protected-access + upload._Initialize( # pylint: disable=protected-access + http, info['url']) upload.RefreshResumableUploadState() upload.EnsureInitialized() if upload.auto_transfer: @@ -663,8 +665,9 @@ class Upload(_Transfer): msg.set_payload(self.stream.read()) msg_root.attach(msg) - # NOTE: We encode the body, but can't use `email.message.Message.as_string` - # because it prepends `> ` to `From ` lines. + # NOTE: We encode the body, but can't use + # `email.message.Message.as_string` because it prepends + # `> ` to `From ` lines. # NOTE: We must use six.StringIO() instead of io.StringIO() since the # `email` library uses cStringIO in Py2 and io.StringIO in Py3. fp = six.StringIO() @@ -697,12 +700,14 @@ class Upload(_Transfer): return self.EnsureInitialized() refresh_request = http_wrapper.Request( - url=self.url, http_method='PUT', headers={'Content-Range': 'bytes */*'}) + url=self.url, http_method='PUT', + headers={'Content-Range': 'bytes */*'}) refresh_response = http_wrapper.MakeRequest( - self.http, refresh_request, redirections=0, retries=self.num_retries) + self.http, refresh_request, redirections=0, + retries=self.num_retries) range_header = self._GetRangeHeaderFromResponse(refresh_response) - if refresh_response.status_code in ( - http_client.OK, http_client.CREATED): + if refresh_response.status_code in (http_client.OK, + http_client.CREATED): self.__complete = True self.__progress = self.total_size self.stream.seek(self.progress) @@ -790,8 +795,8 @@ class Upload(_Transfer): if self.progress + 1 != self.stream.tell(): # TODO(craigcitro): Add a better way to recover here. raise exceptions.CommunicationError( - 'Failed to transfer all bytes in chunk, upload paused at byte ' - '%d' % self.progress) + 'Failed to transfer all bytes in chunk, upload paused at ' + 'byte %d' % self.progress) self._ExecuteCallback(callback, response) if self.__complete and hasattr(self.stream, 'seek'): current_pos = self.stream.tell() @@ -832,7 +837,7 @@ class Upload(_Transfer): additional_headers=additional_headers) def __SendMediaRequest(self, request, end): - """Helper function to make the request for SendMediaBody & SendChunk.""" + """Request helper function for SendMediaBody & SendChunk.""" response = http_wrapper.MakeRequest( self.bytes_http, request, retry_func=self.retry_func, retries=self.num_retries) @@ -879,8 +884,8 @@ class Upload(_Transfer): self.EnsureInitialized() no_log_body = self.total_size is None if self.total_size is None: - # For the streaming resumable case, we need to detect when we're at the - # end of the stream. + # For the streaming resumable case, we need to detect when + # we're at the end of the stream. body_stream = buffered_stream.BufferedStream( self.stream, start, self.chunksize) end = body_stream.stream_end_position diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index ad35768..06f45cc 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -120,9 +120,9 @@ def ExpandRelativePath(method_config, params, relative_path=None): def CalculateWaitForRetry(retry_attempt, max_wait=60): """Calculates amount of time to wait before a retry attempt. - Wait time grows exponentially with the number of attempts. - A random amount of jitter is added to spread out retry attempts from different - clients. + Wait time grows exponentially with the number of attempts. A + random amount of jitter is added to spread out retry attempts from + different clients. Args: retry_attempt: Retry attempt counter. @@ -130,6 +130,7 @@ def CalculateWaitForRetry(retry_attempt, max_wait=60): Returns: Amount of time to wait before retrying request. + """ wait_time = 2 ** retry_attempt diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index c6c9e80..384aa4f 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -105,35 +105,17 @@ class UtilTest(unittest2.TestCase): util.Typecheck( instance_of_class1, (Class1, (Class2, Class3)), 'message')) - try: + with self.assertRaises(exceptions.TypecheckError): util.Typecheck(instance_of_class1, Class2) - self.fail( - 'Type mismatch not detected when called with 2 arguments') - except exceptions.TypecheckError: - pass # expected - try: + with self.assertRaises(exceptions.TypecheckError): util.Typecheck(instance_of_class1, (Class2, Class3)) - self.fail( - 'Type mismatch not detected when called with 2 arguments including ' - 'type tuple') - except exceptions.TypecheckError: - pass # expected - try: + with self.assertRaises(exceptions.TypecheckError): util.Typecheck(instance_of_class1, Class2, 'message') - self.fail( - 'Type mismatch not detected when called with 3 arguments') - except exceptions.TypecheckError: - pass # expected - try: + with self.assertRaises(exceptions.TypecheckError): util.Typecheck(instance_of_class1, (Class2, Class3), 'message') - self.fail( - 'Type mismatch not detected when called with 3 arguments including ' - 'type tuple') - except exceptions.TypecheckError: - pass # expected def testAcceptableMimeType(self): valid_pairs = ( diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 8d78d55..8dcdaac 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -57,8 +57,8 @@ class ClientGenerationTest(unittest2.TestCase): '--overwrite', 'client', ] - logging.info( - 'Testing API %s with command line: %s', api, ' '.join(args)) + logging.info('Testing API %s with command line: %s', + api, ' '.join(args)) retcode = subprocess.call(args) if retcode == 128: logging.error('Failed to fetch discovery doc, continuing.') diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 664331c..c9e94be 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -143,8 +143,8 @@ class CommandRegistry(object): )) arg_names.append(name) flags = [] - for extended_field in sorted( - request_type.fields, key=lambda x: x.name): + for extended_field in sorted(request_type.fields, + key=lambda x: x.name): field = extended_field.field_descriptor if extended_field.name in arg_names: continue @@ -159,7 +159,8 @@ class CommandRegistry(object): # determining the filename from the object metadata. upload_flag_info = FlagInfo( name='upload_filename', type='string', default='', - description='Filename to use for upload.', fv='fv', special=True) + description='Filename to use for upload.', fv='fv', + special=True) flags.append(upload_flag_info) mime_description = ( 'MIME type to use for the upload. Only needed if ' @@ -172,7 +173,8 @@ class CommandRegistry(object): if method_info.supports_download: download_flag_info = FlagInfo( name='download_filename', type='string', default='', - description='Filename to use for download.', fv='fv', special=True) + description='Filename to use for download.', fv='fv', + special=True) flags.append(download_flag_info) overwrite_description = ( 'If True, overwrite the existing file when downloading.') @@ -299,15 +301,13 @@ class CommandRegistry(object): printer('flags.DEFINE_multistring(') with printer.Indent(' '): printer("'add_header', [],") - printer( - "'Additional http headers (as key=value strings). Can be '") - printer("'specified multiple times.')") + printer("'Additional http headers (as key=value strings). '") + printer("'Can be specified multiple times.')") printer('flags.DEFINE_string(') with printer.Indent(' '): printer("'service_account_json_keyfile', '',") - printer( - "'Filename for a JSON service account key downloaded from '") - printer("'the Developer Console.')") + printer("'Filename for a JSON service account key downloaded'") + printer("' from the Developer Console.')") for flag_info in self.__global_flags: self.__PrintFlag(printer, flag_info) printer() @@ -337,10 +337,10 @@ class CommandRegistry(object): printer('def GetClientFromFlags():') with printer.Indent(): printer('"""Return a client object, configured from flags."""') - printer( - 'log_request = FLAGS.log_request or FLAGS.log_request_response') - printer( - 'log_response = FLAGS.log_response or FLAGS.log_request_response') + printer('log_request = FLAGS.log_request or ' + 'FLAGS.log_request_response') + printer('log_response = FLAGS.log_response or ' + 'FLAGS.log_request_response') printer('api_endpoint = apitools_base.NormalizeApiEndpoint(' 'FLAGS.api_endpoint)') printer("additional_http_headers = dict(x.split('=', 1) for x in " @@ -352,8 +352,8 @@ class CommandRegistry(object): printer('}') printer('try:') with printer.Indent(): - printer( - 'client = client_lib.%s(', self.__client_info.client_class_name) + printer('client = client_lib.%s(', + self.__client_info.client_class_name) with printer.Indent(indent=' '): printer('api_endpoint, log_request=log_request,') printer('log_response=log_response,') @@ -452,8 +452,8 @@ class CommandRegistry(object): printer('#!/usr/bin/env python') printer('"""CLI for %s, version %s."""', self.__package, self.__version) - printer('# NOTE: This file is autogenerated and should not be edited by ' - 'hand.') + printer('# NOTE: This file is autogenerated and should not be edited ' + 'by hand.') # TODO(craigcitro): Add a build stamp, along with some other # information. printer() @@ -516,8 +516,8 @@ class CommandRegistry(object): printer( 'class %s(apitools_base_cli.NewCmd):', command_info.class_name) with printer.Indent(): - printer( - '"""Command wrapping %s."""', command_info.client_method_path) + printer('"""Command wrapping %s."""', + command_info.client_method_path) printer() printer('usage = """%s%s%s"""', command_info.name, @@ -526,8 +526,8 @@ class CommandRegistry(object): printer() printer('def __init__(self, name, fv):') with printer.Indent(): - printer( - 'super(%s, self).__init__(name, fv)', command_info.class_name) + printer('super(%s, self).__init__(name, fv)', + command_info.class_name) for flag in command_info.flags: self.__PrintFlag(printer, flag) printer() @@ -561,8 +561,8 @@ class CommandRegistry(object): printer('if FLAGS.upload_filename:') with printer.Indent(): printer('upload = apitools_base.Upload.FromFile(') - printer( - ' FLAGS.upload_filename, FLAGS.upload_mime_type,') + printer(' FLAGS.upload_filename, ' + 'FLAGS.upload_mime_type,') printer(' progress_callback=' 'apitools_base.UploadProgressPrinter,') printer(' finish_callback=' @@ -572,8 +572,9 @@ class CommandRegistry(object): printer('download = None') printer('if FLAGS.download_filename:') with printer.Indent(): - printer('download = apitools_base.Download.FromFile(' - 'FLAGS.download_filename, overwrite=FLAGS.overwrite,') + printer('download = apitools_base.Download.' + 'FromFile(FLAGS.download_filename, ' + 'overwrite=FLAGS.overwrite,') printer(' progress_callback=' 'apitools_base.DownloadProgressPrinter,') printer(' finish_callback=' diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index d90533d..abff599 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -177,7 +177,8 @@ def PrintIndentedDescriptions(printer, ls, name, prefix=''): printer(name + ':') for x in ls: description = '%s: %s' % (x.name, x.description) - for line in textwrap.wrap(description, width, initial_indent=' ', + for line in textwrap.wrap(description, width, + initial_indent=' ', subsequent_indent=' '): printer(line) @@ -240,8 +241,8 @@ class _Proto2Printer(ProtoPrinter): def __PrintEnumCommentLines(self, enum_type): description = enum_type.description or '%s enum type.' % enum_type.name - for line in textwrap.wrap( - description, self.__printer.CalculateWidth() - 3): + for line in textwrap.wrap(description, + self.__printer.CalculateWidth() - 3): self.__printer('// %s', line) PrintIndentedDescriptions(self.__printer, enum_type.values, 'Values', prefix='// ') @@ -286,16 +287,16 @@ class _Proto2Printer(ProtoPrinter): width = self.__printer.CalculateWidth() - 3 for line in textwrap.wrap(description, width): self.__printer('// %s', line) - PrintIndentedDescriptions(self.__printer, message_type.enum_types, 'Enums', - prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.enum_types, + 'Enums', prefix='// ') PrintIndentedDescriptions(self.__printer, message_type.message_types, 'Messages', prefix='// ') - PrintIndentedDescriptions(self.__printer, message_type.fields, 'Fields', - prefix='// ') + PrintIndentedDescriptions(self.__printer, message_type.fields, + 'Fields', prefix='// ') def __PrintFieldDescription(self, description): - for line in textwrap.wrap( - description, self.__printer.CalculateWidth() - 3): + for line in textwrap.wrap(description, + self.__printer.CalculateWidth() - 3): self.__printer('// %s', line) def __PrintFields(self, fields): @@ -465,7 +466,8 @@ def _PrintMessages(proto_printer, message_list): _MESSAGE_FIELD_MAP = { - message_types.DateTimeMessage.definition_name(): message_types.DateTimeField, + message_types.DateTimeMessage.definition_name(): ( + message_types.DateTimeField), } @@ -503,8 +505,8 @@ def _PrintFields(fields, printer): printed_field_info['label_format'] = ', repeated=True' if field_type.DEFAULT_VARIANT != field.variant: - printed_field_info['variant_format'] = ', variant=messages.Variant.%s' % ( - field.variant,) + printed_field_info['variant_format'] = ( + ', variant=messages.Variant.%s' % field.variant) if field.default_value: if field_type in [messages.BytesField, messages.StringField]: diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 1c11866..f9feb77 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -5,7 +5,6 @@ Relevant links: https://developers.google.com/discovery/v1/reference/apis#resource """ -import json from six.moves import urllib_parse from apitools.base.py import base_cli @@ -64,7 +63,8 @@ class DescriptorGenerator(object): '//cloud/bigscience/apitools/base/py:apitools_base') self.__names = names self.__base_url, self.__base_path = _ComputePaths( - self.__package, self.__client_info.url_version, self.__discovery_doc) + self.__package, self.__client_info.url_version, + self.__discovery_doc) # Order is important here: we need the schemas before we can # define the services. @@ -88,8 +88,8 @@ class DescriptorGenerator(object): self.__command_registry = command_registry.CommandRegistry( self.__package, self.__version, self.__client_info, - self.__message_registry, self.__root_package, self.__base_files_package, - self.__base_url, self.__names) + self.__message_registry, self.__root_package, + self.__base_files_package, self.__base_url, self.__names) self.__command_registry.AddGlobalParameters( self.__message_registry.LookupDescriptorOrDie( 'StandardQueryParameters')) @@ -113,7 +113,7 @@ class DescriptorGenerator(object): if api_methods: self.__services_registry.AddServiceFromResource( 'api', {'methods': api_methods}) - self.__client_info = self.__client_info._replace( # pylint:disable=protected-access + self.__client_info = self.__client_info._replace( scopes=self.__services_registry.scopes) @property diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index ab3d292..4a7a3e9 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -56,8 +56,9 @@ class MessageRegistry(object): variant=messages.BytesField.DEFAULT_VARIANT), 'date': TypeInfo(type_name='extra_types.DateField', variant=messages.Variant.STRING), - 'date-time': TypeInfo(type_name='protorpc.message_types.DateTimeMessage', - variant=messages.Variant.MESSAGE), + 'date-time': TypeInfo( + type_name='protorpc.message_types.DateTimeMessage', + variant=messages.Variant.MESSAGE), } def __init__(self, client_info, names, description, @@ -139,7 +140,8 @@ class MessageRegistry(object): if isinstance(new_descriptor, extended_descriptor.ExtendedMessageDescriptor): self.__current_env.message_types.append(new_descriptor) - elif isinstance(new_descriptor, extended_descriptor.ExtendedEnumDescriptor): + elif isinstance(new_descriptor, + extended_descriptor.ExtendedEnumDescriptor): self.__current_env.enum_types.append(new_descriptor) self.__unknown_types.discard(full_name) self.__nascent_types.remove(full_name) @@ -243,8 +245,8 @@ class MessageRegistry(object): self.__DeclareMessageAlias(schema, 'extra_types.JsonValue') return if schema.get('type') != 'object': - raise ValueError( - 'Cannot create message descriptors for type %s', schema.get('type')) + raise ValueError('Cannot create message descriptors for type %s', + schema.get('type')) message = extended_descriptor.ExtendedMessageDescriptor() message.name = self.__names.ClassName(schema['id']) message.description = util.CleanDescription(schema.get( @@ -258,7 +260,7 @@ class MessageRegistry(object): message.fields.append(field) if field.name != name: message.field_mappings.append( - extended_descriptor.ExtendedMessageDescriptor.JsonFieldMapping( + type(message).JsonFieldMapping( python_name=field.name, json_name=name)) self.__AddImport( 'from %s import encoding' % self.__base_files_package) @@ -321,7 +323,8 @@ class MessageRegistry(object): if 'default' in attrs: # TODO(craigcitro): Correctly handle non-primitive default values. default = attrs['default'] - if field.type_name != 'string' and field.variant != messages.Variant.ENUM: + if not (field.type_name == 'string' or + field.variant == messages.Variant.ENUM): default = str(json.loads(default)) if field.variant == messages.Variant.ENUM: default = self.__names.NormalizeEnumName(default) @@ -413,8 +416,8 @@ class MessageRegistry(object): items.get('title') or name_hint) entry_type_name = self.__AddEntryType( entry_name_hint, items.get('items'), parent_name) - return TypeInfo( - type_name=entry_type_name, variant=messages.Variant.MESSAGE) + return TypeInfo(type_name=entry_type_name, + variant=messages.Variant.MESSAGE) else: return self.__GetTypeInfo(items, entry_name_hint) elif type_name == 'any': @@ -445,8 +448,8 @@ class MessageRegistry(object): if field.field_descriptor.variant == messages.Variant.MESSAGE: field_type_name = field.field_descriptor.type_name field_type = self.LookupDescriptor(field_type_name) - if isinstance( - field_type, extended_descriptor.ExtendedEnumDescriptor): + if isinstance(field_type, + extended_descriptor.ExtendedEnumDescriptor): field.field_descriptor.variant = messages.Variant.ENUM for submessage_type in message_type.message_types: self._FixupMessage(submessage_type) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index b9afaf6..d4826a6 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -118,8 +118,8 @@ class ServiceRegistry(object): attrs = sorted( x.name for x in upload_config.all_fields()) for attr in attrs: - printer( - '%s=%r,', attr, getattr(upload_config, attr)) + printer('%s=%r,', + attr, getattr(upload_config, attr)) printer('),') printer('}') @@ -138,8 +138,8 @@ class ServiceRegistry(object): printer("config = self.GetMethodConfig('%s')", method_name) upload_config = method_info.upload_config if upload_config is not None: - printer( - "upload_config = self.GetUploadConfig('%s')", method_name) + printer("upload_config = self.GetUploadConfig('%s')", + method_name) arg_lines = [ 'config, request, global_params=global_params'] if method_info.upload_config: @@ -188,8 +188,8 @@ class ServiceRegistry(object): client_info = self.__client_info printer('"""Generated client library for %s version %s."""', client_info.package, client_info.version) - printer('# NOTE: This file is autogenerated and should not be edited by ' - 'hand.') + printer('# NOTE: This file is autogenerated and should not be edited ' + 'by hand.') printer('from %s import base_api', self.__base_files_package) import_prefix = '' printer('%simport %s as messages', import_prefix, @@ -199,8 +199,9 @@ class ServiceRegistry(object): printer('class %s(base_api.BaseApiClient):', client_info.client_class_name) with printer.Indent(): - printer('"""Generated client library for service %s version %s."""', - client_info.package, client_info.version) + printer( + '"""Generated client library for service %s version %s."""', + client_info.package, client_info.version) printer() printer('MESSAGES_MODULE = messages') printer() @@ -223,20 +224,19 @@ class ServiceRegistry(object): printer( 'super(%s, self).__init__(', client_info.client_class_name) printer(' url, credentials=credentials,') - printer( - ' get_credentials=get_credentials, http=http, model=model,') - printer( - ' log_request=log_request, log_response=log_response,') + printer(' get_credentials=get_credentials, http=http, ' + 'model=model,') + printer(' log_request=log_request, ' + 'log_response=log_response,') printer(' credentials_args=credentials_args,') printer(' default_global_params=default_global_params,') printer(' additional_http_headers=additional_http_headers)') for name in self.__service_method_info_map.keys(): printer('self.%s = self.%s(self)', name, self.__GetServiceClassName(name)) - for name, method_info_map in self.__service_method_info_map.items( - ): + for name, method_info in self.__service_method_info_map.items(): self.__WriteSingleService( - printer, name, method_info_map, client_info.client_class_name) + printer, name, method_info, client_info.client_class_name) def __RegisterService(self, service_name, method_info_map): if service_name in self.__service_method_info_map: @@ -292,8 +292,8 @@ class ServiceRegistry(object): """Determine if this method needs a new request type created.""" if not request_type: return True - if method_description.get( - 'id', '') in self.__unelidable_request_methods: + method_id = method_description.get('id', '') + if method_id in self.__unelidable_request_methods: return True message = self.__message_registry.LookupDescriptorOrDie(request_type) if message is None: @@ -354,8 +354,8 @@ class ServiceRegistry(object): method_id = method_description['id'] ordered_params = [] for param_name in method_description.get('parameterOrder', []): - if method_description['parameters'][ - param_name].get('required', False): + param_info = method_description['parameters'][param_name] + if param_info.get('required', False): ordered_params.append(param_name) method_info = base_api.ApiMethodInfo( relative_path=relative_path, @@ -384,8 +384,9 @@ class ServiceRegistry(object): elif location == 'path': method_info.path_params.append(param) else: - raise ValueError('Unknown parameter location %s for parameter %s' % ( - location, param)) + raise ValueError( + 'Unknown parameter location %s for parameter %s' % ( + location, param)) method_info.path_params.sort() method_info.query_params.sort() return method_info diff --git a/apitools/gen/util.py b/apitools/gen/util.py index b3e6c1c..4fb3d10 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -38,7 +38,8 @@ class Names(object): name_convention=None, capitalize_enums=False): self.__strip_prefixes = sorted(strip_prefixes, cmp=_SortLengthFirst) - self.__name_convention = name_convention or self.DEFAULT_NAME_CONVENTION + self.__name_convention = ( + name_convention or self.DEFAULT_NAME_CONVENTION) self.__capitalize_enums = capitalize_enums @staticmethod @@ -172,8 +173,9 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( 'user_agent': user_agent, 'api_key': api_key, } - client_class_name = ''.join( - map(names.ClassName, (client_info['package'], client_info['version']))) + client_class_name = '%s%s' % ( + names.ClassName(client_info['package']), + names.ClassName(client_info['version'])) client_info['client_class_name'] = client_class_name return cls(**client_info) @@ -297,9 +299,11 @@ def FetchDiscoveryDoc(discovery_url, retries=5): discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) break except (urllib2.HTTPError, urllib2.URLError) as last_exception: - logging.warning('Attempting to fetch discovery doc again after "%s"', - last_exception) + logging.warning( + 'Attempting to fetch discovery doc again after "%s"', + last_exception) if discovery_doc is None: - raise CommunicationError('Could not find discovery doc at url "%s": %s' % ( - discovery_url, last_exception)) + raise CommunicationError( + 'Could not find discovery doc at url "%s": %s' % ( + discovery_url, last_exception)) return discovery_doc diff --git a/default.pylintrc b/default.pylintrc new file mode 100644 index 0000000..bdea83a --- /dev/null +++ b/default.pylintrc @@ -0,0 +1,345 @@ +# PyLint config for apitools code. +# +# NOTES: +# +# - Rules for test / demo code are generated into 'pylintrc_reduced' +# as deltas from this configuration by the 'run_pylint.py' script. +# +# - 'RATIONALE: API mapping' as a defense for non-default settings is +# based on the fact that this library maps APIs which are outside our +# control, and adhering to the out-of-the-box defaults would induce +# breakage / complexity in those mappings +# +[MASTER] + +# Specify a configuration file. +# DEFAULT: rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# DEFAULT: init-hook= + +# Profiled execution. +# DEFAULT: profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +# DEFAULT: ignore=CVS +# NOTE: This path must be relative due to the use of +# os.walk in astroid.modutils.get_module_files. + +# Pickle collected data for later comparisons. +# DEFAULT: persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +# DEFAULT: load-plugins= + +# DEPRECATED +# DEFAULT: include-ids=no + +# DEPRECATED +# DEFAULT: symbols=no + + +[MESSAGES CONTROL] + +disable = + fixme, + import-error, + locally-disabled, + locally-enabled, + maybe-no-member, + method-hidden, + no-init, + no-member, + no-self-use, + redefined-builtin, + similarities, + star-args, + super-on-old-class, + too-few-public-methods, + too-many-arguments, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +# DEFAULT: output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +# DEFAULT: files-output=no + +# Tells whether to display a full report or only the messages +# DEFAULT: reports=yes +# RATIONALE: run from Travis / tox, and don't need / want to parse output. +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +# DEFAULT: evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +# DEFAULT: comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +# DEFAULT: min-similarity-lines=4 + +# Ignore comments when computing similarities. +# DEFAULT: ignore-comments=yes + +# Ignore docstrings when computing similarities. +# DEFAULT: ignore-docstrings=yes + +# Ignore imports when computing similarities. +# DEFAULT: ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +# DEFAULT: init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +# DEFAULT: additional-builtins= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +# DEFAULT: logging-modules=logging + + +[FORMAT] + +# Maximum number of characters on a single line. +# DEFAULT: max-line-length=80 + +# Regexp for a line that is allowed to be longer than the limit. +# DEFAULT: ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +# DEFAULT: single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +# DEFAULT: no-space-check=trailing-comma,dict-separator +# RATIONALE: pylint ignores whitespace checks around the +# constructs "dict-separator" (cases like {1:2}) and +# "trailing-comma" (cases like {1: 2, }). +# By setting "no-space-check" to empty whitespace checks will be +# enforced around both constructs. +no-space-check = + +# Maximum number of lines in a module +# DEFAULT: max-module-lines=1000 +max-module-lines=1500 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +# DEFAULT: indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +# DEFAULT: indent-after-paren=4 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +# DEFAULT: notes=FIXME,XXX,TODO + + +[BASIC] +required-attributes= + +no-docstring-rgx=(__.*__|main) + +docstring-min-length=10 + +# Regular expression which should only match correct module names. The +# leading underscore is sanctioned for private modules by Google's style +# guide. +module-rgx=^(_?[a-z][a-z0-9_]*)|__init__$ + +# Regular expression which should only match correct module level names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression which should only match correct class attribute +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression which should only match correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression which should only match correct function names. +# 'camel_case' and 'snake_case' group names are used for consistency of naming +# styles across functions and methods. +function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Regular expression which should only match correct method names. +# 'camel_case' and 'snake_case' group names are used for consistency of naming +# styles across functions and methods. 'exempt' indicates a name which is +# consistent with all naming styles. +method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression which should only match correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression which should only match correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=main,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names= + +# List of builtins function names that should not be used, separated by a comma +# +bad-functions=input,apply,reduce + +# TEMPORARY +no-docstring-rgx=.* + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +# DEFAULT: ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +# DEFAULT: ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +# DEFAULT: ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +# DEFAULT: zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +# DEFAULT: generated-members=REQUEST,acl_users,aq_parent + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +# DEFAULT: deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +# DEFAULT: import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +# DEFAULT: ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +# DEFAULT: int-import-graph= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +# DEFAULT: ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +# DEFAULT: defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +# DEFAULT: valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +# DEFAULT: valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +# DEFAULT: max-args=5 +# RATIONALE: API-mapping +max-args = 10 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +# DEFAULT: ignored-argument-names=_.* + +# Maximum number of locals for function / method body +# DEFAULT: max-locals=15 +max-locals=20 + +# Maximum number of return / yield for function / method body +# DEFAULT: max-returns=6 + +# Maximum number of branch for function / method body +# DEFAULT: max-branches=12 + +# Maximum number of statements in function / method body +# DEFAULT: max-statements=50 + +# Maximum number of parents for a class (see R0901). +# DEFAULT: max-parents=7 + +# Maximum number of attributes for a class (see R0902). +# DEFAULT: max-attributes=7 +# RATIONALE: API mapping +max-attributes=15 + +# Minimum number of public methods for a class (see R0903). +# DEFAULT: min-public-methods=2 +# RATIONALE: context mgrs may have *no* public methods +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +# DEFAULT: max-public-methods=20 +# RATIONALE: API mapping +max-public-methods=40 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +# DEFAULT: overgeneral-exceptions=Exception diff --git a/run_pylint.py b/run_pylint.py new file mode 100644 index 0000000..aa48577 --- /dev/null +++ b/run_pylint.py @@ -0,0 +1,226 @@ +"""Custom script to run PyLint on apitools codebase. + +"Inspired" by the similar script in gcloud-python. + +This runs pylint as a script via subprocess in two different +subprocesses. The first lints the production/library code +using the default rc file (PRODUCTION_RC). The second lints the +demo/test code using an rc file (TEST_RC) which allows more style +violations (hence it has a reduced number of style checks). +""" + +import ConfigParser +import copy +import os +import subprocess +import sys + + +IGNORED_DIRECTORIES = [ + 'samples/storage_sample/storage', +] +IGNORED_FILES = [ + 'ez_setup.py', + 'setup.py', +] +PRODUCTION_RC = 'default.pylintrc' +TEST_RC = 'reduced.pylintrc' +TEST_DISABLED_MESSAGES = [ + 'attribute-defined-outside-init', + 'exec-used', + 'import-error', + 'invalid-name', + 'missing-docstring', + 'no-init', + 'no-self-use', + 'protected-access', + 'superfluous-parens', + 'too-few-public-methods', + 'too-many-locals', + 'too-many-public-methods', + 'unbalanced-tuple-unpacking', +] +TEST_RC_ADDITIONS = { + 'MESSAGES CONTROL': { + 'disable': ', '.join(TEST_DISABLED_MESSAGES), + }, +} + + +def read_config(filename): + """Reads pylintrc config onto native ConfigParser object.""" + config = ConfigParser.ConfigParser() + with open(filename, 'r') as file_obj: + config.readfp(file_obj) + return config + + +def make_test_rc(base_rc_filename, additions_dict, target_filename): + """Combines a base rc and test additions into single file.""" + main_cfg = read_config(base_rc_filename) + + # Create fresh config for test, which must extend production. + test_cfg = ConfigParser.ConfigParser() + test_cfg._sections = copy.deepcopy(main_cfg._sections) + + for section, opts in additions_dict.items(): + curr_section = test_cfg._sections.setdefault( + section, test_cfg._dict()) + for opt, opt_val in opts.items(): + curr_val = curr_section.get(opt) + if curr_val is None: + raise KeyError('Expected to be adding to existing option.') + curr_section[opt] = '%s, %s' % (curr_val, opt_val) + + with open(target_filename, 'w') as file_obj: + test_cfg.write(file_obj) + + +def valid_filename(filename): + """Checks if a file is a Python file and is not ignored.""" + for directory in IGNORED_DIRECTORIES: + if filename.startswith(directory): + return False + return (filename.endswith('.py') and + filename not in IGNORED_FILES) + + +def is_production_filename(filename): + """Checks if the file contains production code. + + :rtype: boolean + :returns: Boolean indicating production status. + """ + return not ('demo' in filename or 'test' in filename or + filename.startswith('regression')) + + +def get_files_for_linting(allow_limited=True): + """Gets a list of files in the repository. + + By default, returns all files via ``git ls-files``. However, in some cases + uses a specific commit or branch (a so-called diff base) to compare + against for changed files. (This requires ``allow_limited=True``.) + + To speed up linting on Travis pull requests against master, we manually + set the diff base to origin/master. We don't do this on non-pull requests + since origin/master will be equivalent to the currently checked out code. + One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but + this value is not dependable. + + To allow faster local ``tox`` runs, the environment variables + ``GCLOUD_REMOTE_FOR_LINT`` and ``GCLOUD_BRANCH_FOR_LINT`` can be set to + specify a remote branch to diff against. + + :type allow_limited: boolean + :param allow_limited: Boolean indicating if a reduced set of files can + be used. + + :rtype: pair + :returns: Tuple of the diff base using the the list of filenames to be + linted. + """ + diff_base = None + if (os.getenv('TRAVIS_BRANCH') == 'master' and + os.getenv('TRAVIS_PULL_REQUEST') != 'false'): + # In the case of a pull request into master, we want to + # diff against HEAD in master. + diff_base = 'origin/master' + elif os.getenv('TRAVIS') is None: + # Only allow specified remote and branch in local dev. + remote = os.getenv('GCLOUD_REMOTE_FOR_LINT') + branch = os.getenv('GCLOUD_BRANCH_FOR_LINT') + if remote is not None and branch is not None: + diff_base = '%s/%s' % (remote, branch) + + if diff_base is not None and allow_limited: + result = subprocess.check_output(['git', 'diff', '--name-only', + diff_base]) + print 'Using files changed relative to %s:' % (diff_base,) + print '-' * 60 + print result.rstrip('\n') # Don't print trailing newlines. + print '-' * 60 + else: + print 'Diff base not specified, listing all files in repository.' + result = subprocess.check_output(['git', 'ls-files']) + + return result.rstrip('\n').split('\n'), diff_base + + +def get_python_files(all_files=None): + """Gets a list of all Python files in the repository that need linting. + + Relies on :func:`get_files_for_linting()` to determine which files should + be considered. + + NOTE: This requires ``git`` to be installed and requires that this + is run within the ``git`` repository. + + :type all_files: list or ``NoneType`` + :param all_files: Optional list of files to be linted. + + :rtype: tuple + :returns: A tuple containing two lists and a boolean. The first list + contains all production files, the next all test/demo files and + the boolean indicates if a restricted fileset was used. + """ + using_restricted = False + if all_files is None: + all_files, diff_base = get_files_for_linting() + using_restricted = diff_base is not None + + library_files = [] + non_library_files = [] + for filename in all_files: + if valid_filename(filename): + if is_production_filename(filename): + library_files.append(filename) + else: + non_library_files.append(filename) + + return library_files, non_library_files, using_restricted + + +def lint_fileset(filenames, rcfile, description): + """Lints a group of files using a given rcfile.""" + # Only lint filenames that exist. For example, 'git diff --name-only' + # could spit out deleted / renamed files. Another alternative could + # be to use 'git diff --name-status' and filter out files with a + # status of 'D'. + filenames = [filename for filename in filenames + if os.path.exists(filename)] + if filenames: + rc_flag = '--rcfile=%s' % (rcfile,) + pylint_shell_command = ['pylint', rc_flag] + filenames + status_code = subprocess.call(pylint_shell_command) + if status_code != 0: + error_message = ('Pylint failed on %s with ' + 'status %d.' % (description, status_code)) + print >> sys.stderr, error_message + sys.exit(status_code) + else: + print 'Skipping %s, no files to lint.' % (description,) + + +def main(): + """Script entry point. Lints both sets of files.""" + make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC) + library_files, non_library_files, using_restricted = get_python_files() + try: + lint_fileset(library_files, PRODUCTION_RC, 'library code') + lint_fileset(non_library_files, TEST_RC, 'test and demo code') + except SystemExit: + if not using_restricted: + raise + + message = 'Restricted lint failed, expanding to full fileset.' + print >> sys.stderr, message + all_files, _ = get_files_for_linting(allow_limited=False) + library_files, non_library_files, _ = get_python_files( + all_files=all_files) + lint_fileset(library_files, PRODUCTION_RC, 'library code') + lint_fileset(non_library_files, TEST_RC, 'test and demo code') + + +if __name__ == '__main__': + main() diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index 4044d80..f6a62e5 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -7,7 +7,6 @@ files in apitools, via GCS. There are no performance tests here yet. import io import json import os -import pkgutil import unittest import apitools.base.py as apitools_base @@ -17,7 +16,7 @@ _CLIENT = None def _GetClient(): - global _CLIENT + global _CLIENT # pylint: disable=global-statement if _CLIENT is None: _CLIENT = storage.StorageV1() return _CLIENT @@ -167,8 +166,8 @@ class DownloadsTest(unittest.TestCase): 'total_size': response.size, 'url': response.mediaLink, }) - self.__download = storage.Download.FromData(self.__buffer, download_data, - http=self.__client.http) + self.__download = storage.Download.FromData( + self.__buffer, download_data, http=self.__client.http) self.__download.StreamInChunks(callback=_ProgressCallback) self.assertEqual(15, self.__buffer.tell()) self.__buffer.seek(0) diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index 0d25a30..ca58dc8 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -18,7 +18,7 @@ _CLIENT = None def _GetClient(): - global _CLIENT + global _CLIENT # pylint: disable=global-statement if _CLIENT is None: _CLIENT = storage.StorageV1() return _CLIENT diff --git a/tox.ini b/tox.ini index 0001af4..fe1d295 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,21 @@ deps = unittest2 commands = nosetests +[pep8] +exclude = samples/storage_sample/storage,samples/storage_sample/testdata,*.egg/,.*/,ez_setup.py +verbose = 1 + +[testenv:lint] +basepython = + python2.7 +commands = + pep8 + python run_pylint.py +deps = + pep8 + pylint + unittest2 + [testenv:cover] basepython = python2.7 -- GitLab From cabd4ddd4110a2c019aaeb3e71a2fc3b5f4efd51 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 4 May 2015 01:20:49 -0700 Subject: [PATCH 101/295] Add in the apitools mock framework. This includes a framework for easily mocking an apitools client, originally written by @skelterjohn. --- apitools/base/py/testing/__init__.py | 1 + apitools/base/py/testing/mock.py | 321 ++++++++++++++++++ apitools/base/py/testing/mock_test.py | 132 +++++++ .../base/py/testing/testclient/__init__.py | 5 + .../testclient/fusiontables_v1_client.py | 78 +++++ .../testclient/fusiontables_v1_messages.py | 93 +++++ run_pylint.py | 1 + tox.ini | 1 + 8 files changed, 632 insertions(+) create mode 100644 apitools/base/py/testing/__init__.py create mode 100644 apitools/base/py/testing/mock.py create mode 100644 apitools/base/py/testing/mock_test.py create mode 100644 apitools/base/py/testing/testclient/__init__.py create mode 100644 apitools/base/py/testing/testclient/fusiontables_v1_client.py create mode 100644 apitools/base/py/testing/testclient/fusiontables_v1_messages.py diff --git a/apitools/base/py/testing/__init__.py b/apitools/base/py/testing/__init__.py new file mode 100644 index 0000000..27e204d --- /dev/null +++ b/apitools/base/py/testing/__init__.py @@ -0,0 +1 @@ +"""Package marker file.""" diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py new file mode 100644 index 0000000..0d9715a --- /dev/null +++ b/apitools/base/py/testing/mock.py @@ -0,0 +1,321 @@ +"""The mock module allows easy mocking of apitools clients. + +This module allows you to mock out the constructor of a particular apitools +client, for a specific API and version. Then, when the client is created, it +will be run against an expected session that you define. This way code that is +not aware of the testing framework can construct new clients as normal, as long +as it's all done within the context of a mock. +""" + +import difflib +import textwrap + +from protorpc import messages +import apitools.base.py as apitools_base + + +class Error(Exception): + + """Exceptions for this module.""" + + +def _MessagesEqual(msg1, msg2): + """Compare two protorpc messages for equality. + + Using python's == operator does not work in all cases, specifically when + there is a list involved. + + Args: + msg1: protorpc.messages.Message or [protorpc.messages.Message] or number + or string, One of the messages to compare. + msg2: protorpc.messages.Message or [protorpc.messages.Message] or number + or string, One of the messages to compare. + + Returns: + If the messages are isomorphic. + """ + if isinstance(msg1, list) and isinstance(msg2, list): + if len(msg1) != len(msg2): + return False + return all(_MessagesEqual(x, y) for x, y in zip(msg1, msg2)) + + if (not isinstance(msg1, messages.Message) or + not isinstance(msg2, messages.Message)): + return msg1 == msg2 + for field in msg1.all_fields(): + field1 = getattr(msg1, field.name) + field2 = getattr(msg2, field.name) + if not _MessagesEqual(field1, field2): + return False + return True + + +class UnexpectedRequestException(Error): + + def __init__(self, received_call, expected_call): + expected_key, expected_request = expected_call + received_key, received_request = received_call + + expected_repr = apitools_base.MessageToRepr( + expected_request, multiline=True) + received_repr = apitools_base.MessageToRepr( + received_request, multiline=True) + + expected_lines = expected_repr.splitlines() + received_lines = received_repr.splitlines() + + diff_lines = difflib.unified_diff(expected_lines, received_lines) + diff = '\n'.join(diff_lines) + + if expected_key != received_key: + msg = textwrap.dedent("""\ + expected: {expected_key}({expected_request}) + received: {received_key}({received_request}) + """).format( + expected_key=expected_key, + expected_request=expected_repr, + received_key=received_key, + received_request=received_repr) + super(UnexpectedRequestException, self).__init__(msg) + else: + msg = textwrap.dedent("""\ + for request to {key}, + expected: {expected_request} + received: {received_request} + diff: {diff} + """).format( + key=expected_key, + expected_request=expected_repr, + received_request=received_repr, + diff=diff) + super(UnexpectedRequestException, self).__init__(msg) + + +class ExpectedRequestsException(Error): + + def __init__(self, expected_calls): + msg = 'expected:\n' + for (key, request) in expected_calls: + msg += '{key}({request})\n'.format( + key=key, + request=apitools_base.MessageToRepr(request, multiline=True)) + super(ExpectedRequestsException, self).__init__(msg) + + +class _ExpectedRequestResponse(object): + + """Encapsulation of an expected request and corresponding response.""" + + def __init__(self, key, request, response=None, exception=None): + self.__key = key + self.__request = request + + if response and exception: + raise apitools_base.ConfigurationValueError( + 'Should specify at most one of response and exception') + if response and isinstance(response, apitools_base.Error): + raise apitools_base.ConfigurationValueError( + 'Responses should not be an instance of Error') + if exception and not isinstance(exception, apitools_base.Error): + raise apitools_base.ConfigurationValueError( + 'Exceptions must be instances of Error') + + self.__response = response + self.__exception = exception + + @property + def key(self): + return self.__key + + @property + def request(self): + return self.__request + + def ValidateAndRespond(self, key, request): + """Validate that key and request match expectations, and respond if so. + + Args: + key: str, Actual key to compare against expectations. + request: protorpc.messages.Message or [protorpc.messages.Message] + or number or string, Actual request to compare againt expectations + + Raises: + UnexpectedRequestException: If key or request dont match + expectations. + apitools_base.Error: If a non-None exception is specified to + be thrown. + + Returns: + The response that was specified to be returned. + + """ + if key != self.__key or not _MessagesEqual(request, self.__request): + raise UnexpectedRequestException((key, request), + (self.__key, self.__request)) + + if self.__exception: + # pylint:disable=raising-bad-type, Can only throw + # apitools_base.Error. + raise self.__exception + + return self.__response + + +class _MockedService(apitools_base.BaseApiService): + + def __init__(self, key, mocked_client, methods, real_service): + self.__dict__.update(real_service.__dict__) + for method in methods: + real_method = None + if real_service: + real_method = getattr(real_service, method) + setattr(self, method, + _MockedMethod(key + '.' + method, + mocked_client, + real_method)) + + +class _MockedMethod(object): + + """A mocked API service method.""" + + def __init__(self, key, mocked_client, real_method): + self.__key = key + self.__mocked_client = mocked_client + self.__real_method = real_method + + def Expect(self, request, response=None, exception=None, **unused_kwargs): + """Add an expectation on the mocked method. + + Exactly one of response and exception should be specified. + + Args: + request: The request that should be expected + response: The response that should be returned or None if + exception is provided. + exception: An exception that should be thrown, or None. + + """ + # TODO(jasmuth): the unused_kwargs provides a placeholder for + # future things that can be passed to Expect(), like special + # params to the method call. + + # pylint:disable=protected-access, Class in same module. + self.__mocked_client._request_responses.append( + _ExpectedRequestResponse(self.__key, + request, + response=response, + exception=exception)) + + def __call__(self, request, **unused_kwargs): + # TODO(jasmuth): allow the testing code to expect certain + # values in these currently unused_kwargs, especially the + # upload parameter used by media-heavy services like bigquery + # or bigstore. + + # pylint:disable=protected-access, Class in same module. + if self.__mocked_client._request_responses: + request_response = self.__mocked_client._request_responses.pop(0) + else: + raise UnexpectedRequestException( + (self.__key, request), (None, None)) + + response = request_response.ValidateAndRespond(self.__key, request) + + if response is None and self.__real_method: + response = self.__real_method(request) + print apitools_base.MessageToRepr( + response, multiline=True, shortstrings=True) + return response + + return response + + +def _MakeMockedServiceConstructor(mocked_service): + def Constructor(unused_self, unused_client): + return mocked_service + return Constructor + + +class Client(object): + + """Mock an apitools client.""" + + def __init__(self, client_class, real_client=None): + """Mock an apitools API, given its class. + + Args: + client_class: The class for the API. eg, if you + from apis.sqladmin import v1beta3 + then you can pass v1beta3.SqladminV1beta3 to this class + and anything within its context will use your mocked + version. + real_client: apitools Client, The client to make requests + against when the expected response is None. + + """ + + if not real_client: + real_client = client_class(get_credentials=False) + + self.__client_class = client_class + self.__real_service_classes = {} + self.__real_client = real_client + + self._request_responses = [] + + def __enter__(self): + return self.Mock() + + def Mock(self): + """Stub out the client class with mocked services.""" + client = self.__real_client or self.__client_class( + get_credentials=False) + for name in dir(self.__client_class): + service_class = getattr(self.__client_class, name) + if not isinstance(service_class, type): + continue + if not issubclass(service_class, apitools_base.BaseApiService): + continue + self.__real_service_classes[name] = service_class + service = service_class(client) + # pylint:disable=protected-access, Some liberty is allowed with + # mocking. + collection_name = service_class._NAME + api_name = '%s_%s' % (self.__client_class._PACKAGE, + self.__client_class._URL_VERSION) + mocked_service = _MockedService( + api_name + '.' + collection_name, self, + service._method_configs.keys(), + service if self.__real_client else None) + mocked_constructor = _MakeMockedServiceConstructor(mocked_service) + setattr(self.__client_class, name, mocked_constructor) + + setattr(self, collection_name, mocked_service) + + self.__real_include_fields = self.__client_class.IncludeFields + self.__client_class.IncludeFields = self.IncludeFields + + return self + + def __exit__(self, exc_type, value, traceback): + self.Unmock() + if value: + raise exc_type, value, traceback + return True + + def Unmock(self): + for name, service_class in self.__real_service_classes.iteritems(): + setattr(self.__client_class, name, service_class) + + if self._request_responses: + raise ExpectedRequestsException( + [(rq_rs.key, rq_rs.request) for rq_rs + in self._request_responses]) + + self.__client_class.IncludeFields = self.__real_include_fields + + def IncludeFields(self, include_fields): + if self.__real_client: + return self.__real_include_fields(self.__real_client, + include_fields) diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py new file mode 100644 index 0000000..395b7a7 --- /dev/null +++ b/apitools/base/py/testing/mock_test.py @@ -0,0 +1,132 @@ +"""Tests for apitools.base.py.testing.mock.""" + +from protorpc import messages +import unittest2 + +import apitools.base.py as apitools_base +from apitools.base.py.testing import mock +from apitools.base.py.testing import testclient as fusiontables + + +class MockTest(unittest2.TestCase): + + def testMockFusionBasic(self): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect(request=1, response=2) + client = fusiontables.FusiontablesV1(get_credentials=False) + self.assertEqual(client.column.List(1), 2) + with self.assertRaises(mock.UnexpectedRequestException): + client.column.List(3) + + def testMockFusionException(self): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect( + request=1, + exception=apitools_base.HttpError({'status': 404}, '', '')) + client = fusiontables.FusiontablesV1(get_credentials=False) + with self.assertRaises(apitools_base.HttpError): + client.column.List(1) + + def testMockFusionOrder(self): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect(request=1, response=2) + client_class.column.List.Expect(request=2, response=1) + client = fusiontables.FusiontablesV1(get_credentials=False) + self.assertEqual(client.column.List(1), 2) + self.assertEqual(client.column.List(2), 1) + + def testMockFusionWrongOrder(self): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect(request=1, response=2) + client_class.column.List.Expect(request=2, response=1) + client = fusiontables.FusiontablesV1(get_credentials=False) + with self.assertRaises(mock.UnexpectedRequestException): + self.assertEqual(client.column.List(2), 1) + with self.assertRaises(mock.UnexpectedRequestException): + self.assertEqual(client.column.List(1), 2) + + def testMockFusionTooMany(self): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect(request=1, response=2) + client = fusiontables.FusiontablesV1(get_credentials=False) + self.assertEqual(client.column.List(1), 2) + with self.assertRaises(mock.UnexpectedRequestException): + self.assertEqual(client.column.List(2), 1) + + def testMockFusionTooFew(self): + with self.assertRaises(mock.ExpectedRequestsException): + with mock.Client(fusiontables.FusiontablesV1) as client_class: + client_class.column.List.Expect(request=1, response=2) + client_class.column.List.Expect(request=2, response=1) + client = fusiontables.FusiontablesV1(get_credentials=False) + self.assertEqual(client.column.List(1), 2) + + def testFusionUnmock(self): + with mock.Client(fusiontables.FusiontablesV1): + client = fusiontables.FusiontablesV1(get_credentials=False) + mocked_service_type = type(client.column) + client = fusiontables.FusiontablesV1(get_credentials=False) + self.assertNotEqual(type(client.column), mocked_service_type) + + +class _NestedMessage(messages.Message): + nested = messages.StringField(1) + + +class _NestedListMessage(messages.Message): + nested_list = messages.MessageField(_NestedMessage, 1, repeated=True) + + +class _NestedNestedMessage(messages.Message): + nested = messages.MessageField(_NestedMessage, 1) + + +class UtilTest(unittest2.TestCase): + + def testMessagesEqual(self): + self.assertFalse(mock._MessagesEqual( + _NestedNestedMessage( + nested=_NestedMessage( + nested='foo')), + _NestedNestedMessage( + nested=_NestedMessage( + nested='bar')))) + + self.assertTrue(mock._MessagesEqual( + _NestedNestedMessage( + nested=_NestedMessage( + nested='foo')), + _NestedNestedMessage( + nested=_NestedMessage( + nested='foo')))) + + def testListedMessagesEqual(self): + self.assertTrue(mock._MessagesEqual( + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo')]), + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo')]))) + + self.assertTrue(mock._MessagesEqual( + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo'), + _NestedMessage(nested='foo2')]), + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo'), + _NestedMessage(nested='foo2')]))) + + self.assertFalse(mock._MessagesEqual( + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo')]), + _NestedListMessage( + nested_list=[_NestedMessage(nested='bar')]))) + + self.assertFalse(mock._MessagesEqual( + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo')]), + _NestedListMessage( + nested_list=[_NestedMessage(nested='foo'), + _NestedMessage(nested='foo')]))) + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/testing/testclient/__init__.py b/apitools/base/py/testing/testclient/__init__.py new file mode 100644 index 0000000..20aa374 --- /dev/null +++ b/apitools/base/py/testing/testclient/__init__.py @@ -0,0 +1,5 @@ +"""Common imports for generated fusiontables client library.""" +# pylint:disable=wildcard-import + +from apitools.base.py.testing.testclient.fusiontables_v1_client import * +from apitools.base.py.testing.testclient.fusiontables_v1_messages import * diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_client.py b/apitools/base/py/testing/testclient/fusiontables_v1_client.py new file mode 100644 index 0000000..1be17b3 --- /dev/null +++ b/apitools/base/py/testing/testclient/fusiontables_v1_client.py @@ -0,0 +1,78 @@ +"""Generated client library for fusiontables version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. + +from apitools.base.py import base_api +from apitools.base.py.testing.testclient import fusiontables_v1_messages + + +class FusiontablesV1(base_api.BaseApiClient): + + """Generated client library for service fusiontables version v1.""" + + MESSAGES_MODULE = fusiontables_v1_messages + + _PACKAGE = u'fusiontables' + _SCOPES = [u'https://www.googleapis.com/auth/fusiontables', + u'https://www.googleapis.com/auth/fusiontables.readonly'] + _VERSION = u'v1' + _CLIENT_ID = '1042881264118.apps.googleusercontent.com' + _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _USER_AGENT = '' + _CLIENT_CLASS_NAME = u'FusiontablesV1' + _URL_VERSION = u'v1' + + def __init__(self, url='', credentials=None, + get_credentials=True, http=None, model=None, + log_request=False, log_response=False, + credentials_args=None, default_global_params=None, + additional_http_headers=None): + """Create a new fusiontables handle.""" + url = url or u'https://www.googleapis.com/fusiontables/v1/' + super(FusiontablesV1, self).__init__( + url, credentials=credentials, + get_credentials=get_credentials, http=http, model=model, + log_request=log_request, log_response=log_response, + credentials_args=credentials_args, + default_global_params=default_global_params, + additional_http_headers=additional_http_headers) + self.column = self.ColumnService(self) + + class ColumnService(base_api.BaseApiService): + + """Service class for the column resource.""" + + _NAME = u'column' + + def __init__(self, client): + super(FusiontablesV1.ColumnService, self).__init__(client) + self._method_configs = { + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/columns', + request_field='', + request_type_name=u'FusiontablesColumnListRequest', + response_type_name=u'ColumnList', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def List(self, request, global_params=None): + """Retrieves a list of columns. + + Args: + request: (FusiontablesColumnListRequest) input message + global_params: (StandardQueryParameters, default: None) global + arguments + Returns: + (ColumnList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py new file mode 100644 index 0000000..1148f62 --- /dev/null +++ b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py @@ -0,0 +1,93 @@ +"""Generated message classes for fusiontables version v1. + +API for working with Fusion Tables data. +""" +# NOTE: This file is autogenerated and should not be edited by hand. + +from protorpc import messages + + +package = 'fusiontables' + + +class Column(messages.Message): + + """Specifies the id, name and type of a column in a table. + + Messages: + BaseColumnValue: Optional identifier of the base column. If present, this + column is derived from the specified base column. + + Fields: + baseColumn: Optional identifier of the base column. If present, this + column is derived from the specified base column. + columnId: Identifier for the column. + description: Optional column description. + graph_predicate: Optional column predicate. Used to map table to + graph data model (subject,predicate,object) See + http://www.w3.org/TR/2014/REC- + rdf11-concepts-20140225/#data-model + kind: Type name: a template for an individual column. + name: Required name of the column. + type: Required type of the column. + + """ + + class BaseColumnValue(messages.Message): + + """Optional identifier of the base column. If present, this column is + derived from the specified base column. + + Fields: + columnId: The id of the column in the base table from which + this column is derived. + tableIndex: Offset to the entry in the list of base tables + in the table definition. + + """ + + columnId = messages.IntegerField(1, variant=messages.Variant.INT32) + tableIndex = messages.IntegerField(2, variant=messages.Variant.INT32) + + baseColumn = messages.MessageField('BaseColumnValue', 1) + columnId = messages.IntegerField(2, variant=messages.Variant.INT32) + description = messages.StringField(3) + graph_predicate = messages.StringField(4) + kind = messages.StringField(5, default=u'fusiontables#column') + name = messages.StringField(6) + type = messages.StringField(7) + + +class ColumnList(messages.Message): + + """Represents a list of columns in a table. + + Fields: + items: List of all requested columns. + kind: Type name: a list of all columns. + nextPageToken: Token used to access the next page of this + result. No token is displayed if there are no more pages left. + totalItems: Total number of columns for the table. + + """ + + items = messages.MessageField('Column', 1, repeated=True) + kind = messages.StringField(2, default=u'fusiontables#columnList') + nextPageToken = messages.StringField(3) + totalItems = messages.IntegerField(4, variant=messages.Variant.INT32) + + +class FusiontablesColumnListRequest(messages.Message): + + """A FusiontablesColumnListRequest object. + + Fields: + maxResults: Maximum number of columns to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + tableId: Table whose columns are being listed. + """ + + maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) + pageToken = messages.StringField(2) + tableId = messages.StringField(3, required=True) diff --git a/run_pylint.py b/run_pylint.py index aa48577..d6b89d2 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -21,6 +21,7 @@ IGNORED_DIRECTORIES = [ ] IGNORED_FILES = [ 'ez_setup.py', + 'run_pylint.py', 'setup.py', ] PRODUCTION_RC = 'default.pylintrc' diff --git a/tox.ini b/tox.ini index fe1d295..708d795 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ commands = nosetests --with-xunit --with-xcoverage --cover-package=apitools --nocapture --cover-erase --cover-tests --cover-branches deps = google-apputils + python-gflags mock nose unittest2 -- GitLab From ec43a856d16cff62ba4be861e645c9e03e8ac86f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 4 May 2015 01:41:58 -0700 Subject: [PATCH 102/295] Fix some python2-isms in mock, and de-lint. This fixes the last three python2-isms (iteritems, three-arg raise, print statement) in mock.py, and cleans up the remaining lint warnings. --- apitools/base/py/testing/mock.py | 52 ++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index 0d9715a..75a8b07 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -8,9 +8,10 @@ as it's all done within the context of a mock. """ import difflib -import textwrap from protorpc import messages +import six + import apitools.base.py as apitools_base @@ -68,22 +69,24 @@ class UnexpectedRequestException(Error): diff = '\n'.join(diff_lines) if expected_key != received_key: - msg = textwrap.dedent("""\ - expected: {expected_key}({expected_request}) - received: {received_key}({received_request}) - """).format( + msg = '\n'.join(( + 'expected: {expected_key}({expected_request})', + 'received: {received_key}({received_request})', + '', + )).format( expected_key=expected_key, expected_request=expected_repr, received_key=received_key, received_request=received_repr) super(UnexpectedRequestException, self).__init__(msg) else: - msg = textwrap.dedent("""\ - for request to {key}, - expected: {expected_request} - received: {received_request} - diff: {diff} - """).format( + msg = '\n'.join(( + 'for request to {key},', + 'expected: {expected_request}', + 'received: {received_request}', + 'diff: {diff}', + '', + )).format( key=expected_key, expected_request=expected_repr, received_request=received_repr, @@ -154,9 +157,8 @@ class _ExpectedRequestResponse(object): (self.__key, self.__request)) if self.__exception: - # pylint:disable=raising-bad-type, Can only throw - # apitools_base.Error. - raise self.__exception + # Can only throw apitools_base.Error. + raise self.__exception # pylint: disable=raising-bad-type return self.__response @@ -164,6 +166,7 @@ class _ExpectedRequestResponse(object): class _MockedService(apitools_base.BaseApiService): def __init__(self, key, mocked_client, methods, real_service): + super(_MockedService, self).__init__(mocked_client) self.__dict__.update(real_service.__dict__) for method in methods: real_method = None @@ -200,12 +203,14 @@ class _MockedMethod(object): # future things that can be passed to Expect(), like special # params to the method call. - # pylint:disable=protected-access, Class in same module. + # pylint: disable=protected-access + # Class in same module. self.__mocked_client._request_responses.append( _ExpectedRequestResponse(self.__key, request, response=response, exception=exception)) + # pylint: enable=protected-access def __call__(self, request, **unused_kwargs): # TODO(jasmuth): allow the testing code to expect certain @@ -213,19 +218,21 @@ class _MockedMethod(object): # upload parameter used by media-heavy services like bigquery # or bigstore. - # pylint:disable=protected-access, Class in same module. + # pylint: disable=protected-access + # Class in same module. if self.__mocked_client._request_responses: request_response = self.__mocked_client._request_responses.pop(0) else: raise UnexpectedRequestException( (self.__key, request), (None, None)) + # pylint: enable=protected-access response = request_response.ValidateAndRespond(self.__key, request) if response is None and self.__real_method: response = self.__real_method(request) - print apitools_base.MessageToRepr( - response, multiline=True, shortstrings=True) + print(apitools_base.MessageToRepr( + response, multiline=True, shortstrings=True)) return response return response @@ -279,9 +286,10 @@ class Client(object): continue self.__real_service_classes[name] = service_class service = service_class(client) - # pylint:disable=protected-access, Some liberty is allowed with - # mocking. + # pylint: disable=protected-access + # Some liberty is allowed with mocking. collection_name = service_class._NAME + # pylint: enable=protected-access api_name = '%s_%s' % (self.__client_class._PACKAGE, self.__client_class._URL_VERSION) mocked_service = _MockedService( @@ -301,11 +309,11 @@ class Client(object): def __exit__(self, exc_type, value, traceback): self.Unmock() if value: - raise exc_type, value, traceback + six.reraise(exc_type, value, traceback) return True def Unmock(self): - for name, service_class in self.__real_service_classes.iteritems(): + for name, service_class in self.__real_service_classes.items(): setattr(self.__client_class, name, service_class) if self._request_responses: -- GitLab From 2894e54115c470eff61552843e733d44f42fd5be Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 4 May 2015 01:45:18 -0700 Subject: [PATCH 103/295] Update tox and travis envs. Travis still doesn't have cover, pending next PR. --- .travis.yml | 4 +++- tox.ini | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9d3d31..a44cab5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ language: python env: + - TOX_ENV=py26 - TOX_ENV=py27 - - TOX_ENV=pypy - TOX_ENV=py34 + - TOX_ENV=pypy + - TOX_ENV=lint install: - pip install tox - pip install . --allow-external argparse diff --git a/tox.ini b/tox.ini index 708d795..7080887 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pypy,py33,py34 +envlist = py26,py27,pypy,py33,py34,lint,cover [testenv] deps = nose -- GitLab From f83dbc79dab2e10f8aebb1196da842616b2e8473 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 4 May 2015 01:30:57 -0700 Subject: [PATCH 104/295] Fix credentials_lib tests to not rely on sys.argv. Previously, some tests in credentials_lib were implicitly reading argv (via `argparse`); this cleans them up to have an explicit argv passed in. --- apitools/base/py/credentials_lib.py | 4 ++-- apitools/base/py/credentials_lib_test.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 6651aba..76629ce 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -403,10 +403,10 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): self.access_token = token -def _GetRunFlowFlags(): +def _GetRunFlowFlags(args=None): parser = argparse.ArgumentParser(parents=[tools.argparser]) # Get command line argparse flags. - flags = parser.parse_args() + flags = parser.parse_args(args=args) # Allow `gflags` and `argparse` to be used side-by-side. if hasattr(FLAGS, 'auth_host_name'): diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 12f71c3..1bca25a 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -74,8 +74,8 @@ class TestGetRunFlowFlags(unittest2.TestCase): credentials_lib.FLAGS = self._flags_actual def test_with_gflags(self): - HOST = object() - PORT = object() + HOST = 'myhostname' + PORT = '144169' class MockFlags(object): auth_host_name = HOST @@ -83,7 +83,11 @@ class TestGetRunFlowFlags(unittest2.TestCase): auth_local_webserver = False credentials_lib.FLAGS = MockFlags - flags = credentials_lib._GetRunFlowFlags() + flags = credentials_lib._GetRunFlowFlags([ + '--auth_host_name=%s' % HOST, + '--auth_host_port=%s' % PORT, + '--noauth_local_webserver', + ]) self.assertEqual(flags.auth_host_name, HOST) self.assertEqual(flags.auth_host_port, PORT) self.assertEqual(flags.logging_level, 'ERROR') @@ -91,7 +95,7 @@ class TestGetRunFlowFlags(unittest2.TestCase): def test_without_gflags(self): credentials_lib.FLAGS = None - flags = credentials_lib._GetRunFlowFlags() + flags = credentials_lib._GetRunFlowFlags([]) self.assertEqual(flags.auth_host_name, 'localhost') self.assertEqual(flags.auth_host_port, [8080, 8090]) self.assertEqual(flags.logging_level, 'ERROR') -- GitLab From 3e23d74e4ee21fd4dcd16f18ef15b7b95fb6918b Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 4 May 2015 14:17:02 -0700 Subject: [PATCH 105/295] Update version for 0.4.4 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d9ba756..690b1af 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.3' +_APITOOLS_VERSION = '0.4.4' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 4c0ad578d73184568df8594e232011116e302a88 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 5 May 2015 16:29:04 -0700 Subject: [PATCH 106/295] Validate user-provided MIME types on uploads. Currently, we'll allow any string as the MIME type -- but valid MIME types must contain a `/`. The code implicitly assumes this, so we might as well check it and raise a comprehensible error. --- apitools/base/py/util.py | 3 +++ apitools/base/py/util_test.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 06f45cc..779dd97 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -157,6 +157,9 @@ def AcceptableMimeType(accept_patterns, mime_type): Returns: Whether or not mime_type matches (at least) one of these patterns. """ + if '/' not in mime_type: + raise exceptions.InvalidUserInputError( + 'Invalid MIME type: "%s"' % mime_type) unsupported_patterns = [p for p in accept_patterns if ';' in p] if unsupported_patterns: raise exceptions.GeneratedClientError( diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 384aa4f..16767a9 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -142,6 +142,11 @@ class UtilTest(unittest2.TestCase): self.assertFalse(util.AcceptableMimeType(['application/json', 'img/*'], 'text/plain')) + def testMalformedMimeType(self): + self.assertRaises( + exceptions.InvalidUserInputError, + util.AcceptableMimeType, ['*/*'], 'abcd') + def testUnsupportedMimeType(self): self.assertRaises( exceptions.GeneratedClientError, -- GitLab From 222af1f10d80938cbec5ff0165c5a25456e014b0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 13 May 2015 16:46:03 -0700 Subject: [PATCH 107/295] Set a minimum required oauth2client version. This ensures we pick up recent fixes for better `body` handling on 401 when `body` is a stream. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 690b1af..bacc0f8 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ except ImportError: # Python version and OS. REQUIRED_PACKAGES = [ 'httplib2>=0.8', - 'oauth2client>=1.2', + 'oauth2client>=1.4.8', 'protorpc>=0.9.1', 'six>=1.8.0', ] -- GitLab From 0b6a9f5a30f8dc73e4f5fc8176b94d3e96d19f78 Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Wed, 13 May 2015 16:53:35 -0700 Subject: [PATCH 108/295] Stop retrying 401s, since oauth2client handles them properly for streams and expired tokens. --- apitools/base/py/http_wrapper.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index e5fd2fe..d9add5a 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -199,11 +199,6 @@ def CheckResponse(response): elif (response.status_code >= 500 or response.status_code == TOO_MANY_REQUESTS): raise exceptions.BadStatusCodeError.FromResponse(response) - elif response.status_code == http_client.UNAUTHORIZED: - # Sometimes we get a 401 after a connection break. - # TODO(craigcitro): this shouldn't be a retryable exception, but - # for now we retry. - raise exceptions.BadStatusCodeError.FromResponse(response) elif response.retry_after: raise exceptions.RetryAfterError.FromResponse(response) -- GitLab From 7154ce4e372eba7a9ff33530a1c347a42952f301 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 13 May 2015 17:04:30 -0700 Subject: [PATCH 109/295] Update version for v0.4.5 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bacc0f8..32553cd 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.4' +_APITOOLS_VERSION = '0.4.5' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From d23d050feb4d2dacad9486acb16a1d1e164a4954 Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Thu, 14 May 2015 12:54:07 -0700 Subject: [PATCH 110/295] Fix TypeError in credentials_lib This fixes an error introduced with lint fixing where computing the token_expiry of a credential with an expires_in value would result in an exception. --- apitools/base/py/credentials_lib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 76629ce..d49da5a 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -335,8 +335,9 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): self.access_token = credential_info['access_token'] if 'expires_in' in credential_info: expires_in = int(credential_info['expires_in']) - self.token_expiry = datetime.timedelta( - seconds=expires_in + datetime.datetime.utcnow()) + self.token_expiry = ( + datetime.timedelta(seconds=expires_in) + + datetime.datetime.utcnow()) else: self.token_expiry = None self.invalid = False -- GitLab From 375e1b4541c696fba160bc8c94a61559d61325e7 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 14 May 2015 13:04:36 -0700 Subject: [PATCH 111/295] Update version for v0.4.6 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 32553cd..8edfe64 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.5' +_APITOOLS_VERSION = '0.4.6' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 1bfeb8c2aef0af78a1737288064b4b28192a2651 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 20 May 2015 11:48:49 -0700 Subject: [PATCH 112/295] Conditionally import argparse in credentials_lib. This is a bit odd, but avoids a much nastier workaround in gsutil. --- apitools/base/py/credentials_lib.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index d49da5a..3078943 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -2,7 +2,6 @@ """Common credentials classes and constructors.""" from __future__ import print_function -import argparse import datetime import json import os @@ -405,6 +404,15 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): def _GetRunFlowFlags(args=None): + # There's one rare situation where gsutil will not have argparse + # available, but doesn't need anything depending on argparse anyway, + # since they're bringing their own credentials. So we just allow this + # to fail with an ImportError in those cases. + # + # TODO(craigcitro): Move this import back to the top when we drop + # python 2.6 support (eg when gsutil does). + import argparse + parser = argparse.ArgumentParser(parents=[tools.argparser]) # Get command line argparse flags. flags = parser.parse_args(args=args) -- GitLab From a9602587c4f6bff542d80edc7ca624739a3f5c43 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 20 May 2015 11:57:14 -0700 Subject: [PATCH 113/295] Update version for v0.4.7 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8edfe64..4b323a6 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.6' +_APITOOLS_VERSION = '0.4.7' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 0a349349237c2837477bc92662c75a584986bce8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 24 May 2015 14:28:59 -0700 Subject: [PATCH 114/295] Some coverage-related cleanup. --- .travis.yml | 2 +- README.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a44cab5..61f21e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,4 @@ install: - pip install . --allow-external argparse script: tox -e $TOX_ENV after_success: - - tox -e coveralls + - if [[ "${TOX_ENV}" == "py27" ]]; then tox -e coveralls; fi diff --git a/README.rst b/README.rst index 3c8e901..e9eae94 100644 --- a/README.rst +++ b/README.rst @@ -43,5 +43,5 @@ Then run the tests:: :target: https://travis-ci.org/google/apitools .. |pypi| image:: https://img.shields.io/pypi/v/google-apitools.svg :target: https://pypi.python.org/pypi/google-apitools -.. |coverage| image:: https://coveralls.io/repos/google/apitools/badge.png?branch=master +.. |coverage| image:: https://coveralls.io/repos/google/apitools/badge.svg?branch=master :target: https://coveralls.io/r/google/apitools?branch=master -- GitLab From 51e433e06c009052e11a5a07027dd46e4ebfefce Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 24 May 2015 14:38:41 -0700 Subject: [PATCH 115/295] Add tweak for travis+coveralls+tox. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 7080887..bdde060 100644 --- a/tox.ini +++ b/tox.ini @@ -60,3 +60,4 @@ commands = deps = {[testenv:cover]deps} coveralls +passenv = TRAVIS* -- GitLab From 3edef91ef1d37bb57c68804ccd3fb772d22fdfbf Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 24 May 2015 23:26:45 -0700 Subject: [PATCH 116/295] Add a hook for command-line args in tox. This allows us to call `tox -e py27 -- foo.bar` and test just `foo.bar`. --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index bdde060..5cd3c74 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist = py26,py27,pypy,py33,py34,lint,cover deps = nose commands = pip install google-apitools[testing] - nosetests + nosetests [] [testenv:py33] basepython = python3.3 @@ -13,7 +13,7 @@ deps = mock nose unittest2 -commands = nosetests +commands = nosetests [] [testenv:py34] basepython = python3.4 @@ -21,7 +21,7 @@ deps = mock nose unittest2 -commands = nosetests +commands = nosetests [] [pep8] exclude = samples/storage_sample/storage,samples/storage_sample/testdata,*.egg/,.*/,ez_setup.py @@ -42,7 +42,7 @@ deps = basepython = python2.7 commands = - nosetests --with-xunit --with-xcoverage --cover-package=apitools --nocapture --cover-erase --cover-tests --cover-branches + nosetests --with-xunit --with-xcoverage --cover-package=apitools --nocapture --cover-erase --cover-tests --cover-branches [] deps = google-apputils python-gflags -- GitLab From d3caffdf506eb0764cafe1aa3c7d281ea59d8665 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 24 May 2015 23:19:04 -0700 Subject: [PATCH 117/295] Batch cleanup: fix a bug, add tests. This finally exports the internal batch tests to this repo, and fixes a newly-introduced bug in the process. Also cleans up some bad unicode handling in python3 for batch, and a corner case with loggable bodies in http_wrapper. --- apitools/base/py/batch.py | 20 +- apitools/base/py/batch_test.py | 513 +++++++++++++++++++++++++++++++ apitools/base/py/http_wrapper.py | 2 +- 3 files changed, 527 insertions(+), 8 deletions(-) create mode 100644 apitools/base/py/batch_test.py diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 9477531..4910d4a 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -7,10 +7,10 @@ import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart import email.parser as email_parser import itertools -import io import time import uuid +import six from six.moves import http_client from six.moves import urllib_parse @@ -100,7 +100,9 @@ class BatchApiRequest(object): @property def terminal_state(self): - response_code = getattr(self.__http_response, 'status_code', 0) + if self.__http_response is None: + return False + response_code = self.__http_response.status_code return response_code not in self.__retryable_codes def HandleResponse(self, http_response, exception): @@ -176,8 +178,8 @@ class BatchApiRequest(object): Returns: List of ApiCalls. """ - requests = [request for request in self.api_requests if not - request.terminal_state] + requests = [request for request in self.api_requests + if not request.terminal_state] for attempt in range(max_retries): if attempt: @@ -290,7 +292,11 @@ class BatchHttpRequest(object): parsed = urllib_parse.urlsplit(request.url) request_line = urllib_parse.urlunsplit( (None, None, parsed.path, parsed.query, None)) - status_line = request.http_method + ' ' + request_line + ' HTTP/1.1\n' + status_line = u' '.join(( + request.http_method, + request_line.decode('utf-8'), + u'HTTP/1.1\n' + )) major, minor = request.headers.get( 'content-type', 'application/json').split('/') msg = mime_nonmultipart.MIMENonMultipart(major, minor) @@ -309,7 +315,7 @@ class BatchHttpRequest(object): msg.set_payload(request.body) # Serialize the mime message. - str_io = io.StringIO() + str_io = six.StringIO() # maxheaderlen=0 means don't line wrap headers. gen = generator.Generator(str_io, maxheaderlen=0) gen.flatten(msg, unixfrom=False) @@ -320,7 +326,7 @@ class BatchHttpRequest(object): if request.body is None: body = body[:-2] - return status_line.encode('utf-8') + body + return status_line + body def _DeserializeResponse(self, payload): """Convert string into Response and content. diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py new file mode 100644 index 0000000..ee421b3 --- /dev/null +++ b/apitools/base/py/batch_test.py @@ -0,0 +1,513 @@ +"""Tests for google3.cloud.bigscience.apitools.base.py.batch.""" + +import textwrap + +import mock +from six.moves import http_client +from six.moves.urllib import parse +import unittest2 + +from apitools.base.py import batch +from apitools.base.py import exceptions +from apitools.base.py import http_wrapper + + +class FakeCredentials(object): + + def __init__(self): + self.num_refreshes = 0 + + def refresh(self, _): + self.num_refreshes += 1 + + +class FakeHttp(object): + + class FakeRequest(object): + + def __init__(self, credentials=None): + if credentials is not None: + self.credentials = credentials + + def __init__(self, credentials=None): + self.request = FakeHttp.FakeRequest(credentials=credentials) + + +class FakeService(object): + + """A service for testing.""" + + def GetMethodConfig(self, _): + return {} + + def GetUploadConfig(self, _): + return {} + + # pylint: disable=unused-argument + def PrepareHttpRequest( + self, method_config, request, global_params, upload_config): + return global_params['desired_request'] + # pylint: enable=unused-argument + + def ProcessHttpResponse(self, _, http_response): + return http_response + + +class BatchTest(unittest2.TestCase): + + def assertUrlEqual(self, expected_url, provided_url): + + def parse_components(url): + parsed = parse.urlsplit(url) + query = parse.parse_qs(parsed.query) + return parsed._replace(query=''), query + + expected_parse, expected_query = parse_components(expected_url) + provided_parse, provided_query = parse_components(provided_url) + + self.assertEqual(expected_parse, provided_parse) + self.assertEqual(expected_query, provided_query) + + def __ConfigureMock(self, mock_request, expected_request, response): + + if isinstance(response, list): + response = list(response) + + def CheckRequest(_, request, **unused_kwds): + self.assertUrlEqual(expected_request.url, request.url) + self.assertEqual(expected_request.http_method, request.http_method) + if isinstance(response, list): + return response.pop(0) + else: + return response + + mock_request.side_effect = CheckRequest + + def testRequestServiceUnavailable(self): + mock_service = FakeService() + + desired_url = 'https://www.example.com' + batch_api_request = batch.BatchApiRequest(batch_url=desired_url, + retryable_codes=[]) + # The request to be added. The actual request sent will be somewhat + # larger, as this is added to a batch. + desired_request = http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80) + + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 419, + }, 'x' * 419), + http_wrapper.Response({ + 'status': '200', + 'content-type': 'multipart/mixed; boundary="boundary"', + }, textwrap.dedent("""\ + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 503 SERVICE UNAVAILABLE + nope + --boundary--"""), None)) + + batch_api_request.Add( + mock_service, 'unused', None, + global_params={'desired_request': desired_request}) + + api_request_responses = batch_api_request.Execute( + FakeHttp(), sleep_between_polls=0) + + self.assertEqual(1, len(api_request_responses)) + + # Make sure we didn't retry non-retryable code 503. + self.assertEqual(1, mock_request.call_count) + + self.assertTrue(api_request_responses[0].is_error) + self.assertIsNone(api_request_responses[0].response) + self.assertIsInstance(api_request_responses[0].exception, + exceptions.HttpError) + + def testSingleRequestInBatch(self): + mock_service = FakeService() + + desired_url = 'https://www.example.com' + batch_api_request = batch.BatchApiRequest(batch_url=desired_url) + # The request to be added. The actual request sent will be somewhat + # larger, as this is added to a batch. + desired_request = http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80) + + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 419, + }, 'x' * 419), + http_wrapper.Response({ + 'status': '200', + 'content-type': 'multipart/mixed; boundary="boundary"', + }, textwrap.dedent("""\ + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 200 OK + content + --boundary--"""), None)) + + batch_api_request.Add(mock_service, 'unused', None, { + 'desired_request': desired_request, + }) + + api_request_responses = batch_api_request.Execute(FakeHttp()) + + self.assertEqual(1, len(api_request_responses)) + self.assertEqual(1, mock_request.call_count) + + self.assertFalse(api_request_responses[0].is_error) + + response = api_request_responses[0].response + self.assertEqual({'status': '200'}, response.info) + self.assertEqual('content', response.content) + self.assertEqual(desired_url, response.request_url) + + def testRefreshOnAuthFailure(self): + mock_service = FakeService() + + desired_url = 'https://www.example.com' + batch_api_request = batch.BatchApiRequest(batch_url=desired_url) + # The request to be added. The actual request sent will be somewhat + # larger, as this is added to a batch. + desired_request = http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80) + + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 419, + }, 'x' * 419), [ + http_wrapper.Response({ + 'status': '200', + 'content-type': 'multipart/mixed; boundary="boundary"', + }, textwrap.dedent("""\ + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 401 UNAUTHORIZED + Invalid grant + + --boundary--"""), None), + http_wrapper.Response({ + 'status': '200', + 'content-type': 'multipart/mixed; boundary="boundary"', + }, textwrap.dedent("""\ + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 200 OK + content + --boundary--"""), None) + ]) + + batch_api_request.Add(mock_service, 'unused', None, { + 'desired_request': desired_request, + }) + + credentials = FakeCredentials() + api_request_responses = batch_api_request.Execute( + FakeHttp(credentials=credentials), sleep_between_polls=0) + + self.assertEqual(1, len(api_request_responses)) + self.assertEqual(2, mock_request.call_count) + self.assertEqual(1, credentials.num_refreshes) + + self.assertFalse(api_request_responses[0].is_error) + + response = api_request_responses[0].response + self.assertEqual({'status': '200'}, response.info) + self.assertEqual('content', response.content) + self.assertEqual(desired_url, response.request_url) + + def testNoAttempts(self): + desired_url = 'https://www.example.com' + batch_api_request = batch.BatchApiRequest(batch_url=desired_url) + batch_api_request.Add(FakeService(), 'unused', None, { + 'desired_request': http_wrapper.Request(desired_url, 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80), + }) + api_request_responses = batch_api_request.Execute(None, max_retries=0) + self.assertEqual(1, len(api_request_responses)) + self.assertIsNone(api_request_responses[0].response) + self.assertIsNone(api_request_responses[0].exception) + + def _DoTestConvertIdToHeader(self, test_id, expected_result): + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertEqual( + expected_result % batch_request._BatchHttpRequest__base_id, + batch_request._ConvertIdToHeader(test_id)) + + def testConvertIdSimple(self): + self._DoTestConvertIdToHeader('blah', '<%s+blah>') + + def testConvertIdThatNeedsEscaping(self): + self._DoTestConvertIdToHeader('~tilde1', '<%s+%%7Etilde1>') + + def _DoTestConvertHeaderToId(self, header, expected_id): + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertEqual(expected_id, + batch_request._ConvertHeaderToId(header)) + + def testConvertHeaderToIdSimple(self): + self._DoTestConvertHeaderToId('', 'blah') + + def testConvertHeaderToIdWithLotsOfPlus(self): + self._DoTestConvertHeaderToId('', 'plus') + + def _DoTestConvertInvalidHeaderToId(self, invalid_header): + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertRaises(exceptions.BatchError, + batch_request._ConvertHeaderToId, invalid_header) + + def testHeaderWithoutAngleBrackets(self): + self._DoTestConvertInvalidHeaderToId('1+1') + + def testHeaderWithoutPlus(self): + self._DoTestConvertInvalidHeaderToId('
') + + def testSerializeRequest(self): + request = http_wrapper.Request(body='Hello World', headers={ + 'content-type': 'protocol/version', + }) + expected_serialized_request = '\n'.join([ + 'GET HTTP/1.1', + 'Content-Type: protocol/version', + 'MIME-Version: 1.0', + 'content-length: 11', + 'Host: ', + '', + 'Hello World', + ]) + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertEqual(expected_serialized_request, + batch_request._SerializeRequest(request)) + + def testSerializeRequestPreservesHeaders(self): + # Now confirm that if an additional, arbitrary header is added + # that it is successfully serialized to the request. Merely + # check that it is included, because the order of the headers + # in the request is arbitrary. + request = http_wrapper.Request(body='Hello World', headers={ + 'content-type': 'protocol/version', + 'key': 'value', + }) + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertTrue( + 'key: value\n' in batch_request._SerializeRequest(request)) + + def testSerializeRequestNoBody(self): + request = http_wrapper.Request(body=None, headers={ + 'content-type': 'protocol/version', + }) + expected_serialized_request = '\n'.join([ + 'GET HTTP/1.1', + 'Content-Type: protocol/version', + 'MIME-Version: 1.0', + 'Host: ', + ]) + batch_request = batch.BatchHttpRequest('https://www.example.com') + self.assertEqual(expected_serialized_request, + batch_request._SerializeRequest(request)) + + def testDeserializeRequest(self): + serialized_payload = '\n'.join([ + 'GET HTTP/1.1', + 'Content-Type: protocol/version', + 'MIME-Version: 1.0', + 'content-length: 11', + 'key: value', + 'Host: ', + '', + 'Hello World', + ]) + example_url = 'https://www.example.com' + expected_response = http_wrapper.Response({ + 'content-length': str(len('Hello World')), + 'Content-Type': 'protocol/version', + 'key': 'value', + 'MIME-Version': '1.0', + 'status': '', + 'Host': '' + }, 'Hello World', example_url) + + batch_request = batch.BatchHttpRequest(example_url) + self.assertEqual( + expected_response, + batch_request._DeserializeResponse(serialized_payload)) + + def testNewId(self): + batch_request = batch.BatchHttpRequest('https://www.example.com') + + for i in range(100): + self.assertEqual(str(i), batch_request._NewId()) + + def testAdd(self): + batch_request = batch.BatchHttpRequest('https://www.example.com') + + for x in range(100): + batch_request.Add(http_wrapper.Request(body=str(x))) + + for key in batch_request._BatchHttpRequest__request_response_handlers: + value = batch_request._BatchHttpRequest__request_response_handlers[ + key] + self.assertEqual(key, value.request.body) + self.assertFalse(value.request.url) + self.assertEqual('GET', value.request.http_method) + self.assertIsNone(value.response) + self.assertIsNone(value.handler) + + def testInternalExecuteWithFailedRequest(self): + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request('https://www.example.com', 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80), + http_wrapper.Response({'status': '300'}, None, None)) + + batch_request = batch.BatchHttpRequest('https://www.example.com') + + self.assertRaises( + exceptions.HttpError, batch_request._Execute, None) + + def testInternalExecuteWithNonMultipartResponse(self): + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request('https://www.example.com', 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 80, + }, 'x' * 80), + http_wrapper.Response({ + 'status': '200', + 'content-type': 'blah/blah' + }, '', None)) + + batch_request = batch.BatchHttpRequest('https://www.example.com') + + self.assertRaises( + exceptions.BatchError, batch_request._Execute, None) + + def testInternalExecute(self): + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as mock_request: + self.__ConfigureMock( + mock_request, + http_wrapper.Request('https://www.example.com', 'POST', { + 'content-type': 'multipart/mixed; boundary="None"', + 'content-length': 583, + }, 'x' * 583), + http_wrapper.Response({ + 'status': '200', + 'content-type': 'multipart/mixed; boundary="boundary"', + }, textwrap.dedent("""\ + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 200 OK + Second response + + --boundary + content-type: text/plain + content-id: + + HTTP/1.1 401 UNAUTHORIZED + First response + + --boundary--"""), None)) + + test_requests = { + '1': batch.RequestResponseAndHandler( + http_wrapper.Request(body='first'), None, None), + '2': batch.RequestResponseAndHandler( + http_wrapper.Request(body='second'), None, None), + } + + batch_request = batch.BatchHttpRequest('https://www.example.com') + batch_request._BatchHttpRequest__request_response_handlers = ( + test_requests) + + batch_request._Execute(FakeHttp()) + + test_responses = ( + batch_request._BatchHttpRequest__request_response_handlers) + + self.assertEqual(http_client.UNAUTHORIZED, + test_responses['1'].response.status_code) + self.assertEqual(http_client.OK, + test_responses['2'].response.status_code) + + self.assertIn( + 'First response', test_responses['1'].response.content) + self.assertIn( + 'Second response', test_responses['2'].response.content) + + def testPublicExecute(self): + + def LocalCallback(response, exception): + self.assertEqual({'status': '418'}, response.info) + self.assertEqual('Teapot', response.content) + self.assertIsNone(response.request_url) + self.assertIsInstance(exception, exceptions.HttpError) + + global_callback = mock.Mock() + batch_request = batch.BatchHttpRequest( + 'https://www.example.com', global_callback) + + with mock.patch.object(batch.BatchHttpRequest, '_Execute', + autospec=True) as mock_execute: + mock_execute.return_value = None + + test_requests = { + '0': batch.RequestResponseAndHandler( + None, + http_wrapper.Response({'status': '200'}, 'Hello!', None), + None), + '1': batch.RequestResponseAndHandler( + None, + http_wrapper.Response({'status': '418'}, 'Teapot', None), + LocalCallback), + } + + batch_request._BatchHttpRequest__request_response_handlers = ( + test_requests) + batch_request.Execute(None) + + # Global callback was called once per handler. + self.assertEqual(len(test_requests), global_callback.call_count) + + +if __name__ == '__main__': + unittest2.main() diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index d9add5a..94c7e32 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -133,7 +133,7 @@ class Request(object): else: self.headers.pop('content-length', None) # This line ensures we don't try to print large requests. - if not isinstance(value, six.string_types): + if not isinstance(value, (type(None), six.string_types)): self.loggable_body = '' -- GitLab From d286130239f8ffb7c15e67414d34684ff4e95af9 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 25 May 2015 01:12:06 -0700 Subject: [PATCH 118/295] Add tests for list_pager. This pulls in the internal list_pager tests, and picks up an internal fix that hadn't yet been mirrored out. --- apitools/base/py/list_pager.py | 10 +- apitools/base/py/list_pager_test.py | 188 ++++++++++++++++++ .../testclient/fusiontables_v1_client.py | 50 ++++- .../testclient/fusiontables_v1_messages.py | 35 ++++ 4 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 apitools/base/py/list_pager_test.py diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 84d60aa..cf90389 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -12,7 +12,8 @@ def YieldFromList( service, request, limit=None, batch_size=100, method='List', field='items', predicate=None, current_token_attribute='pageToken', - next_token_attribute='nextPageToken'): + next_token_attribute='nextPageToken', + batch_size_attribute='maxResults'): """Make a series of List requests, keeping track of page tokens. Args: @@ -32,14 +33,17 @@ def YieldFromList( requested. next_token_attribute: str, The name of the attribute in a response message holding the page token for the next page. + batch_size_attribute: str, The name of the attribute in a + response message holding the maximum number of results to be + returned. Yields: protorpc.message.Message, The resources listed by the service. """ request = copy.deepcopy(request) - request.maxResults = batch_size - request.pageToken = None + setattr(request, batch_size_attribute, batch_size) + setattr(request, current_token_attribute, None) while limit is None or limit: response = getattr(service, method)(request) items = getattr(response, field) diff --git a/apitools/base/py/list_pager_test.py b/apitools/base/py/list_pager_test.py new file mode 100644 index 0000000..a98372f --- /dev/null +++ b/apitools/base/py/list_pager_test.py @@ -0,0 +1,188 @@ +"""Tests for list_pager.""" + +import unittest2 + +from apitools.base.py import list_pager +from apitools.base.py.testing import mock +from apitools.base.py.testing import testclient as fusiontables + + +class ListPagerTest(unittest2.TestCase): + + def _AssertInstanceSequence(self, results, n): + counter = 0 + for instance in results: + self.assertEqual(instance.name, 'c' + str(counter)) + counter += 1 + + self.assertEqual(counter, n) + + def setUp(self): + self.mocked_client = mock.Client(fusiontables.FusiontablesV1) + self.mocked_client.Mock() + self.addCleanup(self.mocked_client.Unmock) + + def testYieldFromList(self): + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken=None, + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c0'), + fusiontables.Column(name='c1'), + fusiontables.Column(name='c2'), + fusiontables.Column(name='c3'), + ], + nextPageToken='x', + )) + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken='x', + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c4'), + fusiontables.Column(name='c5'), + fusiontables.Column(name='c6'), + fusiontables.Column(name='c7'), + ], + )) + + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + results = list_pager.YieldFromList(client.column, request) + + self._AssertInstanceSequence(results, 8) + + def testYieldNoRecords(self): + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + results = list_pager.YieldFromList(client.column, request, limit=False) + self.assertEqual(0, len(list(results))) + + def testYieldFromListPartial(self): + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken=None, + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c0'), + fusiontables.Column(name='c1'), + fusiontables.Column(name='c2'), + fusiontables.Column(name='c3'), + ], + nextPageToken='x', + )) + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken='x', + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c4'), + fusiontables.Column(name='c5'), + fusiontables.Column(name='c6'), + fusiontables.Column(name='c7'), + ], + )) + + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + results = list_pager.YieldFromList(client.column, request, limit=6) + + self._AssertInstanceSequence(results, 6) + + def testYieldFromListEmpty(self): + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken=None, + tableId='mytable', + ), + fusiontables.ColumnList()) + + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + results = list_pager.YieldFromList(client.column, request, limit=6) + + self._AssertInstanceSequence(results, 0) + + def testYieldFromListWithPredicate(self): + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken=None, + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c0'), + fusiontables.Column(name='bad0'), + fusiontables.Column(name='c1'), + fusiontables.Column(name='bad1'), + ], + nextPageToken='x', + )) + self.mocked_client.column.List.Expect( + fusiontables.FusiontablesColumnListRequest( + maxResults=100, + pageToken='x', + tableId='mytable', + ), + fusiontables.ColumnList( + items=[ + fusiontables.Column(name='c2'), + ], + )) + + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + results = list_pager.YieldFromList( + client.column, request, predicate=lambda x: 'c' in x.name) + + self._AssertInstanceSequence(results, 3) + + def testYieldFromListWithAttributes(self): + self.mocked_client.columnalternate.List.Expect( + fusiontables.FusiontablesColumnListAlternateRequest( + pageSize=100, + pageToken=None, + tableId='mytable', + ), + fusiontables.ColumnListAlternate( + columns=[ + fusiontables.Column(name='c0'), + fusiontables.Column(name='c1'), + ], + nextPageToken='x', + )) + self.mocked_client.columnalternate.List.Expect( + fusiontables.FusiontablesColumnListAlternateRequest( + pageSize=100, + pageToken='x', + tableId='mytable', + ), + fusiontables.ColumnListAlternate( + columns=[ + fusiontables.Column(name='c2'), + ], + )) + + client = fusiontables.FusiontablesV1(get_credentials=False) + request = fusiontables.FusiontablesColumnListAlternateRequest( + tableId='mytable') + results = list_pager.YieldFromList( + client.columnalternate, request, + batch_size_attribute='pageSize', field='columns') + + self._AssertInstanceSequence(results, 3) diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_client.py b/apitools/base/py/testing/testclient/fusiontables_v1_client.py index 1be17b3..45db3c3 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_client.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_client.py @@ -1,5 +1,9 @@ -"""Generated client library for fusiontables version v1.""" -# NOTE: This file is autogenerated and should not be edited by hand. +"""Modified generated client library for fusiontables version v1. + +This is a hand-customized and pruned version of the fusiontables v1 +client, designed for use in testing. + +""" from apitools.base.py import base_api from apitools.base.py.testing.testclient import fusiontables_v1_messages @@ -36,6 +40,7 @@ class FusiontablesV1(base_api.BaseApiClient): default_global_params=default_global_params, additional_http_headers=additional_http_headers) self.column = self.ColumnService(self) + self.columnalternate = self.ColumnAlternateService(self) class ColumnService(base_api.BaseApiService): @@ -76,3 +81,44 @@ class FusiontablesV1(base_api.BaseApiClient): config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) + + class ColumnAlternateService(base_api.BaseApiService): + + """Service class for the column resource.""" + + _NAME = u'columnalternate' + + def __init__(self, client): + super(FusiontablesV1.ColumnAlternateService, self).__init__(client) + self._method_configs = { + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.listalternate', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/columns', + request_field='', + request_type_name=( + u'FusiontablesColumnListAlternateRequest'), + response_type_name=u'ColumnListAlternate', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def List(self, request, global_params=None): + """Retrieves a list of columns. + + Args: + request: (FusiontablesColumnListRequest) input message + global_params: (StandardQueryParameters, default: None) global + arguments + Returns: + (ColumnList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py index 1148f62..fd727a0 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py @@ -91,3 +91,38 @@ class FusiontablesColumnListRequest(messages.Message): maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) pageToken = messages.StringField(2) tableId = messages.StringField(3, required=True) + + +class FusiontablesColumnListAlternateRequest(messages.Message): + + """A FusiontablesColumnListRequest object. + + Fields: + pageSize: Maximum number of columns to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + tableId: Table whose columns are being listed. + """ + + pageSize = messages.IntegerField(1, variant=messages.Variant.UINT32) + pageToken = messages.StringField(2) + tableId = messages.StringField(3, required=True) + + +class ColumnListAlternate(messages.Message): + + """Represents a list of columns in a table. + + Fields: + items: List of all requested columns. + kind: Type name: a list of all columns. + nextPageToken: Token used to access the next page of this + result. No token is displayed if there are no more pages left. + totalItems: Total number of columns for the table. + + """ + + columns = messages.MessageField('Column', 1, repeated=True) + kind = messages.StringField(2, default=u'fusiontables#columnList') + nextPageToken = messages.StringField(3) + totalItems = messages.IntegerField(4, variant=messages.Variant.INT32) -- GitLab From 51cc6b18616987b3b9941715e8d5f9046c848779 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 25 May 2015 01:24:51 -0700 Subject: [PATCH 119/295] Drop unittest2.main() blocks. --- apitools/base/py/base_api_test.py | 10 +--------- apitools/base/py/batch_test.py | 4 ---- apitools/base/py/buffered_stream_test.py | 9 ++------- apitools/base/py/credentials_lib_test.py | 9 +-------- apitools/base/py/encoding_test.py | 7 ------- apitools/base/py/extra_types_test.py | 7 ------- apitools/base/py/http_wrapper_test.py | 5 ----- apitools/base/py/stream_slice_test.py | 7 +------ apitools/base/py/testing/mock_test.py | 3 --- apitools/base/py/transfer_test.py | 9 +-------- apitools/base/py/util_test.py | 7 +------ apitools/gen/client_generation_test.py | 8 -------- 12 files changed, 7 insertions(+), 78 deletions(-) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 8e10366..86141a3 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -1,13 +1,9 @@ -#!/usr/bin/env python - - import datetime import sys -from six.moves import urllib_parse - from protorpc import message_types from protorpc import messages +from six.moves import urllib_parse import unittest2 from apitools.base.py import base_api @@ -181,7 +177,3 @@ class BaseApiTest(unittest2.TestCase): expected_url = service.client.url + 'parameters/gonna/remap/ONE/TWO' http_request = service.PrepareHttpRequest(method_config, request) self.assertEqual(expected_url, http_request.url) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py index ee421b3..cf77364 100644 --- a/apitools/base/py/batch_test.py +++ b/apitools/base/py/batch_test.py @@ -507,7 +507,3 @@ class BatchTest(unittest2.TestCase): # Global callback was called once per handler. self.assertEqual(len(test_requests), global_callback.call_count) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py index 02f515f..4dcf62f 100644 --- a/apitools/base/py/buffered_stream_test.py +++ b/apitools/base/py/buffered_stream_test.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python -"""Tests for stream_slice.""" +"""Tests for buffered_stream.""" -import six import string +import six import unittest2 from apitools.base.py import buffered_stream @@ -51,7 +50,3 @@ class BufferedStreamTest(unittest2.TestCase): bs.read() with self.assertRaises(exceptions.NotYetImplementedError): bs.read(size=-1) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 1bca25a..cf4e5df 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,10 +1,7 @@ -#!/usr/bin/env python - - import re -import six import mock +import six from six.moves import http_client import unittest2 @@ -100,7 +97,3 @@ class TestGetRunFlowFlags(unittest2.TestCase): self.assertEqual(flags.auth_host_port, [8080, 8090]) self.assertEqual(flags.logging_level, 'ERROR') self.assertEqual(flags.noauth_local_webserver, False) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 36861a7..0d10d8b 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python - - import base64 import datetime import json @@ -345,7 +342,3 @@ class EncodingTest(unittest2.TestCase): 'TimeMessage(\n ' 'timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, ' 'tzinfo=TimeZoneOffset(datetime.timedelta(0))),\n)') - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 4505d70..d2fd5b0 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python - - import datetime import json import math @@ -175,7 +172,3 @@ class ExtraTypesTest(unittest2.TestCase): msg = DogeMsg(such_string='wow', wow=-1234, very_unsigned=800, much_repeated=[123, 456]) self.assertEqual(msg, DoRoundtrip(DogeMsg, message=msg)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index 98a9194..ddb927a 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -1,5 +1,4 @@ """Tests for http_wrapper.""" - import unittest2 from apitools.base.py import http_wrapper @@ -23,7 +22,3 @@ class HttpWrapperTest(unittest2.TestCase): def testRequestBodyWithLen(self): http_wrapper.Request(body='burrito') - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py index de821e6..f9b13b9 100644 --- a/apitools/base/py/stream_slice_test.py +++ b/apitools/base/py/stream_slice_test.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python """Tests for stream_slice.""" -import six import string +import six import unittest2 from apitools.base.py import exceptions @@ -49,7 +48,3 @@ class StreamSliceTest(unittest2.TestCase): with self.assertRaises(exceptions.StreamExhausted) as e: ss.read(10) self.assertIn('exhausted after %d' % len(self.value), str(e.exception)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py index 395b7a7..4a6c57c 100644 --- a/apitools/base/py/testing/mock_test.py +++ b/apitools/base/py/testing/mock_test.py @@ -127,6 +127,3 @@ class UtilTest(unittest2.TestCase): _NestedListMessage( nested_list=[_NestedMessage(nested='foo'), _NestedMessage(nested='foo')]))) - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 76e54d0..2977118 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,8 +1,5 @@ -#!/usr/bin/env python - - +"""Tests for transfer.py.""" import six - import unittest2 from apitools.base.py import base_api @@ -61,7 +58,3 @@ class TransferTest(unittest2.TestCase): self.assertEqual(url_builder.query_params['uploadType'], 'media') rewritten_upload_contents = http_request.body self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) - - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 16767a9..54dc3e1 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python - - +"""Tests for util.py.""" from protorpc import messages import unittest2 @@ -174,6 +172,3 @@ class UtilTest(unittest2.TestCase): remapped_params = ['str_field', 'enum_field'] self.assertEqual(remapped_params, util.MapParamNames(params, MessageWithRemappings)) - -if __name__ == '__main__': - unittest2.main() diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 8dcdaac..f34d954 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Test gen_client against all the APIs we use regularly.""" import contextlib @@ -38,9 +37,6 @@ class ClientGenerationTest(unittest2.TestCase): super(ClientGenerationTest, self).setUp() self.gen_client_binary = 'gen_client' - # TODO(craigcitro): Make apitools codegen support python 2.6. - # Maybe. - # # unittest in 2.6 doesn't have skipIf. @unittest2.skipUnless(sys.version_info[0] == 2 and sys.version_info[1] == 7, @@ -74,7 +70,3 @@ class ClientGenerationTest(unittest2.TestCase): retcode = subprocess.call(cmdline_args, stdout=out) # appcommands returns 1 on help self.assertEqual(1, retcode) - - -if __name__ == '__main__': - unittest2.main() -- GitLab From 2272981a5a4d9cb19dcf3be514a9d5ea045a122a Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 25 May 2015 01:27:48 -0700 Subject: [PATCH 120/295] Add another (short) test. --- apitools/gen/util_test.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 apitools/gen/util_test.py diff --git a/apitools/gen/util_test.py b/apitools/gen/util_test.py new file mode 100644 index 0000000..93e9596 --- /dev/null +++ b/apitools/gen/util_test.py @@ -0,0 +1,24 @@ +"""Tests for util.""" +import unittest2 + +from apitools.gen import util + + +class NormalizeVersionTest(unittest2.TestCase): + + def testVersions(self): + already_valid = 'v1' + self.assertEqual(already_valid, util.NormalizeVersion(already_valid)) + to_clean = 'v0.1' + self.assertEqual('v0_1', util.NormalizeVersion(to_clean)) + + +class NamesTest(unittest2.TestCase): + + def testKeywords(self): + names = util.Names(['']) + self.assertEqual('in_', names.CleanName('in')) + + def testNormalizeEnumName(self): + names = util.Names(['']) + self.assertEqual('_0', names.NormalizeEnumName('0')) -- GitLab From 0ed9f2afdd5ed1087cd5b7a35c43c9e7f1937e78 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 25 May 2015 01:40:47 -0700 Subject: [PATCH 121/295] Clean up gen/util tests for python3. --- apitools/gen/util.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 4fb3d10..8236de8 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -9,9 +9,10 @@ import keyword import logging import os import re -import urllib2 import six +import six.moves.urllib.error as urllib_error +import six.moves.urllib.request as urllib_request class Error(Exception): @@ -24,8 +25,8 @@ class CommunicationError(Error): """Error in network communication.""" -def _SortLengthFirst(a, b): - return -cmp(len(a), len(b)) or cmp(a, b) +def _SortLengthFirstKey(a): + return -len(a), a class Names(object): @@ -37,7 +38,7 @@ class Names(object): def __init__(self, strip_prefixes, name_convention=None, capitalize_enums=False): - self.__strip_prefixes = sorted(strip_prefixes, cmp=_SortLengthFirst) + self.__strip_prefixes = sorted(strip_prefixes, key=_SortLengthFirstKey) self.__name_convention = ( name_convention or self.DEFAULT_NAME_CONVENTION) self.__capitalize_enums = capitalize_enums @@ -296,9 +297,11 @@ def FetchDiscoveryDoc(discovery_url, retries=5): last_exception = None for _ in range(retries): try: - discovery_doc = json.loads(urllib2.urlopen(discovery_url).read()) + discovery_doc = json.loads( + urllib_request.urlopen(discovery_url).read()) break - except (urllib2.HTTPError, urllib2.URLError) as last_exception: + except (urllib_error.HTTPError, + urllib_error.URLError) as last_exception: logging.warning( 'Attempting to fetch discovery doc again after "%s"', last_exception) -- GitLab From adf09d08d9373a05174b4a4565514260b5bac3c9 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 22 May 2015 01:25:50 -0700 Subject: [PATCH 122/295] Restore chunked download support. We lost support for chunked downloads in a merge; this PR restores chunked download support, and adds tests. In addition, we add a `StreamMedia` method to the `Download` class, following the `Upload` API. This function is equivalent to `StreamInChunks` with an optional `ignore_chunksize` argument -- somehow `StreamInChunks(..., ignore_chunksize=True)` feels a bit awkward. Relatedly, `Download.StreamInChunks` and `Download.GetRange` now have an explicit `ignore_chunksize` argument. I also updated the two integration tests, which really need to be updated to run automatically. --- apitools/base/py/transfer.py | 77 ++++++++++-- apitools/base/py/transfer_test.py | 145 +++++++++++++++++++++++ samples/storage_sample/downloads_test.py | 8 +- samples/storage_sample/uploads_test.py | 5 +- 4 files changed, 221 insertions(+), 14 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index bcdf794..35a4774 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -298,6 +298,8 @@ class Download(_Transfer): http_request.url = client.FinalizeTransferUrl(http_request.url) url = http_request.url if self.auto_transfer: + end_byte = self.__ComputeEndByte(0) + self.__SetRangeHeader(http_request, 0, end_byte) response = http_wrapper.MakeRequest( self.bytes_http or http, http_request) if response.status_code not in self._ACCEPTABLE_STATUSES: @@ -339,14 +341,42 @@ class Download(_Transfer): else: request.headers['range'] = 'bytes=%d-%d' % (start, end) - def __GetChunk(self, start, end=None, additional_headers=None): + def __ComputeEndByte(self, start, end=None, use_chunks=True): + """Compute the last byte to fetch for this request. + + This is all based on the HTTP spec for Range and + Content-Range. + + Note that this is potentially confusing in several ways: + * the value for the last byte is 0-based, eg "fetch 10 bytes + from the beginning" would return 9 here. + * if we have no information about size, and don't want to + use the chunksize, we'll return None. + See the tests for more examples. + + Args: + start: byte to start at. + end: (int or None, default: None) Suggested last byte. + use_chunks: (bool, default: True) If False, ignore self.chunksize. + + Returns: + Last byte to use in a Range header, or None. + + """ + end_byte = end + if use_chunks: + alternate = start + self.chunksize - 1 + end_byte = min(end_byte, alternate) if end_byte else alternate + if self.total_size: + alternate = self.total_size - 1 + end_byte = min(end_byte, alternate) if end_byte else alternate + return end_byte + + def __GetChunk(self, start, end, additional_headers=None): """Retrieve a chunk, and return the full response.""" self.EnsureInitialized() - end_byte = end - if self.total_size and end: - end_byte = min(end, self.total_size) request = http_wrapper.Request(url=self.url) - self.__SetRangeHeader(request, start, end=end_byte) + self.__SetRangeHeader(request, start, end=end) if additional_headers is not None: request.headers.update(additional_headers) return http_wrapper.MakeRequest( @@ -378,7 +408,8 @@ class Download(_Transfer): self.stream.write('') return response - def GetRange(self, start, end=None, additional_headers=None): + def GetRange(self, start, end=None, additional_headers=None, + use_chunks=True): """Retrieve a given byte range from this download, inclusive. Range must be of one of these three forms: @@ -394,6 +425,8 @@ class Download(_Transfer): end: (int, optional) Where to stop fetching bytes. (See above.) additional_headers: (bool, optional) Any additional headers to pass with the request. + use_chunks: (bool, default: True) If False, ignore self.chunksize + and fetch this range in a single request. Returns: None. Streams bytes into self.stream. @@ -406,7 +439,9 @@ class Download(_Transfer): else: progress = start while not progress_end_normalized or progress < end: - response = self.__GetChunk(progress, end=end, + end_byte = self.__ComputeEndByte(progress, end=end, + use_chunks=use_chunks) + response = self.__GetChunk(progress, end_byte, additional_headers=additional_headers) if not progress_end_normalized: self.__SetTotal(response.info) @@ -420,7 +455,28 @@ class Download(_Transfer): def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=None): - """Stream the entire download.""" + """Stream the entire download in chunks.""" + self.StreamMedia(callback=callback, finish_callback=finish_callback, + additional_headers=additional_headers, + use_chunks=True) + + def StreamMedia(self, callback=None, finish_callback=None, + additional_headers=None, use_chunks=True): + """Stream the entire download. + + Args: + callback: (default: None) Callback to call as each chunk is + completed. + finish_callback: (default: None) Callback to call when the + download is complete. + additional_headers: (default: None) Additional headers to + include in fetching bytes. + use_chunks: (bool, default: True) If False, ignore self.chunksize + and stream this download in a single request. + + Returns: + None. Streams bytes into self.stream. + """ callback = callback or self.progress_callback finish_callback = finish_callback or self.finish_callback @@ -430,8 +486,11 @@ class Download(_Transfer): response = self.__initial_response self.__initial_response = None else: + end_byte = self.__ComputeEndByte(self.progress, + use_chunks=use_chunks) response = self.__GetChunk( - self.progress, additional_headers=additional_headers) + self.progress, end_byte, + additional_headers=additional_headers) if self.total_size is None: self.__SetTotal(response.info) response = self.__ProcessResponse(response) diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 2977118..9d58b1e 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,5 +1,9 @@ """Tests for transfer.py.""" +import string + +import mock import six +from six.moves import http_client import unittest2 from apitools.base.py import base_api @@ -9,6 +13,147 @@ from apitools.base.py import transfer class TransferTest(unittest2.TestCase): + def assertRangeAndContentRangeCompatible(self, request, response): + request_prefix = 'bytes=' + self.assertIn('range', request.headers) + self.assertTrue(request.headers['range'].startswith(request_prefix)) + request_range = request.headers['range'][len(request_prefix):] + + response_prefix = 'bytes ' + self.assertIn('content-range', response.info) + response_header = response.info['content-range'] + self.assertTrue(response_header.startswith(response_prefix)) + response_range = ( + response_header[len(response_prefix):].partition('/')[0]) + + msg = ('Request range ({0}) not a prefix of ' + 'response_range ({1})').format( + request_range, response_range) + self.assertTrue(response_range.startswith(request_range), msg=msg) + + def testComputeEndByte(self): + total_size = 100 + chunksize = 10 + download = transfer.Download.FromStream( + six.StringIO(), chunksize=chunksize, total_size=total_size) + self.assertEqual(chunksize - 1, + download._Download__ComputeEndByte(0, end=50)) + + def testComputeEndByteReturnNone(self): + download = transfer.Download.FromStream(six.StringIO()) + self.assertIsNone( + download._Download__ComputeEndByte(0, use_chunks=False)) + + def testComputeEndByteNoChunks(self): + total_size = 100 + download = transfer.Download.FromStream( + six.StringIO(), chunksize=10, total_size=total_size) + for end in (None, 1000): + self.assertEqual( + total_size - 1, + download._Download__ComputeEndByte(0, end=end, + use_chunks=False), + msg='Failed on end={0}'.format(end)) + + def testComputeEndByteNoTotal(self): + download = transfer.Download.FromStream(six.StringIO()) + default_chunksize = download.chunksize + for chunksize in (100, default_chunksize): + download.chunksize = chunksize + for start in (0, 10): + self.assertEqual( + download.chunksize + start - 1, + download._Download__ComputeEndByte(start), + msg='Failed on start={0}, chunksize={1}'.format( + start, chunksize)) + + def testComputeEndByteSmallTotal(self): + total_size = 100 + download = transfer.Download.FromStream(six.StringIO(), + total_size=total_size) + for start in (0, 10): + self.assertEqual(total_size - 1, + download._Download__ComputeEndByte(start), + msg='Failed on start={0}'.format(start)) + + def testNonChunkedDownload(self): + bytes_http = object() + http = object() + download_stream = six.StringIO() + download = transfer.Download.FromStream(download_stream, total_size=52) + download.bytes_http = bytes_http + base_url = 'https://part.one/' + + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as make_request: + make_request.return_value = http_wrapper.Response( + info={ + 'content-range': 'bytes 0-51/52', + 'status': http_client.OK, + }, + content=string.ascii_lowercase * 2, + request_url=base_url, + ) + request = http_wrapper.Request(url='https://part.one/') + download.InitializeDownload(request, http=http) + self.assertEqual(1, make_request.call_count) + received_request = make_request.call_args[0][1] + self.assertEqual(base_url, received_request.url) + self.assertRangeAndContentRangeCompatible( + received_request, make_request.return_value) + download_stream.seek(0) + self.assertEqual(string.ascii_lowercase * 2, + download_stream.getvalue()) + + def testChunkedDownload(self): + bytes_http = object() + http = object() + download_stream = six.StringIO() + download = transfer.Download.FromStream( + download_stream, chunksize=26, total_size=52) + download.bytes_http = bytes_http + + # Setting autospec on a mock with an iterable side_effect is + # currently broken (http://bugs.python.org/issue17826), so + # instead we write a little function. + def _ReturnBytes(unused_http, http_request, + *unused_args, **unused_kwds): + url = http_request.url + if url == 'https://part.one/': + return http_wrapper.Response( + info={ + 'content-location': 'https://part.two/', + 'content-range': 'bytes 0-25/52', + 'status': http_client.PARTIAL_CONTENT, + }, + content=string.ascii_lowercase, + request_url='https://part.one/', + ) + elif url == 'https://part.two/': + return http_wrapper.Response( + info={ + 'content-range': 'bytes 26-51/52', + 'status': http_client.OK, + }, + content=string.ascii_uppercase, + request_url='https://part.two/', + ) + else: + self.fail('Unknown URL requested: %s' % url) + + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as make_request: + make_request.side_effect = _ReturnBytes + request = http_wrapper.Request(url='https://part.one/') + download.InitializeDownload(request, http=http) + self.assertEqual(2, make_request.call_count) + for call in make_request.call_args_list: + self.assertRangeAndContentRangeCompatible( + call[0][1], _ReturnBytes(*call[0])) + download_stream.seek(0) + self.assertEqual(string.ascii_lowercase + string.ascii_uppercase, + download_stream.getvalue()) + def testFromEncoding(self): # Test a specific corner case in multipart encoding. diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index f6a62e5..2276d98 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -4,11 +4,12 @@ These tests exercise most of the corner cases for upload/download of files in apitools, via GCS. There are no performance tests here yet. """ -import io import json import os import unittest +import six + import apitools.base.py as apitools_base import storage @@ -31,7 +32,7 @@ class DownloadsTest(unittest.TestCase): self.__ResetDownload() def __ResetDownload(self, auto_transfer=False): - self.__buffer = io.StringIO() + self.__buffer = six.StringIO() self.__download = storage.Download.FromStream( self.__buffer, auto_transfer=auto_transfer) @@ -62,6 +63,7 @@ class DownloadsTest(unittest.TestCase): self.assertEqual(0, self.__buffer.tell()) def testObjectDoesNotExist(self): + self.__ResetDownload(auto_transfer=True) with self.assertRaises(apitools_base.HttpError): self.__GetFile(self.__GetRequest('nonexistent_file')) @@ -159,7 +161,7 @@ class DownloadsTest(unittest.TestCase): request = storage.StorageObjectsGetRequest( bucket=self._DEFAULT_BUCKET, object=object_name) response = self.__client.objects.Get(request) - self.__buffer = io.StringIO() + self.__buffer = six.StringIO() download_data = json.dumps({ 'auto_transfer': False, 'progress': 0, diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index ca58dc8..ad4416f 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -4,13 +4,14 @@ These tests exercise most of the corner cases for upload/download of files in apitools, via GCS. There are no performance tests here yet. """ -import io import json import os import random import string import unittest +import six + import apitools.base.py as apitools_base import storage @@ -41,7 +42,7 @@ class UploadsTest(unittest.TestCase): def __ResetUpload(self, size, auto_transfer=True): self.__content = ''.join( random.choice(string.ascii_letters) for _ in range(size)) - self.__buffer = io.StringIO(self.__content) + self.__buffer = six.StringIO(self.__content) self.__upload = storage.Upload.FromStream( self.__buffer, 'text/plain', auto_transfer=auto_transfer) -- GitLab From 595532bc6cf5422cd67147b8e6caec155e861ea8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 26 May 2015 12:26:38 -0700 Subject: [PATCH 123/295] Update for v0.4.8 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b323a6..0da6c70 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.7' +_APITOOLS_VERSION = '0.4.8' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From b51d4761125dcc6cf4caf750536ce6a0e5a3d43c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 27 May 2015 15:59:08 -0700 Subject: [PATCH 124/295] Add a script for generating the sample clients. Currently there's just one client. This also updates the storage client. --- apitools/gen/gen_client_lib.py | 16 +- samples/storage_sample/generate_clients.sh | 3 + samples/storage_sample/storage/storage_v1.py | 3362 +++++++++-------- .../storage/storage_v1_client.py | 748 ++-- .../storage/storage_v1_messages.py | 158 +- 5 files changed, 2343 insertions(+), 1944 deletions(-) create mode 100755 samples/storage_sample/generate_clients.sh diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index f9feb77..df7bad4 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -150,13 +150,17 @@ class DescriptorGenerator(object): printer('import pkgutil') printer() printer('from %s import *', self.__base_files_package) + if self.__root_package == '.': + import_prefix = '' + else: + import_prefix = '%s.' % self.__root_package if self.__generate_cli: - printer('from %s.%s import *', - self.__root_package, self.__client_info.cli_rule_name) - printer('from %s.%s import *', - self.__root_package, self.__client_info.client_rule_name) - printer('from %s.%s import *', - self.__root_package, self.__client_info.messages_rule_name) + printer('from %s%s import *', + import_prefix, self.__client_info.cli_rule_name) + printer('from %s%s import *', + import_prefix, self.__client_info.client_rule_name) + printer('from %s%s import *', + import_prefix, self.__client_info.messages_rule_name) printer() printer('__path__ = pkgutil.extend_path(__path__, __name__)') diff --git a/samples/storage_sample/generate_clients.sh b/samples/storage_sample/generate_clients.sh new file mode 100755 index 0000000..5744480 --- /dev/null +++ b/samples/storage_sample/generate_clients.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +gen_client --discovery_url=storage.v1 --overwrite --outdir=storage --root_package=. client diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage/storage_v1.py index 97627e2..5b9e5a2 100755 --- a/samples/storage_sample/storage/storage_v1.py +++ b/samples/storage_sample/storage/storage_v1.py @@ -1,7 +1,9 @@ #!/usr/bin/env python """CLI for storage, version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. import code +import os import platform import sys @@ -33,8 +35,12 @@ def _DeclareStorageFlags(): 'File with interactive shell history.') flags.DEFINE_multistring( 'add_header', [], - 'Additional http headers (as key=value strings). Can be ' - 'specified multiple times.') + 'Additional http headers (as key=value strings). ' + 'Can be specified multiple times.') + flags.DEFINE_string( + 'service_account_json_keyfile', '', + 'Filename for a JSON service account key downloaded' + ' from the Developer Console.') flags.DEFINE_enum( 'alt', u'json', @@ -109,10 +115,14 @@ def GetClientFromFlags(): log_response = FLAGS.log_response or FLAGS.log_request_response api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header) + credentials_args = { + 'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile) + } try: client = client_lib.StorageV1( api_endpoint, log_request=log_request, log_response=log_response, + credentials_args=credentials_args, additional_http_headers=additional_http_headers) except apitools_base.CredentialsError as e: print 'Error creating credentials: %s' % e @@ -153,17 +163,17 @@ class PyShell(appcommands.Cmd): return e.code -class DefaultObjectAccessControlsDelete(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.Delete.""" +class BucketAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Delete.""" - usage = """defaultObjectAccessControls_delete """ + usage = """bucketAccessControls_delete """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsDelete, self).__init__(name, fv) + super(BucketAccessControlsDelete, self).__init__(name, fv) def RunWithArgs(self, bucket, entity): - """Permanently deletes the default object ACL entry for the specified - entity on the specified bucket. + """Permanently deletes the ACL entry for the specified entity on the + specified bucket. Args: bucket: Name of a bucket. @@ -173,26 +183,25 @@ class DefaultObjectAccessControlsDelete(apitools_base_cli.NewCmd): """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsDeleteRequest( + request = messages.StorageBucketAccessControlsDeleteRequest( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) - result = client.defaultObjectAccessControls.Delete( + result = client.bucketAccessControls.Delete( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsGet(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.Get.""" +class BucketAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Get.""" - usage = """defaultObjectAccessControls_get """ + usage = """bucketAccessControls_get """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsGet, self).__init__(name, fv) + super(BucketAccessControlsGet, self).__init__(name, fv) def RunWithArgs(self, bucket, entity): - """Returns the default object ACL entry for the specified entity on the - specified bucket. + """Returns the ACL entry for the specified entity on the specified bucket. Args: bucket: Name of a bucket. @@ -202,22 +211,22 @@ class DefaultObjectAccessControlsGet(apitools_base_cli.NewCmd): """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsGetRequest( + request = messages.StorageBucketAccessControlsGetRequest( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) - result = client.defaultObjectAccessControls.Get( + result = client.bucketAccessControls.Get( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.Insert.""" +class BucketAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Insert.""" - usage = """defaultObjectAccessControls_insert """ + usage = """bucketAccessControls_insert """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsInsert, self).__init__(name, fv) + super(BucketAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -250,11 +259,6 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) - flags.DEFINE_string( - 'generation', - None, - u'The content generation of the object.', - flag_values=fv) flags.DEFINE_string( 'id', None, @@ -262,14 +266,9 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', - flag_values=fv) - flags.DEFINE_string( - 'object', - None, - u'The name of the object.', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -279,7 +278,8 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER or OWNER.', + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -288,7 +288,7 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): flag_values=fv) def RunWithArgs(self, bucket): - """Creates a new default object ACL entry on the specified bucket. + """Creates a new ACL entry on the specified bucket. Args: bucket: The name of the bucket. @@ -306,18 +306,17 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( + request = messages.BucketAccessControl( bucket=bucket.decode('utf8'), ) if FLAGS['domain'].present: @@ -330,78 +329,52 @@ class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') - if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Insert( + result = client.bucketAccessControls.Insert( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsList(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.List.""" +class BucketAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.List.""" - usage = """defaultObjectAccessControls_list """ + usage = """bucketAccessControls_list """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsList, self).__init__(name, fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"If present, only return default ACL listing if the bucket's current" - u' metageneration matches this value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u"If present, only return default ACL listing if the bucket's current" - u' metageneration does not match the given value.', - flag_values=fv) + super(BucketAccessControlsList, self).__init__(name, fv) def RunWithArgs(self, bucket): - """Retrieves default object ACL entries on the specified bucket. + """Retrieves ACL entries on the specified bucket. Args: bucket: Name of a bucket. - - Flags: - ifMetagenerationMatch: If present, only return default ACL listing if - the bucket's current metageneration matches this value. - ifMetagenerationNotMatch: If present, only return default ACL listing if - the bucket's current metageneration does not match the given value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageDefaultObjectAccessControlsListRequest( + request = messages.StorageBucketAccessControlsListRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.defaultObjectAccessControls.List( + result = client.bucketAccessControls.List( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.Patch.""" +class BucketAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Patch.""" - usage = """defaultObjectAccessControls_patch """ + usage = """bucketAccessControls_patch """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsPatch, self).__init__(name, fv) + super(BucketAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -422,11 +395,6 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) - flags.DEFINE_string( - 'generation', - None, - u'The content generation of the object.', - flag_values=fv) flags.DEFINE_string( 'id', None, @@ -434,14 +402,9 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', - flag_values=fv) - flags.DEFINE_string( - 'object', - None, - u'The name of the object.', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -451,7 +414,8 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER or OWNER.', + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -460,8 +424,8 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): flag_values=fv) def RunWithArgs(self, bucket, entity): - """Updates a default object ACL entry on the specified bucket. This method - supports patch semantics. + """Updates an ACL entry on the specified bucket. This method supports + patch semantics. Args: bucket: The name of the bucket. @@ -479,18 +443,17 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): email: The email address associated with the entity, if any. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( + request = messages.BucketAccessControl( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) @@ -502,32 +465,28 @@ class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') - if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Patch( + result = client.bucketAccessControls.Patch( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): - """Command wrapping defaultObjectAccessControls.Update.""" +class BucketAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping bucketAccessControls.Update.""" - usage = """defaultObjectAccessControls_update """ + usage = """bucketAccessControls_update """ def __init__(self, name, fv): - super(DefaultObjectAccessControlsUpdate, self).__init__(name, fv) + super(BucketAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( 'domain', None, @@ -548,11 +507,6 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): None, u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) - flags.DEFINE_string( - 'generation', - None, - u'The content generation of the object.', - flag_values=fv) flags.DEFINE_string( 'id', None, @@ -560,14 +514,9 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): flag_values=fv) flags.DEFINE_string( 'kind', - u'storage#objectAccessControl', - u'The kind of item this is. For object access control entries, this ' - u'is always storage#objectAccessControl.', - flag_values=fv) - flags.DEFINE_string( - 'object', - None, - u'The name of the object.', + u'storage#bucketAccessControl', + u'The kind of item this is. For bucket access control entries, this ' + u'is always storage#bucketAccessControl.', flag_values=fv) flags.DEFINE_string( 'projectTeam', @@ -577,7 +526,8 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): flags.DEFINE_string( 'role', None, - u'The access permission for the entity. Can be READER or OWNER.', + u'The access permission for the entity. Can be READER, WRITER, or ' + u'OWNER.', flag_values=fv) flags.DEFINE_string( 'selfLink', @@ -586,7 +536,7 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): flag_values=fv) def RunWithArgs(self, bucket, entity): - """Updates a default object ACL entry on the specified bucket. + """Updates an ACL entry on the specified bucket. Args: bucket: The name of the bucket. @@ -604,18 +554,17 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): email: The email address associated with the entity, if any. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. - generation: The content generation of the object. id: The ID of the access-control entry. - kind: The kind of item this is. For object access control entries, this - is always storage#objectAccessControl. - object: The name of the object. + kind: The kind of item this is. For bucket access control entries, this + is always storage#bucketAccessControl. projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER or OWNER. + role: The access permission for the entity. Can be READER, WRITER, or + OWNER. selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.ObjectAccessControl( + request = messages.BucketAccessControl( bucket=bucket.decode('utf8'), entity=entity.decode('utf8'), ) @@ -627,481 +576,450 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): request.entityId = FLAGS.entityId.decode('utf8') if FLAGS['etag'].present: request.etag = FLAGS.etag.decode('utf8') - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) if FLAGS['id'].present: request.id = FLAGS.id.decode('utf8') if FLAGS['kind'].present: request.kind = FLAGS.kind.decode('utf8') - if FLAGS['object'].present: - request.object = FLAGS.object.decode('utf8') if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) if FLAGS['role'].present: request.role = FLAGS.role.decode('utf8') if FLAGS['selfLink'].present: request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.defaultObjectAccessControls.Update( + result = client.bucketAccessControls.Update( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsDelete(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.Delete.""" +class BucketsDelete(apitools_base_cli.NewCmd): + """Command wrapping buckets.Delete.""" - usage = """bucketAccessControls_delete """ + usage = """buckets_delete """ def __init__(self, name, fv): - super(BucketAccessControlsDelete, self).__init__(name, fv) + super(BucketsDelete, self).__init__(name, fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u'If set, only deletes the bucket if its metageneration matches this ' + u'value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u'If set, only deletes the bucket if its metageneration does not ' + u'match this value.', + flag_values=fv) - def RunWithArgs(self, bucket, entity): - """Permanently deletes the ACL entry for the specified entity on the - specified bucket. + def RunWithArgs(self, bucket): + """Permanently deletes an empty bucket. Args: bucket: Name of a bucket. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. + + Flags: + ifMetagenerationMatch: If set, only deletes the bucket if its + metageneration matches this value. + ifMetagenerationNotMatch: If set, only deletes the bucket if its + metageneration does not match this value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsDeleteRequest( + request = messages.StorageBucketsDeleteRequest( bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), ) - result = client.bucketAccessControls.Delete( - request, global_params=global_params) - print apitools_base_cli.FormatOutput(result) - - -class BucketAccessControlsGet(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.Get.""" - - usage = """bucketAccessControls_get """ - - def __init__(self, name, fv): - super(BucketAccessControlsGet, self).__init__(name, fv) - - def RunWithArgs(self, bucket, entity): - """Returns the ACL entry for the specified entity on the specified bucket. - - Args: - bucket: Name of a bucket. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsGetRequest( - bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), - ) - result = client.bucketAccessControls.Get( + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.buckets.Delete( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsInsert(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.Insert.""" +class BucketsGet(apitools_base_cli.NewCmd): + """Command wrapping buckets.Get.""" - usage = """bucketAccessControls_insert """ + usage = """buckets_get """ def __init__(self, name, fv): - super(BucketAccessControlsInsert, self).__init__(name, fv) - flags.DEFINE_string( - 'domain', - None, - u'The domain associated with the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'email', - None, - u'The email address associated with the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'entity', - None, - u'The entity holding the permission, in one of the following forms: ' - u'- user-userId - user-email - group-groupId - group-email - ' - u'domain-domain - project-team-projectId - allUsers - ' - u'allAuthenticatedUsers Examples: - The user liz@example.com would ' - u'be user-liz@example.com. - The group example@googlegroups.com ' - u'would be group-example@googlegroups.com. - To refer to all members' - u' of the Google Apps for Business domain example.com, the entity ' - u'would be domain-example.com.', - flag_values=fv) - flags.DEFINE_string( - 'entityId', - None, - u'The ID for the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'etag', - None, - u'HTTP 1.1 Entity tag for the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'id', - None, - u'The ID of the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', - flag_values=fv) + super(BucketsGet, self).__init__(name, fv) flags.DEFINE_string( - 'projectTeam', + 'ifMetagenerationMatch', None, - u'The project team associated with the entity, if any.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( - 'role', + 'ifMetagenerationNotMatch', None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) - flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', flag_values=fv) def RunWithArgs(self, bucket): - """Creates a new ACL entry on the specified bucket. + """Returns metadata for the specified bucket. Args: - bucket: The name of the bucket. + bucket: Name of a bucket. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. - selfLink: The link to this access-control entry. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + projection: Set of properties to return. Defaults to noAcl. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( + request = messages.StorageBucketsGetRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entity'].present: - request.entity = FLAGS.entity.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Insert( + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Get( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsList(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.List.""" +class BucketsInsert(apitools_base_cli.NewCmd): + """Command wrapping buckets.Insert.""" - usage = """bucketAccessControls_list """ + usage = """buckets_insert """ def __init__(self, name, fv): - super(BucketAccessControlsList, self).__init__(name, fv) + super(BucketsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'bucket', + None, + u'A Bucket resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of default object access controls to this ' + u'bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the bucket ' + u'resource specifies acl or defaultObjectAcl properties, when it ' + u'defaults to full.', + flag_values=fv) - def RunWithArgs(self, bucket): - """Retrieves ACL entries on the specified bucket. + def RunWithArgs(self, project): + """Creates a new bucket. Args: - bucket: Name of a bucket. + project: A valid API project identifier. + + Flags: + bucket: A Bucket resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. + projection: Set of properties to return. Defaults to noAcl, unless the + bucket resource specifies acl or defaultObjectAcl properties, when it + defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketAccessControlsListRequest( - bucket=bucket.decode('utf8'), + request = messages.StorageBucketsInsertRequest( + project=project.decode('utf8'), ) - result = client.bucketAccessControls.List( + if FLAGS['bucket'].present: + request.bucket = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucket) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsInsertRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Insert( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsPatch(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.Patch.""" +class BucketsList(apitools_base_cli.NewCmd): + """Command wrapping buckets.List.""" - usage = """bucketAccessControls_patch """ + usage = """buckets_list """ def __init__(self, name, fv): - super(BucketAccessControlsPatch, self).__init__(name, fv) - flags.DEFINE_string( - 'domain', + super(BucketsList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', None, - u'The domain associated with the entity, if any.', + u'Maximum number of buckets to return.', flag_values=fv) flags.DEFINE_string( - 'email', + 'pageToken', None, - u'The email address associated with the entity, if any.', + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', flag_values=fv) flags.DEFINE_string( - 'entityId', + 'prefix', None, - u'The ID for the entity, if any.', + u'Filter results to buckets whose names begin with this prefix.', flag_values=fv) - flags.DEFINE_string( - 'etag', - None, - u'HTTP 1.1 Entity tag for the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'id', - None, - u'The ID of the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', - flag_values=fv) - flags.DEFINE_string( - 'projectTeam', - None, - u'The project team associated with the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'role', - None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', - flag_values=fv) - flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', flag_values=fv) - def RunWithArgs(self, bucket, entity): - """Updates an ACL entry on the specified bucket. This method supports - patch semantics. + def RunWithArgs(self, project): + """Retrieves a list of buckets for a given project. Args: - bucket: The name of the bucket. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. + project: A valid API project identifier. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. - selfLink: The link to this access-control entry. + maxResults: Maximum number of buckets to return. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to buckets whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( - bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), + request = messages.StorageBucketsListRequest( + project=project.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Patch( + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.List( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketAccessControlsUpdate(apitools_base_cli.NewCmd): - """Command wrapping bucketAccessControls.Update.""" +class BucketsPatch(apitools_base_cli.NewCmd): + """Command wrapping buckets.Patch.""" - usage = """bucketAccessControls_update """ + usage = """buckets_patch """ def __init__(self, name, fv): - super(BucketAccessControlsUpdate, self).__init__(name, fv) - flags.DEFINE_string( - 'domain', - None, - u'The domain associated with the entity, if any.', - flag_values=fv) - flags.DEFINE_string( - 'email', - None, - u'The email address associated with the entity, if any.', - flag_values=fv) + super(BucketsPatch, self).__init__(name, fv) flags.DEFINE_string( - 'entityId', + 'bucketResource', None, - u'The ID for the entity, if any.', + u'A Bucket resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( - 'etag', + 'ifMetagenerationMatch', None, - u'HTTP 1.1 Entity tag for the access-control entry.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( - 'id', + 'ifMetagenerationNotMatch', None, - u'The ID of the access-control entry.', - flag_values=fv) - flags.DEFINE_string( - 'kind', - u'storage#bucketAccessControl', - u'The kind of item this is. For bucket access control entries, this ' - u'is always storage#bucketAccessControl.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) - flags.DEFINE_string( - 'projectTeam', - None, - u'The project team associated with the entity, if any.', + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', flag_values=fv) - flags.DEFINE_string( - 'role', - None, - u'The access permission for the entity. Can be READER, WRITER, or ' - u'OWNER.', + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of default object access controls to this ' + u'bucket.', flag_values=fv) - flags.DEFINE_string( - 'selfLink', - None, - u'The link to this access-control entry.', + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', flag_values=fv) - def RunWithArgs(self, bucket, entity): - """Updates an ACL entry on the specified bucket. + def RunWithArgs(self, bucket): + """Updates a bucket. This method supports patch semantics. Args: - bucket: The name of the bucket. - entity: The entity holding the permission, in one of the following - forms: - user-userId - user-email - group-groupId - group-email - - domain-domain - project-team-projectId - allUsers - - allAuthenticatedUsers Examples: - The user liz@example.com would be - user-liz@example.com. - The group example@googlegroups.com would be - group-example@googlegroups.com. - To refer to all members of the - Google Apps for Business domain example.com, the entity would be - domain-example.com. + bucket: Name of a bucket. Flags: - domain: The domain associated with the entity, if any. - email: The email address associated with the entity, if any. - entityId: The ID for the entity, if any. - etag: HTTP 1.1 Entity tag for the access-control entry. - id: The ID of the access-control entry. - kind: The kind of item this is. For bucket access control entries, this - is always storage#bucketAccessControl. - projectTeam: The project team associated with the entity, if any. - role: The access permission for the entity. Can be READER, WRITER, or - OWNER. - selfLink: The link to this access-control entry. + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. + projection: Set of properties to return. Defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.BucketAccessControl( + request = messages.StorageBucketsPatchRequest( bucket=bucket.decode('utf8'), - entity=entity.decode('utf8'), ) - if FLAGS['domain'].present: - request.domain = FLAGS.domain.decode('utf8') - if FLAGS['email'].present: - request.email = FLAGS.email.decode('utf8') - if FLAGS['entityId'].present: - request.entityId = FLAGS.entityId.decode('utf8') - if FLAGS['etag'].present: - request.etag = FLAGS.etag.decode('utf8') - if FLAGS['id'].present: - request.id = FLAGS.id.decode('utf8') - if FLAGS['kind'].present: - request.kind = FLAGS.kind.decode('utf8') - if FLAGS['projectTeam'].present: - request.projectTeam = apitools_base.JsonToMessage(messages.BucketAccessControl.ProjectTeamValue, FLAGS.projectTeam) - if FLAGS['role'].present: - request.role = FLAGS.role.decode('utf8') - if FLAGS['selfLink'].present: - request.selfLink = FLAGS.selfLink.decode('utf8') - result = client.bucketAccessControls.Update( + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsPatchRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Patch( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ChannelsStop(apitools_base_cli.NewCmd): - """Command wrapping channels.Stop.""" +class BucketsUpdate(apitools_base_cli.NewCmd): + """Command wrapping buckets.Update.""" - usage = """channels_stop""" + usage = """buckets_update """ def __init__(self, name, fv): - super(ChannelsStop, self).__init__(name, fv) + super(BucketsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'address', + 'bucketResource', None, - u'The address where notifications are delivered for this channel.', + u'A Bucket resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( - 'expiration', + 'ifMetagenerationMatch', None, - u'Date and time of notification channel expiration, expressed as a ' - u'Unix timestamp, in milliseconds. Optional.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration matches the given value.", flag_values=fv) flags.DEFINE_string( - 'id', + 'ifMetagenerationNotMatch', None, - u'A UUID or similar unique string that identifies this channel.', + u'Makes the return of the bucket metadata conditional on whether the ' + u"bucket's current metageneration does not match the given value.", flag_values=fv) - flags.DEFINE_string( - 'kind', - u'api#channel', - u'Identifies this as a notification channel used to watch for changes' - u' to a resource. Value: the fixed string "api#channel".', + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], + u'Apply a predefined set of access controls to this bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedDefaultObjectAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of default object access controls to this ' + u'bucket.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to full.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Updates a bucket. + + Args: + bucket: Name of a bucket. + + Flags: + bucketResource: A Bucket resource to be passed as the request body. + ifMetagenerationMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration matches the + given value. + ifMetagenerationNotMatch: Makes the return of the bucket metadata + conditional on whether the bucket's current metageneration does not + match the given value. + predefinedAcl: Apply a predefined set of access controls to this bucket. + predefinedDefaultObjectAcl: Apply a predefined set of default object + access controls to this bucket. + projection: Set of properties to return. Defaults to full. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsUpdateRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['bucketResource'].present: + request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageBucketsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['predefinedDefaultObjectAcl'].present: + request.predefinedDefaultObjectAcl = messages.StorageBucketsUpdateRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageBucketsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.buckets.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ChannelsStop(apitools_base_cli.NewCmd): + """Command wrapping channels.Stop.""" + + usage = """channels_stop""" + + def __init__(self, name, fv): + super(ChannelsStop, self).__init__(name, fv) + flags.DEFINE_string( + 'address', + None, + u'The address where notifications are delivered for this channel.', + flag_values=fv) + flags.DEFINE_string( + 'expiration', + None, + u'Date and time of notification channel expiration, expressed as a ' + u'Unix timestamp, in milliseconds. Optional.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'A UUID or similar unique string that identifies this channel.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'api#channel', + u'Identifies this as a notification channel used to watch for changes' + u' to a resource. Value: the fixed string "api#channel".', flag_values=fv) flags.DEFINE_string( 'params', @@ -1187,804 +1105,591 @@ class ChannelsStop(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) -class ObjectsCompose(apitools_base_cli.NewCmd): - """Command wrapping objects.Compose.""" +class DefaultObjectAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Delete.""" - usage = """objects_compose """ + usage = """defaultObjectAccessControls_delete """ def __init__(self, name, fv): - super(ObjectsCompose, self).__init__(name, fv) - flags.DEFINE_string( - 'composeRequest', - None, - u'A ComposeRequest resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'destinationPredefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to the destination ' - u'object.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', - flag_values=fv) + super(DefaultObjectAccessControlsDelete, self).__init__(name, fv) - def RunWithArgs(self, destinationBucket, destinationObject): - """Concatenates a list of existing objects into a new object in the same - bucket. + def RunWithArgs(self, bucket, entity): + """Permanently deletes the default object ACL entry for the specified + entity on the specified bucket. Args: - destinationBucket: Name of the bucket in which to store the new object. - destinationObject: Name of the new object. + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageDefaultObjectAccessControlsDeleteRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), + ) + result = client.defaultObjectAccessControls.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) - Flags: - composeRequest: A ComposeRequest resource to be passed as the request - body. - destinationPredefinedAcl: Apply a predefined set of access controls to - the destination object. - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + +class DefaultObjectAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Get.""" + + usage = """defaultObjectAccessControls_get """ + + def __init__(self, name, fv): + super(DefaultObjectAccessControlsGet, self).__init__(name, fv) + + def RunWithArgs(self, bucket, entity): + """Returns the default object ACL entry for the specified entity on the + specified bucket. + + Args: + bucket: Name of a bucket. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsComposeRequest( - destinationBucket=destinationBucket.decode('utf8'), - destinationObject=destinationObject.decode('utf8'), + request = messages.StorageDefaultObjectAccessControlsGetRequest( + bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['composeRequest'].present: - request.composeRequest = apitools_base.JsonToMessage(messages.ComposeRequest, FLAGS.composeRequest) - if FLAGS['destinationPredefinedAcl'].present: - request.destinationPredefinedAcl = messages.StorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Compose( - request, global_params=global_params, download=download) + result = client.defaultObjectAccessControls.Get( + request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsCopy(apitools_base_cli.NewCmd): - """Command wrapping objects.Copy.""" +class DefaultObjectAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Insert.""" - usage = """objects_copy """ + usage = """defaultObjectAccessControls_insert """ def __init__(self, name, fv): - super(ObjectsCopy, self).__init__(name, fv) - flags.DEFINE_enum( - 'destinationPredefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to the destination ' - u'object.', - flag_values=fv) + super(DefaultObjectAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( - 'ifGenerationMatch', + 'domain', None, - u"Makes the operation conditional on whether the destination object's" - u' current generation matches the given value.', + u'The domain associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifGenerationNotMatch', + 'email', None, - u"Makes the operation conditional on whether the destination object's" - u' current generation does not match the given value.', + u'The email address associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'entity', None, - u"Makes the operation conditional on whether the destination object's" - u' current metageneration matches the given value.', + u'The entity holding the permission, in one of the following forms: ' + u'- user-userId - user-email - group-groupId - group-email - ' + u'domain-domain - project-team-projectId - allUsers - ' + u'allAuthenticatedUsers Examples: - The user liz@example.com would ' + u'be user-liz@example.com. - The group example@googlegroups.com ' + u'would be group-example@googlegroups.com. - To refer to all members' + u' of the Google Apps for Business domain example.com, the entity ' + u'would be domain-example.com.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'entityId', None, - u"Makes the operation conditional on whether the destination object's" - u' current metageneration does not match the given value.', + u'The ID for the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifSourceGenerationMatch', + 'etag', None, - u"Makes the operation conditional on whether the source object's " - u'generation matches the given value.', + u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) flags.DEFINE_string( - 'ifSourceGenerationNotMatch', + 'generation', None, - u"Makes the operation conditional on whether the source object's " - u'generation does not match the given value.', + u'The content generation of the object.', flag_values=fv) flags.DEFINE_string( - 'ifSourceMetagenerationMatch', + 'id', None, - u"Makes the operation conditional on whether the source object's " - u'current metageneration matches the given value.', + u'The ID of the access-control entry.', flag_values=fv) flags.DEFINE_string( - 'ifSourceMetagenerationNotMatch', - None, - u"Makes the operation conditional on whether the source object's " - u'current metageneration does not match the given value.', + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', flag_values=fv) flags.DEFINE_string( 'object', None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the object ' - u'resource specifies the acl property, when it defaults to full.', + u'The name of the object.', flag_values=fv) flags.DEFINE_string( - 'sourceGeneration', + 'projectTeam', None, - u'If present, selects a specific revision of the source object (as ' - u'opposed to the latest version, the default).', + u'The project team associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): - """Copies an object to a specified location. Optionally overrides - metadata. + def RunWithArgs(self, bucket): + """Creates a new default object ACL entry on the specified bucket. Args: - sourceBucket: Name of the bucket in which to find the source object. - sourceObject: Name of the source object. - destinationBucket: Name of the bucket in which to store the new object. - Overrides the provided object metadata's bucket value, if any. - destinationObject: Name of the new object. Required when the object - metadata is not otherwise provided. Overrides the object metadata's - name value, if any. + bucket: The name of the bucket. Flags: - destinationPredefinedAcl: Apply a predefined set of access controls to - the destination object. - ifGenerationMatch: Makes the operation conditional on whether the - destination object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - destination object's current generation does not match the given - value. - ifMetagenerationMatch: Makes the operation conditional on whether the - destination object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - destination object's current metageneration does not match the given - value. - ifSourceGenerationMatch: Makes the operation conditional on whether the - source object's generation matches the given value. - ifSourceGenerationNotMatch: Makes the operation conditional on whether - the source object's generation does not match the given value. - ifSourceMetagenerationMatch: Makes the operation conditional on whether - the source object's current metageneration matches the given value. - ifSourceMetagenerationNotMatch: Makes the operation conditional on - whether the source object's current metageneration does not match the - given value. - object: A Object resource to be passed as the request body. - projection: Set of properties to return. Defaults to noAcl, unless the - object resource specifies the acl property, when it defaults to full. - sourceGeneration: If present, selects a specific revision of the source - object (as opposed to the latest version, the default). - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsCopyRequest( - sourceBucket=sourceBucket.decode('utf8'), - sourceObject=sourceObject.decode('utf8'), - destinationBucket=destinationBucket.decode('utf8'), - destinationObject=destinationObject.decode('utf8'), + request = messages.ObjectAccessControl( + bucket=bucket.decode('utf8'), ) - if FLAGS['destinationPredefinedAcl'].present: - request.destinationPredefinedAcl = messages.StorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['ifSourceGenerationMatch'].present: - request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) - if FLAGS['ifSourceGenerationNotMatch'].present: - request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) - if FLAGS['ifSourceMetagenerationMatch'].present: - request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) - if FLAGS['ifSourceMetagenerationNotMatch'].present: - request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entity'].present: + request.entity = FLAGS.entity.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') if FLAGS['object'].present: - request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsCopyRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['sourceGeneration'].present: - request.sourceGeneration = int(FLAGS.sourceGeneration) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Copy( - request, global_params=global_params, download=download) + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Insert( + request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsDelete(apitools_base_cli.NewCmd): - """Command wrapping objects.Delete.""" +class DefaultObjectAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.List.""" - usage = """objects_delete """ + usage = """defaultObjectAccessControls_list """ def __init__(self, name, fv): - super(ObjectsDelete, self).__init__(name, fv) - flags.DEFINE_string( - 'generation', - None, - u'If present, permanently deletes a specific revision of this object ' - u'(as opposed to the latest version, the default).', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', - flag_values=fv) + super(DefaultObjectAccessControlsList, self).__init__(name, fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u"If present, only return default ACL listing if the bucket's current" + u' metageneration matches this value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', + u"If present, only return default ACL listing if the bucket's current" + u' metageneration does not match the given value.', flag_values=fv) - def RunWithArgs(self, bucket, object): - """Deletes an object and its metadata. Deletions are permanent if - versioning is not enabled for the bucket, or if the generation parameter - is used. + def RunWithArgs(self, bucket): + """Retrieves default object ACL entries on the specified bucket. Args: - bucket: Name of the bucket in which the object resides. - object: Name of the object. + bucket: Name of a bucket. Flags: - generation: If present, permanently deletes a specific revision of this - object (as opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. + ifMetagenerationMatch: If present, only return default ACL listing if + the bucket's current metageneration matches this value. + ifMetagenerationNotMatch: If present, only return default ACL listing if + the bucket's current metageneration does not match the given value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsDeleteRequest( + request = messages.StorageDefaultObjectAccessControlsListRequest( bucket=bucket.decode('utf8'), - object=object.decode('utf8'), ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.objects.Delete( + result = client.defaultObjectAccessControls.List( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsGet(apitools_base_cli.NewCmd): - """Command wrapping objects.Get.""" +class DefaultObjectAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Patch.""" - usage = """objects_get """ + usage = """defaultObjectAccessControls_patch """ def __init__(self, name, fv): - super(ObjectsGet, self).__init__(name, fv) + super(DefaultObjectAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'domain', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u'The domain associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifGenerationMatch', + 'email', None, - u"Makes the operation conditional on whether the object's generation " - u'matches the given value.', + u'The email address associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifGenerationNotMatch', + 'entityId', None, - u"Makes the operation conditional on whether the object's generation " - u'does not match the given value.', + u'The ID for the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'etag', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'generation', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', + u'The content generation of the object.', flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', + flags.DEFINE_string( + 'id', + None, + u'The ID of the access-control entry.', flag_values=fv) flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + flags.DEFINE_string( + 'object', + None, + u'The name of the object.', + flag_values=fv) + flags.DEFINE_string( + 'projectTeam', + None, + u'The project team associated with the entity, if any.', + flag_values=fv) + flags.DEFINE_string( + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, bucket, object): - """Retrieves an object or its metadata. + def RunWithArgs(self, bucket, entity): + """Updates a default object ACL entry on the specified bucket. This method + supports patch semantics. Args: - bucket: Name of the bucket in which the object resides. - object: Name of the object. + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - projection: Set of properties to return. Defaults to noAcl. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsGetRequest( + request = messages.ObjectAccessControl( bucket=bucket.decode('utf8'), - object=object.decode('utf8'), + entity=entity.decode('utf8'), ) + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Get( - request, global_params=global_params, download=download) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object'].present: + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Patch( + request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsInsert(apitools_base_cli.NewCmd): - """Command wrapping objects.Insert.""" +class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping defaultObjectAccessControls.Update.""" - usage = """objects_insert """ + usage = """defaultObjectAccessControls_update """ def __init__(self, name, fv): - super(ObjectsInsert, self).__init__(name, fv) + super(DefaultObjectAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'contentEncoding', + 'domain', None, - u'If set, sets the contentEncoding property of the final object to ' - u'this value. Setting this parameter is equivalent to setting the ' - u'contentEncoding metadata property. This can be useful when ' - u'uploading an object with uploadType=media to indicate the encoding ' - u'of the content being uploaded.', + u'The domain associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifGenerationMatch', + 'email', None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', + u'The email address associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifGenerationNotMatch', + 'entityId', None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', + u'The ID for the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'etag', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', + u'HTTP 1.1 Entity tag for the access-control entry.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'generation', None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', + u'The content generation of the object.', flag_values=fv) flags.DEFINE_string( - 'name', + 'id', None, - u'Name of the object. Required when the object metadata is not ' - u"otherwise provided. Overrides the object metadata's name value, if " - u'any.', + u'The ID of the access-control entry.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#objectAccessControl', + u'The kind of item this is. For object access control entries, this ' + u'is always storage#objectAccessControl.', flag_values=fv) flags.DEFINE_string( 'object', None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the object ' - u'resource specifies the acl property, when it defaults to full.', + u'The name of the object.', flag_values=fv) flags.DEFINE_string( - 'upload_filename', - '', - 'Filename to use for upload.', + 'projectTeam', + None, + u'The project team associated with the entity, if any.', flag_values=fv) flags.DEFINE_string( - 'upload_mime_type', - '', - 'MIME type to use for the upload. Only needed if the extension on ' - '--upload_filename does not determine the correct (or any) MIME ' - 'type.', + 'role', + None, + u'The access permission for the entity. Can be READER or OWNER.', flag_values=fv) flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + 'selfLink', + None, + u'The link to this access-control entry.', flag_values=fv) - def RunWithArgs(self, bucket): - """Stores a new object and metadata. + def RunWithArgs(self, bucket, entity): + """Updates a default object ACL entry on the specified bucket. Args: - bucket: Name of the bucket in which to store the new object. Overrides - the provided object metadata's bucket value, if any. + bucket: The name of the bucket. + entity: The entity holding the permission, in one of the following + forms: - user-userId - user-email - group-groupId - group-email - + domain-domain - project-team-projectId - allUsers - + allAuthenticatedUsers Examples: - The user liz@example.com would be + user-liz@example.com. - The group example@googlegroups.com would be + group-example@googlegroups.com. - To refer to all members of the + Google Apps for Business domain example.com, the entity would be + domain-example.com. Flags: - contentEncoding: If set, sets the contentEncoding property of the final - object to this value. Setting this parameter is equivalent to setting - the contentEncoding metadata property. This can be useful when - uploading an object with uploadType=media to indicate the encoding of - the content being uploaded. - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - name: Name of the object. Required when the object metadata is not - otherwise provided. Overrides the object metadata's name value, if - any. - object: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. - projection: Set of properties to return. Defaults to noAcl, unless the - object resource specifies the acl property, when it defaults to full. - upload_filename: Filename to use for upload. - upload_mime_type: MIME type to use for the upload. Only needed if the - extension on --upload_filename does not determine the correct (or any) - MIME type. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsInsertRequest( + domain: The domain associated with the entity, if any. + email: The email address associated with the entity, if any. + entityId: The ID for the entity, if any. + etag: HTTP 1.1 Entity tag for the access-control entry. + generation: The content generation of the object. + id: The ID of the access-control entry. + kind: The kind of item this is. For object access control entries, this + is always storage#objectAccessControl. + object: The name of the object. + projectTeam: The project team associated with the entity, if any. + role: The access permission for the entity. Can be READER or OWNER. + selfLink: The link to this access-control entry. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.ObjectAccessControl( bucket=bucket.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['contentEncoding'].present: - request.contentEncoding = FLAGS.contentEncoding.decode('utf8') - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['name'].present: - request.name = FLAGS.name.decode('utf8') + if FLAGS['domain'].present: + request.domain = FLAGS.domain.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['entityId'].present: + request.entityId = FLAGS.entityId.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') if FLAGS['object'].present: - request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) - upload = None - if FLAGS.upload_filename: - upload = apitools_base.Upload.FromFile( - FLAGS.upload_filename, FLAGS.upload_mime_type) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Insert( - request, global_params=global_params, upload=upload, download=download) + request.object = FLAGS.object.decode('utf8') + if FLAGS['projectTeam'].present: + request.projectTeam = apitools_base.JsonToMessage(messages.ObjectAccessControl.ProjectTeamValue, FLAGS.projectTeam) + if FLAGS['role'].present: + request.role = FLAGS.role.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + result = client.defaultObjectAccessControls.Update( + request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsList(apitools_base_cli.NewCmd): - """Command wrapping objects.List.""" +class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Delete.""" - usage = """objects_list """ + usage = """objectAccessControls_delete """ def __init__(self, name, fv): - super(ObjectsList, self).__init__(name, fv) - flags.DEFINE_string( - 'delimiter', - None, - u'Returns results in a directory-like mode. items will contain only ' - u'objects whose names, aside from the prefix, do not contain ' - u'delimiter. Objects whose names, aside from the prefix, contain ' - u'delimiter will have their name, truncated after the delimiter, ' - u'returned in prefixes. Duplicate prefixes are omitted.', - flag_values=fv) - flags.DEFINE_integer( - 'maxResults', - None, - u'Maximum number of items plus prefixes to return. As duplicate ' - u'prefixes are omitted, fewer total results may be returned than ' - u'requested.', - flag_values=fv) - flags.DEFINE_string( - 'pageToken', - None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', - flag_values=fv) + super(ObjectAccessControlsDelete, self).__init__(name, fv) flags.DEFINE_string( - 'prefix', - None, - u'Filter results to objects whose names begin with this prefix.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) - flags.DEFINE_boolean( - 'versions', + 'generation', None, - u'If true, lists all versions of a file as distinct results.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) - def RunWithArgs(self, bucket): - """Retrieves a list of objects matching the criteria. + def RunWithArgs(self, bucket, object, entity): + """Permanently deletes the ACL entry for the specified entity on the + specified object. Args: - bucket: Name of the bucket in which to look for objects. + bucket: Name of a bucket. + object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. Flags: - delimiter: Returns results in a directory-like mode. items will contain - only objects whose names, aside from the prefix, do not contain - delimiter. Objects whose names, aside from the prefix, contain - delimiter will have their name, truncated after the delimiter, - returned in prefixes. Duplicate prefixes are omitted. - maxResults: Maximum number of items plus prefixes to return. As - duplicate prefixes are omitted, fewer total results may be returned - than requested. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - prefix: Filter results to objects whose names begin with this prefix. - projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsListRequest( + request = messages.StorageObjectAccessControlsDeleteRequest( bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), ) - if FLAGS['delimiter'].present: - request.delimiter = FLAGS.delimiter.decode('utf8') - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['prefix'].present: - request.prefix = FLAGS.prefix.decode('utf8') - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsListRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['versions'].present: - request.versions = FLAGS.versions - result = client.objects.List( + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.Delete( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsPatch(apitools_base_cli.NewCmd): - """Command wrapping objects.Patch.""" +class ObjectAccessControlsGet(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Get.""" - usage = """objects_patch """ + usage = """objectAccessControls_get """ def __init__(self, name, fv): - super(ObjectsPatch, self).__init__(name, fv) + super(ObjectAccessControlsGet, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, u'If present, selects a specific revision of this object (as opposed ' u'to the latest version, the default).', flag_values=fv) - flags.DEFINE_string( - 'ifGenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'objectResource', - None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', - flag_values=fv) - def RunWithArgs(self, bucket, object): - """Updates an object's metadata. This method supports patch semantics. + def RunWithArgs(self, bucket, object, entity): + """Returns the ACL entry for the specified entity on the specified object. Args: - bucket: Name of the bucket in which the object resides. + bucket: Name of a bucket. object: Name of the object. + entity: The entity holding the permission. Can be user-userId, user- + emailAddress, group-groupId, group-emailAddress, allUsers, or + allAuthenticatedUsers. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - objectResource: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. - projection: Set of properties to return. Defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsPatchRequest( + request = messages.StorageObjectAccessControlsGetRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), + entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['objectResource'].present: - request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.objects.Patch( + result = client.objectAccessControls.Get( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsUpdate(apitools_base_cli.NewCmd): - """Command wrapping objects.Update.""" +class ObjectAccessControlsInsert(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Insert.""" - usage = """objects_update """ + usage = """objectAccessControls_insert """ def __init__(self, name, fv): - super(ObjectsUpdate, self).__init__(name, fv) + super(ObjectAccessControlsInsert, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, @@ -1992,225 +1697,99 @@ class ObjectsUpdate(apitools_base_cli.NewCmd): u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'ifGenerationMatch', + 'objectAccessControl', None, - u"Makes the operation conditional on whether the object's current " - u'generation matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifGenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'generation does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration matches the given value.', - flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', - None, - u"Makes the operation conditional on whether the object's current " - u'metageneration does not match the given value.', - flag_values=fv) - flags.DEFINE_string( - 'objectResource', - None, - u'A Object resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of access controls to this object.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', - flag_values=fv) - flags.DEFINE_string( - 'download_filename', - '', - 'Filename to use for download.', - flag_values=fv) - flags.DEFINE_boolean( - 'overwrite', - 'False', - 'If True, overwrite the existing file when downloading.', + u'A ObjectAccessControl resource to be passed as the request body.', flag_values=fv) def RunWithArgs(self, bucket, object): - """Updates an object's metadata. + """Creates a new ACL entry on the specified object. Args: - bucket: Name of the bucket in which the object resides. + bucket: Name of a bucket. object: Name of the object. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - ifGenerationMatch: Makes the operation conditional on whether the - object's current generation matches the given value. - ifGenerationNotMatch: Makes the operation conditional on whether the - object's current generation does not match the given value. - ifMetagenerationMatch: Makes the operation conditional on whether the - object's current metageneration matches the given value. - ifMetagenerationNotMatch: Makes the operation conditional on whether the - object's current metageneration does not match the given value. - objectResource: A Object resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this object. - projection: Set of properties to return. Defaults to full. - download_filename: Filename to use for download. - overwrite: If True, overwrite the existing file when downloading. + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsUpdateRequest( + request = messages.StorageObjectAccessControlsInsertRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['ifGenerationMatch'].present: - request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) - if FLAGS['ifGenerationNotMatch'].present: - request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['objectResource'].present: - request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageObjectsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) - download = None - if FLAGS.download_filename: - download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite) - result = client.objects.Update( - request, global_params=global_params, download=download) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Insert( + request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectsWatchAll(apitools_base_cli.NewCmd): - """Command wrapping objects.WatchAll.""" +class ObjectAccessControlsList(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.List.""" - usage = """objects_watchAll """ + usage = """objectAccessControls_list """ def __init__(self, name, fv): - super(ObjectsWatchAll, self).__init__(name, fv) - flags.DEFINE_string( - 'channel', - None, - u'A Channel resource to be passed as the request body.', - flag_values=fv) - flags.DEFINE_string( - 'delimiter', - None, - u'Returns results in a directory-like mode. items will contain only ' - u'objects whose names, aside from the prefix, do not contain ' - u'delimiter. Objects whose names, aside from the prefix, contain ' - u'delimiter will have their name, truncated after the delimiter, ' - u'returned in prefixes. Duplicate prefixes are omitted.', - flag_values=fv) - flags.DEFINE_integer( - 'maxResults', - None, - u'Maximum number of items plus prefixes to return. As duplicate ' - u'prefixes are omitted, fewer total results may be returned than ' - u'requested.', - flag_values=fv) - flags.DEFINE_string( - 'pageToken', - None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', - flag_values=fv) + super(ObjectAccessControlsList, self).__init__(name, fv) flags.DEFINE_string( - 'prefix', - None, - u'Filter results to objects whose names begin with this prefix.', - flag_values=fv) - flags.DEFINE_enum( - 'projection', - u'full', - [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', - flag_values=fv) - flags.DEFINE_boolean( - 'versions', + 'generation', None, - u'If true, lists all versions of a file as distinct results.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', flag_values=fv) - def RunWithArgs(self, bucket): - """Watch for changes on all objects in a bucket. + def RunWithArgs(self, bucket, object): + """Retrieves ACL entries on the specified object. Args: - bucket: Name of the bucket in which to look for objects. + bucket: Name of a bucket. + object: Name of the object. Flags: - channel: A Channel resource to be passed as the request body. - delimiter: Returns results in a directory-like mode. items will contain - only objects whose names, aside from the prefix, do not contain - delimiter. Objects whose names, aside from the prefix, contain - delimiter will have their name, truncated after the delimiter, - returned in prefixes. Duplicate prefixes are omitted. - maxResults: Maximum number of items plus prefixes to return. As - duplicate prefixes are omitted, fewer total results may be returned - than requested. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - prefix: Filter results to objects whose names begin with this prefix. - projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectsWatchAllRequest( + request = messages.StorageObjectAccessControlsListRequest( bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['channel'].present: - request.channel = apitools_base.JsonToMessage(messages.Channel, FLAGS.channel) - if FLAGS['delimiter'].present: - request.delimiter = FLAGS.delimiter.decode('utf8') - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['prefix'].present: - request.prefix = FLAGS.prefix.decode('utf8') - if FLAGS['projection'].present: - request.projection = messages.StorageObjectsWatchAllRequest.ProjectionValueValuesEnum(FLAGS.projection) - if FLAGS['versions'].present: - request.versions = FLAGS.versions - result = client.objects.WatchAll( + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objectAccessControls.List( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.Delete.""" +class ObjectAccessControlsPatch(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Patch.""" - usage = """objectAccessControls_delete """ + usage = """objectAccessControls_patch """ def __init__(self, name, fv): - super(ObjectAccessControlsDelete, self).__init__(name, fv) + super(ObjectAccessControlsPatch, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, u'If present, selects a specific revision of this object (as opposed ' u'to the latest version, the default).', flag_values=fv) + flags.DEFINE_string( + 'objectAccessControl', + None, + u'A ObjectAccessControl resource to be passed as the request body.', + flag_values=fv) def RunWithArgs(self, bucket, object, entity): - """Permanently deletes the ACL entry for the specified entity on the - specified object. + """Updates an ACL entry on the specified object. This method supports + patch semantics. Args: bucket: Name of a bucket. @@ -2222,37 +1801,46 @@ class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsDeleteRequest( + request = messages.StorageObjectAccessControlsPatchRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - result = client.objectAccessControls.Delete( + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Patch( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsGet(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.Get.""" +class ObjectAccessControlsUpdate(apitools_base_cli.NewCmd): + """Command wrapping objectAccessControls.Update.""" - usage = """objectAccessControls_get """ + usage = """objectAccessControls_update """ def __init__(self, name, fv): - super(ObjectAccessControlsGet, self).__init__(name, fv) + super(ObjectAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, u'If present, selects a specific revision of this object (as opposed ' u'to the latest version, the default).', flag_values=fv) + flags.DEFINE_string( + 'objectAccessControl', + None, + u'A ObjectAccessControl resource to be passed as the request body.', + flag_values=fv) def RunWithArgs(self, bucket, object, entity): - """Returns the ACL entry for the specified entity on the specified object. + """Updates an ACL entry on the specified object. Args: bucket: Name of a bucket. @@ -2264,113 +1852,374 @@ class ObjectAccessControlsGet(apitools_base_cli.NewCmd): Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). + objectAccessControl: A ObjectAccessControl resource to be passed as the + request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectAccessControlsUpdateRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + entity=entity.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['objectAccessControl'].present: + request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) + result = client.objectAccessControls.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ObjectsCompose(apitools_base_cli.NewCmd): + """Command wrapping objects.Compose.""" + + usage = """objects_compose """ + + def __init__(self, name, fv): + super(ObjectsCompose, self).__init__(name, fv) + flags.DEFINE_string( + 'composeRequest', + None, + u'A ComposeRequest resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, destinationBucket, destinationObject): + """Concatenates a list of existing objects into a new object in the same + bucket. + + Args: + destinationBucket: Name of the bucket in which to store the new object. + destinationObject: Name of the new object. + + Flags: + composeRequest: A ComposeRequest resource to be passed as the request + body. + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsComposeRequest( + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), + ) + if FLAGS['composeRequest'].present: + request.composeRequest = apitools_base.JsonToMessage(messages.ComposeRequest, FLAGS.composeRequest) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.objects.Compose( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) + + +class ObjectsCopy(apitools_base_cli.NewCmd): + """Command wrapping objects.Copy.""" + + usage = """objects_copy """ + + def __init__(self, name, fv): + super(ObjectsCopy, self).__init__(name, fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'sourceGeneration', + None, + u'If present, selects a specific revision of the source object (as ' + u'opposed to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): + """Copies a source object to a destination object. Optionally overrides + metadata. + + Args: + sourceBucket: Name of the bucket in which to find the source object. + sourceObject: Name of the source object. + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's + name value, if any. + + Flags: + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given + value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether + the source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on + whether the source object's current metageneration does not match the + given value. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsGetRequest( - bucket=bucket.decode('utf8'), - object=object.decode('utf8'), - entity=entity.decode('utf8'), + request = messages.StorageObjectsCopyRequest( + sourceBucket=sourceBucket.decode('utf8'), + sourceObject=sourceObject.decode('utf8'), + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - result = client.objectAccessControls.Get( - request, global_params=global_params) + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['ifSourceGenerationMatch'].present: + request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) + if FLAGS['ifSourceGenerationNotMatch'].present: + request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) + if FLAGS['ifSourceMetagenerationMatch'].present: + request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) + if FLAGS['ifSourceMetagenerationNotMatch'].present: + request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsCopyRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['sourceGeneration'].present: + request.sourceGeneration = int(FLAGS.sourceGeneration) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.objects.Copy( + request, global_params=global_params, download=download) print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsInsert(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.Insert.""" +class ObjectsDelete(apitools_base_cli.NewCmd): + """Command wrapping objects.Delete.""" - usage = """objectAccessControls_insert """ + usage = """objects_delete """ def __init__(self, name, fv): - super(ObjectAccessControlsInsert, self).__init__(name, fv) + super(ObjectsDelete, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u'If present, permanently deletes a specific revision of this object ' + u'(as opposed to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'objectAccessControl', + 'ifGenerationMatch', None, - u'A ObjectAccessControl resource to be passed as the request body.', + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', flag_values=fv) - - def RunWithArgs(self, bucket, object): - """Creates a new ACL entry on the specified object. - - Args: - bucket: Name of a bucket. - object: Name of the object. - - Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsInsertRequest( - bucket=bucket.decode('utf8'), - object=object.decode('utf8'), - ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Insert( - request, global_params=global_params) - print apitools_base_cli.FormatOutput(result) - - -class ObjectAccessControlsList(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.List.""" - - usage = """objectAccessControls_list """ - - def __init__(self, name, fv): - super(ObjectAccessControlsList, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'ifGenerationNotMatch', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', flag_values=fv) def RunWithArgs(self, bucket, object): - """Retrieves ACL entries on the specified object. + """Deletes an object and its metadata. Deletions are permanent if + versioning is not enabled for the bucket, or if the generation parameter + is used. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. object: Name of the object. Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). + generation: If present, permanently deletes a specific revision of this + object (as opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsListRequest( + request = messages.StorageObjectsDeleteRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - result = client.objectAccessControls.List( + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + result = client.objects.Delete( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class ObjectAccessControlsPatch(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.Patch.""" +class ObjectsGet(apitools_base_cli.NewCmd): + """Command wrapping objects.Get.""" - usage = """objectAccessControls_patch """ + usage = """objects_get """ def __init__(self, name, fv): - super(ObjectAccessControlsPatch, self).__init__(name, fv) + super(ObjectsGet, self).__init__(name, fv) flags.DEFINE_string( 'generation', None, @@ -2378,159 +2227,291 @@ class ObjectAccessControlsPatch(apitools_base_cli.NewCmd): u'to the latest version, the default).', flag_values=fv) flags.DEFINE_string( - 'objectAccessControl', + 'ifGenerationMatch', None, - u'A ObjectAccessControl resource to be passed as the request body.', + u"Makes the operation conditional on whether the object's generation " + u'matches the given value.', flag_values=fv) - - def RunWithArgs(self, bucket, object, entity): - """Updates an ACL entry on the specified object. This method supports - patch semantics. - - Args: - bucket: Name of a bucket. - object: Name of the object. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. - - Flags: - generation: If present, selects a specific revision of this object (as - opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. - """ - client = GetClientFromFlags() - global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsPatchRequest( - bucket=bucket.decode('utf8'), - object=object.decode('utf8'), - entity=entity.decode('utf8'), - ) - if FLAGS['generation'].present: - request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Patch( - request, global_params=global_params) - print apitools_base_cli.FormatOutput(result) - - -class ObjectAccessControlsUpdate(apitools_base_cli.NewCmd): - """Command wrapping objectAccessControls.Update.""" - - usage = """objectAccessControls_update """ - - def __init__(self, name, fv): - super(ObjectAccessControlsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'generation', + 'ifGenerationNotMatch', None, - u'If present, selects a specific revision of this object (as opposed ' - u'to the latest version, the default).', + u"Makes the operation conditional on whether the object's generation " + u'does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'objectAccessControl', + 'ifMetagenerationMatch', None, - u'A ObjectAccessControl resource to be passed as the request body.', + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', flag_values=fv) - def RunWithArgs(self, bucket, object, entity): - """Updates an ACL entry on the specified object. + def RunWithArgs(self, bucket, object): + """Retrieves an object or its metadata. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. object: Name of the object. - entity: The entity holding the permission. Can be user-userId, user- - emailAddress, group-groupId, group-emailAddress, allUsers, or - allAuthenticatedUsers. Flags: generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - objectAccessControl: A ObjectAccessControl resource to be passed as the - request body. + ifGenerationMatch: Makes the operation conditional on whether the + object's generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + projection: Set of properties to return. Defaults to noAcl. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageObjectAccessControlsUpdateRequest( + request = messages.StorageObjectsGetRequest( bucket=bucket.decode('utf8'), object=object.decode('utf8'), - entity=entity.decode('utf8'), ) if FLAGS['generation'].present: request.generation = int(FLAGS.generation) - if FLAGS['objectAccessControl'].present: - request.objectAccessControl = apitools_base.JsonToMessage(messages.ObjectAccessControl, FLAGS.objectAccessControl) - result = client.objectAccessControls.Update( - request, global_params=global_params) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.objects.Get( + request, global_params=global_params, download=download) print apitools_base_cli.FormatOutput(result) -class BucketsDelete(apitools_base_cli.NewCmd): - """Command wrapping buckets.Delete.""" +class ObjectsInsert(apitools_base_cli.NewCmd): + """Command wrapping objects.Insert.""" - usage = """buckets_delete """ + usage = """objects_insert """ def __init__(self, name, fv): - super(BucketsDelete, self).__init__(name, fv) + super(ObjectsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'contentEncoding', + None, + u'If set, sets the contentEncoding property of the final object to ' + u'this value. Setting this parameter is equivalent to setting the ' + u'contentEncoding metadata property. This can be useful when ' + u'uploading an object with uploadType=media to indicate the encoding ' + u'of the content being uploaded.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u'If set, only deletes the bucket if its metageneration matches this ' - u'value.', + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u'If set, only deletes the bucket if its metageneration does not ' - u'match this value.', + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Name of the object. Required when the object metadata is not ' + u"otherwise provided. Overrides the object metadata's name value, if " + u'any.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'A Object resource to be passed as the request body.', + flag_values=fv) + flags.DEFINE_enum( + 'predefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to this object.', + flag_values=fv) + flags.DEFINE_enum( + 'projection', + u'full', + [u'full', u'noAcl'], + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'upload_filename', + '', + 'Filename to use for upload.', + flag_values=fv) + flags.DEFINE_string( + 'upload_mime_type', + '', + 'MIME type to use for the upload. Only needed if the extension on ' + '--upload_filename does not determine the correct (or any) MIME ' + 'type.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', flag_values=fv) def RunWithArgs(self, bucket): - """Permanently deletes an empty bucket. + """Stores a new object and metadata. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which to store the new object. Overrides + the provided object metadata's bucket value, if any. Flags: - ifMetagenerationMatch: If set, only deletes the bucket if its - metageneration matches this value. - ifMetagenerationNotMatch: If set, only deletes the bucket if its - metageneration does not match this value. + contentEncoding: If set, sets the contentEncoding property of the final + object to this value. Setting this parameter is equivalent to setting + the contentEncoding metadata property. This can be useful when + uploading an object with uploadType=media to indicate the encoding of + the content being uploaded. + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + name: Name of the object. Required when the object metadata is not + otherwise provided. Overrides the object metadata's name value, if + any. + object: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + upload_filename: Filename to use for upload. + upload_mime_type: MIME type to use for the upload. Only needed if the + extension on --upload_filename does not determine the correct (or any) + MIME type. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsDeleteRequest( + request = messages.StorageObjectsInsertRequest( bucket=bucket.decode('utf8'), ) + if FLAGS['contentEncoding'].present: + request.contentEncoding = FLAGS.contentEncoding.decode('utf8') + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - result = client.buckets.Delete( - request, global_params=global_params) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) + if FLAGS['predefinedAcl'].present: + request.predefinedAcl = messages.StorageObjectsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) + if FLAGS['projection'].present: + request.projection = messages.StorageObjectsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) + upload = None + if FLAGS.upload_filename: + upload = apitools_base.Upload.FromFile( + FLAGS.upload_filename, FLAGS.upload_mime_type, + progress_callback=apitools_base.UploadProgressPrinter, + finish_callback=apitools_base.UploadCompletePrinter) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.objects.Insert( + request, global_params=global_params, upload=upload, download=download) print apitools_base_cli.FormatOutput(result) -class BucketsGet(apitools_base_cli.NewCmd): - """Command wrapping buckets.Get.""" +class ObjectsList(apitools_base_cli.NewCmd): + """Command wrapping objects.List.""" - usage = """buckets_get """ + usage = """objects_list """ def __init__(self, name, fv): - super(BucketsGet, self).__init__(name, fv) + super(ObjectsList, self).__init__(name, fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'delimiter', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested. The default value of this parameter is 1,000 items.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationNotMatch', + 'pageToken', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', + flag_values=fv) + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', flag_values=fv) flags.DEFINE_enum( 'projection', @@ -2538,202 +2519,405 @@ class BucketsGet(apitools_base_cli.NewCmd): [u'full', u'noAcl'], u'Set of properties to return. Defaults to noAcl.', flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of an object as distinct results. The ' + u'default is false. For more information, see Object Versioning.', + flag_values=fv) def RunWithArgs(self, bucket): - """Returns metadata for the specified bucket. + """Retrieves a list of objects matching the criteria. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which to look for objects. Flags: - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. The default value of this parameter is 1,000 items. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of an object as distinct results. + The default is false. For more information, see Object Versioning. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsGetRequest( + request = messages.StorageObjectsListRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') if FLAGS['projection'].present: - request.projection = messages.StorageBucketsGetRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Get( + request.projection = messages.StorageObjectsListRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.List( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketsInsert(apitools_base_cli.NewCmd): - """Command wrapping buckets.Insert.""" +class ObjectsPatch(apitools_base_cli.NewCmd): + """Command wrapping objects.Patch.""" - usage = """buckets_insert """ + usage = """objects_patch """ def __init__(self, name, fv): - super(BucketsInsert, self).__init__(name, fv) + super(ObjectsPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) flags.DEFINE_string( - 'bucket', + 'ifGenerationMatch', None, - u'A Bucket resource to be passed as the request body.', + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', flag_values=fv) flags.DEFINE_enum( - 'predefinedDefaultObjectAcl', + 'predefinedAcl', u'authenticatedRead', [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of default object access controls to this ' - u'bucket.', + u'Apply a predefined set of access controls to this object.', flag_values=fv) flags.DEFINE_enum( 'projection', u'full', [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl, unless the bucket ' - u'resource specifies acl or defaultObjectAcl properties, when it ' - u'defaults to full.', + u'Set of properties to return. Defaults to full.', flag_values=fv) - def RunWithArgs(self, project): - """Creates a new bucket. + def RunWithArgs(self, bucket, object): + """Updates an object's metadata. This method supports patch semantics. Args: - project: A valid API project identifier. + bucket: Name of the bucket in which the object resides. + object: Name of the object. Flags: - bucket: A Bucket resource to be passed as the request body. - predefinedAcl: Apply a predefined set of access controls to this bucket. - predefinedDefaultObjectAcl: Apply a predefined set of default object - access controls to this bucket. - projection: Set of properties to return. Defaults to noAcl, unless the - bucket resource specifies acl or defaultObjectAcl properties, when it - defaults to full. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. + projection: Set of properties to return. Defaults to full. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsInsertRequest( - project=project.decode('utf8'), + request = messages.StorageObjectsPatchRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['bucket'].present: - request.bucket = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucket) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsInsertRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['predefinedDefaultObjectAcl'].present: - request.predefinedDefaultObjectAcl = messages.StorageBucketsInsertRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + request.predefinedAcl = messages.StorageObjectsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) if FLAGS['projection'].present: - request.projection = messages.StorageBucketsInsertRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Insert( + request.projection = messages.StorageObjectsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) + result = client.objects.Patch( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketsList(apitools_base_cli.NewCmd): - """Command wrapping buckets.List.""" +class ObjectsRewrite(apitools_base_cli.NewCmd): + """Command wrapping objects.Rewrite.""" - usage = """buckets_list """ + usage = """objects_rewrite """ def __init__(self, name, fv): - super(BucketsList, self).__init__(name, fv) - flags.DEFINE_integer( - 'maxResults', + super(ObjectsRewrite, self).__init__(name, fv) + flags.DEFINE_enum( + 'destinationPredefinedAcl', + u'authenticatedRead', + [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], + u'Apply a predefined set of access controls to the destination ' + u'object.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', None, - u'Maximum number of buckets to return.', + u"Makes the operation conditional on whether the destination object's" + u' current generation matches the given value.', flag_values=fv) flags.DEFINE_string( - 'pageToken', + 'ifGenerationNotMatch', None, - u'A previously-returned page token representing part of the larger ' - u'set of results to view.', + u"Makes the operation conditional on whether the destination object's" + u' current generation does not match the given value.', flag_values=fv) flags.DEFINE_string( - 'prefix', + 'ifMetagenerationMatch', None, - u'Filter results to buckets whose names begin with this prefix.', + u"Makes the operation conditional on whether the destination object's" + u' current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the destination object's" + u' current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceGenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'generation does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifSourceMetagenerationNotMatch', + None, + u"Makes the operation conditional on whether the source object's " + u'current metageneration does not match the given value.', + flag_values=fv) + flags.DEFINE_string( + 'maxBytesRewrittenPerCall', + None, + u'The maximum number of bytes that will be rewritten per rewrite ' + u"request. Most callers shouldn't need to specify this parameter - it" + u' is primarily in place to support testing. If specified the value ' + u'must be an integral multiple of 1 MiB (1048576). Also, this only ' + u'applies to requests where the source and destination span locations' + u' and/or storage classes. Finally, this value must not change across' + u" rewrite calls else you'll get an error that the rewriteToken is " + u'invalid.', + flag_values=fv) + flags.DEFINE_string( + 'object', + None, + u'A Object resource to be passed as the request body.', flag_values=fv) flags.DEFINE_enum( 'projection', u'full', [u'full', u'noAcl'], - u'Set of properties to return. Defaults to noAcl.', + u'Set of properties to return. Defaults to noAcl, unless the object ' + u'resource specifies the acl property, when it defaults to full.', + flag_values=fv) + flags.DEFINE_string( + 'rewriteToken', + None, + u'Include this field (from the previous rewrite response) on each ' + u'rewrite request after the first one, until the rewrite response ' + u"'done' flag is true. Calls that provide a rewriteToken can omit all" + u' other request fields, but if included those fields must match the ' + u'values provided in the first rewrite request.', + flag_values=fv) + flags.DEFINE_string( + 'sourceGeneration', + None, + u'If present, selects a specific revision of the source object (as ' + u'opposed to the latest version, the default).', flag_values=fv) - def RunWithArgs(self, project): - """Retrieves a list of buckets for a given project. + def RunWithArgs(self, sourceBucket, sourceObject, destinationBucket, destinationObject): + """Rewrites a source object to a destination object. Optionally overrides + metadata. Args: - project: A valid API project identifier. + sourceBucket: Name of the bucket in which to find the source object. + sourceObject: Name of the source object. + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's + name value, if any. Flags: - maxResults: Maximum number of buckets to return. - pageToken: A previously-returned page token representing part of the - larger set of results to view. - prefix: Filter results to buckets whose names begin with this prefix. - projection: Set of properties to return. Defaults to noAcl. + destinationPredefinedAcl: Apply a predefined set of access controls to + the destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given + value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether + the source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on + whether the source object's current metageneration does not match the + given value. + maxBytesRewrittenPerCall: The maximum number of bytes that will be + rewritten per rewrite request. Most callers shouldn't need to specify + this parameter - it is primarily in place to support testing. If + specified the value must be an integral multiple of 1 MiB (1048576). + Also, this only applies to requests where the source and destination + span locations and/or storage classes. Finally, this value must not + change across rewrite calls else you'll get an error that the + rewriteToken is invalid. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + rewriteToken: Include this field (from the previous rewrite response) on + each rewrite request after the first one, until the rewrite response + 'done' flag is true. Calls that provide a rewriteToken can omit all + other request fields, but if included those fields must match the + values provided in the first rewrite request. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsListRequest( - project=project.decode('utf8'), + request = messages.StorageObjectsRewriteRequest( + sourceBucket=sourceBucket.decode('utf8'), + sourceObject=sourceObject.decode('utf8'), + destinationBucket=destinationBucket.decode('utf8'), + destinationObject=destinationObject.decode('utf8'), ) - if FLAGS['maxResults'].present: - request.maxResults = FLAGS.maxResults - if FLAGS['pageToken'].present: - request.pageToken = FLAGS.pageToken.decode('utf8') - if FLAGS['prefix'].present: - request.prefix = FLAGS.prefix.decode('utf8') + if FLAGS['destinationPredefinedAcl'].present: + request.destinationPredefinedAcl = messages.StorageObjectsRewriteRequest.DestinationPredefinedAclValueValuesEnum(FLAGS.destinationPredefinedAcl) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) + if FLAGS['ifMetagenerationMatch'].present: + request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) + if FLAGS['ifMetagenerationNotMatch'].present: + request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['ifSourceGenerationMatch'].present: + request.ifSourceGenerationMatch = int(FLAGS.ifSourceGenerationMatch) + if FLAGS['ifSourceGenerationNotMatch'].present: + request.ifSourceGenerationNotMatch = int(FLAGS.ifSourceGenerationNotMatch) + if FLAGS['ifSourceMetagenerationMatch'].present: + request.ifSourceMetagenerationMatch = int(FLAGS.ifSourceMetagenerationMatch) + if FLAGS['ifSourceMetagenerationNotMatch'].present: + request.ifSourceMetagenerationNotMatch = int(FLAGS.ifSourceMetagenerationNotMatch) + if FLAGS['maxBytesRewrittenPerCall'].present: + request.maxBytesRewrittenPerCall = int(FLAGS.maxBytesRewrittenPerCall) + if FLAGS['object'].present: + request.object = apitools_base.JsonToMessage(messages.Object, FLAGS.object) if FLAGS['projection'].present: - request.projection = messages.StorageBucketsListRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.List( + request.projection = messages.StorageObjectsRewriteRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['rewriteToken'].present: + request.rewriteToken = FLAGS.rewriteToken.decode('utf8') + if FLAGS['sourceGeneration'].present: + request.sourceGeneration = int(FLAGS.sourceGeneration) + result = client.objects.Rewrite( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) -class BucketsPatch(apitools_base_cli.NewCmd): - """Command wrapping buckets.Patch.""" +class ObjectsUpdate(apitools_base_cli.NewCmd): + """Command wrapping objects.Update.""" - usage = """buckets_patch """ + usage = """objects_update """ def __init__(self, name, fv): - super(BucketsPatch, self).__init__(name, fv) + super(ObjectsUpdate, self).__init__(name, fv) flags.DEFINE_string( - 'bucketResource', + 'generation', None, - u'A Bucket resource to be passed as the request body.', + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation matches the given value.', + flag_values=fv) + flags.DEFINE_string( + 'ifGenerationNotMatch', + None, + u"Makes the operation conditional on whether the object's current " + u'generation does not match the given value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationMatch', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", + u"Makes the operation conditional on whether the object's current " + u'metageneration matches the given value.', flag_values=fv) flags.DEFINE_string( 'ifMetagenerationNotMatch', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", + u"Makes the operation conditional on whether the object's current " + u'metageneration does not match the given value.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + flags.DEFINE_string( + 'objectResource', + None, + u'A Object resource to be passed as the request body.', flag_values=fv) flags.DEFINE_enum( - 'predefinedDefaultObjectAcl', + 'predefinedAcl', u'authenticatedRead', [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of default object access controls to this ' - u'bucket.', + u'Apply a predefined set of access controls to this object.', flag_values=fv) flags.DEFINE_enum( 'projection', @@ -2741,148 +2925,199 @@ class BucketsPatch(apitools_base_cli.NewCmd): [u'full', u'noAcl'], u'Set of properties to return. Defaults to full.', flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) - def RunWithArgs(self, bucket): - """Updates a bucket. This method supports patch semantics. + def RunWithArgs(self, bucket, object): + """Updates an object's metadata. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which the object resides. + object: Name of the object. Flags: - bucketResource: A Bucket resource to be passed as the request body. - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. - predefinedAcl: Apply a predefined set of access controls to this bucket. - predefinedDefaultObjectAcl: Apply a predefined set of default object - access controls to this bucket. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + ifGenerationMatch: Makes the operation conditional on whether the + object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + object's current metageneration does not match the given value. + objectResource: A Object resource to be passed as the request body. + predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsPatchRequest( + request = messages.StorageObjectsUpdateRequest( bucket=bucket.decode('utf8'), + object=object.decode('utf8'), ) - if FLAGS['bucketResource'].present: - request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['ifGenerationMatch'].present: + request.ifGenerationMatch = int(FLAGS.ifGenerationMatch) + if FLAGS['ifGenerationNotMatch'].present: + request.ifGenerationNotMatch = int(FLAGS.ifGenerationNotMatch) if FLAGS['ifMetagenerationMatch'].present: request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) if FLAGS['ifMetagenerationNotMatch'].present: request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) + if FLAGS['objectResource'].present: + request.objectResource = apitools_base.JsonToMessage(messages.Object, FLAGS.objectResource) if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsPatchRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['predefinedDefaultObjectAcl'].present: - request.predefinedDefaultObjectAcl = messages.StorageBucketsPatchRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + request.predefinedAcl = messages.StorageObjectsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) if FLAGS['projection'].present: - request.projection = messages.StorageBucketsPatchRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Patch( - request, global_params=global_params) + request.projection = messages.StorageObjectsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.objects.Update( + request, global_params=global_params, download=download) print apitools_base_cli.FormatOutput(result) -class BucketsUpdate(apitools_base_cli.NewCmd): - """Command wrapping buckets.Update.""" +class ObjectsWatchAll(apitools_base_cli.NewCmd): + """Command wrapping objects.WatchAll.""" - usage = """buckets_update """ + usage = """objects_watchAll """ def __init__(self, name, fv): - super(BucketsUpdate, self).__init__(name, fv) + super(ObjectsWatchAll, self).__init__(name, fv) flags.DEFINE_string( - 'bucketResource', + 'channel', None, - u'A Bucket resource to be passed as the request body.', + u'A Channel resource to be passed as the request body.', flag_values=fv) flags.DEFINE_string( - 'ifMetagenerationMatch', + 'delimiter', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration matches the given value.", + u'Returns results in a directory-like mode. items will contain only ' + u'objects whose names, aside from the prefix, do not contain ' + u'delimiter. Objects whose names, aside from the prefix, contain ' + u'delimiter will have their name, truncated after the delimiter, ' + u'returned in prefixes. Duplicate prefixes are omitted.', flag_values=fv) - flags.DEFINE_string( - 'ifMetagenerationNotMatch', + flags.DEFINE_integer( + 'maxResults', None, - u'Makes the return of the bucket metadata conditional on whether the ' - u"bucket's current metageneration does not match the given value.", + u'Maximum number of items plus prefixes to return. As duplicate ' + u'prefixes are omitted, fewer total results may be returned than ' + u'requested. The default value of this parameter is 1,000 items.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedAcl', - u'authenticatedRead', - [u'authenticatedRead', u'private', u'projectPrivate', u'publicRead', u'publicReadWrite'], - u'Apply a predefined set of access controls to this bucket.', + flags.DEFINE_string( + 'pageToken', + None, + u'A previously-returned page token representing part of the larger ' + u'set of results to view.', flag_values=fv) - flags.DEFINE_enum( - 'predefinedDefaultObjectAcl', - u'authenticatedRead', - [u'authenticatedRead', u'bucketOwnerFullControl', u'bucketOwnerRead', u'private', u'projectPrivate', u'publicRead'], - u'Apply a predefined set of default object access controls to this ' - u'bucket.', + flags.DEFINE_string( + 'prefix', + None, + u'Filter results to objects whose names begin with this prefix.', flag_values=fv) flags.DEFINE_enum( 'projection', u'full', [u'full', u'noAcl'], - u'Set of properties to return. Defaults to full.', + u'Set of properties to return. Defaults to noAcl.', + flag_values=fv) + flags.DEFINE_boolean( + 'versions', + None, + u'If true, lists all versions of an object as distinct results. The ' + u'default is false. For more information, see Object Versioning.', flag_values=fv) def RunWithArgs(self, bucket): - """Updates a bucket. + """Watch for changes on all objects in a bucket. Args: - bucket: Name of a bucket. + bucket: Name of the bucket in which to look for objects. Flags: - bucketResource: A Bucket resource to be passed as the request body. - ifMetagenerationMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration matches the - given value. - ifMetagenerationNotMatch: Makes the return of the bucket metadata - conditional on whether the bucket's current metageneration does not - match the given value. - predefinedAcl: Apply a predefined set of access controls to this bucket. - predefinedDefaultObjectAcl: Apply a predefined set of default object - access controls to this bucket. - projection: Set of properties to return. Defaults to full. + channel: A Channel resource to be passed as the request body. + delimiter: Returns results in a directory-like mode. items will contain + only objects whose names, aside from the prefix, do not contain + delimiter. Objects whose names, aside from the prefix, contain + delimiter will have their name, truncated after the delimiter, + returned in prefixes. Duplicate prefixes are omitted. + maxResults: Maximum number of items plus prefixes to return. As + duplicate prefixes are omitted, fewer total results may be returned + than requested. The default value of this parameter is 1,000 items. + pageToken: A previously-returned page token representing part of the + larger set of results to view. + prefix: Filter results to objects whose names begin with this prefix. + projection: Set of properties to return. Defaults to noAcl. + versions: If true, lists all versions of an object as distinct results. + The default is false. For more information, see Object Versioning. """ client = GetClientFromFlags() global_params = GetGlobalParamsFromFlags() - request = messages.StorageBucketsUpdateRequest( + request = messages.StorageObjectsWatchAllRequest( bucket=bucket.decode('utf8'), ) - if FLAGS['bucketResource'].present: - request.bucketResource = apitools_base.JsonToMessage(messages.Bucket, FLAGS.bucketResource) - if FLAGS['ifMetagenerationMatch'].present: - request.ifMetagenerationMatch = int(FLAGS.ifMetagenerationMatch) - if FLAGS['ifMetagenerationNotMatch'].present: - request.ifMetagenerationNotMatch = int(FLAGS.ifMetagenerationNotMatch) - if FLAGS['predefinedAcl'].present: - request.predefinedAcl = messages.StorageBucketsUpdateRequest.PredefinedAclValueValuesEnum(FLAGS.predefinedAcl) - if FLAGS['predefinedDefaultObjectAcl'].present: - request.predefinedDefaultObjectAcl = messages.StorageBucketsUpdateRequest.PredefinedDefaultObjectAclValueValuesEnum(FLAGS.predefinedDefaultObjectAcl) + if FLAGS['channel'].present: + request.channel = apitools_base.JsonToMessage(messages.Channel, FLAGS.channel) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['prefix'].present: + request.prefix = FLAGS.prefix.decode('utf8') if FLAGS['projection'].present: - request.projection = messages.StorageBucketsUpdateRequest.ProjectionValueValuesEnum(FLAGS.projection) - result = client.buckets.Update( + request.projection = messages.StorageObjectsWatchAllRequest.ProjectionValueValuesEnum(FLAGS.projection) + if FLAGS['versions'].present: + request.versions = FLAGS.versions + result = client.objects.WatchAll( request, global_params=global_params) print apitools_base_cli.FormatOutput(result) def main(_): appcommands.AddCmd('pyshell', PyShell) - appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) - appcommands.AddCmd('defaultObjectAccessControls_get', DefaultObjectAccessControlsGet) - appcommands.AddCmd('defaultObjectAccessControls_insert', DefaultObjectAccessControlsInsert) - appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) - appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) - appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) appcommands.AddCmd('bucketAccessControls_delete', BucketAccessControlsDelete) appcommands.AddCmd('bucketAccessControls_get', BucketAccessControlsGet) appcommands.AddCmd('bucketAccessControls_insert', BucketAccessControlsInsert) appcommands.AddCmd('bucketAccessControls_list', BucketAccessControlsList) appcommands.AddCmd('bucketAccessControls_patch', BucketAccessControlsPatch) appcommands.AddCmd('bucketAccessControls_update', BucketAccessControlsUpdate) + appcommands.AddCmd('buckets_delete', BucketsDelete) + appcommands.AddCmd('buckets_get', BucketsGet) + appcommands.AddCmd('buckets_insert', BucketsInsert) + appcommands.AddCmd('buckets_list', BucketsList) + appcommands.AddCmd('buckets_patch', BucketsPatch) + appcommands.AddCmd('buckets_update', BucketsUpdate) appcommands.AddCmd('channels_stop', ChannelsStop) + appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) + appcommands.AddCmd('defaultObjectAccessControls_get', DefaultObjectAccessControlsGet) + appcommands.AddCmd('defaultObjectAccessControls_insert', DefaultObjectAccessControlsInsert) + appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) + appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) + appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) + appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) + appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) + appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) + appcommands.AddCmd('objectAccessControls_list', ObjectAccessControlsList) + appcommands.AddCmd('objectAccessControls_patch', ObjectAccessControlsPatch) + appcommands.AddCmd('objectAccessControls_update', ObjectAccessControlsUpdate) appcommands.AddCmd('objects_compose', ObjectsCompose) appcommands.AddCmd('objects_copy', ObjectsCopy) appcommands.AddCmd('objects_delete', ObjectsDelete) @@ -2890,20 +3125,9 @@ def main(_): appcommands.AddCmd('objects_insert', ObjectsInsert) appcommands.AddCmd('objects_list', ObjectsList) appcommands.AddCmd('objects_patch', ObjectsPatch) + appcommands.AddCmd('objects_rewrite', ObjectsRewrite) appcommands.AddCmd('objects_update', ObjectsUpdate) appcommands.AddCmd('objects_watchAll', ObjectsWatchAll) - appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) - appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) - appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) - appcommands.AddCmd('objectAccessControls_list', ObjectAccessControlsList) - appcommands.AddCmd('objectAccessControls_patch', ObjectAccessControlsPatch) - appcommands.AddCmd('objectAccessControls_update', ObjectAccessControlsUpdate) - appcommands.AddCmd('buckets_delete', BucketsDelete) - appcommands.AddCmd('buckets_get', BucketsGet) - appcommands.AddCmd('buckets_insert', BucketsInsert) - appcommands.AddCmd('buckets_list', BucketsList) - appcommands.AddCmd('buckets_patch', BucketsPatch) - appcommands.AddCmd('buckets_update', BucketsUpdate) apitools_base_cli.SetupLogger() if hasattr(appcommands, 'SetDefaultCommand'): diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage/storage_v1_client.py index 4d5024d..18bd33d 100644 --- a/samples/storage_sample/storage/storage_v1_client.py +++ b/samples/storage_sample/storage/storage_v1_client.py @@ -1,4 +1,5 @@ """Generated client library for storage version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api import storage_v1_messages as messages @@ -32,91 +33,91 @@ class StorageV1(base_api.BaseApiClient): credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers) - self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) self.bucketAccessControls = self.BucketAccessControlsService(self) + self.buckets = self.BucketsService(self) self.channels = self.ChannelsService(self) - self.objects = self.ObjectsService(self) + self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) self.objectAccessControls = self.ObjectAccessControlsService(self) - self.buckets = self.BucketsService(self) + self.objects = self.ObjectsService(self) - class DefaultObjectAccessControlsService(base_api.BaseApiService): - """Service class for the defaultObjectAccessControls resource.""" + class BucketAccessControlsService(base_api.BaseApiService): + """Service class for the bucketAccessControls resource.""" - _NAME = u'defaultObjectAccessControls' + _NAME = u'bucketAccessControls' def __init__(self, client): - super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) + super(StorageV1.BucketAccessControlsService, self).__init__(client) self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.defaultObjectAccessControls.delete', + method_id=u'storage.bucketAccessControls.delete', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', - response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', + request_type_name=u'StorageBucketAccessControlsDeleteRequest', + response_type_name=u'StorageBucketAccessControlsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.get', + method_id=u'storage.bucketAccessControls.get', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', - response_type_name=u'ObjectAccessControl', + request_type_name=u'StorageBucketAccessControlsGetRequest', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.defaultObjectAccessControls.insert', + method_id=u'storage.bucketAccessControls.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl', + relative_path=u'b/{bucket}/acl', request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.list', + method_id=u'storage.bucketAccessControls.list', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/defaultObjectAcl', + query_params=[], + relative_path=u'b/{bucket}/acl', request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsListRequest', - response_type_name=u'ObjectAccessControls', + request_type_name=u'StorageBucketAccessControlsListRequest', + response_type_name=u'BucketAccessControls', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.defaultObjectAccessControls.patch', + method_id=u'storage.bucketAccessControls.patch', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.defaultObjectAccessControls.update', + method_id=u'storage.bucketAccessControls.update', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + relative_path=u'b/{bucket}/acl/{entity}', request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', supports_download=False, ), } @@ -125,161 +126,161 @@ class StorageV1(base_api.BaseApiClient): } def Delete(self, request, global_params=None): - """Permanently deletes the default object ACL entry for the specified entity on the specified bucket. + """Permanently deletes the ACL entry for the specified entity on the specified bucket. Args: - request: (StorageDefaultObjectAccessControlsDeleteRequest) input message + request: (StorageBucketAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageDefaultObjectAccessControlsDeleteResponse) The response message. + (StorageBucketAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) def Get(self, request, global_params=None): - """Returns the default object ACL entry for the specified entity on the specified bucket. + """Returns the ACL entry for the specified entity on the specified bucket. Args: - request: (StorageDefaultObjectAccessControlsGetRequest) input message + request: (StorageBucketAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) def Insert(self, request, global_params=None): - """Creates a new default object ACL entry on the specified bucket. + """Creates a new ACL entry on the specified bucket. Args: - request: (ObjectAccessControl) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves default object ACL entries on the specified bucket. + """Retrieves ACL entries on the specified bucket. Args: - request: (StorageDefaultObjectAccessControlsListRequest) input message + request: (StorageBucketAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControls) The response message. + (BucketAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. + """Updates an ACL entry on the specified bucket. This method supports patch semantics. Args: - request: (ObjectAccessControl) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) def Update(self, request, global_params=None): - """Updates a default object ACL entry on the specified bucket. + """Updates an ACL entry on the specified bucket. Args: - request: (ObjectAccessControl) input message + request: (BucketAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (ObjectAccessControl) The response message. + (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) - class BucketAccessControlsService(base_api.BaseApiService): - """Service class for the bucketAccessControls resource.""" + class BucketsService(base_api.BaseApiService): + """Service class for the buckets resource.""" - _NAME = u'bucketAccessControls' + _NAME = u'buckets' def __init__(self, client): - super(StorageV1.BucketAccessControlsService, self).__init__(client) + super(StorageV1.BucketsService, self).__init__(client) self._method_configs = { 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.bucketAccessControls.delete', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + method_id=u'storage.buckets.delete', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}', request_field='', - request_type_name=u'StorageBucketAccessControlsDeleteRequest', - response_type_name=u'StorageBucketAccessControlsDeleteResponse', + request_type_name=u'StorageBucketsDeleteRequest', + response_type_name=u'StorageBucketsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.bucketAccessControls.get', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', + method_id=u'storage.buckets.get', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}', request_field='', - request_type_name=u'StorageBucketAccessControlsGetRequest', - response_type_name=u'BucketAccessControl', + request_type_name=u'StorageBucketsGetRequest', + response_type_name=u'Bucket', supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.bucketAccessControls.insert', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/acl', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + method_id=u'storage.buckets.insert', + ordered_params=[u'project'], + path_params=[], + query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection'], + relative_path=u'b', + request_field=u'bucket', + request_type_name=u'StorageBucketsInsertRequest', + response_type_name=u'Bucket', supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.bucketAccessControls.list', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/acl', + method_id=u'storage.buckets.list', + ordered_params=[u'project'], + path_params=[], + query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection'], + relative_path=u'b', request_field='', - request_type_name=u'StorageBucketAccessControlsListRequest', - response_type_name=u'BucketAccessControls', + request_type_name=u'StorageBucketsListRequest', + response_type_name=u'Buckets', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.bucketAccessControls.patch', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + method_id=u'storage.buckets.patch', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsPatchRequest', + response_type_name=u'Bucket', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.bucketAccessControls.update', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', + method_id=u'storage.buckets.update', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsUpdateRequest', + response_type_name=u'Bucket', supports_download=False, ), } @@ -288,78 +289,78 @@ class StorageV1(base_api.BaseApiClient): } def Delete(self, request, global_params=None): - """Permanently deletes the ACL entry for the specified entity on the specified bucket. + """Permanently deletes an empty bucket. Args: - request: (StorageBucketAccessControlsDeleteRequest) input message + request: (StorageBucketsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageBucketAccessControlsDeleteResponse) The response message. + (StorageBucketsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) def Get(self, request, global_params=None): - """Returns the ACL entry for the specified entity on the specified bucket. + """Returns metadata for the specified bucket. Args: - request: (StorageBucketAccessControlsGetRequest) input message + request: (StorageBucketsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) def Insert(self, request, global_params=None): - """Creates a new ACL entry on the specified bucket. + """Creates a new bucket. Args: - request: (BucketAccessControl) input message + request: (StorageBucketsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves ACL entries on the specified bucket. + """Retrieves a list of buckets for a given project. Args: - request: (StorageBucketAccessControlsListRequest) input message + request: (StorageBucketsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControls) The response message. + (Buckets) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates an ACL entry on the specified bucket. This method supports patch semantics. + """Updates a bucket. This method supports patch semantics. Args: - request: (BucketAccessControl) input message + request: (StorageBucketsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) def Update(self, request, global_params=None): - """Updates an ACL entry on the specified bucket. + """Updates a bucket. Args: - request: (BucketAccessControl) input message + request: (StorageBucketsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (BucketAccessControl) The response message. + (Bucket) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( @@ -403,268 +404,166 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) - class ObjectsService(base_api.BaseApiService): - """Service class for the objects resource.""" + class DefaultObjectAccessControlsService(base_api.BaseApiService): + """Service class for the defaultObjectAccessControls resource.""" - _NAME = u'objects' + _NAME = u'defaultObjectAccessControls' def __init__(self, client): - super(StorageV1.ObjectsService, self).__init__(client) + super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) self._method_configs = { - 'Compose': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.compose', - ordered_params=[u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], - relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', - request_field=u'composeRequest', - request_type_name=u'StorageObjectsComposeRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'Copy': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.copy', - ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], - relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', - request_field=u'object', - request_type_name=u'StorageObjectsCopyRequest', - response_type_name=u'Object', - supports_download=True, - ), 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.objects.delete', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/o/{object}', + method_id=u'storage.defaultObjectAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'StorageObjectsDeleteRequest', - response_type_name=u'StorageObjectsDeleteResponse', + request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', + response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.objects.get', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}/o/{object}', + method_id=u'storage.defaultObjectAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', - request_type_name=u'StorageObjectsGetRequest', - response_type_name=u'Object', - supports_download=True, + request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.objects.insert', + method_id=u'storage.defaultObjectAccessControls.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o', - request_field=u'object', - request_type_name=u'StorageObjectsInsertRequest', - response_type_name=u'Object', - supports_download=True, + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.objects.list', + method_id=u'storage.defaultObjectAccessControls.list', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o', + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/defaultObjectAcl', request_field='', - request_type_name=u'StorageObjectsListRequest', - response_type_name=u'Objects', + request_type_name=u'StorageDefaultObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.objects.patch', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsPatchRequest', - response_type_name=u'Object', + method_id=u'storage.defaultObjectAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.objects.update', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsUpdateRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'WatchAll': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.watchAll', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o/watch', - request_field=u'channel', - request_type_name=u'StorageObjectsWatchAllRequest', - response_type_name=u'Channel', + method_id=u'storage.defaultObjectAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', supports_download=False, ), } self._upload_configs = { - 'Insert': base_api.ApiUploadInfo( - accept=['*/*'], - max_size=None, - resumable_multipart=True, - resumable_path=u'/resumable/upload/storage/v1/b/{bucket}/o', - simple_multipart=True, - simple_path=u'/upload/storage/v1/b/{bucket}/o', - ), } - def Compose(self, request, global_params=None, download=None): - """Concatenates a list of existing objects into a new object in the same bucket. - - Args: - request: (StorageObjectsComposeRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. - Returns: - (Object) The response message. - """ - config = self.GetMethodConfig('Compose') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - - def Copy(self, request, global_params=None, download=None): - """Copies an object to a specified location. Optionally overrides metadata. - - Args: - request: (StorageObjectsCopyRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. - Returns: - (Object) The response message. - """ - config = self.GetMethodConfig('Copy') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - def Delete(self, request, global_params=None): - """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. + """Permanently deletes the default object ACL entry for the specified entity on the specified bucket. Args: - request: (StorageObjectsDeleteRequest) input message + request: (StorageDefaultObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageObjectsDeleteResponse) The response message. + (StorageDefaultObjectAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) - def Get(self, request, global_params=None, download=None): - """Retrieves an object or its metadata. + def Get(self, request, global_params=None): + """Returns the default object ACL entry for the specified entity on the specified bucket. Args: - request: (StorageObjectsGetRequest) input message + request: (StorageDefaultObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( - config, request, global_params=global_params, - download=download) + config, request, global_params=global_params) - def Insert(self, request, global_params=None, upload=None, download=None): - """Stores a new object and metadata. + def Insert(self, request, global_params=None): + """Creates a new default object ACL entry on the specified bucket. Args: - request: (StorageObjectsInsertRequest) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments - upload: (Upload, default: None) If present, upload - this stream with the request. - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Insert') - upload_config = self.GetUploadConfig('Insert') return self._RunMethod( - config, request, global_params=global_params, - upload=upload, upload_config=upload_config, - download=download) + config, request, global_params=global_params) def List(self, request, global_params=None): - """Retrieves a list of objects matching the criteria. + """Retrieves default object ACL entries on the specified bucket. Args: - request: (StorageObjectsListRequest) input message + request: (StorageDefaultObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Objects) The response message. + (ObjectAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates an object's metadata. This method supports patch semantics. + """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. Args: - request: (StorageObjectsPatchRequest) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Object) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) - def Update(self, request, global_params=None, download=None): - """Updates an object's metadata. + def Update(self, request, global_params=None): + """Updates a default object ACL entry on the specified bucket. Args: - request: (StorageObjectsUpdateRequest) input message + request: (ObjectAccessControl) input message global_params: (StandardQueryParameters, default: None) global arguments - download: (Download, default: None) If present, download - data from the request via this stream. Returns: - (Object) The response message. + (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Update') - return self._RunMethod( - config, request, global_params=global_params, - download=download) - - def WatchAll(self, request, global_params=None): - """Watch for changes on all objects in a bucket. - - Args: - request: (StorageObjectsWatchAllRequest) input message - global_params: (StandardQueryParameters, default: None) global arguments - Returns: - (Channel) The response message. - """ - config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) @@ -831,165 +730,292 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) - class BucketsService(base_api.BaseApiService): - """Service class for the buckets resource.""" + class ObjectsService(base_api.BaseApiService): + """Service class for the objects resource.""" - _NAME = u'buckets' + _NAME = u'objects' def __init__(self, client): - super(StorageV1.BucketsService, self).__init__(client) + super(StorageV1.ObjectsService, self).__init__(client) self._method_configs = { + 'Compose': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.compose', + ordered_params=[u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], + relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', + request_field=u'composeRequest', + request_type_name=u'StorageObjectsComposeRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'Copy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.copy', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsCopyRequest', + response_type_name=u'Object', + supports_download=True, + ), 'Delete': base_api.ApiMethodInfo( http_method=u'DELETE', - method_id=u'storage.buckets.delete', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}', + method_id=u'storage.objects.delete', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/o/{object}', request_field='', - request_type_name=u'StorageBucketsDeleteRequest', - response_type_name=u'StorageBucketsDeleteResponse', + request_type_name=u'StorageObjectsDeleteRequest', + response_type_name=u'StorageObjectsDeleteResponse', supports_download=False, ), 'Get': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.buckets.get', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}', + method_id=u'storage.objects.get', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}/o/{object}', request_field='', - request_type_name=u'StorageBucketsGetRequest', - response_type_name=u'Bucket', - supports_download=False, + request_type_name=u'StorageObjectsGetRequest', + response_type_name=u'Object', + supports_download=True, ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', - method_id=u'storage.buckets.insert', - ordered_params=[u'project'], - path_params=[], - query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection'], - relative_path=u'b', - request_field=u'bucket', - request_type_name=u'StorageBucketsInsertRequest', - response_type_name=u'Bucket', - supports_download=False, + method_id=u'storage.objects.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o', + request_field=u'object', + request_type_name=u'StorageObjectsInsertRequest', + response_type_name=u'Object', + supports_download=True, ), 'List': base_api.ApiMethodInfo( http_method=u'GET', - method_id=u'storage.buckets.list', - ordered_params=[u'project'], - path_params=[], - query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection'], - relative_path=u'b', + method_id=u'storage.objects.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o', request_field='', - request_type_name=u'StorageBucketsListRequest', - response_type_name=u'Buckets', + request_type_name=u'StorageObjectsListRequest', + response_type_name=u'Objects', supports_download=False, ), 'Patch': base_api.ApiMethodInfo( http_method=u'PATCH', - method_id=u'storage.buckets.patch', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsPatchRequest', - response_type_name=u'Bucket', + method_id=u'storage.objects.patch', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsPatchRequest', + response_type_name=u'Object', + supports_download=False, + ), + 'Rewrite': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.rewrite', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'maxBytesRewrittenPerCall', u'projection', u'rewriteToken', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsRewriteRequest', + response_type_name=u'RewriteResponse', supports_download=False, ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', - method_id=u'storage.buckets.update', + method_id=u'storage.objects.update', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsUpdateRequest', + response_type_name=u'Object', + supports_download=True, + ), + 'WatchAll': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.watchAll', ordered_params=[u'bucket'], path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsUpdateRequest', - response_type_name=u'Bucket', + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o/watch', + request_field=u'channel', + request_type_name=u'StorageObjectsWatchAllRequest', + response_type_name=u'Channel', supports_download=False, ), } self._upload_configs = { + 'Insert': base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload/storage/v1/b/{bucket}/o', + simple_multipart=True, + simple_path=u'/upload/storage/v1/b/{bucket}/o', + ), } + def Compose(self, request, global_params=None, download=None): + """Concatenates a list of existing objects into a new object in the same bucket. + + Args: + request: (StorageObjectsComposeRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Compose') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def Copy(self, request, global_params=None, download=None): + """Copies a source object to a destination object. Optionally overrides metadata. + + Args: + request: (StorageObjectsCopyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. + """ + config = self.GetMethodConfig('Copy') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + def Delete(self, request, global_params=None): - """Permanently deletes an empty bucket. + """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. Args: - request: (StorageBucketsDeleteRequest) input message + request: (StorageObjectsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (StorageBucketsDeleteResponse) The response message. + (StorageObjectsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) - def Get(self, request, global_params=None): - """Returns metadata for the specified bucket. + def Get(self, request, global_params=None, download=None): + """Retrieves an object or its metadata. Args: - request: (StorageBucketsGetRequest) input message + request: (StorageObjectsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. Returns: - (Bucket) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( - config, request, global_params=global_params) + config, request, global_params=global_params, + download=download) - def Insert(self, request, global_params=None): - """Creates a new bucket. + def Insert(self, request, global_params=None, upload=None, download=None): + """Stores a new object and metadata. Args: - request: (StorageBucketsInsertRequest) input message + request: (StorageObjectsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments + upload: (Upload, default: None) If present, upload + this stream with the request. + download: (Download, default: None) If present, download + data from the request via this stream. Returns: - (Bucket) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Insert') + upload_config = self.GetUploadConfig('Insert') return self._RunMethod( - config, request, global_params=global_params) + config, request, global_params=global_params, + upload=upload, upload_config=upload_config, + download=download) def List(self, request, global_params=None): - """Retrieves a list of buckets for a given project. + """Retrieves a list of objects matching the criteria. Args: - request: (StorageBucketsListRequest) input message + request: (StorageObjectsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Buckets) The response message. + (Objects) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) def Patch(self, request, global_params=None): - """Updates a bucket. This method supports patch semantics. + """Updates an object's metadata. This method supports patch semantics. Args: - request: (StorageBucketsPatchRequest) input message + request: (StorageObjectsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (Object) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) - def Update(self, request, global_params=None): - """Updates a bucket. + def Rewrite(self, request, global_params=None): + """Rewrites a source object to a destination object. Optionally overrides metadata. Args: - request: (StorageBucketsUpdateRequest) input message + request: (StorageObjectsRewriteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: - (Bucket) The response message. + (RewriteResponse) The response message. + """ + config = self.GetMethodConfig('Rewrite') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None, download=None): + """Updates an object's metadata. + + Args: + request: (StorageObjectsUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Object) The response message. """ config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def WatchAll(self, request, global_params=None): + """Watch for changes on all objects in a bucket. + + Args: + request: (StorageObjectsWatchAllRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Channel) The response message. + """ + config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py index 0e7b585..212f089 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -2,6 +2,7 @@ Lets you store and retrieve potentially-large, immutable data objects. """ +# NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import encoding from apitools.base.py import extra_types @@ -50,9 +51,9 @@ class Bucket(messages.Message): projectNumber: The project number of the project the bucket belongs to. selfLink: The URI of this bucket. storageClass: The bucket's storage class. This defines how objects in the - bucket are stored and determines the SLA and the cost of storage. - Typical values are STANDARD and DURABLE_REDUCED_AVAILABILITY. Defaults - to STANDARD. See the developer's guide for the authoritative list. + bucket are stored and determines the SLA and the cost of storage. Values + include STANDARD, NEARLINE and DURABLE_REDUCED_AVAILABILITY. Defaults to + STANDARD. For more information, see storage classes. timeCreated: Creation time of the bucket in RFC 3339 format. versioning: The bucket's versioning configuration. website: The bucket's website configuration. @@ -416,7 +417,7 @@ class Object(messages.Message): contentLanguage: Content-Language of the object data. contentType: Content-Type of the object data. crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded - using base64. + using base64 in big-endian byte order. etag: HTTP 1.1 Entity tag for the object. generation: The content generation of this object. Used for object versioning. @@ -593,6 +594,33 @@ class Objects(messages.Message): prefixes = messages.StringField(4, repeated=True) +class RewriteResponse(messages.Message): + """A rewrite response. + + Fields: + done: true if the copy is finished; otherwise, false if the copy is in + progress. This property is always present in the response. + kind: The kind of item this is. + objectSize: The total size of the object being copied in bytes. This + property is always present in the response. + resource: A resource containing the metadata for the copied-to object. + This property is present in the response only when copying completes. + rewriteToken: A token to use in subsequent requests to continue copying + data. This token is present in the response only when there is more data + to copy. + totalBytesRewritten: The total bytes written so far, which can be used to + provide a waiting user with a progress indicator. This property is + always present in the response. + """ + + done = messages.BooleanField(1) + kind = messages.StringField(2, default=u'storage#rewriteResponse') + objectSize = messages.IntegerField(3, variant=messages.Variant.UINT64) + resource = messages.MessageField('Object', 4) + rewriteToken = messages.StringField(5) + totalBytesRewritten = messages.IntegerField(6, variant=messages.Variant.UINT64) + + class StandardQueryParameters(messages.Message): """Query parameters accepted by all methods. @@ -1507,12 +1535,13 @@ class StorageObjectsListRequest(messages.Message): prefixes. Duplicate prefixes are omitted. maxResults: Maximum number of items plus prefixes to return. As duplicate prefixes are omitted, fewer total results may be returned than - requested. + requested. The default value of this parameter is 1,000 items. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. + versions: If true, lists all versions of an object as distinct results. + The default is false. For more information, see Object Versioning. """ class ProjectionValueValuesEnum(messages.Enum): @@ -1605,6 +1634,118 @@ class StorageObjectsPatchRequest(messages.Message): projection = messages.EnumField('ProjectionValueValuesEnum', 10) +class StorageObjectsRewriteRequest(messages.Message): + """A StorageObjectsRewriteRequest object. + + Enums: + DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access + controls to the destination object. + ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, + unless the object resource specifies the acl property, when it defaults + to full. + + Fields: + destinationBucket: Name of the bucket in which to store the new object. + Overrides the provided object metadata's bucket value, if any. + destinationObject: Name of the new object. Required when the object + metadata is not otherwise provided. Overrides the object metadata's name + value, if any. + destinationPredefinedAcl: Apply a predefined set of access controls to the + destination object. + ifGenerationMatch: Makes the operation conditional on whether the + destination object's current generation matches the given value. + ifGenerationNotMatch: Makes the operation conditional on whether the + destination object's current generation does not match the given value. + ifMetagenerationMatch: Makes the operation conditional on whether the + destination object's current metageneration matches the given value. + ifMetagenerationNotMatch: Makes the operation conditional on whether the + destination object's current metageneration does not match the given + value. + ifSourceGenerationMatch: Makes the operation conditional on whether the + source object's generation matches the given value. + ifSourceGenerationNotMatch: Makes the operation conditional on whether the + source object's generation does not match the given value. + ifSourceMetagenerationMatch: Makes the operation conditional on whether + the source object's current metageneration matches the given value. + ifSourceMetagenerationNotMatch: Makes the operation conditional on whether + the source object's current metageneration does not match the given + value. + maxBytesRewrittenPerCall: The maximum number of bytes that will be + rewritten per rewrite request. Most callers shouldn't need to specify + this parameter - it is primarily in place to support testing. If + specified the value must be an integral multiple of 1 MiB (1048576). + Also, this only applies to requests where the source and destination + span locations and/or storage classes. Finally, this value must not + change across rewrite calls else you'll get an error that the + rewriteToken is invalid. + object: A Object resource to be passed as the request body. + projection: Set of properties to return. Defaults to noAcl, unless the + object resource specifies the acl property, when it defaults to full. + rewriteToken: Include this field (from the previous rewrite response) on + each rewrite request after the first one, until the rewrite response + 'done' flag is true. Calls that provide a rewriteToken can omit all + other request fields, but if included those fields must match the values + provided in the first rewrite request. + sourceBucket: Name of the bucket in which to find the source object. + sourceGeneration: If present, selects a specific revision of the source + object (as opposed to the latest version, the default). + sourceObject: Name of the source object. + """ + + class DestinationPredefinedAclValueValuesEnum(messages.Enum): + """Apply a predefined set of access controls to the destination object. + + Values: + authenticatedRead: Object owner gets OWNER access, and + allAuthenticatedUsers get READER access. + bucketOwnerFullControl: Object owner gets OWNER access, and project team + owners get OWNER access. + bucketOwnerRead: Object owner gets OWNER access, and project team owners + get READER access. + private: Object owner gets OWNER access. + projectPrivate: Object owner gets OWNER access, and project team members + get access according to their roles. + publicRead: Object owner gets OWNER access, and allUsers get READER + access. + """ + authenticatedRead = 0 + bucketOwnerFullControl = 1 + bucketOwnerRead = 2 + private = 3 + projectPrivate = 4 + publicRead = 5 + + class ProjectionValueValuesEnum(messages.Enum): + """Set of properties to return. Defaults to noAcl, unless the object + resource specifies the acl property, when it defaults to full. + + Values: + full: Include all properties. + noAcl: Omit the acl property. + """ + full = 0 + noAcl = 1 + + destinationBucket = messages.StringField(1, required=True) + destinationObject = messages.StringField(2, required=True) + destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) + ifGenerationMatch = messages.IntegerField(4) + ifGenerationNotMatch = messages.IntegerField(5) + ifMetagenerationMatch = messages.IntegerField(6) + ifMetagenerationNotMatch = messages.IntegerField(7) + ifSourceGenerationMatch = messages.IntegerField(8) + ifSourceGenerationNotMatch = messages.IntegerField(9) + ifSourceMetagenerationMatch = messages.IntegerField(10) + ifSourceMetagenerationNotMatch = messages.IntegerField(11) + maxBytesRewrittenPerCall = messages.IntegerField(12) + object = messages.MessageField('Object', 13) + projection = messages.EnumField('ProjectionValueValuesEnum', 14) + rewriteToken = messages.StringField(15) + sourceBucket = messages.StringField(16, required=True) + sourceGeneration = messages.IntegerField(17) + sourceObject = messages.StringField(18, required=True) + + class StorageObjectsUpdateRequest(messages.Message): """A StorageObjectsUpdateRequest object. @@ -1692,12 +1833,13 @@ class StorageObjectsWatchAllRequest(messages.Message): prefixes. Duplicate prefixes are omitted. maxResults: Maximum number of items plus prefixes to return. As duplicate prefixes are omitted, fewer total results may be returned than - requested. + requested. The default value of this parameter is 1,000 items. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. - versions: If true, lists all versions of a file as distinct results. + versions: If true, lists all versions of an object as distinct results. + The default is false. For more information, see Object Versioning. """ class ProjectionValueValuesEnum(messages.Enum): -- GitLab From 6d2be21753318dd091fbd69eeb79d057e18c6fe5 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 27 May 2015 15:43:37 -0700 Subject: [PATCH 125/295] Update the `protorpc.messages` import name. In generated messages files, importing `messages` can cause conflicts when messages themselves have a field called `messages`. In order to avoid this, we just rename to `_messages`. (Already reviewed internally.) Fixes #28. --- .../testclient/fusiontables_v1_messages.py | 60 +++++++++---------- apitools/gen/extended_descriptor.py | 10 ++-- apitools/gen/message_registry.py | 5 +- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py index fd727a0..68284b5 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py @@ -4,13 +4,13 @@ API for working with Fusion Tables data. """ # NOTE: This file is autogenerated and should not be edited by hand. -from protorpc import messages +from protorpc import messages as _messages package = 'fusiontables' -class Column(messages.Message): +class Column(_messages.Message): """Specifies the id, name and type of a column in a table. @@ -33,7 +33,7 @@ class Column(messages.Message): """ - class BaseColumnValue(messages.Message): + class BaseColumnValue(_messages.Message): """Optional identifier of the base column. If present, this column is derived from the specified base column. @@ -46,19 +46,19 @@ class Column(messages.Message): """ - columnId = messages.IntegerField(1, variant=messages.Variant.INT32) - tableIndex = messages.IntegerField(2, variant=messages.Variant.INT32) + columnId = _messages.IntegerField(1, variant=_messages.Variant.INT32) + tableIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32) - baseColumn = messages.MessageField('BaseColumnValue', 1) - columnId = messages.IntegerField(2, variant=messages.Variant.INT32) - description = messages.StringField(3) - graph_predicate = messages.StringField(4) - kind = messages.StringField(5, default=u'fusiontables#column') - name = messages.StringField(6) - type = messages.StringField(7) + baseColumn = _messages.MessageField('BaseColumnValue', 1) + columnId = _messages.IntegerField(2, variant=_messages.Variant.INT32) + description = _messages.StringField(3) + graph_predicate = _messages.StringField(4) + kind = _messages.StringField(5, default=u'fusiontables#column') + name = _messages.StringField(6) + type = _messages.StringField(7) -class ColumnList(messages.Message): +class ColumnList(_messages.Message): """Represents a list of columns in a table. @@ -71,13 +71,13 @@ class ColumnList(messages.Message): """ - items = messages.MessageField('Column', 1, repeated=True) - kind = messages.StringField(2, default=u'fusiontables#columnList') - nextPageToken = messages.StringField(3) - totalItems = messages.IntegerField(4, variant=messages.Variant.INT32) + items = _messages.MessageField('Column', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#columnList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) -class FusiontablesColumnListRequest(messages.Message): +class FusiontablesColumnListRequest(_messages.Message): """A FusiontablesColumnListRequest object. @@ -88,12 +88,12 @@ class FusiontablesColumnListRequest(messages.Message): tableId: Table whose columns are being listed. """ - maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) - pageToken = messages.StringField(2) - tableId = messages.StringField(3, required=True) + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + tableId = _messages.StringField(3, required=True) -class FusiontablesColumnListAlternateRequest(messages.Message): +class FusiontablesColumnListAlternateRequest(_messages.Message): """A FusiontablesColumnListRequest object. @@ -104,12 +104,12 @@ class FusiontablesColumnListAlternateRequest(messages.Message): tableId: Table whose columns are being listed. """ - pageSize = messages.IntegerField(1, variant=messages.Variant.UINT32) - pageToken = messages.StringField(2) - tableId = messages.StringField(3, required=True) + pageSize = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + tableId = _messages.StringField(3, required=True) -class ColumnListAlternate(messages.Message): +class ColumnListAlternate(_messages.Message): """Represents a list of columns in a table. @@ -122,7 +122,7 @@ class ColumnListAlternate(messages.Message): """ - columns = messages.MessageField('Column', 1, repeated=True) - kind = messages.StringField(2, default=u'fusiontables#columnList') - nextPageToken = messages.StringField(3) - totalItems = messages.IntegerField(4, variant=messages.Variant.INT32) + columns = _messages.MessageField('Column', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#columnList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index abff599..103aa7b 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -364,7 +364,7 @@ class _ProtoRpcPrinter(ProtoPrinter): self.__printer('"""') def PrintEnum(self, enum_type): - self.__printer('class %s(messages.Enum):', enum_type.name) + self.__printer('class %s(_messages.Enum):', enum_type.name) with self.__printer.Indent(): self.__PrintEnumDocstringLines(enum_type) enum_values = sorted( @@ -440,7 +440,7 @@ class _ProtoRpcPrinter(ProtoPrinter): return for decorator in message_type.decorators: self.__printer('@%s', decorator) - self.__printer('class %s(messages.Message):', message_type.name) + self.__printer('class %s(_messages.Message):', message_type.name) with self.__printer.Indent(): self.__PrintMessageDocstringLines(message_type) _PrintEnums(self, message_type.enum_types) @@ -476,7 +476,7 @@ def _PrintFields(fields, printer): field = extended_field.field_descriptor printed_field_info = { 'name': field.name, - 'module': 'messages', + 'module': '_messages', 'type_name': '', 'type_format': '', 'number': field.number, @@ -487,7 +487,7 @@ def _PrintFields(fields, printer): message_field = _MESSAGE_FIELD_MAP.get(field.type_name) if message_field: - printed_field_info['module'] = 'message_types' + printed_field_info['module'] = '_message_types' field_type = message_field elif field.type_name == 'extra_types.DateField': printed_field_info['module'] = 'extra_types' @@ -506,7 +506,7 @@ def _PrintFields(fields, printer): if field_type.DEFAULT_VARIANT != field.variant: printed_field_info['variant_format'] = ( - ', variant=messages.Variant.%s' % field.variant) + ', variant=_messages.Variant.%s' % field.variant) if field.default_value: if field_type in [messages.BytesField, messages.StringField]: diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 4a7a3e9..d467e9b 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -73,7 +73,7 @@ class MessageRegistry(object): package=self.__package, description=self.__description) # Add required imports self.__file_descriptor.additional_imports = [ - 'from protorpc import messages', + 'from protorpc import messages as _messages', ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() @@ -394,7 +394,8 @@ class MessageRegistry(object): attrs['format'], type_name)) if (type_info.type_name.startswith('protorpc.message_types.') or type_info.type_name.startswith('message_types.')): - self.__AddImport('from protorpc import message_types') + self.__AddImport( + 'from protorpc import message_types as _message_types') if type_info.type_name.startswith('extra_types.'): self.__AddImport( 'from %s import extra_types' % self.__base_files_package) -- GitLab From 7bc411f22ff152d3306bc6bbd9fc0c3bd28b1f41 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 27 May 2015 16:05:34 -0700 Subject: [PATCH 126/295] Update storage client. --- .../storage/storage_v1_messages.py | 822 +++++++++--------- 1 file changed, 411 insertions(+), 411 deletions(-) diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py index 212f089..7148d0a 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -6,14 +6,14 @@ Lets you store and retrieve potentially-large, immutable data objects. from apitools.base.py import encoding from apitools.base.py import extra_types -from protorpc import message_types -from protorpc import messages +from protorpc import message_types as _message_types +from protorpc import messages as _messages package = 'storage' -class Bucket(messages.Message): +class Bucket(_messages.Message): """A bucket. Messages: @@ -59,7 +59,7 @@ class Bucket(messages.Message): website: The bucket's website configuration. """ - class CorsValueListEntry(messages.Message): + class CorsValueListEntry(_messages.Message): """A CorsValueListEntry object. Fields: @@ -74,12 +74,12 @@ class Bucket(messages.Message): headers to give permission for the user-agent to share across domains. """ - maxAgeSeconds = messages.IntegerField(1, variant=messages.Variant.INT32) - method = messages.StringField(2, repeated=True) - origin = messages.StringField(3, repeated=True) - responseHeader = messages.StringField(4, repeated=True) + maxAgeSeconds = _messages.IntegerField(1, variant=_messages.Variant.INT32) + method = _messages.StringField(2, repeated=True) + origin = _messages.StringField(3, repeated=True) + responseHeader = _messages.StringField(4, repeated=True) - class LifecycleValue(messages.Message): + class LifecycleValue(_messages.Message): """The bucket's lifecycle configuration. See lifecycle management for more information. @@ -91,7 +91,7 @@ class Bucket(messages.Message): and the condition(s) under which the action will be taken. """ - class RuleValueListEntry(messages.Message): + class RuleValueListEntry(_messages.Message): """A RuleValueListEntry object. Messages: @@ -103,16 +103,16 @@ class Bucket(messages.Message): condition: The condition(s) under which the action will be taken. """ - class ActionValue(messages.Message): + class ActionValue(_messages.Message): """The action to take. Fields: type: Type of the action. Currently, only Delete is supported. """ - type = messages.StringField(1) + type = _messages.StringField(1) - class ConditionValue(messages.Message): + class ConditionValue(_messages.Message): """The condition(s) under which the action will be taken. Fields: @@ -130,17 +130,17 @@ class Bucket(messages.Message): the object. """ - age = messages.IntegerField(1, variant=messages.Variant.INT32) + age = _messages.IntegerField(1, variant=_messages.Variant.INT32) createdBefore = extra_types.DateField(2) - isLive = messages.BooleanField(3) - numNewerVersions = messages.IntegerField(4, variant=messages.Variant.INT32) + isLive = _messages.BooleanField(3) + numNewerVersions = _messages.IntegerField(4, variant=_messages.Variant.INT32) - action = messages.MessageField('ActionValue', 1) - condition = messages.MessageField('ConditionValue', 2) + action = _messages.MessageField('ActionValue', 1) + condition = _messages.MessageField('ConditionValue', 2) - rule = messages.MessageField('RuleValueListEntry', 1, repeated=True) + rule = _messages.MessageField('RuleValueListEntry', 1, repeated=True) - class LoggingValue(messages.Message): + class LoggingValue(_messages.Message): """The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. @@ -150,10 +150,10 @@ class Bucket(messages.Message): logObjectPrefix: A prefix for log object names. """ - logBucket = messages.StringField(1) - logObjectPrefix = messages.StringField(2) + logBucket = _messages.StringField(1) + logObjectPrefix = _messages.StringField(2) - class OwnerValue(messages.Message): + class OwnerValue(_messages.Message): """The owner of the bucket. This is always the project team's owner group. Fields: @@ -161,19 +161,19 @@ class Bucket(messages.Message): entityId: The ID for the entity. """ - entity = messages.StringField(1) - entityId = messages.StringField(2) + entity = _messages.StringField(1) + entityId = _messages.StringField(2) - class VersioningValue(messages.Message): + class VersioningValue(_messages.Message): """The bucket's versioning configuration. Fields: enabled: While set to true, versioning is fully enabled for this bucket. """ - enabled = messages.BooleanField(1) + enabled = _messages.BooleanField(1) - class WebsiteValue(messages.Message): + class WebsiteValue(_messages.Message): """The bucket's website configuration. Fields: @@ -183,30 +183,30 @@ class Bucket(messages.Message): not found. """ - mainPageSuffix = messages.StringField(1) - notFoundPage = messages.StringField(2) - - acl = messages.MessageField('BucketAccessControl', 1, repeated=True) - cors = messages.MessageField('CorsValueListEntry', 2, repeated=True) - defaultObjectAcl = messages.MessageField('ObjectAccessControl', 3, repeated=True) - etag = messages.StringField(4) - id = messages.StringField(5) - kind = messages.StringField(6, default=u'storage#bucket') - lifecycle = messages.MessageField('LifecycleValue', 7) - location = messages.StringField(8) - logging = messages.MessageField('LoggingValue', 9) - metageneration = messages.IntegerField(10) - name = messages.StringField(11) - owner = messages.MessageField('OwnerValue', 12) - projectNumber = messages.IntegerField(13, variant=messages.Variant.UINT64) - selfLink = messages.StringField(14) - storageClass = messages.StringField(15) - timeCreated = message_types.DateTimeField(16) - versioning = messages.MessageField('VersioningValue', 17) - website = messages.MessageField('WebsiteValue', 18) - - -class BucketAccessControl(messages.Message): + mainPageSuffix = _messages.StringField(1) + notFoundPage = _messages.StringField(2) + + acl = _messages.MessageField('BucketAccessControl', 1, repeated=True) + cors = _messages.MessageField('CorsValueListEntry', 2, repeated=True) + defaultObjectAcl = _messages.MessageField('ObjectAccessControl', 3, repeated=True) + etag = _messages.StringField(4) + id = _messages.StringField(5) + kind = _messages.StringField(6, default=u'storage#bucket') + lifecycle = _messages.MessageField('LifecycleValue', 7) + location = _messages.StringField(8) + logging = _messages.MessageField('LoggingValue', 9) + metageneration = _messages.IntegerField(10) + name = _messages.StringField(11) + owner = _messages.MessageField('OwnerValue', 12) + projectNumber = _messages.IntegerField(13, variant=_messages.Variant.UINT64) + selfLink = _messages.StringField(14) + storageClass = _messages.StringField(15) + timeCreated = _message_types.DateTimeField(16) + versioning = _messages.MessageField('VersioningValue', 17) + website = _messages.MessageField('WebsiteValue', 18) + + +class BucketAccessControl(_messages.Message): """An access-control entry. Messages: @@ -234,7 +234,7 @@ class BucketAccessControl(messages.Message): selfLink: The link to this access-control entry. """ - class ProjectTeamValue(messages.Message): + class ProjectTeamValue(_messages.Message): """The project team associated with the entity, if any. Fields: @@ -242,23 +242,23 @@ class BucketAccessControl(messages.Message): team: The team. Can be owners, editors, or viewers. """ - projectNumber = messages.StringField(1) - team = messages.StringField(2) + projectNumber = _messages.StringField(1) + team = _messages.StringField(2) - bucket = messages.StringField(1) - domain = messages.StringField(2) - email = messages.StringField(3) - entity = messages.StringField(4) - entityId = messages.StringField(5) - etag = messages.StringField(6) - id = messages.StringField(7) - kind = messages.StringField(8, default=u'storage#bucketAccessControl') - projectTeam = messages.MessageField('ProjectTeamValue', 9) - role = messages.StringField(10) - selfLink = messages.StringField(11) + bucket = _messages.StringField(1) + domain = _messages.StringField(2) + email = _messages.StringField(3) + entity = _messages.StringField(4) + entityId = _messages.StringField(5) + etag = _messages.StringField(6) + id = _messages.StringField(7) + kind = _messages.StringField(8, default=u'storage#bucketAccessControl') + projectTeam = _messages.MessageField('ProjectTeamValue', 9) + role = _messages.StringField(10) + selfLink = _messages.StringField(11) -class BucketAccessControls(messages.Message): +class BucketAccessControls(_messages.Message): """An access-control list. Fields: @@ -267,11 +267,11 @@ class BucketAccessControls(messages.Message): entries, this is always storage#bucketAccessControls. """ - items = messages.MessageField('BucketAccessControl', 1, repeated=True) - kind = messages.StringField(2, default=u'storage#bucketAccessControls') + items = _messages.MessageField('BucketAccessControl', 1, repeated=True) + kind = _messages.StringField(2, default=u'storage#bucketAccessControls') -class Buckets(messages.Message): +class Buckets(_messages.Message): """A list of buckets. Fields: @@ -283,12 +283,12 @@ class Buckets(messages.Message): of results. """ - items = messages.MessageField('Bucket', 1, repeated=True) - kind = messages.StringField(2, default=u'storage#buckets') - nextPageToken = messages.StringField(3) + items = _messages.MessageField('Bucket', 1, repeated=True) + kind = _messages.StringField(2, default=u'storage#buckets') + nextPageToken = _messages.StringField(3) -class Channel(messages.Message): +class Channel(_messages.Message): """An notification channel used to watch for resource changes. Messages: @@ -314,7 +314,7 @@ class Channel(messages.Message): """ @encoding.MapUnrecognizedFields('additionalProperties') - class ParamsValue(messages.Message): + class ParamsValue(_messages.Message): """Additional parameters controlling delivery channel behavior. Optional. Messages: @@ -324,7 +324,7 @@ class Channel(messages.Message): additionalProperties: Declares a new parameter by name. """ - class AdditionalProperty(messages.Message): + class AdditionalProperty(_messages.Message): """An additional property for a ParamsValue object. Fields: @@ -332,24 +332,24 @@ class Channel(messages.Message): value: A string attribute. """ - key = messages.StringField(1) - value = messages.StringField(2) + key = _messages.StringField(1) + value = _messages.StringField(2) - additionalProperties = messages.MessageField('AdditionalProperty', 1, repeated=True) + additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) - address = messages.StringField(1) - expiration = messages.IntegerField(2) - id = messages.StringField(3) - kind = messages.StringField(4, default=u'api#channel') - params = messages.MessageField('ParamsValue', 5) - payload = messages.BooleanField(6) - resourceId = messages.StringField(7) - resourceUri = messages.StringField(8) - token = messages.StringField(9) - type = messages.StringField(10) + address = _messages.StringField(1) + expiration = _messages.IntegerField(2) + id = _messages.StringField(3) + kind = _messages.StringField(4, default=u'api#channel') + params = _messages.MessageField('ParamsValue', 5) + payload = _messages.BooleanField(6) + resourceId = _messages.StringField(7) + resourceUri = _messages.StringField(8) + token = _messages.StringField(9) + type = _messages.StringField(10) -class ComposeRequest(messages.Message): +class ComposeRequest(_messages.Message): """A Compose request. Messages: @@ -362,7 +362,7 @@ class ComposeRequest(messages.Message): single object. """ - class SourceObjectsValueListEntry(messages.Message): + class SourceObjectsValueListEntry(_messages.Message): """A SourceObjectsValueListEntry object. Messages: @@ -377,7 +377,7 @@ class ComposeRequest(messages.Message): execute. """ - class ObjectPreconditionsValue(messages.Message): + class ObjectPreconditionsValue(_messages.Message): """Conditions that must be met for this operation to execute. Fields: @@ -387,18 +387,18 @@ class ComposeRequest(messages.Message): value or the call will fail. """ - ifGenerationMatch = messages.IntegerField(1) + ifGenerationMatch = _messages.IntegerField(1) - generation = messages.IntegerField(1) - name = messages.StringField(2) - objectPreconditions = messages.MessageField('ObjectPreconditionsValue', 3) + generation = _messages.IntegerField(1) + name = _messages.StringField(2) + objectPreconditions = _messages.MessageField('ObjectPreconditionsValue', 3) - destination = messages.MessageField('Object', 1) - kind = messages.StringField(2, default=u'storage#composeRequest') - sourceObjects = messages.MessageField('SourceObjectsValueListEntry', 3, repeated=True) + destination = _messages.MessageField('Object', 1) + kind = _messages.StringField(2, default=u'storage#composeRequest') + sourceObjects = _messages.MessageField('SourceObjectsValueListEntry', 3, repeated=True) -class Object(messages.Message): +class Object(_messages.Message): """An object. Messages: @@ -445,7 +445,7 @@ class Object(messages.Message): """ @encoding.MapUnrecognizedFields('additionalProperties') - class MetadataValue(messages.Message): + class MetadataValue(_messages.Message): """User-provided metadata, in key/value pairs. Messages: @@ -455,7 +455,7 @@ class Object(messages.Message): additionalProperties: An individual metadata entry. """ - class AdditionalProperty(messages.Message): + class AdditionalProperty(_messages.Message): """An additional property for a MetadataValue object. Fields: @@ -463,12 +463,12 @@ class Object(messages.Message): value: A string attribute. """ - key = messages.StringField(1) - value = messages.StringField(2) + key = _messages.StringField(1) + value = _messages.StringField(2) - additionalProperties = messages.MessageField('AdditionalProperty', 1, repeated=True) + additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) - class OwnerValue(messages.Message): + class OwnerValue(_messages.Message): """The owner of the object. This will always be the uploader of the object. @@ -477,36 +477,36 @@ class Object(messages.Message): entityId: The ID for the entity. """ - entity = messages.StringField(1) - entityId = messages.StringField(2) - - acl = messages.MessageField('ObjectAccessControl', 1, repeated=True) - bucket = messages.StringField(2) - cacheControl = messages.StringField(3) - componentCount = messages.IntegerField(4, variant=messages.Variant.INT32) - contentDisposition = messages.StringField(5) - contentEncoding = messages.StringField(6) - contentLanguage = messages.StringField(7) - contentType = messages.StringField(8) - crc32c = messages.StringField(9) - etag = messages.StringField(10) - generation = messages.IntegerField(11) - id = messages.StringField(12) - kind = messages.StringField(13, default=u'storage#object') - md5Hash = messages.StringField(14) - mediaLink = messages.StringField(15) - metadata = messages.MessageField('MetadataValue', 16) - metageneration = messages.IntegerField(17) - name = messages.StringField(18) - owner = messages.MessageField('OwnerValue', 19) - selfLink = messages.StringField(20) - size = messages.IntegerField(21, variant=messages.Variant.UINT64) - storageClass = messages.StringField(22) - timeDeleted = message_types.DateTimeField(23) - updated = message_types.DateTimeField(24) - - -class ObjectAccessControl(messages.Message): + entity = _messages.StringField(1) + entityId = _messages.StringField(2) + + acl = _messages.MessageField('ObjectAccessControl', 1, repeated=True) + bucket = _messages.StringField(2) + cacheControl = _messages.StringField(3) + componentCount = _messages.IntegerField(4, variant=_messages.Variant.INT32) + contentDisposition = _messages.StringField(5) + contentEncoding = _messages.StringField(6) + contentLanguage = _messages.StringField(7) + contentType = _messages.StringField(8) + crc32c = _messages.StringField(9) + etag = _messages.StringField(10) + generation = _messages.IntegerField(11) + id = _messages.StringField(12) + kind = _messages.StringField(13, default=u'storage#object') + md5Hash = _messages.StringField(14) + mediaLink = _messages.StringField(15) + metadata = _messages.MessageField('MetadataValue', 16) + metageneration = _messages.IntegerField(17) + name = _messages.StringField(18) + owner = _messages.MessageField('OwnerValue', 19) + selfLink = _messages.StringField(20) + size = _messages.IntegerField(21, variant=_messages.Variant.UINT64) + storageClass = _messages.StringField(22) + timeDeleted = _message_types.DateTimeField(23) + updated = _message_types.DateTimeField(24) + + +class ObjectAccessControl(_messages.Message): """An access-control entry. Messages: @@ -535,7 +535,7 @@ class ObjectAccessControl(messages.Message): selfLink: The link to this access-control entry. """ - class ProjectTeamValue(messages.Message): + class ProjectTeamValue(_messages.Message): """The project team associated with the entity, if any. Fields: @@ -543,25 +543,25 @@ class ObjectAccessControl(messages.Message): team: The team. Can be owners, editors, or viewers. """ - projectNumber = messages.StringField(1) - team = messages.StringField(2) - - bucket = messages.StringField(1) - domain = messages.StringField(2) - email = messages.StringField(3) - entity = messages.StringField(4) - entityId = messages.StringField(5) - etag = messages.StringField(6) - generation = messages.IntegerField(7) - id = messages.StringField(8) - kind = messages.StringField(9, default=u'storage#objectAccessControl') - object = messages.StringField(10) - projectTeam = messages.MessageField('ProjectTeamValue', 11) - role = messages.StringField(12) - selfLink = messages.StringField(13) - - -class ObjectAccessControls(messages.Message): + projectNumber = _messages.StringField(1) + team = _messages.StringField(2) + + bucket = _messages.StringField(1) + domain = _messages.StringField(2) + email = _messages.StringField(3) + entity = _messages.StringField(4) + entityId = _messages.StringField(5) + etag = _messages.StringField(6) + generation = _messages.IntegerField(7) + id = _messages.StringField(8) + kind = _messages.StringField(9, default=u'storage#objectAccessControl') + object = _messages.StringField(10) + projectTeam = _messages.MessageField('ProjectTeamValue', 11) + role = _messages.StringField(12) + selfLink = _messages.StringField(13) + + +class ObjectAccessControls(_messages.Message): """An access-control list. Fields: @@ -570,11 +570,11 @@ class ObjectAccessControls(messages.Message): entries, this is always storage#objectAccessControls. """ - items = messages.MessageField('extra_types.JsonValue', 1, repeated=True) - kind = messages.StringField(2, default=u'storage#objectAccessControls') + items = _messages.MessageField('extra_types.JsonValue', 1, repeated=True) + kind = _messages.StringField(2, default=u'storage#objectAccessControls') -class Objects(messages.Message): +class Objects(_messages.Message): """A list of objects. Fields: @@ -588,13 +588,13 @@ class Objects(messages.Message): and including the requested delimiter. """ - items = messages.MessageField('Object', 1, repeated=True) - kind = messages.StringField(2, default=u'storage#objects') - nextPageToken = messages.StringField(3) - prefixes = messages.StringField(4, repeated=True) + items = _messages.MessageField('Object', 1, repeated=True) + kind = _messages.StringField(2, default=u'storage#objects') + nextPageToken = _messages.StringField(3) + prefixes = _messages.StringField(4, repeated=True) -class RewriteResponse(messages.Message): +class RewriteResponse(_messages.Message): """A rewrite response. Fields: @@ -613,15 +613,15 @@ class RewriteResponse(messages.Message): always present in the response. """ - done = messages.BooleanField(1) - kind = messages.StringField(2, default=u'storage#rewriteResponse') - objectSize = messages.IntegerField(3, variant=messages.Variant.UINT64) - resource = messages.MessageField('Object', 4) - rewriteToken = messages.StringField(5) - totalBytesRewritten = messages.IntegerField(6, variant=messages.Variant.UINT64) + done = _messages.BooleanField(1) + kind = _messages.StringField(2, default=u'storage#rewriteResponse') + objectSize = _messages.IntegerField(3, variant=_messages.Variant.UINT64) + resource = _messages.MessageField('Object', 4) + rewriteToken = _messages.StringField(5) + totalBytesRewritten = _messages.IntegerField(6, variant=_messages.Variant.UINT64) -class StandardQueryParameters(messages.Message): +class StandardQueryParameters(_messages.Message): """Query parameters accepted by all methods. Enums: @@ -644,7 +644,7 @@ class StandardQueryParameters(messages.Message): you want to enforce per-user limits. """ - class AltValueValuesEnum(messages.Enum): + class AltValueValuesEnum(_messages.Enum): """Data format for the response. Values: @@ -652,17 +652,17 @@ class StandardQueryParameters(messages.Message): """ json = 0 - alt = messages.EnumField('AltValueValuesEnum', 1, default=u'json') - fields = messages.StringField(2) - key = messages.StringField(3) - oauth_token = messages.StringField(4) - prettyPrint = messages.BooleanField(5, default=True) - quotaUser = messages.StringField(6) - trace = messages.StringField(7) - userIp = messages.StringField(8) + alt = _messages.EnumField('AltValueValuesEnum', 1, default=u'json') + fields = _messages.StringField(2) + key = _messages.StringField(3) + oauth_token = _messages.StringField(4) + prettyPrint = _messages.BooleanField(5, default=True) + quotaUser = _messages.StringField(6) + trace = _messages.StringField(7) + userIp = _messages.StringField(8) -class StorageBucketAccessControlsDeleteRequest(messages.Message): +class StorageBucketAccessControlsDeleteRequest(_messages.Message): """A StorageBucketAccessControlsDeleteRequest object. Fields: @@ -672,15 +672,15 @@ class StorageBucketAccessControlsDeleteRequest(messages.Message): allAuthenticatedUsers. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) -class StorageBucketAccessControlsDeleteResponse(messages.Message): +class StorageBucketAccessControlsDeleteResponse(_messages.Message): """An empty StorageBucketAccessControlsDelete response.""" -class StorageBucketAccessControlsGetRequest(messages.Message): +class StorageBucketAccessControlsGetRequest(_messages.Message): """A StorageBucketAccessControlsGetRequest object. Fields: @@ -690,21 +690,21 @@ class StorageBucketAccessControlsGetRequest(messages.Message): allAuthenticatedUsers. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) -class StorageBucketAccessControlsListRequest(messages.Message): +class StorageBucketAccessControlsListRequest(_messages.Message): """A StorageBucketAccessControlsListRequest object. Fields: bucket: Name of a bucket. """ - bucket = messages.StringField(1, required=True) + bucket = _messages.StringField(1, required=True) -class StorageBucketsDeleteRequest(messages.Message): +class StorageBucketsDeleteRequest(_messages.Message): """A StorageBucketsDeleteRequest object. Fields: @@ -715,16 +715,16 @@ class StorageBucketsDeleteRequest(messages.Message): metageneration does not match this value. """ - bucket = messages.StringField(1, required=True) - ifMetagenerationMatch = messages.IntegerField(2) - ifMetagenerationNotMatch = messages.IntegerField(3) + bucket = _messages.StringField(1, required=True) + ifMetagenerationMatch = _messages.IntegerField(2) + ifMetagenerationNotMatch = _messages.IntegerField(3) -class StorageBucketsDeleteResponse(messages.Message): +class StorageBucketsDeleteResponse(_messages.Message): """An empty StorageBucketsDelete response.""" -class StorageBucketsGetRequest(messages.Message): +class StorageBucketsGetRequest(_messages.Message): """A StorageBucketsGetRequest object. Enums: @@ -740,7 +740,7 @@ class StorageBucketsGetRequest(messages.Message): projection: Set of properties to return. Defaults to noAcl. """ - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl. Values: @@ -750,13 +750,13 @@ class StorageBucketsGetRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - ifMetagenerationMatch = messages.IntegerField(2) - ifMetagenerationNotMatch = messages.IntegerField(3) - projection = messages.EnumField('ProjectionValueValuesEnum', 4) + bucket = _messages.StringField(1, required=True) + ifMetagenerationMatch = _messages.IntegerField(2) + ifMetagenerationNotMatch = _messages.IntegerField(3) + projection = _messages.EnumField('ProjectionValueValuesEnum', 4) -class StorageBucketsInsertRequest(messages.Message): +class StorageBucketsInsertRequest(_messages.Message): """A StorageBucketsInsertRequest object. Enums: @@ -779,7 +779,7 @@ class StorageBucketsInsertRequest(messages.Message): defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this bucket. Values: @@ -799,7 +799,7 @@ class StorageBucketsInsertRequest(messages.Message): publicRead = 3 publicReadWrite = 4 - class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): """Apply a predefined set of default object access controls to this bucket. @@ -823,7 +823,7 @@ class StorageBucketsInsertRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. @@ -835,14 +835,14 @@ class StorageBucketsInsertRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.MessageField('Bucket', 1) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 2) - predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 3) - project = messages.StringField(4, required=True) - projection = messages.EnumField('ProjectionValueValuesEnum', 5) + bucket = _messages.MessageField('Bucket', 1) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 2) + predefinedDefaultObjectAcl = _messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 3) + project = _messages.StringField(4, required=True) + projection = _messages.EnumField('ProjectionValueValuesEnum', 5) -class StorageBucketsListRequest(messages.Message): +class StorageBucketsListRequest(_messages.Message): """A StorageBucketsListRequest object. Enums: @@ -857,7 +857,7 @@ class StorageBucketsListRequest(messages.Message): projection: Set of properties to return. Defaults to noAcl. """ - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl. Values: @@ -867,14 +867,14 @@ class StorageBucketsListRequest(messages.Message): full = 0 noAcl = 1 - maxResults = messages.IntegerField(1, variant=messages.Variant.UINT32) - pageToken = messages.StringField(2) - prefix = messages.StringField(3) - project = messages.StringField(4, required=True) - projection = messages.EnumField('ProjectionValueValuesEnum', 5) + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + prefix = _messages.StringField(3) + project = _messages.StringField(4, required=True) + projection = _messages.EnumField('ProjectionValueValuesEnum', 5) -class StorageBucketsPatchRequest(messages.Message): +class StorageBucketsPatchRequest(_messages.Message): """A StorageBucketsPatchRequest object. Enums: @@ -898,7 +898,7 @@ class StorageBucketsPatchRequest(messages.Message): projection: Set of properties to return. Defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this bucket. Values: @@ -918,7 +918,7 @@ class StorageBucketsPatchRequest(messages.Message): publicRead = 3 publicReadWrite = 4 - class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): """Apply a predefined set of default object access controls to this bucket. @@ -942,7 +942,7 @@ class StorageBucketsPatchRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to full. Values: @@ -952,16 +952,16 @@ class StorageBucketsPatchRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - bucketResource = messages.MessageField('Bucket', 2) - ifMetagenerationMatch = messages.IntegerField(3) - ifMetagenerationNotMatch = messages.IntegerField(4) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) - predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) - projection = messages.EnumField('ProjectionValueValuesEnum', 7) + bucket = _messages.StringField(1, required=True) + bucketResource = _messages.MessageField('Bucket', 2) + ifMetagenerationMatch = _messages.IntegerField(3) + ifMetagenerationNotMatch = _messages.IntegerField(4) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 5) + predefinedDefaultObjectAcl = _messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) + projection = _messages.EnumField('ProjectionValueValuesEnum', 7) -class StorageBucketsUpdateRequest(messages.Message): +class StorageBucketsUpdateRequest(_messages.Message): """A StorageBucketsUpdateRequest object. Enums: @@ -985,7 +985,7 @@ class StorageBucketsUpdateRequest(messages.Message): projection: Set of properties to return. Defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this bucket. Values: @@ -1005,7 +1005,7 @@ class StorageBucketsUpdateRequest(messages.Message): publicRead = 3 publicReadWrite = 4 - class PredefinedDefaultObjectAclValueValuesEnum(messages.Enum): + class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): """Apply a predefined set of default object access controls to this bucket. @@ -1029,7 +1029,7 @@ class StorageBucketsUpdateRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to full. Values: @@ -1039,20 +1039,20 @@ class StorageBucketsUpdateRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - bucketResource = messages.MessageField('Bucket', 2) - ifMetagenerationMatch = messages.IntegerField(3) - ifMetagenerationNotMatch = messages.IntegerField(4) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 5) - predefinedDefaultObjectAcl = messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) - projection = messages.EnumField('ProjectionValueValuesEnum', 7) + bucket = _messages.StringField(1, required=True) + bucketResource = _messages.MessageField('Bucket', 2) + ifMetagenerationMatch = _messages.IntegerField(3) + ifMetagenerationNotMatch = _messages.IntegerField(4) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 5) + predefinedDefaultObjectAcl = _messages.EnumField('PredefinedDefaultObjectAclValueValuesEnum', 6) + projection = _messages.EnumField('ProjectionValueValuesEnum', 7) -class StorageChannelsStopResponse(messages.Message): +class StorageChannelsStopResponse(_messages.Message): """An empty StorageChannelsStop response.""" -class StorageDefaultObjectAccessControlsDeleteRequest(messages.Message): +class StorageDefaultObjectAccessControlsDeleteRequest(_messages.Message): """A StorageDefaultObjectAccessControlsDeleteRequest object. Fields: @@ -1062,15 +1062,15 @@ class StorageDefaultObjectAccessControlsDeleteRequest(messages.Message): allAuthenticatedUsers. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) -class StorageDefaultObjectAccessControlsDeleteResponse(messages.Message): +class StorageDefaultObjectAccessControlsDeleteResponse(_messages.Message): """An empty StorageDefaultObjectAccessControlsDelete response.""" -class StorageDefaultObjectAccessControlsGetRequest(messages.Message): +class StorageDefaultObjectAccessControlsGetRequest(_messages.Message): """A StorageDefaultObjectAccessControlsGetRequest object. Fields: @@ -1080,11 +1080,11 @@ class StorageDefaultObjectAccessControlsGetRequest(messages.Message): allAuthenticatedUsers. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) -class StorageDefaultObjectAccessControlsListRequest(messages.Message): +class StorageDefaultObjectAccessControlsListRequest(_messages.Message): """A StorageDefaultObjectAccessControlsListRequest object. Fields: @@ -1095,12 +1095,12 @@ class StorageDefaultObjectAccessControlsListRequest(messages.Message): the bucket's current metageneration does not match the given value. """ - bucket = messages.StringField(1, required=True) - ifMetagenerationMatch = messages.IntegerField(2) - ifMetagenerationNotMatch = messages.IntegerField(3) + bucket = _messages.StringField(1, required=True) + ifMetagenerationMatch = _messages.IntegerField(2) + ifMetagenerationNotMatch = _messages.IntegerField(3) -class StorageObjectAccessControlsDeleteRequest(messages.Message): +class StorageObjectAccessControlsDeleteRequest(_messages.Message): """A StorageObjectAccessControlsDeleteRequest object. Fields: @@ -1113,17 +1113,17 @@ class StorageObjectAccessControlsDeleteRequest(messages.Message): object: Name of the object. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) - generation = messages.IntegerField(3) - object = messages.StringField(4, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) + generation = _messages.IntegerField(3) + object = _messages.StringField(4, required=True) -class StorageObjectAccessControlsDeleteResponse(messages.Message): +class StorageObjectAccessControlsDeleteResponse(_messages.Message): """An empty StorageObjectAccessControlsDelete response.""" -class StorageObjectAccessControlsGetRequest(messages.Message): +class StorageObjectAccessControlsGetRequest(_messages.Message): """A StorageObjectAccessControlsGetRequest object. Fields: @@ -1136,13 +1136,13 @@ class StorageObjectAccessControlsGetRequest(messages.Message): object: Name of the object. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) - generation = messages.IntegerField(3) - object = messages.StringField(4, required=True) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) + generation = _messages.IntegerField(3) + object = _messages.StringField(4, required=True) -class StorageObjectAccessControlsInsertRequest(messages.Message): +class StorageObjectAccessControlsInsertRequest(_messages.Message): """A StorageObjectAccessControlsInsertRequest object. Fields: @@ -1154,13 +1154,13 @@ class StorageObjectAccessControlsInsertRequest(messages.Message): request body. """ - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - object = messages.StringField(3, required=True) - objectAccessControl = messages.MessageField('ObjectAccessControl', 4) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + object = _messages.StringField(3, required=True) + objectAccessControl = _messages.MessageField('ObjectAccessControl', 4) -class StorageObjectAccessControlsListRequest(messages.Message): +class StorageObjectAccessControlsListRequest(_messages.Message): """A StorageObjectAccessControlsListRequest object. Fields: @@ -1170,12 +1170,12 @@ class StorageObjectAccessControlsListRequest(messages.Message): object: Name of the object. """ - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - object = messages.StringField(3, required=True) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + object = _messages.StringField(3, required=True) -class StorageObjectAccessControlsPatchRequest(messages.Message): +class StorageObjectAccessControlsPatchRequest(_messages.Message): """A StorageObjectAccessControlsPatchRequest object. Fields: @@ -1190,14 +1190,14 @@ class StorageObjectAccessControlsPatchRequest(messages.Message): request body. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) - generation = messages.IntegerField(3) - object = messages.StringField(4, required=True) - objectAccessControl = messages.MessageField('ObjectAccessControl', 5) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) + generation = _messages.IntegerField(3) + object = _messages.StringField(4, required=True) + objectAccessControl = _messages.MessageField('ObjectAccessControl', 5) -class StorageObjectAccessControlsUpdateRequest(messages.Message): +class StorageObjectAccessControlsUpdateRequest(_messages.Message): """A StorageObjectAccessControlsUpdateRequest object. Fields: @@ -1212,14 +1212,14 @@ class StorageObjectAccessControlsUpdateRequest(messages.Message): request body. """ - bucket = messages.StringField(1, required=True) - entity = messages.StringField(2, required=True) - generation = messages.IntegerField(3) - object = messages.StringField(4, required=True) - objectAccessControl = messages.MessageField('ObjectAccessControl', 5) + bucket = _messages.StringField(1, required=True) + entity = _messages.StringField(2, required=True) + generation = _messages.IntegerField(3) + object = _messages.StringField(4, required=True) + objectAccessControl = _messages.MessageField('ObjectAccessControl', 5) -class StorageObjectsComposeRequest(messages.Message): +class StorageObjectsComposeRequest(_messages.Message): """A StorageObjectsComposeRequest object. Enums: @@ -1239,7 +1239,7 @@ class StorageObjectsComposeRequest(messages.Message): object's current metageneration matches the given value. """ - class DestinationPredefinedAclValueValuesEnum(messages.Enum): + class DestinationPredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to the destination object. Values: @@ -1262,15 +1262,15 @@ class StorageObjectsComposeRequest(messages.Message): projectPrivate = 4 publicRead = 5 - composeRequest = messages.MessageField('ComposeRequest', 1) - destinationBucket = messages.StringField(2, required=True) - destinationObject = messages.StringField(3, required=True) - destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 4) - ifGenerationMatch = messages.IntegerField(5) - ifMetagenerationMatch = messages.IntegerField(6) + composeRequest = _messages.MessageField('ComposeRequest', 1) + destinationBucket = _messages.StringField(2, required=True) + destinationObject = _messages.StringField(3, required=True) + destinationPredefinedAcl = _messages.EnumField('DestinationPredefinedAclValueValuesEnum', 4) + ifGenerationMatch = _messages.IntegerField(5) + ifMetagenerationMatch = _messages.IntegerField(6) -class StorageObjectsCopyRequest(messages.Message): +class StorageObjectsCopyRequest(_messages.Message): """A StorageObjectsCopyRequest object. Enums: @@ -1315,7 +1315,7 @@ class StorageObjectsCopyRequest(messages.Message): sourceObject: Name of the source object. """ - class DestinationPredefinedAclValueValuesEnum(messages.Enum): + class DestinationPredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to the destination object. Values: @@ -1338,7 +1338,7 @@ class StorageObjectsCopyRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. @@ -1349,25 +1349,25 @@ class StorageObjectsCopyRequest(messages.Message): full = 0 noAcl = 1 - destinationBucket = messages.StringField(1, required=True) - destinationObject = messages.StringField(2, required=True) - destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) - ifGenerationMatch = messages.IntegerField(4) - ifGenerationNotMatch = messages.IntegerField(5) - ifMetagenerationMatch = messages.IntegerField(6) - ifMetagenerationNotMatch = messages.IntegerField(7) - ifSourceGenerationMatch = messages.IntegerField(8) - ifSourceGenerationNotMatch = messages.IntegerField(9) - ifSourceMetagenerationMatch = messages.IntegerField(10) - ifSourceMetagenerationNotMatch = messages.IntegerField(11) - object = messages.MessageField('Object', 12) - projection = messages.EnumField('ProjectionValueValuesEnum', 13) - sourceBucket = messages.StringField(14, required=True) - sourceGeneration = messages.IntegerField(15) - sourceObject = messages.StringField(16, required=True) - - -class StorageObjectsDeleteRequest(messages.Message): + destinationBucket = _messages.StringField(1, required=True) + destinationObject = _messages.StringField(2, required=True) + destinationPredefinedAcl = _messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) + ifGenerationMatch = _messages.IntegerField(4) + ifGenerationNotMatch = _messages.IntegerField(5) + ifMetagenerationMatch = _messages.IntegerField(6) + ifMetagenerationNotMatch = _messages.IntegerField(7) + ifSourceGenerationMatch = _messages.IntegerField(8) + ifSourceGenerationNotMatch = _messages.IntegerField(9) + ifSourceMetagenerationMatch = _messages.IntegerField(10) + ifSourceMetagenerationNotMatch = _messages.IntegerField(11) + object = _messages.MessageField('Object', 12) + projection = _messages.EnumField('ProjectionValueValuesEnum', 13) + sourceBucket = _messages.StringField(14, required=True) + sourceGeneration = _messages.IntegerField(15) + sourceObject = _messages.StringField(16, required=True) + + +class StorageObjectsDeleteRequest(_messages.Message): """A StorageObjectsDeleteRequest object. Fields: @@ -1385,20 +1385,20 @@ class StorageObjectsDeleteRequest(messages.Message): object: Name of the object. """ - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - ifGenerationMatch = messages.IntegerField(3) - ifGenerationNotMatch = messages.IntegerField(4) - ifMetagenerationMatch = messages.IntegerField(5) - ifMetagenerationNotMatch = messages.IntegerField(6) - object = messages.StringField(7, required=True) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + ifGenerationMatch = _messages.IntegerField(3) + ifGenerationNotMatch = _messages.IntegerField(4) + ifMetagenerationMatch = _messages.IntegerField(5) + ifMetagenerationNotMatch = _messages.IntegerField(6) + object = _messages.StringField(7, required=True) -class StorageObjectsDeleteResponse(messages.Message): +class StorageObjectsDeleteResponse(_messages.Message): """An empty StorageObjectsDelete response.""" -class StorageObjectsGetRequest(messages.Message): +class StorageObjectsGetRequest(_messages.Message): """A StorageObjectsGetRequest object. Enums: @@ -1420,7 +1420,7 @@ class StorageObjectsGetRequest(messages.Message): projection: Set of properties to return. Defaults to noAcl. """ - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl. Values: @@ -1430,17 +1430,17 @@ class StorageObjectsGetRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - ifGenerationMatch = messages.IntegerField(3) - ifGenerationNotMatch = messages.IntegerField(4) - ifMetagenerationMatch = messages.IntegerField(5) - ifMetagenerationNotMatch = messages.IntegerField(6) - object = messages.StringField(7, required=True) - projection = messages.EnumField('ProjectionValueValuesEnum', 8) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + ifGenerationMatch = _messages.IntegerField(3) + ifGenerationNotMatch = _messages.IntegerField(4) + ifMetagenerationMatch = _messages.IntegerField(5) + ifMetagenerationNotMatch = _messages.IntegerField(6) + object = _messages.StringField(7, required=True) + projection = _messages.EnumField('ProjectionValueValuesEnum', 8) -class StorageObjectsInsertRequest(messages.Message): +class StorageObjectsInsertRequest(_messages.Message): """A StorageObjectsInsertRequest object. Enums: @@ -1474,7 +1474,7 @@ class StorageObjectsInsertRequest(messages.Message): object resource specifies the acl property, when it defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this object. Values: @@ -1497,7 +1497,7 @@ class StorageObjectsInsertRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. @@ -1508,19 +1508,19 @@ class StorageObjectsInsertRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - contentEncoding = messages.StringField(2) - ifGenerationMatch = messages.IntegerField(3) - ifGenerationNotMatch = messages.IntegerField(4) - ifMetagenerationMatch = messages.IntegerField(5) - ifMetagenerationNotMatch = messages.IntegerField(6) - name = messages.StringField(7) - object = messages.MessageField('Object', 8) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) - projection = messages.EnumField('ProjectionValueValuesEnum', 10) + bucket = _messages.StringField(1, required=True) + contentEncoding = _messages.StringField(2) + ifGenerationMatch = _messages.IntegerField(3) + ifGenerationNotMatch = _messages.IntegerField(4) + ifMetagenerationMatch = _messages.IntegerField(5) + ifMetagenerationNotMatch = _messages.IntegerField(6) + name = _messages.StringField(7) + object = _messages.MessageField('Object', 8) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = _messages.EnumField('ProjectionValueValuesEnum', 10) -class StorageObjectsListRequest(messages.Message): +class StorageObjectsListRequest(_messages.Message): """A StorageObjectsListRequest object. Enums: @@ -1544,7 +1544,7 @@ class StorageObjectsListRequest(messages.Message): The default is false. For more information, see Object Versioning. """ - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl. Values: @@ -1554,16 +1554,16 @@ class StorageObjectsListRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - delimiter = messages.StringField(2) - maxResults = messages.IntegerField(3, variant=messages.Variant.UINT32) - pageToken = messages.StringField(4) - prefix = messages.StringField(5) - projection = messages.EnumField('ProjectionValueValuesEnum', 6) - versions = messages.BooleanField(7) + bucket = _messages.StringField(1, required=True) + delimiter = _messages.StringField(2) + maxResults = _messages.IntegerField(3, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(4) + prefix = _messages.StringField(5) + projection = _messages.EnumField('ProjectionValueValuesEnum', 6) + versions = _messages.BooleanField(7) -class StorageObjectsPatchRequest(messages.Message): +class StorageObjectsPatchRequest(_messages.Message): """A StorageObjectsPatchRequest object. Enums: @@ -1589,7 +1589,7 @@ class StorageObjectsPatchRequest(messages.Message): projection: Set of properties to return. Defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this object. Values: @@ -1612,7 +1612,7 @@ class StorageObjectsPatchRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to full. Values: @@ -1622,19 +1622,19 @@ class StorageObjectsPatchRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - ifGenerationMatch = messages.IntegerField(3) - ifGenerationNotMatch = messages.IntegerField(4) - ifMetagenerationMatch = messages.IntegerField(5) - ifMetagenerationNotMatch = messages.IntegerField(6) - object = messages.StringField(7, required=True) - objectResource = messages.MessageField('Object', 8) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) - projection = messages.EnumField('ProjectionValueValuesEnum', 10) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + ifGenerationMatch = _messages.IntegerField(3) + ifGenerationNotMatch = _messages.IntegerField(4) + ifMetagenerationMatch = _messages.IntegerField(5) + ifMetagenerationNotMatch = _messages.IntegerField(6) + object = _messages.StringField(7, required=True) + objectResource = _messages.MessageField('Object', 8) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = _messages.EnumField('ProjectionValueValuesEnum', 10) -class StorageObjectsRewriteRequest(messages.Message): +class StorageObjectsRewriteRequest(_messages.Message): """A StorageObjectsRewriteRequest object. Enums: @@ -1692,7 +1692,7 @@ class StorageObjectsRewriteRequest(messages.Message): sourceObject: Name of the source object. """ - class DestinationPredefinedAclValueValuesEnum(messages.Enum): + class DestinationPredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to the destination object. Values: @@ -1715,7 +1715,7 @@ class StorageObjectsRewriteRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. @@ -1726,27 +1726,27 @@ class StorageObjectsRewriteRequest(messages.Message): full = 0 noAcl = 1 - destinationBucket = messages.StringField(1, required=True) - destinationObject = messages.StringField(2, required=True) - destinationPredefinedAcl = messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) - ifGenerationMatch = messages.IntegerField(4) - ifGenerationNotMatch = messages.IntegerField(5) - ifMetagenerationMatch = messages.IntegerField(6) - ifMetagenerationNotMatch = messages.IntegerField(7) - ifSourceGenerationMatch = messages.IntegerField(8) - ifSourceGenerationNotMatch = messages.IntegerField(9) - ifSourceMetagenerationMatch = messages.IntegerField(10) - ifSourceMetagenerationNotMatch = messages.IntegerField(11) - maxBytesRewrittenPerCall = messages.IntegerField(12) - object = messages.MessageField('Object', 13) - projection = messages.EnumField('ProjectionValueValuesEnum', 14) - rewriteToken = messages.StringField(15) - sourceBucket = messages.StringField(16, required=True) - sourceGeneration = messages.IntegerField(17) - sourceObject = messages.StringField(18, required=True) - - -class StorageObjectsUpdateRequest(messages.Message): + destinationBucket = _messages.StringField(1, required=True) + destinationObject = _messages.StringField(2, required=True) + destinationPredefinedAcl = _messages.EnumField('DestinationPredefinedAclValueValuesEnum', 3) + ifGenerationMatch = _messages.IntegerField(4) + ifGenerationNotMatch = _messages.IntegerField(5) + ifMetagenerationMatch = _messages.IntegerField(6) + ifMetagenerationNotMatch = _messages.IntegerField(7) + ifSourceGenerationMatch = _messages.IntegerField(8) + ifSourceGenerationNotMatch = _messages.IntegerField(9) + ifSourceMetagenerationMatch = _messages.IntegerField(10) + ifSourceMetagenerationNotMatch = _messages.IntegerField(11) + maxBytesRewrittenPerCall = _messages.IntegerField(12) + object = _messages.MessageField('Object', 13) + projection = _messages.EnumField('ProjectionValueValuesEnum', 14) + rewriteToken = _messages.StringField(15) + sourceBucket = _messages.StringField(16, required=True) + sourceGeneration = _messages.IntegerField(17) + sourceObject = _messages.StringField(18, required=True) + + +class StorageObjectsUpdateRequest(_messages.Message): """A StorageObjectsUpdateRequest object. Enums: @@ -1772,7 +1772,7 @@ class StorageObjectsUpdateRequest(messages.Message): projection: Set of properties to return. Defaults to full. """ - class PredefinedAclValueValuesEnum(messages.Enum): + class PredefinedAclValueValuesEnum(_messages.Enum): """Apply a predefined set of access controls to this object. Values: @@ -1795,7 +1795,7 @@ class StorageObjectsUpdateRequest(messages.Message): projectPrivate = 4 publicRead = 5 - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to full. Values: @@ -1805,19 +1805,19 @@ class StorageObjectsUpdateRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - generation = messages.IntegerField(2) - ifGenerationMatch = messages.IntegerField(3) - ifGenerationNotMatch = messages.IntegerField(4) - ifMetagenerationMatch = messages.IntegerField(5) - ifMetagenerationNotMatch = messages.IntegerField(6) - object = messages.StringField(7, required=True) - objectResource = messages.MessageField('Object', 8) - predefinedAcl = messages.EnumField('PredefinedAclValueValuesEnum', 9) - projection = messages.EnumField('ProjectionValueValuesEnum', 10) + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + ifGenerationMatch = _messages.IntegerField(3) + ifGenerationNotMatch = _messages.IntegerField(4) + ifMetagenerationMatch = _messages.IntegerField(5) + ifMetagenerationNotMatch = _messages.IntegerField(6) + object = _messages.StringField(7, required=True) + objectResource = _messages.MessageField('Object', 8) + predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 9) + projection = _messages.EnumField('ProjectionValueValuesEnum', 10) -class StorageObjectsWatchAllRequest(messages.Message): +class StorageObjectsWatchAllRequest(_messages.Message): """A StorageObjectsWatchAllRequest object. Enums: @@ -1842,7 +1842,7 @@ class StorageObjectsWatchAllRequest(messages.Message): The default is false. For more information, see Object Versioning. """ - class ProjectionValueValuesEnum(messages.Enum): + class ProjectionValueValuesEnum(_messages.Enum): """Set of properties to return. Defaults to noAcl. Values: @@ -1852,13 +1852,13 @@ class StorageObjectsWatchAllRequest(messages.Message): full = 0 noAcl = 1 - bucket = messages.StringField(1, required=True) - channel = messages.MessageField('Channel', 2) - delimiter = messages.StringField(3) - maxResults = messages.IntegerField(4, variant=messages.Variant.UINT32) - pageToken = messages.StringField(5) - prefix = messages.StringField(6) - projection = messages.EnumField('ProjectionValueValuesEnum', 7) - versions = messages.BooleanField(8) + bucket = _messages.StringField(1, required=True) + channel = _messages.MessageField('Channel', 2) + delimiter = _messages.StringField(3) + maxResults = _messages.IntegerField(4, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(5) + prefix = _messages.StringField(6) + projection = _messages.EnumField('ProjectionValueValuesEnum', 7) + versions = _messages.BooleanField(8) -- GitLab From c5dd3bcb434945e1ef91ec110ee8ce4595675528 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 1 Jun 2015 15:21:26 -0700 Subject: [PATCH 127/295] Fix an issue with non-unique keys for custom JSON encoding. Currently, JSON encoding uses `type.definition_name()` as the key for custom JSON encodings. However, this makes use of the top-level `package` attribute, which we purposely set to just the api name -- meaning we're guaranteed conflicts in the case of simultaneous API versions for a given API. The fix is straightforward -- we do a little extra work to pull out the original module name, which we know will be unique. --- apitools/base/py/encoding.py | 38 ++++++++++++++++++++++++++--- apitools/base/py/encoding_test.py | 26 ++++++++++++++++++++ apitools/gen/extended_descriptor.py | 24 ++++++++++-------- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index c35e4cf..7c2cac0 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -520,7 +520,30 @@ _JSON_ENUM_MAPPINGS = {} _JSON_FIELD_MAPPINGS = {} -def AddCustomJsonEnumMapping(enum_type, python_name, json_name): +def _GetTypeKey(message_type, package): + """Get the prefix for this message type in mapping dicts.""" + key = message_type.definition_name() + if package and key.startswith(package + '.'): + module_name = message_type.__module__ + # We normalize '__main__' to something unique, if possible. + if module_name == '__main__': + try: + file_name = sys.modules[module_name].__file__ + except (AttributeError, KeyError): + pass + else: + base_name = os.path.basename(file_name) + split_name = os.path.splitext(base_name) + if len(split_name) == 1: + module_name = unicode(base_name) + else: + module_name = u'.'.join(split_name[:-1]) + key = module_name + '.' + key.partition('.')[2] + return key + + +def AddCustomJsonEnumMapping(enum_type, python_name, json_name, + package=''): """Add a custom wire encoding for a given enum value. This is primarily used in generated code, to handle enum values @@ -530,11 +553,14 @@ def AddCustomJsonEnumMapping(enum_type, python_name, json_name): enum_type: (messages.Enum) An enum type python_name: (string) Python name for this value. json_name: (string) JSON name to be used on the wire. + package: (basestring, optional) Package prefix for this enum, if + present. We strip this off the enum name in order to generate + unique keys. """ if not issubclass(enum_type, messages.Enum): raise exceptions.TypecheckError( 'Cannot set JSON enum mapping for non-enum "%s"' % enum_type) - enum_name = enum_type.definition_name() + enum_name = _GetTypeKey(enum_type, package) if python_name not in enum_type.names(): raise exceptions.InvalidDataError( 'Enum value %s not a value for type %s' % (python_name, enum_type)) @@ -543,7 +569,8 @@ def AddCustomJsonEnumMapping(enum_type, python_name, json_name): field_mappings[python_name] = json_name -def AddCustomJsonFieldMapping(message_type, python_name, json_name): +def AddCustomJsonFieldMapping(message_type, python_name, json_name, + package=''): """Add a custom wire encoding for a given message field. This is primarily used in generated code, to handle enum values @@ -553,12 +580,15 @@ def AddCustomJsonFieldMapping(message_type, python_name, json_name): message_type: (messages.Message) A message type python_name: (string) Python name for this value. json_name: (string) JSON name to be used on the wire. + package: (basestring, optional) Package prefix for this message, if + present. We strip this off the message name in order to generate + unique keys. """ if not issubclass(message_type, messages.Message): raise exceptions.TypecheckError( 'Cannot set JSON field mapping for ' 'non-message "%s"' % message_type) - message_name = message_type.definition_name() + message_name = _GetTypeKey(message_type, package) try: _ = message_type.field_by_name(python_name) except KeyError: diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 0d10d8b..7ec69de 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -1,6 +1,7 @@ import base64 import datetime import json +import sys from protorpc import message_types from protorpc import messages @@ -342,3 +343,28 @@ class EncodingTest(unittest2.TestCase): 'TimeMessage(\n ' 'timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, ' 'tzinfo=TimeZoneOffset(datetime.timedelta(0))),\n)') + + def testPackageMappingsNoPackage(self): + this_module_name = util.get_package_for_module(__name__) + full_type_name = 'MessageWithEnum.ThisEnum' + full_key = '%s.%s' % (this_module_name, full_type_name) + self.assertEqual(full_key, + encoding._GetTypeKey(MessageWithEnum.ThisEnum, '')) + + def testPackageMappingsWithPackage(self): + this_module_name = util.get_package_for_module(__name__) + full_type_name = 'MessageWithEnum.ThisEnum' + full_key = '%s.%s' % (this_module_name, full_type_name) + this_module = sys.modules[__name__] + new_package = 'new_package' + try: + setattr(this_module, 'package', new_package) + new_key = '%s.%s' % (new_package, full_type_name) + self.assertEqual( + new_key, + encoding._GetTypeKey(MessageWithEnum.ThisEnum, '')) + self.assertEqual( + full_key, + encoding._GetTypeKey(MessageWithEnum.ThisEnum, new_package)) + finally: + delattr(this_module, 'package') diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 103aa7b..5274100 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -149,9 +149,11 @@ def _WriteFile(file_descriptor, package, version, proto_printer): proto_printer.PrintPreamble(package, version, file_descriptor) _PrintEnums(proto_printer, file_descriptor.enum_types) _PrintMessages(proto_printer, file_descriptor.message_types) - custom_json_mappings = _FetchCustomMappings(file_descriptor.enum_types) + custom_json_mappings = _FetchCustomMappings( + file_descriptor.enum_types, file_descriptor.package) custom_json_mappings.extend( - _FetchCustomMappings(file_descriptor.message_types)) + _FetchCustomMappings( + file_descriptor.message_types, file_descriptor.package)) for mapping in custom_json_mappings: proto_printer.PrintCustomJsonMapping(mapping) @@ -183,29 +185,31 @@ def PrintIndentedDescriptions(printer, ls, name, prefix=''): printer(line) -def _FetchCustomMappings(descriptor_ls): +def _FetchCustomMappings(descriptor_ls, package): """Find and return all custom mappings for descriptors in descriptor_ls.""" custom_mappings = [] for descriptor in descriptor_ls: if isinstance(descriptor, ExtendedEnumDescriptor): custom_mappings.extend( - _FormatCustomJsonMapping('Enum', m, descriptor) + _FormatCustomJsonMapping('Enum', m, descriptor, package) for m in descriptor.enum_mappings) elif isinstance(descriptor, ExtendedMessageDescriptor): custom_mappings.extend( - _FormatCustomJsonMapping('Field', m, descriptor) + _FormatCustomJsonMapping('Field', m, descriptor, package) for m in descriptor.field_mappings) - custom_mappings.extend(_FetchCustomMappings(descriptor.enum_types)) custom_mappings.extend( - _FetchCustomMappings(descriptor.message_types)) + _FetchCustomMappings(descriptor.enum_types, package)) + custom_mappings.extend( + _FetchCustomMappings(descriptor.message_types, package)) return custom_mappings -def _FormatCustomJsonMapping(mapping_type, mapping, descriptor): +def _FormatCustomJsonMapping(mapping_type, mapping, descriptor, package): return '\n'.join(( 'encoding.AddCustomJson%sMapping(' % mapping_type, - " %s, '%s', '%s')" % (descriptor.full_name, mapping.python_name, - mapping.json_name) + " %s, '%s', '%s'," % (descriptor.full_name, mapping.python_name, + mapping.json_name), + ' package=%r)' % package, )) -- GitLab From 046fd2689d90ebd9c2abb7512a213fc0d7c13b69 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 1 Jun 2015 15:27:48 -0700 Subject: [PATCH 128/295] Fix a pair of missing imports. --- apitools/base/py/encoding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 7c2cac0..5325481 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -6,7 +6,8 @@ import collections import datetime import json import logging - +import os +import sys from protorpc import message_types from protorpc import messages -- GitLab From 206a06da14e122a1ad9c33b4f3702dfc61ae6791 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 3 Jun 2015 12:52:59 -0700 Subject: [PATCH 129/295] Switch to containers in Travis-CI. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 61f21e0..5bbdd50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +sudo: false env: - TOX_ENV=py26 - TOX_ENV=py27 -- GitLab From 7bdd56e1984898fbc242495d4d4870455fd27d50 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 2 Jun 2015 17:17:36 -0700 Subject: [PATCH 130/295] Add oauth2l as a custom script. This adds oauth2l, a small tool for testing/playing with OAuth2 for Google APIs. For more information, $ pip install google-apitools[cli] $ oauth2l help Also includes tests for oauth2l. Note that oauth2l is unsupported in python3 (until we drop flags/appcommands). --- apitools/base/py/credentials_lib.py | 10 +- apitools/data/__init__.py | 5 + apitools/data/apitools_client_secrets.json | 15 + apitools/scripts/__init__.py | 5 + apitools/scripts/oauth2l.py | 309 +++++++++++++++ apitools/scripts/oauth2l_test.py | 356 ++++++++++++++++++ .../scripts/testdata/fake_client_secrets.json | 15 + .../testdata/noninstalled_client_secrets.json | 3 + setup.py | 8 +- 9 files changed, 721 insertions(+), 5 deletions(-) create mode 100644 apitools/data/__init__.py create mode 100644 apitools/data/apitools_client_secrets.json create mode 100644 apitools/scripts/__init__.py create mode 100644 apitools/scripts/oauth2l.py create mode 100644 apitools/scripts/oauth2l_test.py create mode 100644 apitools/scripts/testdata/fake_client_secrets.json create mode 100644 apitools/scripts/testdata/noninstalled_client_secrets.json diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 3078943..11fc525 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -45,7 +45,8 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, service_account_name=None, service_account_keyfile=None, service_account_json_keyfile=None, api_key=None, # pylint: disable=unused-argument - client=None): # pylint: disable=unused-argument + client=None, # pylint: disable=unused-argument + oauth2client_args=None): """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) if ((service_account_name and not service_account_keyfile) or @@ -94,7 +95,8 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, return credentials credentials_filename = credentials_filename or os.path.expanduser( '~/.apitools.token') - credentials = CredentialsFromFile(credentials_filename, client_info) + credentials = CredentialsFromFile(credentials_filename, client_info, + oauth2client_args=oauth2client_args) if credentials is not None: return credentials raise exceptions.CredentialsError('Could not create valid credentials') @@ -428,7 +430,7 @@ def _GetRunFlowFlags(args=None): # TODO(craigcitro): Switch this from taking a path to taking a stream. -def CredentialsFromFile(path, client_info): +def CredentialsFromFile(path, client_info, oauth2client_args=None): """Read credentials from a file.""" credential_store = oauth2client.multistore_file.get_credential_storage( path, @@ -446,7 +448,7 @@ def CredentialsFromFile(path, client_info): # retry loop, they can ^C. try: flow = oauth2client.client.OAuth2WebServerFlow(**client_info) - flags = _GetRunFlowFlags() + flags = _GetRunFlowFlags(args=oauth2client_args) credentials = tools.run_flow(flow, credential_store, flags) break except (oauth2client.client.FlowExchangeError, SystemExit) as e: diff --git a/apitools/data/__init__.py b/apitools/data/__init__.py new file mode 100644 index 0000000..54fa3d5 --- /dev/null +++ b/apitools/data/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +"""Shared __init__.py for apitools.""" + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/apitools/data/apitools_client_secrets.json b/apitools/data/apitools_client_secrets.json new file mode 100644 index 0000000..5761d14 --- /dev/null +++ b/apitools/data/apitools_client_secrets.json @@ -0,0 +1,15 @@ +{ + "installed": { + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "client_email": "", + "client_id": "1042881264118.apps.googleusercontent.com", + "client_secret": "x_Tw5K8nnjoRAqULM9PFAC2b", + "client_x509_cert_url": "", + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "oob" + ], + "token_uri": "https://accounts.google.com/o/oauth2/token" + } +} diff --git a/apitools/scripts/__init__.py b/apitools/scripts/__init__.py new file mode 100644 index 0000000..54fa3d5 --- /dev/null +++ b/apitools/scripts/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +"""Shared __init__.py for apitools.""" + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/apitools/scripts/oauth2l.py b/apitools/scripts/oauth2l.py new file mode 100644 index 0000000..44bb9bc --- /dev/null +++ b/apitools/scripts/oauth2l.py @@ -0,0 +1,309 @@ +"""Command-line utility for fetching/inspecting credentials. + +oauth2l (pronounced "oauthtool") is a small utility for fetching +credentials, or inspecting existing credentials. Here we demonstrate +some sample use: + + $ oauth2l fetch userinfo.email bigquery compute + Fetched credentials of type: + oauth2client.client.OAuth2Credentials + Access token: + ya29.abcdefghijklmnopqrstuvwxyz123yessirree + $ oauth2l header userinfo.email + Authorization: Bearer ya29.zyxwvutsrqpnmolkjihgfedcba + $ oauth2l validate thisisnotatoken + + $ oauth2l validate ya29.zyxwvutsrqpnmolkjihgfedcba + $ oauth2l scopes ya29.abcdefghijklmnopqrstuvwxyz123yessirree + https://www.googleapis.com/auth/bigquery + https://www.googleapis.com/auth/compute + https://www.googleapis.com/auth/userinfo.email + +The `header` command is designed to be easy to use with `curl`: + + $ curl "$(oauth2l header bigquery)" \ + 'https://www.googleapis.com/bigquery/v2/projects' + +The token can also be printed in other formats, for easy chaining +into other programs: + + $ oauth2l fetch -f json_compact userinfo.email + + $ oauth2l fetch -f bare drive + ya29.suchT0kenManyCredentialsW0Wokyougetthepoint + +""" + +import httplib +import json +import logging +import os +import pkgutil +import sys +import textwrap + +import gflags as flags +from google.apputils import appcommands +import oauth2client.client + +import apitools.base.py as apitools_base +from apitools.base.py import cli as apitools_cli + +FLAGS = flags.FLAGS +# We could use a generated client here, but it's used for precisely +# one URL, with one parameter and no worries about URL encoding. Let's +# go with simple. +_OAUTH2_TOKENINFO_TEMPLATE = ( + 'https://www.googleapis.com/oauth2/v2/tokeninfo' + '?access_token={access_token}' +) + + +flags.DEFINE_string( + 'client_secrets', '', + 'If specified, use the client ID/secret from the named ' + 'file, which should be a client_secrets.json file as downloaded ' + 'from the Developer Console.') +flags.DEFINE_string( + 'credentials_filename', '', + '(optional) Filename for fetching/storing credentials.') +flags.DEFINE_string( + 'service_account_json_keyfile', '', + 'Filename for a JSON service account key downloaded from the Developer ' + 'Console.') + + +def GetDefaultClientInfo(): + client_secrets = json.loads(pkgutil.get_data( + 'apitools.data', 'apitools_client_secrets.json'))['installed'] + return { + 'client_id': client_secrets['client_id'], + 'client_secret': client_secrets['client_secret'], + 'user_agent': 'apitools/0.2 oauth2l/0.1', + } + + +def GetClientInfoFromFlags(): + """Fetch client info from FLAGS.""" + if FLAGS.client_secrets: + client_secrets_path = os.path.expanduser(FLAGS.client_secrets) + if not os.path.exists(client_secrets_path): + raise ValueError('Cannot find file: %s' % FLAGS.client_secrets) + with open(client_secrets_path) as client_secrets_file: + client_secrets = json.load(client_secrets_file) + if 'installed' not in client_secrets: + raise ValueError('Provided client ID must be for an installed app') + client_secrets = client_secrets['installed'] + return { + 'client_id': client_secrets['client_id'], + 'client_secret': client_secrets['client_secret'], + 'user_agent': 'apitools/0.2 oauth2l/0.1', + } + else: + return GetDefaultClientInfo() + + +def _ExpandScopes(scopes): + scope_prefix = 'https://www.googleapis.com/auth/' + return [s if s.startswith('https://') else scope_prefix + s + for s in scopes] + + +def _PrettyJson(data): + return json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')) + + +def _CompactJson(data): + return json.dumps(data, sort_keys=True, separators=(',', ':')) + + +def _Format(fmt, credentials): + """Format credentials according to fmt.""" + if fmt == 'bare': + return credentials.access_token + elif fmt == 'header': + return 'Authorization: Bearer %s' % credentials.access_token + elif fmt == 'json': + return _PrettyJson(json.loads(credentials.to_json())) + elif fmt == 'json_compact': + return _CompactJson(json.loads(credentials.to_json())) + elif fmt == 'pretty': + format_str = textwrap.dedent('\n'.join([ + 'Fetched credentials of type:', + ' {credentials_type.__module__}.{credentials_type.__name__}', + 'Access token:', + ' {credentials.access_token}', + ])) + return format_str.format(credentials=credentials, + credentials_type=type(credentials)) + raise ValueError('Unknown format: {}'.format(fmt)) + +_FORMATS = set(('bare', 'header', 'json', 'json_compact', 'pretty')) + + +def _GetTokenScopes(access_token): + """Return the list of valid scopes for the given token as a list.""" + url = _OAUTH2_TOKENINFO_TEMPLATE.format(access_token=access_token) + response = apitools_base.MakeRequest( + apitools_base.GetHttp(), apitools_base.Request(url)) + if response.status_code not in [httplib.OK, httplib.BAD_REQUEST]: + raise apitools_base.HttpError.FromResponse(response) + if response.status_code == httplib.BAD_REQUEST: + return [] + return json.loads(response.content)['scope'].split(' ') + + +def _ValidateToken(access_token): + """Return True iff the provided access token is valid.""" + return bool(_GetTokenScopes(access_token)) + + +def FetchCredentials(scopes, client_info=None, credentials_filename=None): + """Fetch a credential for the given client_info and scopes.""" + client_info = client_info or GetClientInfoFromFlags() + scopes = _ExpandScopes(scopes) + if not scopes: + raise ValueError('No scopes provided') + credentials_filename = credentials_filename or FLAGS.credentials_filename + # TODO(craigcitro): Remove this logging nonsense once we quiet the + # spurious logging in oauth2client. + old_level = logging.getLogger().level + logging.getLogger().setLevel(logging.ERROR) + credentials = apitools_base.GetCredentials( + 'oauth2l', scopes, credentials_filename=credentials_filename, + service_account_json_keyfile=FLAGS.service_account_json_keyfile, + oauth2client_args='', **client_info) + logging.getLogger().setLevel(old_level) + if not _ValidateToken(credentials.access_token): + credentials.refresh(apitools_base.GetHttp()) + return credentials + + +class _Email(apitools_cli.NewCmd): + + """Get user email.""" + + usage = 'email ' + + def RunWithArgs(self, access_token): + """Print the email address for this token, if possible.""" + userinfo = apitools_base.GetUserinfo( + oauth2client.client.AccessTokenCredentials(access_token, + 'oauth2l/1.0')) + user_email = userinfo.get('email') + if user_email: + print user_email + + +class _Fetch(apitools_cli.NewCmd): + + """Fetch credentials.""" + + usage = 'fetch [ ...]' + + def __init__(self, name, flag_values): + super(_Fetch, self).__init__(name, flag_values) + flags.DEFINE_enum( + 'credentials_format', 'pretty', sorted(_FORMATS), + 'Output format for token.', + short_name='f', flag_values=flag_values) + + def RunWithArgs(self, *scopes): + """Fetch a valid access token and display it.""" + credentials = FetchCredentials(scopes) + print _Format(FLAGS.credentials_format.lower(), credentials) + + +class _Header(apitools_cli.NewCmd): + + """Print credentials for a header.""" + + usage = 'header [ ...]' + + def RunWithArgs(self, *scopes): + """Fetch a valid access token and display it formatted for a header.""" + print _Format('header', FetchCredentials(scopes)) + + +class _Scopes(apitools_cli.NewCmd): + + """Get the list of scopes for a token.""" + + usage = 'scopes ' + + def RunWithArgs(self, access_token): + """Print the list of scopes for a valid token.""" + scopes = _GetTokenScopes(access_token) + if not scopes: + return 1 + for scope in sorted(scopes): + print scope + + +class _Userinfo(apitools_cli.NewCmd): + + """Get userinfo.""" + + usage = 'userinfo ' + + def __init__(self, name, flag_values): + super(_Userinfo, self).__init__(name, flag_values) + flags.DEFINE_enum( + 'format', 'json', sorted(('json', 'json_compact')), + 'Output format for userinfo.', + short_name='f', flag_values=flag_values) + + def RunWithArgs(self, access_token): + """Print the userinfo for this token (if we have the right scopes).""" + userinfo = apitools_base.GetUserinfo( + oauth2client.client.AccessTokenCredentials(access_token, + 'oauth2l/1.0')) + if FLAGS.format == 'json': + print _PrettyJson(userinfo) + else: + print _CompactJson(userinfo) + + +class _Validate(apitools_cli.NewCmd): + + """Validate a token.""" + + usage = 'validate ' + + def RunWithArgs(self, access_token): + """Validate an access token. Exits with 0 if valid, 1 otherwise.""" + return 1 - (_ValidateToken(access_token)) + + +def run_main(): # pylint:disable=invalid-name + """Function to be used as setuptools script entry point.""" + # Put the flags for this module somewhere the flags module will look + # for them. + + # pylint:disable=protected-access + new_name = flags._GetMainModule() + sys.modules[new_name] = sys.modules['__main__'] + for flag in FLAGS.FlagsByModuleDict().get(__name__, []): + FLAGS._RegisterFlagByModule(new_name, flag) + for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): + FLAGS._RegisterKeyFlagForModule(new_name, key_flag) + # pylint:enable=protected-access + + # Now set __main__ appropriately so that appcommands will be + # happy. + sys.modules['__main__'] = sys.modules[__name__] + appcommands.Run() + sys.modules['__main__'] = sys.modules.pop(new_name) + + +def main(unused_argv): + appcommands.AddCmd('email', _Email) + appcommands.AddCmd('fetch', _Fetch) + appcommands.AddCmd('header', _Header) + appcommands.AddCmd('scopes', _Scopes) + appcommands.AddCmd('userinfo', _Userinfo) + appcommands.AddCmd('validate', _Validate) + + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/scripts/oauth2l_test.py b/apitools/scripts/oauth2l_test.py new file mode 100644 index 0000000..8f25b35 --- /dev/null +++ b/apitools/scripts/oauth2l_test.py @@ -0,0 +1,356 @@ +"""Tests for oauth2l.""" + +import json +import os +import sys + +import mock +import oauth2client.client +import six +from six.moves import http_client +import unittest2 + +import apitools.base.py as apitools_base + +_OAUTH2L_MAIN_RUN = False + +if six.PY2: + import gflags as flags + from google.apputils import appcommands + from apitools.scripts import oauth2l + FLAGS = flags.FLAGS + + +class _FakeResponse(object): + + def __init__(self, status_code, scopes=None): + self.status_code = status_code + if self.status_code == http_client.OK: + self.content = json.dumps({'scope': ' '.join(scopes or [])}) + else: + self.content = 'Error' + self.info = str(http_client.responses[self.status_code]) + self.request_url = 'some-url' + + +def _GetCommandOutput(t, command_name, command_argv): + global _OAUTH2L_MAIN_RUN # pylint: disable=global-statement + if not _OAUTH2L_MAIN_RUN: + oauth2l.main(None) + _OAUTH2L_MAIN_RUN = True + command = appcommands.GetCommandByName(command_name) + if command is None: + t.fail('Unknown command: %s' % command_name) + orig_stdout = sys.stdout + new_stdout = six.StringIO() + try: + sys.stdout = new_stdout + command.CommandRun([command_name] + command_argv) + finally: + sys.stdout = orig_stdout + FLAGS.Reset() + new_stdout.seek(0) + return new_stdout.getvalue().rstrip() + + +@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') +class TestTest(unittest2.TestCase): + + def testOutput(self): + self.assertRaises(AssertionError, + _GetCommandOutput, self, 'foo', []) + + +@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') +class Oauth2lFormattingTest(unittest2.TestCase): + + def setUp(self): + # Set up an access token to use + self.access_token = 'ya29.abdefghijklmnopqrstuvwxyz' + self.user_agent = 'oauth2l/1.0' + self.credentials = oauth2client.client.AccessTokenCredentials( + self.access_token, self.user_agent) + + def _Args(self, credentials_format): + return ['--credentials_format=' + credentials_format, 'userinfo.email'] + + def testFormatBare(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'fetch', self._Args('bare')) + self.assertEqual(self.access_token, output) + self.assertEqual(1, mock_credentials.call_count) + + def testFormatHeader(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'fetch', self._Args('header')) + header = 'Authorization: Bearer %s' % self.access_token + self.assertEqual(header, output) + self.assertEqual(1, mock_credentials.call_count) + + def testHeaderCommand(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'header', ['userinfo.email']) + header = 'Authorization: Bearer %s' % self.access_token + self.assertEqual(header, output) + self.assertEqual(1, mock_credentials.call_count) + + def testFormatJson(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'fetch', self._Args('json')) + output_lines = [l.strip() for l in output.splitlines()] + expected_lines = [ + '"_class": "AccessTokenCredentials",', + '"access_token": "%s",' % self.access_token, + ] + for line in expected_lines: + self.assertIn(line, output_lines) + self.assertEqual(1, mock_credentials.call_count) + + def testFormatJsonCompact(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'fetch', + self._Args('json_compact')) + expected_clauses = [ + '"_class":"AccessTokenCredentials",', + '"access_token":"%s",' % self.access_token, + ] + for clause in expected_clauses: + self.assertIn(clause, output) + self.assertEqual(1, len(output.splitlines())) + self.assertEqual(1, mock_credentials.call_count) + + def testFormatPretty(self): + with mock.patch.object(oauth2l, 'FetchCredentials', + return_value=self.credentials, + autospec=True) as mock_credentials: + output = _GetCommandOutput(self, 'fetch', self._Args('pretty')) + expecteds = ['oauth2client.client.AccessTokenCredentials', + self.access_token] + for expected in expecteds: + self.assertIn(expected, output) + self.assertEqual(1, mock_credentials.call_count) + + def testFakeFormat(self): + self.assertRaises(ValueError, + oauth2l._Format, 'xml', self.credentials) + + +@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') +class TestFetch(unittest2.TestCase): + + def setUp(self): + # Set up an access token to use + self.access_token = 'ya29.abdefghijklmnopqrstuvwxyz' + self.user_agent = 'oauth2l/1.0' + self.credentials = oauth2client.client.AccessTokenCredentials( + self.access_token, self.user_agent) + + def testNoScopes(self): + output = _GetCommandOutput(self, 'fetch', []) + self.assertEqual( + 'Exception raised in fetch operation: No scopes provided', + output) + + def testScopes(self): + expected_scopes = [ + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/cloud-platform', + ] + with mock.patch.object(apitools_base, 'GetCredentials', + return_value=self.credentials, + autospec=True) as mock_fetch: + with mock.patch.object(oauth2l, '_GetTokenScopes', + return_value=expected_scopes, + autospec=True) as mock_get_scopes: + output = _GetCommandOutput( + self, 'fetch', ['userinfo.email', 'cloud-platform']) + self.assertIn(self.access_token, output) + self.assertEqual(1, mock_fetch.call_count) + args, _ = mock_fetch.call_args + self.assertEqual(expected_scopes, args[-1]) + self.assertEqual(1, mock_get_scopes.call_count) + self.assertEqual((self.access_token,), + mock_get_scopes.call_args[0]) + + def testCredentialsRefreshed(self): + with mock.patch.object(apitools_base, 'GetCredentials', + return_value=self.credentials, + autospec=True) as mock_fetch: + with mock.patch.object(oauth2l, '_ValidateToken', + return_value=False, + autospec=True) as mock_validate: + with mock.patch.object(self.credentials, 'refresh', + return_value=None, + autospec=True) as mock_refresh: + output = _GetCommandOutput(self, 'fetch', + ['userinfo.email']) + self.assertIn(self.access_token, output) + self.assertEqual(1, mock_fetch.call_count) + self.assertEqual(1, mock_validate.call_count) + self.assertEqual(1, mock_refresh.call_count) + + def testDefaultClientInfo(self): + with mock.patch.object(apitools_base, 'GetCredentials', + return_value=self.credentials, + autospec=True) as mock_fetch: + with mock.patch.object(oauth2l, '_ValidateToken', + return_value=True, + autospec=True) as mock_validate: + output = _GetCommandOutput(self, 'fetch', ['userinfo.email']) + self.assertIn(self.access_token, output) + self.assertEqual(1, mock_fetch.call_count) + _, kwargs = mock_fetch.call_args + self.assertEqual( + '1042881264118.apps.googleusercontent.com', + kwargs['client_id']) + self.assertEqual(1, mock_validate.call_count) + + def testMissingClientSecrets(self): + try: + FLAGS.client_secrets = '/non/existent/file' + self.assertRaises( + ValueError, + oauth2l.GetClientInfoFromFlags) + finally: + FLAGS.Reset() + + def testWrongClientSecretsFormat(self): + client_secrets_path = os.path.join( + os.path.dirname(__file__), + 'testdata/noninstalled_client_secrets.json') + try: + FLAGS.client_secrets = client_secrets_path + self.assertRaises( + ValueError, + oauth2l.GetClientInfoFromFlags) + finally: + FLAGS.Reset() + + def testCustomClientInfo(self): + client_secrets_path = os.path.join( + os.path.dirname(__file__), 'testdata/fake_client_secrets.json') + with mock.patch.object(apitools_base, 'GetCredentials', + return_value=self.credentials, + autospec=True) as mock_fetch: + with mock.patch.object(oauth2l, '_ValidateToken', + return_value=True, + autospec=True) as mock_validate: + fetch_args = [ + '--client_secrets=' + client_secrets_path, + 'userinfo.email'] + output = _GetCommandOutput(self, 'fetch', fetch_args) + self.assertIn(self.access_token, output) + self.assertEqual(1, mock_fetch.call_count) + _, kwargs = mock_fetch.call_args + self.assertEqual('144169.apps.googleusercontent.com', + kwargs['client_id']) + self.assertEqual('awesomesecret', + kwargs['client_secret']) + self.assertEqual(1, mock_validate.call_count) + + +@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') +class TestOtherCommands(unittest2.TestCase): + + def setUp(self): + # Set up an access token to use + self.access_token = 'ya29.abdefghijklmnopqrstuvwxyz' + self.user_agent = 'oauth2l/1.0' + self.credentials = oauth2client.client.AccessTokenCredentials( + self.access_token, self.user_agent) + + def testEmail(self): + user_info = {'email': 'foo@example.com'} + with mock.patch.object(apitools_base, 'GetUserinfo', + return_value=user_info, + autospec=True) as mock_get_userinfo: + output = _GetCommandOutput(self, 'email', [self.access_token]) + self.assertEqual(user_info['email'], output) + self.assertEqual(1, mock_get_userinfo.call_count) + self.assertEqual(self.access_token, + mock_get_userinfo.call_args[0][0].access_token) + + def testNoEmail(self): + with mock.patch.object(apitools_base, 'GetUserinfo', + return_value={}, + autospec=True) as mock_get_userinfo: + output = _GetCommandOutput(self, 'email', [self.access_token]) + self.assertEqual('', output) + self.assertEqual(1, mock_get_userinfo.call_count) + + def testUserinfo(self): + user_info = {'email': 'foo@example.com'} + with mock.patch.object(apitools_base, 'GetUserinfo', + return_value=user_info, + autospec=True) as mock_get_userinfo: + output = _GetCommandOutput(self, 'userinfo', [self.access_token]) + self.assertEqual(json.dumps(user_info, indent=4), output) + self.assertEqual(1, mock_get_userinfo.call_count) + self.assertEqual(self.access_token, + mock_get_userinfo.call_args[0][0].access_token) + + def testUserinfoCompact(self): + user_info = {'email': 'foo@example.com'} + with mock.patch.object(apitools_base, 'GetUserinfo', + return_value=user_info, + autospec=True) as mock_get_userinfo: + output = _GetCommandOutput( + self, 'userinfo', ['--format=json_compact', self.access_token]) + self.assertEqual(json.dumps(user_info, separators=(',', ':')), + output) + self.assertEqual(1, mock_get_userinfo.call_count) + self.assertEqual(self.access_token, + mock_get_userinfo.call_args[0][0].access_token) + + def testScopes(self): + scopes = [u'https://www.googleapis.com/auth/userinfo.email', + u'https://www.googleapis.com/auth/cloud-platform'] + response = _FakeResponse(http_client.OK, scopes=scopes) + with mock.patch.object(apitools_base, 'MakeRequest', + return_value=response, + autospec=True) as mock_make_request: + output = _GetCommandOutput(self, 'scopes', [self.access_token]) + self.assertEqual(sorted(scopes), output.splitlines()) + self.assertEqual(1, mock_make_request.call_count) + + def testValidate(self): + scopes = [u'https://www.googleapis.com/auth/userinfo.email', + u'https://www.googleapis.com/auth/cloud-platform'] + response = _FakeResponse(http_client.OK, scopes=scopes) + with mock.patch.object(apitools_base, 'MakeRequest', + return_value=response, + autospec=True) as mock_make_request: + output = _GetCommandOutput(self, 'validate', [self.access_token]) + self.assertEqual('', output) + self.assertEqual(1, mock_make_request.call_count) + + def testBadResponseCode(self): + response = _FakeResponse(http_client.BAD_REQUEST) + with mock.patch.object(apitools_base, 'MakeRequest', + return_value=response, + autospec=True) as mock_make_request: + output = _GetCommandOutput(self, 'scopes', [self.access_token]) + self.assertEqual('', output) + self.assertEqual(1, mock_make_request.call_count) + + def testUnexpectedResponseCode(self): + response = _FakeResponse(http_client.INTERNAL_SERVER_ERROR) + with mock.patch.object(apitools_base, 'MakeRequest', + return_value=response, + autospec=True) as mock_make_request: + output = _GetCommandOutput(self, 'scopes', [self.access_token]) + self.assertIn(str(http_client.responses[response.status_code]), + output) + self.assertIn('Exception raised in scopes operation: HttpError', + output) + self.assertEqual(1, mock_make_request.call_count) diff --git a/apitools/scripts/testdata/fake_client_secrets.json b/apitools/scripts/testdata/fake_client_secrets.json new file mode 100644 index 0000000..f1fabe6 --- /dev/null +++ b/apitools/scripts/testdata/fake_client_secrets.json @@ -0,0 +1,15 @@ +{ + "installed": { + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "client_email": "", + "client_id": "144169.apps.googleusercontent.com", + "client_secret": "awesomesecret", + "client_x509_cert_url": "", + "redirect_uris": [ + "urn:ietf:wg:oauth:2.0:oob", + "oob" + ], + "token_uri": "https://accounts.google.com/o/oauth2/token" + } +} diff --git a/apitools/scripts/testdata/noninstalled_client_secrets.json b/apitools/scripts/testdata/noninstalled_client_secrets.json new file mode 100644 index 0000000..6e67027 --- /dev/null +++ b/apitools/scripts/testdata/noninstalled_client_secrets.json @@ -0,0 +1,3 @@ +{ + "webapp": {} +} diff --git a/setup.py b/setup.py index 0da6c70..85f9630 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,8 @@ TESTING_PACKAGES = [ CONSOLE_SCRIPTS = [ 'gen_client = apitools.gen.gen_client:run_main', - ] + 'oauth2l = apitools.scripts.oauth2l:run_main [cli]', +] py_version = platform.python_version() @@ -78,6 +79,11 @@ setuptools.setup( 'cli': CLI_PACKAGES, 'testing': TESTING_PACKAGES, }, + # Add in any packaged data. + include_package_data=True, + package_data={ + 'apitools.data': ['*'], + }, # PyPI package information. classifiers=[ 'License :: OSI Approved :: Apache Software License', -- GitLab From 15543534fb4ab1bebb38385a61beb4b5c1090d7f Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Wed, 10 Jun 2015 14:20:11 -0700 Subject: [PATCH 131/295] Protect GCE metadata server cache file with a configurable lock This avoids IOErrors/OSErrors caused by multiple threads opening the cache file simultaneously using different file descriptors. It also allows callers to override the lock with a multi-process-safe lock via SetCredentialCacheFileLock. --- apitools/base/py/credentials_lib.py | 80 ++++++++++++++++++----------- 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 11fc525..d7e6133 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -5,6 +5,7 @@ from __future__ import print_function import datetime import json import os +import threading import httplib2 import oauth2client @@ -38,6 +39,15 @@ __all__ = [ ] +# Lock when accessing the cache file to avoid resource contention. +cache_file_lock = threading.Lock() + + +def SetCredentialsCacheFileLock(lock): + global cache_file_lock # pylint: disable=global-statement + cache_file_lock = lock + + # TODO(craigcitro): Expose the extra args here somewhere higher up, # possibly as flags in the generated CLI. def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, @@ -199,20 +209,23 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): 'scopes': sorted(list(scopes)) if scopes else None, 'svc_acct_name': self.__service_account_name, } - if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( - cache_filename, 'r+b', 'rb') - try: - locked_file.open_and_lock() - cached_creds_str = locked_file.file_handle().read() - if cached_creds_str: - # Cached credentials metadata dict. - cached_creds = json.loads(cached_creds_str) - if creds['svc_acct_name'] == cached_creds['svc_acct_name']: - if creds['scopes'] in (None, cached_creds['scopes']): - scopes = cached_creds['scopes'] - finally: - locked_file.unlock_and_close() + with cache_file_lock: + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + cached_creds_str = locked_file.file_handle().read() + if cached_creds_str: + # Cached credentials metadata dict. + cached_creds = json.loads(cached_creds_str) + if (creds['svc_acct_name'] == + cached_creds['svc_acct_name']): + if (creds['scopes'] in + (None, cached_creds['scopes'])): + scopes = cached_creds['scopes'] + finally: + locked_file.unlock_and_close() return scopes def _WriteCacheFile(self, cache_filename, scopes): @@ -225,22 +238,23 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): cache_filename: Cache filename to check. scopes: Scopes for the desired credentials. """ - if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( - cache_filename, 'r+b', 'rb') - try: - locked_file.open_and_lock() - if locked_file.is_locked(): - creds = { # Credentials metadata dict. - 'scopes': sorted(list(scopes)), - 'svc_acct_name': self.__service_account_name} - locked_file.file_handle().write( - json.dumps(creds, encoding='ascii')) - # If it's not locked, the locking process will - # write the same data to the file, so just - # continue. - finally: - locked_file.unlock_and_close() + with cache_file_lock: + if _EnsureFileExists(cache_filename): + locked_file = oauth2client.locked_file.LockedFile( + cache_filename, 'r+b', 'rb') + try: + locked_file.open_and_lock() + if locked_file.is_locked(): + creds = { # Credentials metadata dict. + 'scopes': sorted(list(scopes)), + 'svc_acct_name': self.__service_account_name} + locked_file.file_handle().write( + json.dumps(creds, encoding='ascii')) + # If it's not locked, the locking process will + # write the same data to the file, so just + # continue. + finally: + locked_file.unlock_and_close() def _ScopesFromMetadataServer(self, scopes): if not util.DetectGce(): @@ -348,7 +362,11 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): @classmethod def from_json(cls, json_data): data = json.loads(json_data) - credentials = GceAssertionCredentials(scopes=[data['scope']]) + kwargs = {} + if 'cache_filename' in data.get('kwargs', []): + kwargs['cache_filename'] = data['kwargs']['cache_filename'] + credentials = GceAssertionCredentials(scopes=[data['scope']], + **kwargs) if 'access_token' in data: credentials.access_token = data['access_token'] if 'token_expiry' in data: -- GitLab From 31ba3a16342bea79118960160b67357a9ea43b1e Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 12 Jun 2015 10:58:43 -0700 Subject: [PATCH 132/295] Small formatting cleanup in batch. Fix a not-actually-needed bit of stripping in batch message formatting. --- apitools/base/py/batch.py | 5 ----- apitools/base/py/batch_test.py | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 4910d4a..2cf03a4 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -321,11 +321,6 @@ class BatchHttpRequest(object): gen.flatten(msg, unixfrom=False) body = str_io.getvalue() - # Strip off the \n\n that the MIME lib tacks onto the end of the - # payload. - if request.body is None: - body = body[:-2] - return status_line + body def _DeserializeResponse(self, payload): diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py index cf77364..7c20171 100644 --- a/apitools/base/py/batch_test.py +++ b/apitools/base/py/batch_test.py @@ -332,6 +332,8 @@ class BatchTest(unittest2.TestCase): 'Content-Type: protocol/version', 'MIME-Version: 1.0', 'Host: ', + '', + '', ]) batch_request = batch.BatchHttpRequest('https://www.example.com') self.assertEqual(expected_serialized_request, -- GitLab From 5f1b64ef568a550d856ed286bbc1155db6884b44 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 17 Jun 2015 00:07:52 -0700 Subject: [PATCH 133/295] Fix a bug in enum handling and dictionaries. Previously, we didn't correctly handle decoding Enum fields as value types in a dictionary; the root cause here was that I forgot that EnumField isn't a subclass of MessageField. :) Adds a fix and a test. (Reviewed internally.) --- apitools/base/py/encoding.py | 2 ++ apitools/base/py/encoding_test.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 5325481..07e2f68 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -400,6 +400,8 @@ def _DecodeUnrecognizedFields(message, pair_type): value_type = pair_type.field_by_name('value') if isinstance(value_type, messages.MessageField): decoded_value = DictToMessage(value, pair_type.value.message_type) + elif isinstance(value_type, messages.EnumField): + decoded_value = pair_type.value.type(value) else: decoded_value = value new_pair = pair_type(key=str(unknown_field), value=decoded_value) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 7ec69de..77bbe0a 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -33,6 +33,21 @@ class AdditionalPropertiesMessage(messages.Message): key = messages.StringField(1) value = messages.StringField(2) + additional_properties = messages.MessageField( + 'AdditionalProperty', 1, repeated=True) + + +@encoding.MapUnrecognizedFields('additional_properties') +class UnrecognizedEnumMessage(messages.Message): + + class ThisEnum(messages.Enum): + VALUE_ONE = 1 + VALUE_TWO = 2 + + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.EnumField('UnrecognizedEnumMessage.ThisEnum', 2) + additional_properties = messages.MessageField( AdditionalProperty, 1, repeated=True) @@ -188,6 +203,14 @@ class EncodingTest(unittest2.TestCase): self.assertEqual(1, len(result.additional_properties)) self.assertEqual(0, result.additional_properties[0].value.index) + def testUnrecognizedEnum(self): + json_msg = '{"input": "VALUE_ONE"}' + result = encoding.JsonToMessage( + UnrecognizedEnumMessage, json_msg) + self.assertEqual(1, len(result.additional_properties)) + self.assertEqual(UnrecognizedEnumMessage.ThisEnum.VALUE_ONE, + result.additional_properties[0].value) + def testNestedFieldMapping(self): nested_msg = AdditionalPropertiesMessage() nested_msg.additional_properties = [ -- GitLab From b431a72ccc02dd8a5c360367b5f2edd2154240de Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Mon, 22 Jun 2015 10:08:55 -0700 Subject: [PATCH 134/295] Allow clients to set max_retry_wait. Use float for jitter This change allows clients to specify a max_retry_wait to provide a custom upper bound for automatic client retries. It also changes the granularity of retry jitter to a floating-point value, further decreasing the likelihood of clustering retry requests. --- apitools/base/py/base_api.py | 19 +++++++++++++++++-- apitools/base/py/http_wrapper.py | 17 ++++++++++++----- apitools/base/py/util.py | 13 +++++-------- apitools/base/py/util_test.py | 25 +++++++++++++++++-------- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index d10314b..391c1b4 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -198,7 +198,7 @@ class BaseApiClient(object): def __init__(self, url, credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, - num_retries=5, credentials_args=None, + num_retries=5, max_retry_wait=60, credentials_args=None, default_global_params=None, additional_http_headers=None): _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) if default_global_params is not None: @@ -207,8 +207,10 @@ class BaseApiClient(object): self.log_request = log_request self.log_response = log_response self.__num_retries = 5 + self.__max_retry_wait = 60 # We let the @property machinery below do our validation. self.num_retries = num_retries + self.max_retry_wait = max_retry_wait self._credentials = credentials if get_credentials and not credentials: credentials_args = credentials_args or {} @@ -334,6 +336,18 @@ class BaseApiClient(object): 'Cannot have negative value for num_retries') self.__num_retries = value + @property + def max_retry_wait(self): + return self.__max_retry_wait + + @max_retry_wait.setter + def max_retry_wait(self, value): + util.Typecheck(value, six.integer_types) + if value <= 0: + raise exceptions.InvalidDataError( + 'max_retry_wait must be a postiive integer') + self.__max_retry_wait = value + @contextlib.contextmanager def WithRetries(self, num_retries): old_num_retries = self.num_retries @@ -617,7 +631,8 @@ class BaseApiService(object): if upload and upload.bytes_http: http = upload.bytes_http http_response = http_wrapper.MakeRequest( - http, http_request, retries=self.__client.num_retries) + http, http_request, retries=self.__client.num_retries, + max_retry_wait=self.__client.max_retry_wait) return self.ProcessHttpResponse(method_config, http_response) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 94c7e32..94c639e 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -47,7 +47,8 @@ _REDIRECT_STATUS_CODES = ( # exc: Exception being raised. # num_retries: Number of retries consumed; used for exponential backoff. ExceptionRetryArgs = collections.namedtuple( - 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries']) + 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries', + 'max_retry_wait']) @contextlib.contextmanager @@ -276,10 +277,12 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): logging.debug('Retrying request to url %s after exception %s', retry_args.http_request.url, retry_args.exc) time.sleep( - retry_after or util.CalculateWaitForRetry(retry_args.num_retries)) + retry_after or util.CalculateWaitForRetry( + retry_args.num_retries, max_wait=retry_args.max_retry_wait)) -def MakeRequest(http, http_request, retries=7, redirections=5, +def MakeRequest(http, http_request, retries=7, max_retry_wait=60, + redirections=5, retry_func=HandleExceptionsAndRebuildHttpConnections, check_response_func=CheckResponse): """Send http_request via the given http, performing error/retry handling. @@ -288,7 +291,10 @@ def MakeRequest(http, http_request, retries=7, redirections=5, http: An httplib2.Http instance, or a http multiplexer that delegates to an underlying http, for example, HTTPMultiplexer. http_request: A Request to send. - retries: (int, default 5) Number of retries to attempt on 5XX replies. + retries: (int, default 7) Number of retries to attempt on retryable + replies (such as 429 or 5XX). + max_retry_wait: (int, default 60) Maximum number of seconds to wait + when retrying. redirections: (int, default 5) Number of redirects to follow. retry_func: Function to handle retries on exceptions. Arguments are (Httplib2.Http, Request, Exception, int num_retries). @@ -315,7 +321,8 @@ def MakeRequest(http, http_request, retries=7, redirections=5, if retry >= retries: raise else: - retry_func(ExceptionRetryArgs(http, http_request, e, retry)) + retry_func(ExceptionRetryArgs( + http, http_request, e, retry, max_retry_wait)) def _MakeRequestNoRetry(http, http_request, redirections=5, diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 779dd97..b92c9f8 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -126,20 +126,17 @@ def CalculateWaitForRetry(retry_attempt, max_wait=60): Args: retry_attempt: Retry attempt counter. - max_wait: Upper bound for wait time. + max_wait: Upper bound for wait time [seconds]. Returns: - Amount of time to wait before retrying request. + Number of seconds to wait before retrying request. """ wait_time = 2 ** retry_attempt - # randrange requires a nonzero interval, so we want to drop it if - # the range is too small for jitter. - if retry_attempt: - max_jitter = (2 ** retry_attempt) / 2 - wait_time += random.randrange(-max_jitter, max_jitter) - return min(wait_time, max_wait) + max_jitter = wait_time / 4.0 + wait_time += random.uniform(-max_jitter, max_jitter) + return max(1, min(wait_time, max_wait)) def AcceptableMimeType(accept_patterns, mime_type): diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 54dc3e1..6cb551d 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -56,14 +56,23 @@ class UtilTest(unittest2.TestCase): method_config_no_reserved, {'x': 'foo/:bar:'})) def testCalculateWaitForRetry(self): - self.assertTrue(util.CalculateWaitForRetry(1) in range(1, 4)) - self.assertTrue(util.CalculateWaitForRetry(2) in range(2, 7)) - self.assertTrue(util.CalculateWaitForRetry(3) in range(4, 13)) - self.assertTrue(util.CalculateWaitForRetry(4) in range(8, 25)) - - self.assertEquals(10, util.CalculateWaitForRetry(5, max_wait=10)) - - self.assertGreater(util.CalculateWaitForRetry(0), 0) + try0 = util.CalculateWaitForRetry(0) + self.assertTrue(try0 >= 1.0) + self.assertTrue(try0 <= 1.5) + try1 = util.CalculateWaitForRetry(1) + self.assertTrue(try1 >= 1.0) + self.assertTrue(try1 <= 3.0) + try2 = util.CalculateWaitForRetry(2) + self.assertTrue(try2 >= 2.0) + self.assertTrue(try2 <= 6.0) + try3 = util.CalculateWaitForRetry(3) + self.assertTrue(try3 >= 4.0) + self.assertTrue(try3 <= 12.0) + try4 = util.CalculateWaitForRetry(4) + self.assertTrue(try4 >= 8.0) + self.assertTrue(try4 <= 24.0) + + self.assertAlmostEqual(10, util.CalculateWaitForRetry(5, max_wait=10)) def testTypecheck(self): -- GitLab From db5efc27c51ee2422bdb2ee794897531bdef3ac5 Mon Sep 17 00:00:00 2001 From: Brandon Salmon Date: Wed, 24 Jun 2015 12:06:59 -0700 Subject: [PATCH 135/295] Fixed off by one error in GetRange. --- apitools/base/py/transfer.py | 20 ++++++++++++++------ apitools/base/py/transfer_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 35a4774..f1802f0 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -366,10 +366,18 @@ class Download(_Transfer): end_byte = end if use_chunks: alternate = start + self.chunksize - 1 - end_byte = min(end_byte, alternate) if end_byte else alternate + if end_byte is not None: + end_byte = min(end_byte, alternate) + else: + end_byte = alternate + if self.total_size: alternate = self.total_size - 1 - end_byte = min(end_byte, alternate) if end_byte else alternate + if end_byte is not None: + end_byte = min(end_byte, alternate) + else: + end_byte = alternate + return end_byte def __GetChunk(self, start, end, additional_headers=None): @@ -434,18 +442,18 @@ class Download(_Transfer): self.EnsureInitialized() progress_end_normalized = False if self.total_size is not None: - progress, end = self.__NormalizeStartEnd(start, end) + progress, end_byte = self.__NormalizeStartEnd(start, end) progress_end_normalized = True else: progress = start - while not progress_end_normalized or progress < end: - end_byte = self.__ComputeEndByte(progress, end=end, + while not progress_end_normalized or progress <= end_byte: + end_byte = self.__ComputeEndByte(progress, end=end_byte, use_chunks=use_chunks) response = self.__GetChunk(progress, end_byte, additional_headers=additional_headers) if not progress_end_normalized: self.__SetTotal(response.info) - progress, end = self.__NormalizeStartEnd(start, end) + progress, end_byte = self.__NormalizeStartEnd(start, end) progress_end_normalized = True response = self.__ProcessResponse(response) progress += response.length diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 9d58b1e..8e29f12 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -76,6 +76,36 @@ class TransferTest(unittest2.TestCase): download._Download__ComputeEndByte(start), msg='Failed on start={0}'.format(start)) + def testGetRange(self): + for (start_byte, end_byte) in [(0, 25), (5, 15), (0, 0), (25, 25)]: + bytes_http = object() + http = object() + download_stream = six.StringIO() + download = transfer.Download.FromStream(download_stream, + total_size=26, + auto_transfer=False) + download.bytes_http = bytes_http + base_url = 'https://part.one/' + with mock.patch.object(http_wrapper, 'MakeRequest', + autospec=True) as make_request: + make_request.return_value = http_wrapper.Response( + info={ + 'content-range': 'bytes %d-%d/26' % + (start_byte, end_byte), + 'status': http_client.OK, + }, + content=string.ascii_lowercase[start_byte:end_byte+1], + request_url=base_url, + ) + request = http_wrapper.Request(url='https://part.one/') + download.InitializeDownload(request, http=http) + download.GetRange(start_byte, end_byte) + self.assertEqual(1, make_request.call_count) + received_request = make_request.call_args[0][1] + self.assertEqual(base_url, received_request.url) + self.assertRangeAndContentRangeCompatible( + received_request, make_request.return_value) + def testNonChunkedDownload(self): bytes_http = object() http = object() -- GitLab From 88b3b9b052dd03b7d2717e56c138a3d91bf0ab23 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 28 Jun 2015 00:28:50 -0700 Subject: [PATCH 136/295] Fix a bug in apitools for missing bodies on POSTs. It turns out that for some API calls (eg storage.v1.objects.copy), an empty body is perfectly valid. Currently, we fail in this case with a terrible error message. (reviewed internally) --- apitools/base/py/base_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 391c1b4..b10aa9a 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -561,6 +561,9 @@ class BaseApiService(object): util.Typecheck(body_field, messages.MessageField) body_type = body_field.type + # If there was no body provided, we use an empty message of the + # appropriate type. + body_value = body_value or body_type() if upload and not body_value: # We're going to fill in the body later. return -- GitLab From 7132ea95843d1516c22e3743d01780f4432781c8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 29 Jun 2015 15:04:56 -0700 Subject: [PATCH 137/295] Fix a bug in url-encoding of bytes fields. Previously, we encoded url parameters based solely on *value* -- meaning that we would try to normalize bytes fields to utf8, even when they should remain bytes. This just moves the logic up to encode while we're still holding on to type information, so that we can check field types first. (reviewed internally) --- apitools/base/py/base_api.py | 35 +++++++++++++++++++++---------- apitools/base/py/base_api_test.py | 30 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index b10aa9a..f12d9f6 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -466,6 +466,18 @@ class BaseApiService(object): query_info['pp'] = 0 return query_info + def __FinalUrlValue(self, value, field): + """Encode value for the URL, using field to skip encoding for bytes.""" + if isinstance(field, messages.BytesField): + return value + elif isinstance(value, six.text_type): + return value.encode('utf8') + elif isinstance(value, six.binary_type): + return value.decode('utf8') + elif isinstance(value, datetime.datetime): + return value.isoformat() + return value + def __ConstructQueryParams(self, query_params, request, global_params): """Construct a dictionary of query parameters for this request.""" # First, handle the global params. @@ -474,23 +486,24 @@ class BaseApiService(object): global_param_names = util.MapParamNames( [x.name for x in self.__client.params_type.all_fields()], self.__client.params_type) - query_info = dict((param, getattr(global_params, param)) - for param in global_param_names) + global_params_type = type(global_params) + query_info = dict( + (param, + self.__FinalUrlValue(getattr(global_params, param), + getattr(global_params_type, param))) + for param in global_param_names) # Next, add the query params. query_param_names = util.MapParamNames(query_params, type(request)) - query_info.update((param, getattr(request, param, None)) - for param in query_param_names) + request_type = type(request) + query_info.update( + (param, + self.__FinalUrlValue(getattr(request, param, None), + getattr(request_type, param))) + for param in query_param_names) query_info = dict((k, v) for k, v in query_info.items() if v is not None) query_info = self.__EncodePrettyPrint(query_info) query_info = util.MapRequestParams(query_info, type(request)) - for k, v in query_info.items(): - if isinstance(v, six.text_type): - query_info[k] = v.encode('utf8') - elif isinstance(v, str): - query_info[k] = v.decode('utf8') - elif isinstance(v, datetime.datetime): - query_info[k] = v.isoformat() return query_info def __ConstructRelativePath(self, method_config, request, diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 86141a3..876fb91 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -3,6 +3,7 @@ import sys from protorpc import message_types from protorpc import messages +import six from six.moves import urllib_parse import unittest2 @@ -13,6 +14,7 @@ from apitools.base.py import http_wrapper class SimpleMessage(messages.Message): field = messages.StringField(1) + bytes_field = messages.BytesField(2) class MessageWithTime(messages.Message): @@ -40,6 +42,7 @@ class StandardQueryParameters(messages.Message): prettyPrint = messages.BooleanField( 5, default=True) # pylint: disable=invalid-name pp = messages.BooleanField(6, default=True) + nextPageToken = messages.BytesField(7) # pylint:disable=invalid-name class FakeCredentials(object): @@ -147,6 +150,33 @@ class BaseApiTest(unittest2.TestCase): self.assertTrue('prettyPrint=0' in http_request.url) self.assertTrue('pp=0' in http_request.url) + def testQueryBytesRequest(self): + method_config = base_api.ApiMethodInfo( + request_type_name='SimpleMessage', query_params=['bytes_field']) + service = FakeService() + non_unicode_message = b''.join((six.int2byte(100), + six.int2byte(200))) + request = SimpleMessage(bytes_field=non_unicode_message) + global_params = StandardQueryParameters() + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + want = urllib_parse.urlencode({'bytes_field': non_unicode_message}) + self.assertIn(want, http_request.url) + + def testQueryBytesGlobalParams(self): + method_config = base_api.ApiMethodInfo( + request_type_name='SimpleMessage', query_params=['bytes_field']) + service = FakeService() + non_unicode_message = b''.join((six.int2byte(100), + six.int2byte(200))) + request = SimpleMessage() + global_params = StandardQueryParameters( + nextPageToken=non_unicode_message) + http_request = service.PrepareHttpRequest(method_config, request, + global_params=global_params) + want = urllib_parse.urlencode({'nextPageToken': non_unicode_message}) + self.assertIn(want, http_request.url) + def testQueryRemapping(self): method_config = base_api.ApiMethodInfo( request_type_name='MessageWithRemappings', -- GitLab From 4ad7475ebc1f0289b57041a575cc3fb3d38b72ab Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 29 Jun 2015 15:13:47 -0700 Subject: [PATCH 138/295] Fix : before / in paths for apitools. Currently, if we have a top-level methodPath of the form `path:methodName`, we'll pass this to `urlparse.urljoin`, where this is interpreted as `scheme:netloc`. In cases other than media upload, this is **not** what we want. This fixes the issue by adding our own custom `urljoin`, which detects and handles this case. We also add a test. (This is basically a copy of https://www.google.com/url?sa=D&q=https%3A%2F%2Fgithub.com%2Fgoogle%2Fgoogle-api-python-client%2Fcommit%2F7ee535d14007b3b05ca9cab1a35e93fb01657537 tweaked for apitools.) (reviewed internally) --- apitools/base/py/base_api.py | 19 ++++++++++++++++++- apitools/base/py/base_api_test.py | 10 ++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index f12d9f6..1208f79 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -126,12 +126,29 @@ def NormalizeApiEndpoint(api_endpoint): return api_endpoint +def _urljoin(base, url): # pylint: disable=invalid-name + """Custom urljoin replacement supporting : before / in url.""" + # In general, it's unsafe to simply join base and url. However, for + # the case of discovery documents, we know: + # * base will never contain params, query, or fragment + # * url will never contain a scheme or net_loc. + # In general, this means we can safely join on /; we just need to + # ensure we end up with precisely one / joining base and url. The + # exception here is the case of media uploads, where url will be an + # absolute url. + if url.startswith('http://') or url.startswith('https://'): + return urllib.parse.urljoin(base, url) + new_base = base if base.endswith('/') else base + '/' + new_url = url[1:] if url.startswith('/') else url + return new_base + new_url + + class _UrlBuilder(object): """Convenient container for url data.""" def __init__(self, base_url, relative_path=None, query_params=None): - components = urllib.parse.urlsplit(urllib.parse.urljoin( + components = urllib.parse.urlsplit(_urljoin( base_url, relative_path or '')) if components.fragment: raise exceptions.ConfigurationValueError( diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 876fb91..5267684 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -207,3 +207,13 @@ class BaseApiTest(unittest2.TestCase): expected_url = service.client.url + 'parameters/gonna/remap/ONE/TWO' http_request = service.PrepareHttpRequest(method_config, request) self.assertEqual(expected_url, http_request.url) + + def testColonInRelativePath(self): + method_config = base_api.ApiMethodInfo( + relative_path='path:withJustColon', + request_type_name='SimpleMessage') + service = FakeService() + request = SimpleMessage() + http_request = service.PrepareHttpRequest(method_config, request) + self.assertEqual('http://www.example.com/path:withJustColon', + http_request.url) -- GitLab From 9f7339ee00e0698e8e3315131aabecdacab1b793 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 29 Jun 2015 22:45:58 -0700 Subject: [PATCH 139/295] Better arg handling in argparse. Swaps `argparse.parse_args` for `argparse.parse_known_args` when processing `oauth2client` args, to avoid throwing exceptions on the other args for generated CLIs. --- apitools/base/py/credentials_lib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index d7e6133..3549de1 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -435,7 +435,7 @@ def _GetRunFlowFlags(args=None): parser = argparse.ArgumentParser(parents=[tools.argparser]) # Get command line argparse flags. - flags = parser.parse_args(args=args) + flags, _ = parser.parse_known_args(args=args) # Allow `gflags` and `argparse` to be used side-by-side. if hasattr(FLAGS, 'auth_host_name'): @@ -460,7 +460,7 @@ def CredentialsFromFile(path, client_info, oauth2client_args=None): credentials = credential_store.get() if credentials is None or credentials.invalid: print('Generating new OAuth credentials ...') - while True: + for _ in range(20): # If authorization fails, we want to retry, rather than let this # cascade up and get caught elsewhere. If users want out of the # retry loop, they can ^C. @@ -471,8 +471,8 @@ def CredentialsFromFile(path, client_info, oauth2client_args=None): break except (oauth2client.client.FlowExchangeError, SystemExit) as e: # Here SystemExit is "no credential at all", and the - # FlowExchangeError is "invalid" -- usually because you reused - # a token. + # FlowExchangeError is "invalid" -- usually because + # you reused a token. print('Invalid authorization: %s' % (e,)) except httplib2.HttpLib2Error as e: print('Communication error: %s' % (e,)) -- GitLab From 3f9968e2bb51472575c5ada689f703bad139e856 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 19 Jul 2015 20:46:11 -0700 Subject: [PATCH 140/295] Encode bytes in url params using urlsafe_base64. (Reviewed internally.) --- apitools/base/py/base_api.py | 5 +++-- apitools/base/py/base_api_test.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 1208f79..91e7f49 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Base class for api services.""" +import base64 import contextlib import datetime import logging @@ -485,8 +486,8 @@ class BaseApiService(object): def __FinalUrlValue(self, value, field): """Encode value for the URL, using field to skip encoding for bytes.""" - if isinstance(field, messages.BytesField): - return value + if isinstance(field, messages.BytesField) and value is not None: + return base64.urlsafe_b64encode(value) elif isinstance(value, six.text_type): return value.encode('utf8') elif isinstance(value, six.binary_type): diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 5267684..7758c43 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -1,3 +1,4 @@ +import base64 import datetime import sys @@ -160,7 +161,9 @@ class BaseApiTest(unittest2.TestCase): global_params = StandardQueryParameters() http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) - want = urllib_parse.urlencode({'bytes_field': non_unicode_message}) + want = urllib_parse.urlencode({ + 'bytes_field': base64.urlsafe_b64encode(non_unicode_message), + }) self.assertIn(want, http_request.url) def testQueryBytesGlobalParams(self): @@ -174,7 +177,9 @@ class BaseApiTest(unittest2.TestCase): nextPageToken=non_unicode_message) http_request = service.PrepareHttpRequest(method_config, request, global_params=global_params) - want = urllib_parse.urlencode({'nextPageToken': non_unicode_message}) + want = urllib_parse.urlencode({ + 'nextPageToken': base64.urlsafe_b64encode(non_unicode_message), + }) self.assertIn(want, http_request.url) def testQueryRemapping(self): -- GitLab From 4b147c4e572dc83d6d8c5eb7bffe4d7d2c45237c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 19 Jul 2015 21:03:45 -0700 Subject: [PATCH 141/295] Fix a bug in handling for repeated any values. If an API had a repeated any value marked as an additionalProperty, we'd fail to decode, because we can't treat a JsonArray or a JsonValue with an array_value as a list in Python. We work around this in a painful way; in particular, there's a nasty inline import because encoding.py needs to be split in two. (reviewed internally) --- apitools/base/py/encoding.py | 27 +++++++++++++++++++++++++++ apitools/base/py/encoding_test.py | 17 +++++++++++++++++ default.pylintrc | 2 ++ 3 files changed, 46 insertions(+) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 07e2f68..82562e2 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -383,6 +383,8 @@ def _DecodeUnknownMessages(message, encoded_message, pair_type): if name in all_field_names: continue value = PyValueToMessage(field_type, value_dict) + if pair_type.value.repeated: + value = _AsMessageList(value) new_pair = pair_type(key=name, value=value) new_values.append(new_pair) return new_values @@ -680,3 +682,28 @@ def _DecodeCustomFieldNames(message_type, encoded_message): decoded_message[python_name] = decoded_message.pop(json_name) encoded_message = json.dumps(decoded_message) return encoded_message + + +def _AsMessageList(msg): + """Convert the provided list-as-JsonValue to a list.""" + # This really needs to live in extra_types, but extra_types needs + # to import this file to be able to register codecs. + # TODO(craigcitro): Split out a codecs module and fix this ugly + # import. + from apitools.base.py import extra_types + + def _IsRepeatedJsonValue(msg): + """Return True if msg is a repeated value as a JsonValue.""" + if isinstance(msg, extra_types.JsonArray): + return True + if isinstance(msg, extra_types.JsonValue) and msg.array_value: + return True + return False + + if not _IsRepeatedJsonValue(msg): + raise ValueError('invalid argument to _AsMessageList') + if isinstance(msg, extra_types.JsonValue): + msg = msg.array_value + if isinstance(msg, extra_types.JsonArray): + msg = msg.entries + return msg diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 77bbe0a..0389af1 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -10,6 +10,7 @@ import unittest2 from apitools.base.py import encoding from apitools.base.py import exceptions +from apitools.base.py import extra_types class SimpleMessage(messages.Message): @@ -101,6 +102,17 @@ class MessageWithRemappings(messages.Message): repeated_field = messages.StringField(5, repeated=True) +@encoding.MapUnrecognizedFields('additional_properties') +class RepeatedJsonValueMessage(messages.Message): + + class AdditionalProperty(messages.Message): + key = messages.StringField(1) + value = messages.MessageField(extra_types.JsonValue, 2, repeated=True) + + additional_properties = messages.MessageField('AdditionalProperty', 1, + repeated=True) + + encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, 'enum_value', 'wire_name') encoding.AddCustomJsonFieldMapping(MessageWithRemappings, @@ -391,3 +403,8 @@ class EncodingTest(unittest2.TestCase): encoding._GetTypeKey(MessageWithEnum.ThisEnum, new_package)) finally: delattr(this_module, 'package') + + def testRepeatedJsonValuesAsRepeatedProperty(self): + encoded_msg = '{"a": [{"one": 1}]}' + msg = encoding.JsonToMessage(RepeatedJsonValueMessage, encoded_msg) + self.assertEqual(encoded_msg, encoding.MessageToJson(msg)) diff --git a/default.pylintrc b/default.pylintrc index bdea83a..1540a51 100644 --- a/default.pylintrc +++ b/default.pylintrc @@ -44,7 +44,9 @@ [MESSAGES CONTROL] +# TODO: remove cyclic-import. disable = + cyclic-import, fixme, import-error, locally-disabled, -- GitLab From 9c0603dbb8ae10fcd51d7413c767682996dccd36 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 19 Jul 2015 21:09:10 -0700 Subject: [PATCH 142/295] Look for installed app IDs in client_secrets. Previously we only looked for web IDs. --- apitools/gen/gen_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 7b7b648..3bb44f8 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -122,7 +122,7 @@ def _GetCodegenFromFlags(): try: with open(FLAGS.client_json) as client_json: f = json.loads(client_json.read()) - web = f.get('web', {}) + web = f.get('installed', f.get('web', {})) client_id = web.get('client_id') client_secret = web.get('client_secret') except IOError: -- GitLab From 997f07d244dd740de08c98abeb7eb8b87130bc02 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 19 Jul 2015 23:19:48 -0700 Subject: [PATCH 143/295] Add a `pip_package` command to gen_client. This is the most basic form of "generate a PyPI-viable package for a given API". There's still some work to be done (namely around letting the caller thread in version numbers), but this gets the job done in the simplest case. --- apitools/gen/gen_client.py | 72 ++++++++++++++++++++++++++++----- apitools/gen/gen_client_lib.py | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 3bb44f8..a8e32f7 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -103,16 +103,29 @@ def _CopyLocalFile(filename): out.write(src_data) +_DISCOVERY_DOC = None + + +def _GetDiscoveryDocFromFlags(): + """Get the discovery doc from flags.""" + global _DISCOVERY_DOC # pylint: disable=global-statement + if _DISCOVERY_DOC is None: + if FLAGS.discovery_url: + try: + discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) + except exceptions.CommunicationError: + raise exceptions.GeneratedClientError( + 'Could not fetch discovery doc') + else: + infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' + discovery_doc = json.load(open(infile)) + _DISCOVERY_DOC = discovery_doc + return _DISCOVERY_DOC + + def _GetCodegenFromFlags(): """Create a codegen object from flags.""" - if FLAGS.discovery_url: - try: - discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) - except exceptions.CommunicationError: - return None - else: - infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' - discovery_doc = json.load(open(infile)) + discovery_doc = _GetDiscoveryDocFromFlags() names = util.Names( FLAGS.strip_prefix, FLAGS.experimental_name_convention, @@ -148,9 +161,10 @@ def _GetCodegenFromFlags(): raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') + if not os.path.exists(outdir): + os.makedirs(outdir) - root_package = FLAGS.root_package or util.GetPackage( - outdir) # pylint: disable=line-too-long + root_package = FLAGS.root_package or util.GetPackage(outdir) return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, base_package=FLAGS.base_package, @@ -169,6 +183,11 @@ def _WriteBaseFiles(codegen): _CopyLocalFile('exceptions.py') +def _WriteIntermediateInit(codegen): + with open('__init__.py', 'w') as out: + codegen.WriteIntermediateInit(out) + + def _WriteProtoFiles(codegen): with util.Chdir(codegen.outdir): with open(codegen.client_info.messages_proto_file_name, 'w') as out: @@ -197,6 +216,11 @@ def _WriteInit(codegen): codegen.WriteInit(out) +def _WriteSetupPy(codegen): + with open('setup.py', 'w') as out: + codegen.WriteSetupPy(out) + + class GenerateClient(appcommands.Cmd): """Driver for client code generation.""" @@ -211,6 +235,33 @@ class GenerateClient(appcommands.Cmd): _WriteInit(codegen) +class GeneratePipPackage(appcommands.Cmd): + + """Generate a client as a pip-installable tarball.""" + + def Run(self, _): + """Create a client in a pip package.""" + discovery_doc = _GetDiscoveryDocFromFlags() + package = discovery_doc['name'] + original_outdir = os.path.expanduser(FLAGS.outdir) + FLAGS.outdir = os.path.join( + FLAGS.outdir, 'apitools/clients/%s' % package) + FLAGS.root_package = 'apitools.clients.%s' % package + FLAGS.generate_cli = False + codegen = _GetCodegenFromFlags() + if codegen is None: + logging.error('Failed to create codegen, exiting.') + return 1 + _WriteGeneratedFiles(codegen) + _WriteInit(codegen) + with util.Chdir(original_outdir): + _WriteSetupPy(codegen) + with util.Chdir('apitools'): + _WriteIntermediateInit(codegen) + with util.Chdir('clients'): + _WriteIntermediateInit(codegen) + + class GenerateProto(appcommands.Cmd): """Generate just the two proto files for a given API.""" @@ -247,6 +298,7 @@ def run_main(): def main(_): appcommands.AddCmd('client', GenerateClient) + appcommands.AddCmd('pip_package', GeneratePipPackage) appcommands.AddCmd('proto', GenerateProto) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index df7bad4..1f8131f 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -5,6 +5,8 @@ Relevant links: https://developers.google.com/discovery/v1/reference/apis#resource """ +import datetime + from six.moves import urllib_parse from apitools.base.py import base_cli @@ -132,6 +134,10 @@ class DescriptorGenerator(object): def outdir(self): return self.__outdir + @property + def package(self): + return self.__package + @property def use_proto2(self): return self.__use_proto2 @@ -164,6 +170,73 @@ class DescriptorGenerator(object): printer() printer('__path__ = pkgutil.extend_path(__path__, __name__)') + def WriteIntermediateInit(self, out): + """Write a simple __init__.py for an intermediate directory.""" + printer = self._GetPrinter(out) + printer('#!/usr/bin/env python') + printer('"""Shared __init__.py for apitools."""') + printer() + printer('from pkgutil import extend_path') + printer('__path__ = extend_path(__path__, __name__)') + + def WriteSetupPy(self, out): + """Write a setup.py for upload to PyPI.""" + printer = self._GetPrinter(out) + year = datetime.datetime.now().year + printer('# Copyright %s Google Inc. All Rights Reserved.' % year) + printer('#') + printer('# Licensed under the Apache License, Version 2.0 (the' + '"License");') + printer('# you may not use this file except in compliance with ' + 'the License.') + printer('# You may obtain a copy of the License at') + printer('#') + printer('# http://www.apache.org/licenses/LICENSE-2.0') + printer('#') + printer('# Unless required by applicable law or agreed to in writing, ' + 'software') + printer('# distributed under the License is distributed on an "AS IS" ' + 'BASIS,') + printer('# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either ' + 'express or implied.') + printer('# See the License for the specific language governing ' + 'permissions and') + printer('# limitations under the License.') + printer() + printer('import setuptools') + printer('REQUIREMENTS = [') + with printer.Indent(indent=' '): + # TODO(craigcitro): Have this track apitools' version. + printer('"google-apitools>=0.4.8",') + printer('"httplib2>=0.9",') + printer('"oauth2client>=1.4.12",') + printer('"protorpc>=0.10.0",') + printer(']') + printer('_PACKAGE = "apitools.clients.%s"' % self.__package) + printer() + printer('setuptools.setup(') + # TODO(craigcitro): Allow customization of these options. + with printer.Indent(indent=' '): + printer('name="google-apitools-%s",' % self.__package) + printer('version="0.4.8",') + printer('description="Autogenerated apitools library for %s",' % ( + self.__package,)) + printer('url="https://github.com/google/apitools",') + printer('author="Craig Citro",') + printer('author_email="craigcitro@google.com",') + printer('packages=setuptools.find_packages(),') + printer('install_requires=REQUIREMENTS,') + printer('classifiers=[') + with printer.Indent(indent=' '): + printer('"Programming Language :: Python :: 2.7",') + printer('"License :: OSI Approved :: Apache Software ' + 'License",') + printer('],') + printer('license="Apache 2.0",') + printer('keywords="apitools apitools-%s %s",' % ( + self.__package, self.__package)) + printer(')') + def WriteMessagesFile(self, out): self.__message_registry.WriteFile(self._GetPrinter(out)) -- GitLab From 92f87ab04d1835597215a655be579956f7a91c8f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 26 Jul 2015 13:27:29 -0700 Subject: [PATCH 144/295] Add html coverage dir to .gitignore. --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b51fd87..02eda25 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,12 @@ distribute-* # Test files .tox/ +nosetests.xml + +# Coverage related .coverage coverage.xml -nosetests.xml +htmlcov/ # Make sure a generated file isn't accidentally committed. reduced.pylintrc -- GitLab From 09d780474d00f3a8f4c2295154d74dae2023c1d3 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 26 Jul 2015 13:28:20 -0700 Subject: [PATCH 145/295] Drop the CLI from the sample storage client imports. --- samples/storage_sample/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/storage_sample/storage/__init__.py b/samples/storage_sample/storage/__init__.py index 6426fac..2c8e598 100644 --- a/samples/storage_sample/storage/__init__.py +++ b/samples/storage_sample/storage/__init__.py @@ -4,7 +4,6 @@ import pkgutil from apitools.base.py import * -from storage_v1 import * from storage_v1_client import * from storage_v1_messages import * -- GitLab From a68025654ccbf0070cb6ec94ec2b05bcf23be32b Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 26 Jul 2015 13:29:21 -0700 Subject: [PATCH 146/295] Fix a bug in upload support, and do some cleanup. A recent cleanup introduced a NameError in a corner case; this was covered by integration tests (yay!) that aren't run by default (boo!). We clean this up (and fix a revealed corner-case bug), and clean up the tests to run independent of the directory they're invoked from. Last, we add a tox env (still for testing purposes) that computes coverage for the storage integration tests; we're currently at 75%, covering most of the relevant code in transfer.py. --- apitools/base/py/transfer.py | 10 ++++++++-- samples/storage_sample/downloads_test.py | 4 +++- samples/storage_sample/uploads_test.py | 14 ++++++++++++++ tox.ini | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index f1802f0..0da8412 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -331,7 +331,7 @@ class Download(_Transfer): else: if start < 0: start = max(0, start + self.total_size) - return start, self.total_size + return start, self.total_size - 1 def __SetRangeHeader(self, request, start, end=None): if start < 0: @@ -364,6 +364,10 @@ class Download(_Transfer): """ end_byte = end + + if start < 0 and not self.total_size: + return end_byte + if use_chunks: alternate = start + self.chunksize - 1 if end_byte is not None: @@ -446,7 +450,9 @@ class Download(_Transfer): progress_end_normalized = True else: progress = start - while not progress_end_normalized or progress <= end_byte: + end_byte = end + while (not progress_end_normalized or end_byte is None or + progress <= end_byte): end_byte = self.__ComputeEndByte(progress, end=end_byte, use_chunks=use_chunks) response = self.__GetChunk(progress, end_byte, diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index 2276d98..72a4821 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -37,7 +37,9 @@ class DownloadsTest(unittest.TestCase): self.__buffer, auto_transfer=auto_transfer) def __GetTestdataFileContents(self, filename): - file_contents = open('testdata/%s' % filename).read() + file_path = os.path.join( + os.path.dirname(__file__), self._TESTDATA_PREFIX, filename) + file_contents = open(file_path).read() self.assertIsNotNone( file_contents, msg=('Could not read file %s' % filename)) return file_contents diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index ad4416f..4eb5aba 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -103,6 +103,20 @@ class UploadsTest(unittest.TestCase): response = self.__InsertFile(filename, request=request) self.assertEqual(size, response.size) + def testStreamMedia(self): + filename = 'ten_meg_file' + size = 10 << 20 + self.__ResetUpload(size, auto_transfer=False) + self.__upload.strategy = 'resumable' + self.__upload.total_size = size + request = self.__InsertRequest(filename) + initial_response = self.__client.objects.Insert( + request, upload=self.__upload) + self.assertIsNotNone(initial_response) + self.assertEqual(0, self.__buffer.tell()) + self.__upload.StreamMedia() + self.assertEqual(size, self.__buffer.tell()) + def testBreakAndResumeUpload(self): filename = ('ten_meg_file_' + ''.join(random.sample(string.ascii_letters, 5))) diff --git a/tox.ini b/tox.ini index 5cd3c74..8baa1c8 100644 --- a/tox.ini +++ b/tox.ini @@ -61,3 +61,18 @@ deps = {[testenv:cover]deps} coveralls passenv = TRAVIS* + +[testenv:transfer_coverage] +basepython = + python2.7 +deps = + mock + nose + unittest2 + coverage +commands = + coverage run --branch -p samples/storage_sample/downloads_test.py + coverage run --branch -p samples/storage_sample/uploads_test.py + coverage run --branch -p apitools/base/py/transfer_test.py + coverage combine + coverage html -- GitLab From 7a3e28f93ba368342b92d561d31768e1f88ae543 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 26 Jul 2015 16:32:53 -0700 Subject: [PATCH 147/295] Minor code cleanup in credentials_lib. (Also update some lint rules.) --- apitools/base/py/credentials_lib.py | 76 ++++++++++-------------- apitools/base/py/credentials_lib_test.py | 14 +++-- default.pylintrc | 2 + 3 files changed, 41 insertions(+), 51 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 3549de1..778adf1 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -142,15 +142,26 @@ def _EnsureFileExists(filename): return True -def _OpenNoProxy(request): - """Wrapper around urllib2.open that ignores proxies.""" +def _GceMetadataRequest(relative_url, use_metadata_ip=False): + """Request the given url from the GCE metadata service.""" + if use_metadata_ip: + base_url = 'http://169.254.169.254/' + else: + base_url = 'http://metadata.google.internal/' + url = base_url + 'computeMetadata/v1/' + relative_url + # Extra header requirement can be found here: + # https://developers.google.com/compute/docs/metadata + headers = {'Metadata-Flavor': 'Google'} + request = urllib.request.Request(url, headers=headers) opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - return opener.open(request) + try: + response = opener.open(request) + except urllib.error.URLError as e: + raise exceptions.CommunicationError( + 'Could not reach metadata service: %s' % e.reason) + return response -# TODO(craigcitro): We override to add some utility code, and to -# update the old refresh implementation. Push this code into -# oauth2client. class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """Assertion credentials for GCE instances.""" @@ -171,13 +182,10 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): # identified these scopes in the same execution. However, the # available scopes don't change once an instance is created, # so there is no reason to perform more than one query. - # - # TODO(craigcitro): Move this into oauth2client. self.__service_account_name = service_account_name - cache_filename = None cached_scopes = None - if 'cache_filename' in kwds: - cache_filename = kwds['cache_filename'] + cache_filename = kwds.get('cache_filename') + if cache_filename: cached_scopes = self._CheckCacheFileForMatch( cache_filename, scopes) @@ -276,35 +284,16 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): return scopes def GetServiceAccount(self, account): - account_uri = ( - 'http://metadata.google.internal/computeMetadata/' - 'v1/instance/service-accounts') - additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request( - account_uri, headers=additional_headers) - try: - response = _OpenNoProxy(request) - except urllib.error.URLError as e: - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) + relative_url = 'instance/service-accounts' + response = _GceMetadataRequest(relative_url) response_lines = [line.rstrip('/\n\r') for line in response.readlines()] return account in response_lines def GetInstanceScopes(self): - # Extra header requirement can be found here: - # https://developers.google.com/compute/docs/metadata - scopes_uri = ( - 'http://metadata.google.internal/computeMetadata/v1/instance/' - 'service-accounts/%s/scopes') % self.__service_account_name - additional_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request( - scopes_uri, headers=additional_headers) - try: - response = _OpenNoProxy(request) - except urllib.error.URLError as e: - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) + relative_url = 'instance/service-accounts/{0}/scopes'.format( + self.__service_account_name) + response = _GceMetadataRequest(relative_url) return util.NormalizeScopes(scope.strip() for scope in response.readlines()) @@ -328,24 +317,21 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): If self.store is initialized, store acquired credentials there. """ - token_uri = ( - 'http://metadata.google.internal/computeMetadata/v1/instance/' - 'service-accounts/%s/token') % self.__service_account_name - extra_headers = {'X-Google-Metadata-Request': 'True'} - request = urllib.request.Request(token_uri, headers=extra_headers) + relative_url = 'instance/service-accounts/{0}/token'.format( + self.__service_account_name) try: - content = _OpenNoProxy(request).read() - except urllib.error.URLError as e: + response = _GceMetadataRequest(relative_url) + except exceptions.CommunicationError: self.invalid = True if self.store: self.store.locked_put(self) - raise exceptions.CommunicationError( - 'Could not reach metadata service: %s' % e.reason) + raise + content = response.read() try: credential_info = json.loads(content) except ValueError: raise exceptions.CredentialsError( - 'Invalid credentials response: uri %s' % token_uri) + 'Could not parse response as JSON: %s' % content) self.access_token = credential_info['access_token'] if 'expires_in' in credential_info: diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index cf4e5df..067b874 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -26,16 +26,15 @@ def CreateUriValidator(uri_regexp, content=''): class CredentialsLibTest(unittest2.TestCase): def _GetServiceCreds(self, service_account_name=None, scopes=None): - scopes = scopes or ['scope1'] kwargs = {} if service_account_name is not None: kwargs['service_account_name'] = service_account_name service_account_name = service_account_name or 'default' - def MockMetadataCalls(request): - request_url = request.get_full_url() + def MockMetadataCalls(request_url): + default_scopes = scopes or ['scope1'] if request_url.endswith('scopes'): - return six.StringIO(''.join(scopes)) + return six.StringIO(''.join(default_scopes)) elif request_url.endswith('service-accounts'): return six.StringIO(service_account_name) elif request_url.endswith( @@ -43,7 +42,7 @@ class CredentialsLibTest(unittest2.TestCase): return six.StringIO('{"access_token": "token"}') self.fail('Unexpected HTTP request to %s' % request_url) - with mock.patch.object(credentials_lib, '_OpenNoProxy', + with mock.patch.object(credentials_lib, '_GceMetadataRequest', side_effect=MockMetadataCalls, autospec=True) as opener_mock: with mock.patch.object(util, 'DetectGce', @@ -58,8 +57,11 @@ class CredentialsLibTest(unittest2.TestCase): self.assertEqual(3, opener_mock.call_count) def testGceServiceAccounts(self): + scopes = ['scope1'] self._GetServiceCreds() - self._GetServiceCreds(service_account_name='my_service_account') + self._GetServiceCreds(scopes=scopes) + self._GetServiceCreds(service_account_name='my_service_account', + scopes=scopes) class TestGetRunFlowFlags(unittest2.TestCase): diff --git a/default.pylintrc b/default.pylintrc index 1540a51..eb0266f 100644 --- a/default.pylintrc +++ b/default.pylintrc @@ -57,12 +57,14 @@ disable = no-member, no-self-use, redefined-builtin, + redundant-keyword-arg, similarities, star-args, super-on-old-class, too-few-public-methods, too-many-arguments, too-many-branches, + too-many-function-args, too-many-instance-attributes, too-many-locals, too-many-public-methods, -- GitLab From b78556b007ec74f8273b0364acf42c3fcf59edaa Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 4 Aug 2015 15:18:06 -0700 Subject: [PATCH 148/295] Ensure we only use resumable uploads when they're available. Previously we'd blindly set an upload to use the resumable path, even if no resumable option was available. No good. --- apitools/base/py/transfer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 0da8412..b0e3e18 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -673,6 +673,8 @@ class Upload(_Transfer): Returns: None. """ + if upload_config.resumable_path is None: + self.strategy = SIMPLE_UPLOAD if self.strategy is not None: return strategy = SIMPLE_UPLOAD -- GitLab From df62164751eef37f3655be0948d58d19a7e4822c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 11 Aug 2015 10:17:44 -0700 Subject: [PATCH 149/295] Two updates to pip packaging: 1. Update pip packages to include the version in the name. 2. Use the revision number from the discovery document for package versioning, if available. --- apitools/gen/gen_client_lib.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 1f8131f..c7f4bee 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -58,6 +58,7 @@ class DescriptorGenerator(object): self.__discovery_doc.get('description', '')) self.__package = self.__client_info.package self.__version = self.__client_info.version + self.__revision = discovery_doc.get('revision', '1') self.__generate_cli = generate_cli self.__root_package = root_package self.__base_files_package = base_package @@ -217,8 +218,9 @@ class DescriptorGenerator(object): printer('setuptools.setup(') # TODO(craigcitro): Allow customization of these options. with printer.Indent(indent=' '): - printer('name="google-apitools-%s",' % self.__package) - printer('version="0.4.8",') + printer('name="google-apitools-%s-%s",', + self.__package, self.__version) + printer('version="0.4.%s",', self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') -- GitLab From 7a2882b48d61c11d9dda0cb2159727cc0ff3220f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 12 Aug 2015 08:39:07 -0700 Subject: [PATCH 150/295] Update minimum six dep. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 85f9630..2c8d674 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ REQUIRED_PACKAGES = [ 'httplib2>=0.8', 'oauth2client>=1.4.8', 'protorpc>=0.9.1', - 'six>=1.8.0', + 'six>=1.9.0', ] CLI_PACKAGES = [ -- GitLab From b62b00fe72f8be25990a9fbd3b77b119e86dc399 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 12 Aug 2015 23:03:28 -0700 Subject: [PATCH 151/295] Use the Application Default Credential if available. Previously, we'd skip checking for the "well-known file" variant of the ADC if present; now, we check for this before falling back to our file cache. In particular, this involves adding a heuristic for when the set of scopes used by gcloud are sufficient for our needs. --- apitools/base/py/credentials_lib.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 778adf1..cd4a4cf 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -54,6 +54,7 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, credentials_filename=None, service_account_name=None, service_account_keyfile=None, service_account_json_keyfile=None, + skip_application_default_credentials=False, api_key=None, # pylint: disable=unused-argument client=None, # pylint: disable=unused-argument oauth2client_args=None): @@ -103,6 +104,10 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, credentials = GceAssertionCredentials.Get(scopes) if credentials is not None: return credentials + if not skip_application_default_credentials: + credentials = _GetApplicationDefaultCredentials(scopes) + if credentials is not None: + return credentials credentials_filename = credentials_filename or os.path.expanduser( '~/.apitools.token') credentials = CredentialsFromFile(credentials_filename, client_info, @@ -129,6 +134,27 @@ def ServiceAccountCredentials(service_account_name, private_key, scopes, service_account_name, private_key, scopes, **service_account_kwargs) +def _GetApplicationDefaultCredentials(scopes): + gc = oauth2client.client.GoogleCredentials + with cache_file_lock: + try: + # pylint: disable=protected-access + # We've already done our own check for GAE/GCE + # credentials, we don't want to pay for checking again. + credentials = gc._implicit_credentials_from_files() + except oauth2client.client.ApplicationDefaultCredentialsError: + return None + # If we got back a non-service-account credential, we need to use + # a heuristic to decide whether or not the application default + # credential will work for us. We assume that if we're requesting + # cloud-platform, our scopes are a subset of cloud scopes, and the + # ADC will work. + cp = 'https://www.googleapis.com/auth/cloud-platform' + if not isinstance(credentials, gc) or cp in scopes: + return credentials + return None + + def _EnsureFileExists(filename): """Touches a file; returns False on error, True on success.""" if not os.path.exists(filename): -- GitLab From 62bece2fa709007d001f21272523d8d25f0b0355 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 13 Aug 2015 11:45:42 -0700 Subject: [PATCH 152/295] Update for v0.4.9 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2c8d674..d5a17d8 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.8' +_APITOOLS_VERSION = '0.4.9' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From f67705527eab2af7717c2519bbca4219853a3b08 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 14 Aug 2015 14:46:06 -0700 Subject: [PATCH 153/295] Reset the logging level regardless of exceptions thrown. (internal fix) --- apitools/base/py/encoding.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 82562e2..972a95f 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -266,10 +266,12 @@ class _ProtoJsonApiTools(protojson.ProtoJson): # remove this later. old_level = logging.getLogger().level logging.getLogger().setLevel(logging.ERROR) - result = _DecodeCustomFieldNames(message_type, encoded_message) - result = super(_ProtoJsonApiTools, self).decode_message( - message_type, result) - logging.getLogger().setLevel(old_level) + try: + result = _DecodeCustomFieldNames(message_type, encoded_message) + result = super(_ProtoJsonApiTools, self).decode_message( + message_type, result) + finally: + logging.getLogger().setLevel(old_level) result = _ProcessUnknownEnums(result, encoded_message) result = _ProcessUnknownMessages(result, encoded_message) return _DecodeUnknownFields(result, encoded_message) -- GitLab From 3ef0fdd4e121fd8f59440c90a8995f92822548b5 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 14 Aug 2015 15:42:56 -0700 Subject: [PATCH 154/295] YieldFromList now uses encoding.CopyProtoMessage instead of copy.deepcopy. (reviewed internally) --- apitools/base/py/list_pager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index cf90389..438d011 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """A helper function that executes a series of List queries for many APIs.""" -import copy +import encoding __all__ = [ 'YieldFromList', @@ -41,7 +41,7 @@ def YieldFromList( protorpc.message.Message, The resources listed by the service. """ - request = copy.deepcopy(request) + request = encoding.CopyProtoMessage(request) setattr(request, batch_size_attribute, batch_size) setattr(request, current_token_attribute, None) while limit is None or limit: -- GitLab From 5dca8372da3bc796fb313008716bb7b929572384 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 14 Aug 2015 15:49:53 -0700 Subject: [PATCH 155/295] Path fixup for last patch. --- apitools/base/py/list_pager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 438d011..85c2594 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """A helper function that executes a series of List queries for many APIs.""" -import encoding +from apitools.base.py import encoding __all__ = [ 'YieldFromList', -- GitLab From b39116b2f383b4d17a4e9e3b31247027ab168212 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 16 Aug 2015 00:21:20 -0700 Subject: [PATCH 156/295] Add a system for registering credentials fetching methods. This allows us to add new ways to fetch credentials that apitools itself need not know about. --- apitools/base/py/credentials_lib.py | 177 +++++++++++++++++----------- 1 file changed, 109 insertions(+), 68 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index cd4a4cf..9a8f36e 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -48,64 +48,55 @@ def SetCredentialsCacheFileLock(lock): cache_file_lock = lock -# TODO(craigcitro): Expose the extra args here somewhere higher up, -# possibly as flags in the generated CLI. +# List of additional methods we use when attempting to construct +# credentials. Users can register their own methods here, which we try +# before the defaults. +_CREDENTIALS_METHODS = [] + + +def _RegisterCredentialsMethod(method, position=None): + """Register a new method for fetching credentials. + + This new method should be a function with signature: + client_info, **kwds -> Credentials or None + This method can be used as a decorator, unless position needs to + be supplied. + + Note that method must *always* accept arbitrary keyword arguments. + + Args: + method: New credential-fetching method. + position: (default: None) Where in the list of methods to + add this; if None, we append. In all but rare cases, + this should be either 0 or None. + Returns: + method, for use as a decorator. + + """ + if position is None: + position = len(_CREDENTIALS_METHODS) + else: + position = min(position, len(_CREDENTIALS_METHODS)) + _CREDENTIALS_METHODS.insert(position, method) + return method + + def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, credentials_filename=None, - service_account_name=None, service_account_keyfile=None, - service_account_json_keyfile=None, - skip_application_default_credentials=False, api_key=None, # pylint: disable=unused-argument client=None, # pylint: disable=unused-argument - oauth2client_args=None): + oauth2client_args=None, + **kwds): """Attempt to get credentials, using an oauth dance as the last resort.""" scopes = util.NormalizeScopes(scopes) - if ((service_account_name and not service_account_keyfile) or - (service_account_keyfile and not service_account_name)): - raise exceptions.CredentialsError( - 'Service account name or keyfile provided without the other') - # TODO(craigcitro): Error checking. client_info = { 'client_id': client_id, 'client_secret': client_secret, - 'scope': ' '.join(sorted(util.NormalizeScopes(scopes))), + 'scope': ' '.join(sorted(scopes)), 'user_agent': user_agent or '%s-generated/0.1' % package_name, } - service_account_kwargs = { - 'user_agent': client_info['user_agent'], - } - if service_account_json_keyfile: - with open(service_account_json_keyfile) as keyfile: - service_account_info = json.load(keyfile) - account_type = service_account_info.get('type') - if account_type != oauth2client.client.SERVICE_ACCOUNT: - raise exceptions.CredentialsError( - 'Invalid service account credentials: %s' % ( - service_account_json_keyfile,)) - # pylint: disable=protected-access - credentials = oauth2client.service_account._ServiceAccountCredentials( - service_account_id=service_account_info['client_id'], - service_account_email=service_account_info['client_email'], - private_key_id=service_account_info['private_key_id'], - private_key_pkcs8_text=service_account_info['private_key'], - scopes=scopes, - **service_account_kwargs) - # pylint: enable=protected-access - return credentials - if service_account_name is not None: - credentials = ServiceAccountCredentialsFromFile( - service_account_name, service_account_keyfile, scopes, - service_account_kwargs=service_account_kwargs) - if credentials is not None: - return credentials - credentials = GaeAssertionCredentials.Get(scopes) - if credentials is not None: - return credentials - credentials = GceAssertionCredentials.Get(scopes) - if credentials is not None: - return credentials - if not skip_application_default_credentials: - credentials = _GetApplicationDefaultCredentials(scopes) + for method in _CREDENTIALS_METHODS: + credentials = method(client_info, **kwds) if credentials is not None: return credentials credentials_filename = credentials_filename or os.path.expanduser( @@ -134,27 +125,6 @@ def ServiceAccountCredentials(service_account_name, private_key, scopes, service_account_name, private_key, scopes, **service_account_kwargs) -def _GetApplicationDefaultCredentials(scopes): - gc = oauth2client.client.GoogleCredentials - with cache_file_lock: - try: - # pylint: disable=protected-access - # We've already done our own check for GAE/GCE - # credentials, we don't want to pay for checking again. - credentials = gc._implicit_credentials_from_files() - except oauth2client.client.ApplicationDefaultCredentialsError: - return None - # If we got back a non-service-account credential, we need to use - # a heuristic to decide whether or not the application default - # credential will work for us. We assume that if we're requesting - # cloud-platform, our scopes are a subset of cloud scopes, and the - # ADC will work. - cp = 'https://www.googleapis.com/auth/cloud-platform' - if not isinstance(credentials, gc) or cp in scopes: - return credentials - return None - - def _EnsureFileExists(filename): """Touches a file; returns False on error, True on success.""" if not os.path.exists(filename): @@ -519,3 +489,74 @@ def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name credentials.refresh(http) response, content = http.request(url) return json.loads(content or '{}') # Save ourselves from an empty reply. + + +@_RegisterCredentialsMethod +def _GetServiceAccountCredentials( + client_info, service_account_name=None, service_account_keyfile=None, + service_account_json_keyfile=None, **unused_kwds): + if ((service_account_name and not service_account_keyfile) or + (service_account_keyfile and not service_account_name)): + raise exceptions.CredentialsError( + 'Service account name or keyfile provided without the other') + scopes = client_info['scope'].split() + user_agent = client_info['user_agent'] + if service_account_json_keyfile: + with open(service_account_json_keyfile) as keyfile: + service_account_info = json.load(keyfile) + account_type = service_account_info.get('type') + if account_type != oauth2client.client.SERVICE_ACCOUNT: + raise exceptions.CredentialsError( + 'Invalid service account credentials: %s' % ( + service_account_json_keyfile,)) + # pylint: disable=protected-access + credentials = oauth2client.service_account._ServiceAccountCredentials( + service_account_id=service_account_info['client_id'], + service_account_email=service_account_info['client_email'], + private_key_id=service_account_info['private_key_id'], + private_key_pkcs8_text=service_account_info['private_key'], + scopes=scopes, user_agent=user_agent) + # pylint: enable=protected-access + return credentials + if service_account_name is not None: + credentials = ServiceAccountCredentialsFromFile( + service_account_name, service_account_keyfile, scopes, + service_account_kwargs={'user_agent': user_agent}) + if credentials is not None: + return credentials + + +@_RegisterCredentialsMethod +def _GetGaeServiceAccount(unused_client_info, scopes, **unused_kwds): + return GaeAssertionCredentials.Get(scopes=scopes) + + +@_RegisterCredentialsMethod +def _GetGceServiceAccount(unused_client_info, scopes, **unused_kwds): + return GceAssertionCredentials.Get(scopes=scopes) + + +@_RegisterCredentialsMethod +def _GetApplicationDefaultCredentials( + unused_client_info, scopes, skip_application_default_credentials=False, + **unused_kwds): + if skip_application_default_credentials: + return None + gc = oauth2client.client.GoogleCredentials + with cache_file_lock: + try: + # pylint: disable=protected-access + # We've already done our own check for GAE/GCE + # credentials, we don't want to pay for checking again. + credentials = gc._implicit_credentials_from_files() + except oauth2client.client.ApplicationDefaultCredentialsError: + return None + # If we got back a non-service account credential, we need to use + # a heuristic to decide whether or not the application default + # credential will work for us. We assume that if we're requesting + # cloud-platform, our scopes are a subset of cloud scopes, and the + # ADC will work. + cp = 'https://www.googleapis.com/auth/cloud-platform' + if not isinstance(credentials, gc) or cp in scopes: + return credentials + return None -- GitLab From 58f12b90d207dc1268d370e8a6b90818e434262e Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 20 Aug 2015 22:08:29 -0700 Subject: [PATCH 157/295] Add an internal hook for creating Http instances. This is primarily for internal use. --- apitools/base/py/http_wrapper.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 94c639e..03a094d 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -372,5 +372,16 @@ def _MakeRequestNoRetry(http, http_request, redirections=5, return response +_HTTP_FACTORIES = [] + + +def _RegisterHttpFactory(factory): + _HTTP_FACTORIES.append(factory) + + def GetHttp(**kwds): + for factory in _HTTP_FACTORIES: + http = factory(**kwds) + if http is not None: + return http return httplib2.Http(**kwds) -- GitLab From 6ce14297201d81cf0e2c2639fda3d63bbae3245f Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 20 Aug 2015 22:35:56 -0700 Subject: [PATCH 158/295] Add two more hooks for internal use. --- apitools/base/py/__init__.py | 6 ++++++ apitools/base/py/base_api.py | 6 ++++++ apitools/base/py/cli.py | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py index 53b34d0..0bbcf9f 100644 --- a/apitools/base/py/__init__.py +++ b/apitools/base/py/__init__.py @@ -12,3 +12,9 @@ from apitools.base.py.http_wrapper import * from apitools.base.py.list_pager import * from apitools.base.py.transfer import * from apitools.base.py.util import * + +try: + # pylint:disable=no-name-in-module + from apitools.base.py.internal import * +except ImportError: + pass diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 91e7f49..97bbb6f 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -202,6 +202,11 @@ class _UrlBuilder(object): self.__scheme, self.__netloc, self.relative_path, self.query, '')) +def _SkipGetCredentials(): + """Hook for skipping credentials. For internal use.""" + return False + + class BaseApiClient(object): """Base class for client libraries.""" @@ -230,6 +235,7 @@ class BaseApiClient(object): self.num_retries = num_retries self.max_retry_wait = max_retry_wait self._credentials = credentials + get_credentials = get_credentials and not _SkipGetCredentials() if get_credentials and not credentials: credentials_args = credentials_args or {} self._SetCredentials(**credentials_args) diff --git a/apitools/base/py/cli.py b/apitools/base/py/cli.py index 6f7aa32..eccd66b 100644 --- a/apitools/base/py/cli.py +++ b/apitools/base/py/cli.py @@ -12,3 +12,9 @@ cause pain. from apitools.base.py.app2 import * from apitools.base.py.base_cli import * + +try: + # pylint:disable=no-name-in-module + from apitools.base.py.internal.cli import * +except ImportError: + pass -- GitLab From 7db2f673370438159044d02d2d3fc0100ccb17d9 Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Wed, 26 Aug 2015 14:17:37 -0700 Subject: [PATCH 159/295] Fix OverflowError in GetRange on 32-bit Python --- apitools/base/py/transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index b0e3e18..aba55ca 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -463,7 +463,7 @@ class Download(_Transfer): progress_end_normalized = True response = self.__ProcessResponse(response) progress += response.length - if not response: + if response.length == 0: raise exceptions.TransferRetryError( 'Zero bytes unexpectedly returned in download response') -- GitLab From d21a5b121474aedcde2e65435987f73217b48448 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 26 Aug 2015 14:23:24 -0700 Subject: [PATCH 160/295] Update for v0.4.10 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d5a17d8..19a6d58 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.9' +_APITOOLS_VERSION = '0.4.10' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 88811c654e0365b5f90f6c5ade981c922a540f40 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 27 Aug 2015 13:12:00 -0700 Subject: [PATCH 161/295] Fix a bug in GetCredentials. A "cleanup" was making calls to registered functions with the wrong number of arguments. --- apitools/base/py/credentials_lib.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 9a8f36e..ba49e0d 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -527,19 +527,22 @@ def _GetServiceAccountCredentials( @_RegisterCredentialsMethod -def _GetGaeServiceAccount(unused_client_info, scopes, **unused_kwds): +def _GetGaeServiceAccount(client_info, **unused_kwds): + scopes = client_info['scope'].split(' ') return GaeAssertionCredentials.Get(scopes=scopes) @_RegisterCredentialsMethod -def _GetGceServiceAccount(unused_client_info, scopes, **unused_kwds): +def _GetGceServiceAccount(client_info, **unused_kwds): + scopes = client_info['scope'].split(' ') return GceAssertionCredentials.Get(scopes=scopes) @_RegisterCredentialsMethod def _GetApplicationDefaultCredentials( - unused_client_info, scopes, skip_application_default_credentials=False, + client_info, skip_application_default_credentials=False, **unused_kwds): + scopes = client_info['scope'].split() if skip_application_default_credentials: return None gc = oauth2client.client.GoogleCredentials -- GitLab From 4540ecb4c5bdd4ef88f1207a8672d11e12d2020c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 27 Aug 2015 13:15:00 -0700 Subject: [PATCH 162/295] Update for v0.4.11 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19a6d58..9fe7ecd 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.10' +_APITOOLS_VERSION = '0.4.11' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From aa3f20da081c156facc677dabcd536b4e3bfaf74 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 4 Sep 2015 14:45:17 -0400 Subject: [PATCH 163/295] Let gen_client run on windows Remove trivial dependence of gen_client on base_cli module, as the former depends on readline which is not supported on windows. --- apitools/base/py/base_cli.py | 5 ----- apitools/gen/gen_client_lib.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index 3fe3b28..7d6ed05 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -58,11 +58,6 @@ def DeclareBaseFlags(): _BASE_FLAGS_DECLARED = True -# NOTE: This is specified here so that it can be read by other files -# without depending on the flag to be registered. -TRACE_HELP = ( - 'A tracing token of the form "token:" ' - 'to include in api requests.') FLAGS = flags.FLAGS diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index c7f4bee..e39e2fd 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -9,7 +9,6 @@ import datetime from six.moves import urllib_parse -from apitools.base.py import base_cli from apitools.gen import command_registry from apitools.gen import message_registry from apitools.gen import service_registry @@ -27,7 +26,8 @@ def _StandardQueryParametersSchema(discovery_doc): # We add an entry for the trace, since Discovery doesn't. standard_query_schema['properties']['trace'] = { 'type': 'string', - 'description': base_cli.TRACE_HELP, + 'description': ('A tracing token of the form "token:" ' + 'to include in api requests.'), 'location': 'query', } return standard_query_schema -- GitLab From 6c912f680e9daac319d765ae3b4ef91249e5c886 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 4 Sep 2015 17:58:59 -0400 Subject: [PATCH 164/295] Remove unused DescriptorGenerator attribute --- apitools/gen/gen_client_lib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index c7f4bee..cb97ae6 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -62,8 +62,6 @@ class DescriptorGenerator(object): self.__generate_cli = generate_cli self.__root_package = root_package self.__base_files_package = base_package - self.__base_files_target = ( - '//cloud/bigscience/apitools/base/py:apitools_base') self.__names = names self.__base_url, self.__base_path = _ComputePaths( self.__package, self.__client_info.url_version, -- GitLab From b3470469df289e469ee5406a17437c4c5de56b9b Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 10 Sep 2015 14:15:37 -0700 Subject: [PATCH 165/295] Ensure apitools.gen.util.Chdir cleans up. Previously, if a test failed, later tests using `Chdir` would fail, since they're in a now-nonexistent directory, and `os.getcwd()` fails. --- apitools/gen/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 8236de8..bdc570e 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -141,9 +141,11 @@ def Chdir(dirname, create=True): else: os.mkdir(dirname) previous_directory = os.getcwd() - os.chdir(dirname) - yield - os.chdir(previous_directory) + try: + os.chdir(dirname) + yield + finally: + os.chdir(previous_directory) def NormalizeVersion(version): -- GitLab From 5d7512dddf326b5b0593cebc0a08fde99ed767eb Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 10 Sep 2015 15:08:16 -0700 Subject: [PATCH 166/295] Update our use of protorpc metaclasses. We need to subclass a given protorpc metaclass in extra_types.py; as luck would have it, that had to be tweaked and updated for python3, which means we've got to do something here, too. This updates us to require a new minimum protorpc version, and switches from `protorpc.messages.Field.__metaclass__` to `protorpc.messages._FieldMeta` for the metaclass we inherit from. --- apitools/base/py/extra_types.py | 32 +++++++++++++++++--------------- setup.py | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 68fc716..403ceda 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -35,23 +35,25 @@ DateTimeMessage = message_types.DateTimeMessage # pylint:enable=invalid-name -class DateField(messages.Field): +# We insert our own metaclass here to avoid letting ProtoRPC +# register this as the default field type for strings. +# * since ProtoRPC does this via metaclasses, we don't have any +# choice but to use one ourselves +# * since a subclass's metaclass must inherit from its superclass's +# metaclass, we're forced to have this hard-to-read inheritance. +# +# pylint: disable=protected-access +class _FieldMeta(messages._FieldMeta): + + def __init__(cls, name, bases, dct): # pylint: disable=no-self-argument + # pylint: disable=super-init-not-called,non-parent-init-called + type.__init__(cls, name, bases, dct) +# pylint: enable=protected-access - """Field definition for Date values.""" - # We insert our own metaclass here to avoid letting ProtoRPC - # register this as the default field type for strings. - # * since ProtoRPC does this via metaclasses, we don't have any - # choice but to use one ourselves - # * since a subclass's metaclass must inherit from its superclass's - # metaclass, we're forced to have this hard-to-read inheritance. - # - # pylint: disable=invalid-name - class __metaclass__(messages.Field.__metaclass__): - - def __init__(cls, name, bases, dct): - super(messages.Field.__metaclass__, cls).__init__(name, bases, dct) - # pylint: enable=invalid-name +class DateField(six.with_metaclass(_FieldMeta, messages.Field)): + + """Field definition for Date values.""" VARIANTS = frozenset([messages.Variant.STRING]) DEFAULT_VARIANT = messages.Variant.STRING diff --git a/setup.py b/setup.py index 9fe7ecd..4e72511 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ except ImportError: REQUIRED_PACKAGES = [ 'httplib2>=0.8', 'oauth2client>=1.4.8', - 'protorpc>=0.9.1', + 'protorpc>=0.11.0', 'six>=1.9.0', ] -- GitLab From 3dedd3518a82e5591571e4a02227028bf01f3ff0 Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Fri, 11 Sep 2015 13:09:10 -0700 Subject: [PATCH 167/295] Fix double-POST for auto_transfer=False uploads --- apitools/base/py/transfer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index aba55ca..f144582 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -831,6 +831,8 @@ class Upload(_Transfer): # go ahead and pump the bytes now. if self.auto_transfer: return self.StreamInChunks() + else: + return http_response def __GetLastByte(self, range_header): _, _, end = range_header.partition('-') -- GitLab From a2117ba7b0507521ddc132c35c1eec9a664acb89 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Thu, 10 Sep 2015 13:28:11 -0400 Subject: [PATCH 168/295] Remove google.apputils/gflags usage in favor of argparse. This PR converts apitools.gen.gen/gen_client.py to use python standard argparse for command line argument parsing. Note that now type of generation argument must occur last on the command line. --- apitools/gen/client_generation_test.py | 23 +- apitools/gen/gen_client.py | 352 ++++++------ apitools/gen/gen_client_test.py | 68 +++ apitools/gen/test_utils.py | 40 ++ apitools/gen/testdata/dns_v1.json | 707 +++++++++++++++++++++++++ setup.py | 2 +- 6 files changed, 998 insertions(+), 194 deletions(-) create mode 100644 apitools/gen/gen_client_test.py create mode 100644 apitools/gen/test_utils.py create mode 100644 apitools/gen/testdata/dns_v1.json diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index f34d954..cb58bb4 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -1,13 +1,11 @@ """Test gen_client against all the APIs we use regularly.""" -import contextlib import logging import os -import shutil import subprocess -import sys import tempfile +from apitools.gen import test_utils import unittest2 @@ -19,31 +17,16 @@ _API_LIST = [ ] -@contextlib.contextmanager -def TempDir(): - original_dir = os.getcwd() - path = tempfile.mkdtemp() - try: - os.chdir(path) - yield path - finally: - os.chdir(original_dir) - shutil.rmtree(path) - - class ClientGenerationTest(unittest2.TestCase): def setUp(self): super(ClientGenerationTest, self).setUp() self.gen_client_binary = 'gen_client' - # unittest in 2.6 doesn't have skipIf. - @unittest2.skipUnless(sys.version_info[0] == 2 and - sys.version_info[1] == 7, - 'Only runs in Python 2.7') + @test_utils.RunOnlyOnPython27 def testGeneration(self): for api in _API_LIST: - with TempDir(): + with test_utils.TempDir(change_to=True): args = [ self.gen_client_binary, '--client_id=12345', diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index a8e32f7..d1de17c 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Command-line interface to gen_client.""" +import argparse import contextlib import json import logging @@ -8,90 +9,10 @@ import os import pkgutil import sys -from google.apputils import appcommands -import gflags as flags - from apitools.base.py import exceptions from apitools.gen import gen_client_lib from apitools.gen import util -flags.DEFINE_string( - 'infile', '', - 'Filename for the discovery document. Mutually exclusive with ' - '--discovery_url.') -flags.DEFINE_string( - 'discovery_url', '', - 'URL (or "name.version") of the discovery document to use. ' - 'Mutually exclusive with --infile.') - -flags.DEFINE_string( - 'base_package', - 'apitools.base.py', - 'Base package path of apitools (defaults to ' - 'apitools.base.py)' -) -flags.DEFINE_string( - 'outdir', '', - 'Directory name for output files. (Defaults to the API name.)') -flags.DEFINE_boolean( - 'overwrite', False, - 'Only overwrite the output directory if this flag is specified.') -flags.DEFINE_string( - 'root_package', '', - 'Python import path for where these modules should be imported from.') - - -flags.DEFINE_multistring( - 'strip_prefix', [], - 'Prefix to strip from type names in the discovery document. (May ' - 'be specified multiple times.)') -flags.DEFINE_string( - 'api_key', None, - 'API key to use for API access.') -flags.DEFINE_string( - 'client_json', None, - 'Use the given file downloaded from the dev. console for client_id ' - 'and client_secret.') -flags.DEFINE_string( - 'client_id', '1042881264118.apps.googleusercontent.com', - 'Client ID to use for the generated client.') -flags.DEFINE_string( - 'client_secret', 'x_Tw5K8nnjoRAqULM9PFAC2b', - 'Client secret for the generated client.') -flags.DEFINE_multistring( - 'scope', [], - 'Scopes to request in the generated client. May be specified more than ' - 'once.') -flags.DEFINE_string( - 'user_agent', '', - 'User agent for the generated client. Defaults to -generated/0.1.') -flags.DEFINE_boolean( - 'generate_cli', True, 'If True, a CLI is also generated.') -flags.DEFINE_list( - 'unelidable_request_methods', [], - 'Full method IDs of methods for which we should NOT try to elide ' - 'the request type. (Should be a comma-separated list.)') - -flags.DEFINE_boolean( - 'experimental_capitalize_enums', False, - 'Dangerous: attempt to rewrite enum values to be uppercase.') -flags.DEFINE_enum( - 'experimental_name_convention', util.Names.DEFAULT_NAME_CONVENTION, - util.Names.NAME_CONVENTIONS, - 'Dangerous: use a particular style for generated names.') -flags.DEFINE_boolean( - 'experimental_proto2_output', False, - 'Dangerous: also output a proto2 message file.') - -FLAGS = flags.FLAGS - -flags.RegisterValidator( - 'infile', lambda i: not (i and FLAGS.discovery_url), - 'Cannot specify both --infile and --discovery_url') -flags.RegisterValidator( - 'discovery_url', lambda i: not (i and FLAGS.infile), - 'Cannot specify both --infile and --discovery_url') - def _CopyLocalFile(filename): with contextlib.closing(open(filename, 'w')) as out: @@ -106,44 +27,44 @@ def _CopyLocalFile(filename): _DISCOVERY_DOC = None -def _GetDiscoveryDocFromFlags(): +def _GetDiscoveryDocFromFlags(args): """Get the discovery doc from flags.""" global _DISCOVERY_DOC # pylint: disable=global-statement if _DISCOVERY_DOC is None: - if FLAGS.discovery_url: + if args.discovery_url: try: - discovery_doc = util.FetchDiscoveryDoc(FLAGS.discovery_url) + discovery_doc = util.FetchDiscoveryDoc(args.discovery_url) except exceptions.CommunicationError: raise exceptions.GeneratedClientError( 'Could not fetch discovery doc') else: - infile = os.path.expanduser(FLAGS.infile) or '/dev/stdin' + infile = os.path.expanduser(args.infile) or '/dev/stdin' discovery_doc = json.load(open(infile)) _DISCOVERY_DOC = discovery_doc return _DISCOVERY_DOC -def _GetCodegenFromFlags(): +def _GetCodegenFromFlags(args): """Create a codegen object from flags.""" - discovery_doc = _GetDiscoveryDocFromFlags() + discovery_doc = _GetDiscoveryDocFromFlags(args) names = util.Names( - FLAGS.strip_prefix, - FLAGS.experimental_name_convention, - FLAGS.experimental_capitalize_enums) + args.strip_prefix, + args.experimental_name_convention, + args.experimental_capitalize_enums) - if FLAGS.client_json: + if args.client_json: try: - with open(FLAGS.client_json) as client_json: + with open(args.client_json) as client_json: f = json.loads(client_json.read()) web = f.get('installed', f.get('web', {})) client_id = web.get('client_id') client_secret = web.get('client_secret') except IOError: raise exceptions.NotFoundError( - 'Failed to open client json file: %s' % FLAGS.client_json) + 'Failed to open client json file: %s' % args.client_json) else: - client_id = FLAGS.client_id - client_secret = FLAGS.client_secret + client_id = args.client_id + client_secret = args.client_secret if not client_id: logging.warning('No client ID supplied') @@ -154,23 +75,23 @@ def _GetCodegenFromFlags(): client_secret = '' client_info = util.ClientInfo.Create( - discovery_doc, FLAGS.scope, client_id, client_secret, - FLAGS.user_agent, names, FLAGS.api_key) - outdir = os.path.expanduser(FLAGS.outdir) or client_info.default_directory - if os.path.exists(outdir) and not FLAGS.overwrite: + discovery_doc, args.scope, client_id, client_secret, + args.user_agent, names, args.api_key) + outdir = os.path.expanduser(args.outdir) or client_info.default_directory + if os.path.exists(outdir) and not args.overwrite: raise exceptions.ConfigurationValueError( 'Output directory exists, pass --overwrite to replace ' 'the existing files.') if not os.path.exists(outdir): os.makedirs(outdir) - root_package = FLAGS.root_package or util.GetPackage(outdir) + root_package = args.root_package or util.GetPackage(outdir) return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, root_package, outdir, - base_package=FLAGS.base_package, - generate_cli=FLAGS.generate_cli, - use_proto2=FLAGS.experimental_proto2_output, - unelidable_request_methods=FLAGS.unelidable_request_methods) + base_package=args.base_package, + generate_cli=args.generate_cli, + use_proto2=args.experimental_proto2_output, + unelidable_request_methods=args.unelidable_request_methods) # TODO(craigcitro): Delete this if we don't need this functionality. @@ -196,7 +117,7 @@ def _WriteProtoFiles(codegen): codegen.WriteServicesProtoFile(out) -def _WriteGeneratedFiles(codegen): +def _WriteGeneratedFiles(args, codegen): if codegen.use_proto2: _WriteProtoFiles(codegen) with util.Chdir(codegen.outdir): @@ -204,7 +125,7 @@ def _WriteGeneratedFiles(codegen): codegen.WriteMessagesFile(out) with open(codegen.client_info.client_file_name, 'w') as out: codegen.WriteClientLibrary(out) - if FLAGS.generate_cli: + if args.generate_cli: with open(codegen.client_info.cli_file_name, 'w') as out: codegen.WriteCli(out) os.chmod(codegen.client_info.cli_file_name, 0o755) @@ -221,86 +142,171 @@ def _WriteSetupPy(codegen): codegen.WriteSetupPy(out) -class GenerateClient(appcommands.Cmd): +def GenerateClient(args): """Driver for client code generation.""" - def Run(self, _): - """Create a client library.""" - codegen = _GetCodegenFromFlags() - if codegen is None: - logging.error('Failed to create codegen, exiting.') - return 128 - _WriteGeneratedFiles(codegen) - _WriteInit(codegen) + codegen = _GetCodegenFromFlags(args) + if codegen is None: + logging.error('Failed to create codegen, exiting.') + return 128 + _WriteGeneratedFiles(args, codegen) + _WriteInit(codegen) -class GeneratePipPackage(appcommands.Cmd): +def GeneratePipPackage(args): """Generate a client as a pip-installable tarball.""" - def Run(self, _): - """Create a client in a pip package.""" - discovery_doc = _GetDiscoveryDocFromFlags() - package = discovery_doc['name'] - original_outdir = os.path.expanduser(FLAGS.outdir) - FLAGS.outdir = os.path.join( - FLAGS.outdir, 'apitools/clients/%s' % package) - FLAGS.root_package = 'apitools.clients.%s' % package - FLAGS.generate_cli = False - codegen = _GetCodegenFromFlags() - if codegen is None: - logging.error('Failed to create codegen, exiting.') - return 1 - _WriteGeneratedFiles(codegen) - _WriteInit(codegen) - with util.Chdir(original_outdir): - _WriteSetupPy(codegen) - with util.Chdir('apitools'): + discovery_doc = _GetDiscoveryDocFromFlags(args) + package = discovery_doc['name'] + original_outdir = os.path.expanduser(args.outdir) + args.outdir = os.path.join( + args.outdir, 'apitools/clients/%s' % package) + args.root_package = 'apitools.clients.%s' % package + args.generate_cli = False + codegen = _GetCodegenFromFlags(args) + if codegen is None: + logging.error('Failed to create codegen, exiting.') + return 1 + _WriteGeneratedFiles(args, codegen) + _WriteInit(codegen) + with util.Chdir(original_outdir): + _WriteSetupPy(codegen) + with util.Chdir('apitools'): + _WriteIntermediateInit(codegen) + with util.Chdir('clients'): _WriteIntermediateInit(codegen) - with util.Chdir('clients'): - _WriteIntermediateInit(codegen) -class GenerateProto(appcommands.Cmd): - +def GenerateProto(args): """Generate just the two proto files for a given API.""" - def Run(self, _): - """Create proto definitions for an API.""" - codegen = _GetCodegenFromFlags() - _WriteProtoFiles(codegen) - - -# pylint:disable=invalid-name - - -def run_main(): - """Function to be used as setuptools script entry point.""" - # Put the flags for this module somewhere the flags module will look - # for them. - - # pylint:disable=protected-access - new_name = flags._GetMainModule() - sys.modules[new_name] = sys.modules['__main__'] - for flag in FLAGS.FlagsByModuleDict().get(__name__, []): - FLAGS._RegisterFlagByModule(new_name, flag) - for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): - FLAGS._RegisterKeyFlagForModule(new_name, key_flag) - # pylint:enable=protected-access - - # Now set __main__ appropriately so that appcommands will be - # happy. - sys.modules['__main__'] = sys.modules[__name__] - appcommands.Run() - sys.modules['__main__'] = sys.modules.pop(new_name) - - -def main(_): - appcommands.AddCmd('client', GenerateClient) - appcommands.AddCmd('pip_package', GeneratePipPackage) - appcommands.AddCmd('proto', GenerateProto) - + codegen = _GetCodegenFromFlags(args) + _WriteProtoFiles(codegen) + + +def main(argv=None): + if argv is None: + argv = sys.argv + parser = argparse.ArgumentParser( + description='Apitools Client Code Generator') + + discovery_group = parser.add_mutually_exclusive_group() + discovery_group.add_argument( + '--infile', + help=('Filename for the discovery document. Mutually exclusive with ' + '--discovery_url')) + + discovery_group.add_argument( + '--discovery_url', + help=('URL (or "name.version") of the discovery document to use. ' + 'Mutually exclusive with --infile.')) + + parser.add_argument( + '--base_package', + default='apitools.base.py', + help='Base package path of apitools (defaults to apitools.base.py') + + parser.add_argument( + '--outdir', + default='', + help='Directory name for output files. (Defaults to the API name.)') + + parser.add_argument( + '--overwrite', + default=False, action='store_true', + help='Only overwrite the output directory if this flag is specified.') + + parser.add_argument( + '--root_package', + default='', + help=('Python import path for where these modules ' + 'should be imported from.')) + + parser.add_argument( + '--strip_prefix', nargs='*', + default=[], + help=('Prefix to strip from type names in the discovery document. ' + '(May be specified multiple times.)')) + + parser.add_argument( + '--api_key', + help=('API key to use for API access.')) + + parser.add_argument( + '--client_json', + help=('Use the given file downloaded from the dev. console for ' + 'client_id and client_secret.')) + + parser.add_argument( + '--client_id', + default='1042881264118.apps.googleusercontent.com', + help='Client ID to use for the generated client.') + + parser.add_argument( + '--client_secret', + default='x_Tw5K8nnjoRAqULM9PFAC2b', + help='Client secret for the generated client.') + + parser.add_argument( + '--scope', nargs='*', + default=[], + help=('Scopes to request in the generated client. ' + 'May be specified more than once.')) + + parser.add_argument( + '--user_agent', + default='x_Tw5K8nnjoRAqULM9PFAC2b', + help=('User agent for the generated client. ' + 'Defaults to -generated/0.1.')) + + parser.add_argument( + '--generate_cli', dest='generate_cli', action='store_true', + help='If specified (default), a CLI is also generated.') + parser.add_argument( + '--nogenerate_cli', dest='generate_cli', action='store_false', + help='CLI will not be generated.') + parser.set_defaults(generate_cli=True) + + parser.add_argument( + '--unelidable_request_methods', nargs='*', + default=[], + help=('Full method IDs of methods for which we should NOT try to ' + 'elide the request type. (Should be a comma-separated list.')) + + parser.add_argument( + '--experimental_capitalize_enums', + default=False, action='store_true', + help='Dangerous: attempt to rewrite enum values to be uppercase.') + + parser.add_argument( + '--experimental_name_convention', + choices=util.Names.NAME_CONVENTIONS, + default=util.Names.DEFAULT_NAME_CONVENTION, + help='Dangerous: use a particular style for generated names.') + + parser.add_argument( + '--experimental_proto2_output', + default=False, action='store_true', + help='Dangerous: also output a proto2 message file.') + + subparsers = parser.add_subparsers(help='Type of generated code') + + client_parser = subparsers.add_parser( + 'client', help='Generate apitools client in destination folder') + client_parser.set_defaults(func=GenerateClient) + + pip_package_parser = subparsers.add_parser( + 'pip_package', help='Generate apitools client pip package') + pip_package_parser.set_defaults(func=GeneratePipPackage) + + proto_parser = subparsers.add_parser( + 'proto', help='Generate apitools client protos') + proto_parser.set_defaults(func=GenerateProto) + + args = parser.parse_args(argv[1:]) + return args.func(args) or 0 if __name__ == '__main__': - appcommands.Run() + sys.exit(main()) diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py new file mode 100644 index 0000000..435c42d --- /dev/null +++ b/apitools/gen/gen_client_test.py @@ -0,0 +1,68 @@ +"""Test for gen_client module.""" + +from apitools.gen import gen_client +from apitools.gen import test_utils + +import unittest2 +import os + + +def GetDocPath(name): + return os.path.join(os.path.dirname(__file__), 'testdata', name) + + +@test_utils.RunOnlyOnPython27 +class ClientGenCliTest(unittest2.TestCase): + + def testHelp_NotEnoughArguments(self): + with self.assertRaisesRegexp(SystemExit, '0'): + with test_utils.CaptureOutput() as (_, err): + gen_client.main([gen_client.__file__, '-h']) + err_output = err.getvalue() + self.assertIn('usage:', err_output) + self.assertIn('error: too few arguments', err_output) + + def testGenClient_SimpleDoc(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--nogenerate_cli', + '--infile', GetDocPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', 'google.apis', + 'client' + ]) + self.assertEquals( + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py']), + set(os.listdir(tmp_dir_path))) + + def testGenPipPackage_SimpleDoc(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--nogenerate_cli', + '--infile', GetDocPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', 'google.apis', + 'pip_package' + ]) + self.assertEquals( + set(['apitools', 'setup.py']), + set(os.listdir(tmp_dir_path))) + + def testGenProto_SimpleDoc(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--nogenerate_cli', + '--infile', GetDocPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', 'google.apis', + 'proto' + ]) + self.assertEquals( + set(['dns_v1_messages.proto', 'dns_v1_services.proto']), + set(os.listdir(tmp_dir_path))) diff --git a/apitools/gen/test_utils.py b/apitools/gen/test_utils.py new file mode 100644 index 0000000..9778823 --- /dev/null +++ b/apitools/gen/test_utils.py @@ -0,0 +1,40 @@ +"""Various utilities used in tests.""" + +import contextlib +import os +import tempfile +import shutil +import sys + +import six +import unittest2 + + +RunOnlyOnPython27 = unittest2.skipUnless( + sys.version_info[:2] == (2, 7), 'Only runs in Python 2.7') + + +@contextlib.contextmanager +def TempDir(change_to=False): + if change_to: + original_dir = os.getcwd() + path = tempfile.mkdtemp() + try: + if change_to: + os.chdir(path) + yield path + finally: + if change_to: + os.chdir(original_dir) + shutil.rmtree(path) + + +@contextlib.contextmanager +def CaptureOutput(): + new_stdout, new_stderr = six.StringIO(), six.StringIO() + old_stdout, old_stderr = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_stdout, new_stderr + yield new_stdout, new_stderr + finally: + sys.stdout, sys.stderr = old_stdout, old_stderr diff --git a/apitools/gen/testdata/dns_v1.json b/apitools/gen/testdata/dns_v1.json new file mode 100644 index 0000000..77c1553 --- /dev/null +++ b/apitools/gen/testdata/dns_v1.json @@ -0,0 +1,707 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "dns:v1", + "name": "dns", + "version": "v1", + "revision": "20150807", + "title": "Google Cloud DNS API", + "description": "The Google Cloud DNS API provides services for configuring and serving authoritative DNS records.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/cloud-dns", + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/dns/v1/projects/", + "basePath": "/dns/v1/projects/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "dns/v1/projects/", + "batchPath": "batch", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/cloud-platform.read-only": { + "description": "MESSAGE UNDER CONSTRUCTION View your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readonly": { + "description": "View your DNS records hosted by Google Cloud DNS" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readwrite": { + "description": "View and manage your DNS records hosted by Google Cloud DNS" + } + } + } + }, + "schemas": { + "Change": { + "id": "Change", + "type": "object", + "description": "An atomic update to a collection of ResourceRecordSets.", + "properties": { + "additions": { + "type": "array", + "description": "Which ResourceRecordSets to add?", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "deletions": { + "type": "array", + "description": "Which ResourceRecordSets to remove? Must match existing data exactly.", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#change\".", + "default": "dns#change" + }, + "startTime": { + "type": "string", + "description": "The time that this operation was started by the server. This is in RFC3339 text format." + }, + "status": { + "type": "string", + "description": "Status of the operation (output only).", + "enum": [ + "done", + "pending" + ], + "enumDescriptions": [ + "", + "" + ] + } + } + }, + "ChangesListResponse": { + "id": "ChangesListResponse", + "type": "object", + "description": "The response to a request to enumerate Changes to a ResourceRecordSets collection.", + "properties": { + "changes": { + "type": "array", + "description": "The requested changes.", + "items": { + "$ref": "Change" + } + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#changesListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a \"snapshot\" of collections larger than the maximum page size." + } + } + }, + "ManagedZone": { + "id": "ManagedZone", + "type": "object", + "description": "A zone is a subtree of the DNS namespace under one administrative responsibility. A ManagedZone is a resource that represents a DNS zone hosted by the Cloud DNS service.", + "properties": { + "creationTime": { + "type": "string", + "description": "The time that this resource was created on the server. This is in RFC3339 text format. Output only." + }, + "description": { + "type": "string", + "description": "A mutable string of at most 1024 characters associated with this resource for the user's convenience. Has no effect on the managed zone's function." + }, + "dnsName": { + "type": "string", + "description": "The DNS name of this managed zone, for instance \"example.com.\"." + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)", + "format": "uint64" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#managedZone\".", + "default": "dns#managedZone" + }, + "name": { + "type": "string", + "description": "User assigned name for this resource. Must be unique within the project. The name must be 1-32 characters long, must begin with a letter, end with a letter or digit, and only contain lowercase letters, digits or dashes." + }, + "nameServerSet": { + "type": "string", + "description": "Optionally specifies the NameServerSet for this ManagedZone. A NameServerSet is a set of DNS name servers that all host the same ManagedZones. Most users will leave this field unset." + }, + "nameServers": { + "type": "array", + "description": "Delegate your managed_zone to these virtual name servers; defined by the server (output only)", + "items": { + "type": "string" + } + } + } + }, + "ManagedZonesListResponse": { + "id": "ManagedZonesListResponse", + "type": "object", + "properties": { + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#managedZonesListResponse" + }, + "managedZones": { + "type": "array", + "description": "The managed zone resources.", + "items": { + "$ref": "ManagedZone" + } + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your page token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + } + } + }, + "Project": { + "id": "Project", + "type": "object", + "description": "A project resource. The project is a top level container for resources including Cloud DNS ManagedZones. Projects can be created only in the APIs console.", + "properties": { + "id": { + "type": "string", + "description": "User assigned unique identifier for the resource (output only)." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#project\".", + "default": "dns#project" + }, + "number": { + "type": "string", + "description": "Unique numeric identifier for the resource; defined by the server (output only).", + "format": "uint64" + }, + "quota": { + "$ref": "Quota", + "description": "Quotas assigned to this project (output only)." + } + } + }, + "Quota": { + "id": "Quota", + "type": "object", + "description": "Limits associated with a Project.", + "properties": { + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#quota\".", + "default": "dns#quota" + }, + "managedZones": { + "type": "integer", + "description": "Maximum allowed number of managed zones in the project.", + "format": "int32" + }, + "resourceRecordsPerRrset": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecords per ResourceRecordSet.", + "format": "int32" + }, + "rrsetAdditionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to add per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetDeletionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to delete per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetsPerManagedZone": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets per zone in the project.", + "format": "int32" + }, + "totalRrdataSizePerChange": { + "type": "integer", + "description": "Maximum allowed size for total rrdata in one ChangesCreateRequest in bytes.", + "format": "int32" + } + } + }, + "ResourceRecordSet": { + "id": "ResourceRecordSet", + "type": "object", + "description": "A unit of data that will be returned by the DNS servers.", + "properties": { + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#resourceRecordSet\".", + "default": "dns#resourceRecordSet" + }, + "name": { + "type": "string", + "description": "For example, www.example.com." + }, + "rrdatas": { + "type": "array", + "description": "As defined in RFC 1035 (section 5) and RFC 1034 (section 3.6.1).", + "items": { + "type": "string" + } + }, + "ttl": { + "type": "integer", + "description": "Number of seconds that this ResourceRecordSet can be cached by resolvers.", + "format": "int32" + }, + "type": { + "type": "string", + "description": "The identifier of a supported record type, for example, A, AAAA, MX, TXT, and so on." + } + } + }, + "ResourceRecordSetsListResponse": { + "id": "ResourceRecordSetsListResponse", + "type": "object", + "properties": { + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#resourceRecordSetsListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + }, + "rrsets": { + "type": "array", + "description": "The resource record set resources.", + "items": { + "$ref": "ResourceRecordSet" + } + } + } + } + }, + "resources": { + "changes": { + "methods": { + "create": { + "id": "dns.changes.create", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "POST", + "description": "Atomically update the ResourceRecordSet collection.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "request": { + "$ref": "Change" + }, + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.changes.get", + "path": "{project}/managedZones/{managedZone}/changes/{changeId}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Change.", + "parameters": { + "changeId": { + "type": "string", + "description": "The identifier of the requested change, from a previous ResourceRecordSetsChangeResponse.", + "required": true, + "location": "path" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone", + "changeId" + ], + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.changes.list", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "GET", + "description": "Enumerate Changes to a ResourceRecordSet collection.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "sortBy": { + "type": "string", + "description": "Sorting criterion. The only supported value is change sequence.", + "default": "changeSequence", + "enum": [ + "changeSequence" + ], + "enumDescriptions": [ + "" + ], + "location": "query" + }, + "sortOrder": { + "type": "string", + "description": "Sorting order direction: 'ascending' or 'descending'.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ChangesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "managedZones": { + "methods": { + "create": { + "id": "dns.managedZones.create", + "path": "{project}/managedZones", + "httpMethod": "POST", + "description": "Create a new ManagedZone.", + "parameters": { + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "request": { + "$ref": "ManagedZone" + }, + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "delete": { + "id": "dns.managedZones.delete", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "DELETE", + "description": "Delete a previously created ManagedZone.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.managedZones.get", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing ManagedZone.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.managedZones.list", + "path": "{project}/managedZones", + "httpMethod": "GET", + "description": "Enumerate ManagedZones that have been created but not yet deleted.", + "parameters": { + "dnsName": { + "type": "string", + "description": "Restricts the list to return only zones with this domain name.", + "location": "query" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "ManagedZonesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "projects": { + "methods": { + "get": { + "id": "dns.projects.get", + "path": "{project}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Project.", + "parameters": { + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "Project" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "resourceRecordSets": { + "methods": { + "list": { + "id": "dns.resourceRecordSets.list", + "path": "{project}/managedZones/{managedZone}/rrsets", + "httpMethod": "GET", + "description": "Enumerate ResourceRecordSets that have been created but not yet deleted.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "name": { + "type": "string", + "description": "Restricts the list to return only records with this fully qualified domain name.", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "type": { + "type": "string", + "description": "Restricts the list to return only records of this type. If present, the \"name\" parameter must also be present.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ResourceRecordSetsListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + } + } +} diff --git a/setup.py b/setup.py index 9fe7ecd..c23dca7 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ TESTING_PACKAGES = [ ] CONSOLE_SCRIPTS = [ - 'gen_client = apitools.gen.gen_client:run_main', + 'gen_client = apitools.gen.gen_client:main', 'oauth2l = apitools.scripts.oauth2l:run_main [cli]', ] -- GitLab From 1e98638e224adb7793a03151e72649cd766cf4ae Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Sun, 13 Sep 2015 15:46:09 -0400 Subject: [PATCH 169/295] Fix import path in the generated client. When --root_package flag is specified for apitools/gen/gen_client.py, for three generated files __init__.py, ..._client.py, ..._messages.py, import paths in the generated client is incorrect/inconsistent. In the generated __init__.py file, messages are imported as from .__messages import ... while in the generated client root_package skipped, producing import __messages as messages. With this PR this import line in client becomes from import __messages as messages If root_package is not specified then one would get old behavior. --- apitools/gen/service_registry.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index d4826a6..20bff15 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -21,7 +21,7 @@ class ServiceRegistry(object): def __init__(self, client_info, message_registry, command_registry, base_url, base_path, names, - root_package_dir, base_files_package, + root_package, base_files_package, unelidable_request_methods): self.__client_info = client_info self.__package = client_info.package @@ -31,7 +31,7 @@ class ServiceRegistry(object): self.__command_registry = command_registry self.__base_url = base_url self.__base_path = base_path - self.__root_package_dir = root_package_dir + self.__root_package = root_package self.__base_files_package = base_files_package self.__unelidable_request_methods = unelidable_request_methods self.__all_scopes = set(self.__client_info.scopes) @@ -191,7 +191,10 @@ class ServiceRegistry(object): printer('# NOTE: This file is autogenerated and should not be edited ' 'by hand.') printer('from %s import base_api', self.__base_files_package) - import_prefix = '' + if self.__root_package: + import_prefix = 'from {0} '.format(self.__root_package) + else: + import_prefix = '' printer('%simport %s as messages', import_prefix, client_info.messages_rule_name) printer() -- GitLab From 85048005e0ba90f586a9ff2926ca91ac03e71fcd Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 11:16:28 -0700 Subject: [PATCH 170/295] Fork protorpc into a protorpclite module. ProtoRPC has a number of large features (remotes and services, WSGI/webapp integration, etc) that we don't use, and are currently causing a bit of friction for (internal) package management. Easy solution: fork. This commit simply pulls in the relevant modules, and strips out some of the bits we don't use. Going forward, it's quite possible that we can now clean up much of apitools_base.encoding and apitools_base.extra_types by simply modifying protorpclite. --- apitools/base/protorpclite/__init__.py | 0 apitools/base/protorpclite/descriptor.py | 613 +++++ apitools/base/protorpclite/descriptor_test.py | 506 ++++ apitools/base/protorpclite/message_types.py | 119 + .../base/protorpclite/message_types_test.py | 88 + apitools/base/protorpclite/messages.py | 1935 +++++++++++++++ apitools/base/protorpclite/messages_test.py | 2084 +++++++++++++++++ apitools/base/protorpclite/protojson.py | 366 +++ apitools/base/protorpclite/protojson_test.py | 441 ++++ apitools/base/protorpclite/test_util.py | 665 ++++++ apitools/base/protorpclite/util.py | 295 +++ apitools/base/protorpclite/util_test.py | 232 ++ apitools/base/py/base_api.py | 4 +- apitools/base/py/base_api_test.py | 4 +- apitools/base/py/encoding.py | 8 +- apitools/base/py/encoding_test.py | 8 +- apitools/base/py/extra_types.py | 6 +- apitools/base/py/extra_types_test.py | 2 +- apitools/base/py/testing/mock.py | 2 +- apitools/base/py/testing/mock_test.py | 3 +- .../testclient/fusiontables_v1_messages.py | 2 +- apitools/base/py/util.py | 2 +- apitools/base/py/util_test.py | 2 +- apitools/gen/command_registry.py | 12 +- apitools/gen/extended_descriptor.py | 6 +- apitools/gen/message_registry.py | 16 +- setup.py | 1 - 27 files changed, 7383 insertions(+), 39 deletions(-) create mode 100644 apitools/base/protorpclite/__init__.py create mode 100644 apitools/base/protorpclite/descriptor.py create mode 100644 apitools/base/protorpclite/descriptor_test.py create mode 100644 apitools/base/protorpclite/message_types.py create mode 100644 apitools/base/protorpclite/message_types_test.py create mode 100644 apitools/base/protorpclite/messages.py create mode 100644 apitools/base/protorpclite/messages_test.py create mode 100644 apitools/base/protorpclite/protojson.py create mode 100644 apitools/base/protorpclite/protojson_test.py create mode 100644 apitools/base/protorpclite/test_util.py create mode 100644 apitools/base/protorpclite/util.py create mode 100644 apitools/base/protorpclite/util_test.py diff --git a/apitools/base/protorpclite/__init__.py b/apitools/base/protorpclite/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apitools/base/protorpclite/descriptor.py b/apitools/base/protorpclite/descriptor.py new file mode 100644 index 0000000..cf5fe12 --- /dev/null +++ b/apitools/base/protorpclite/descriptor.py @@ -0,0 +1,613 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Services descriptor definitions. + +Contains message definitions and functions for converting +service classes into transmittable message format. + +Describing an Enum instance, Enum class, Field class or Message class will +generate an appropriate descriptor object that describes that class. +This message can itself be used to transmit information to clients wishing +to know the description of an enum value, enum, field or message without +needing to download the source code. This format is also compatible with +other, non-Python languages. + +The descriptors are modeled to be binary compatible with: + + http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto + +NOTE: The names of types and fields are not always the same between these +descriptors and the ones defined in descriptor.proto. This was done in order +to make source code files that use these descriptors easier to read. For +example, it is not necessary to prefix TYPE to all the values in +FieldDescriptor.Variant as is done in descriptor.proto FieldDescriptorProto.Type. + +Example: + + class Pixel(messages.Message): + + x = messages.IntegerField(1, required=True) + y = messages.IntegerField(2, required=True) + + color = messages.BytesField(3) + + # Describe Pixel class using message descriptor. + fields = [] + + field = FieldDescriptor() + field.name = 'x' + field.number = 1 + field.label = FieldDescriptor.Label.REQUIRED + field.variant = FieldDescriptor.Variant.INT64 + fields.append(field) + + field = FieldDescriptor() + field.name = 'y' + field.number = 2 + field.label = FieldDescriptor.Label.REQUIRED + field.variant = FieldDescriptor.Variant.INT64 + fields.append(field) + + field = FieldDescriptor() + field.name = 'color' + field.number = 3 + field.label = FieldDescriptor.Label.OPTIONAL + field.variant = FieldDescriptor.Variant.BYTES + fields.append(field) + + message = MessageDescriptor() + message.name = 'Pixel' + message.fields = fields + + # Describing is the equivalent of building the above message. + message == describe_message(Pixel) + +Public Classes: + EnumValueDescriptor: Describes Enum values. + EnumDescriptor: Describes Enum classes. + FieldDescriptor: Describes field instances. + FileDescriptor: Describes a single 'file' unit. + FileSet: Describes a collection of file descriptors. + MessageDescriptor: Describes Message classes. + +Public Functions: + describe_enum_value: Describe an individual enum-value. + describe_enum: Describe an Enum class. + describe_field: Describe a Field definition. + describe_file: Describe a 'file' unit from a Python module or object. + describe_file_set: Describe a file set from a list of modules or objects. + describe_message: Describe a Message definition. +""" +import six + +__author__ = 'rafek@google.com (Rafe Kaplan)' + +import codecs +import types + +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import util + + +__all__ = ['EnumDescriptor', + 'EnumValueDescriptor', + 'FieldDescriptor', + 'MessageDescriptor', + 'FileDescriptor', + 'FileSet', + 'DescriptorLibrary', + + 'describe_enum', + 'describe_enum_value', + 'describe_field', + 'describe_message', + 'describe_file', + 'describe_file_set', + 'describe', + 'import_descriptor_loader', + ] + + +# NOTE: MessageField is missing because message fields cannot have +# a default value at this time. +# TODO(rafek): Support default message values. +# +# Map to functions that convert default values of fields of a given type +# to a string. The function must return a value that is compatible with +# FieldDescriptor.default_value and therefore a unicode string. +_DEFAULT_TO_STRING_MAP = { + messages.IntegerField: six.text_type, + messages.FloatField: six.text_type, + messages.BooleanField: lambda value: value and u'true' or u'false', + messages.BytesField: lambda value: codecs.escape_encode(value)[0], + messages.StringField: lambda value: value, + messages.EnumField: lambda value: six.text_type(value.number), +} + +_DEFAULT_FROM_STRING_MAP = { + messages.IntegerField: int, + messages.FloatField: float, + messages.BooleanField: lambda value: value == u'true', + messages.BytesField: lambda value: codecs.escape_decode(value)[0], + messages.StringField: lambda value: value, + messages.EnumField: int, +} + + +class EnumValueDescriptor(messages.Message): + """Enum value descriptor. + + Fields: + name: Name of enumeration value. + number: Number of enumeration value. + """ + + # TODO(rafek): Why are these listed as optional in descriptor.proto. + # Harmonize? + name = messages.StringField(1, required=True) + number = messages.IntegerField(2, + required=True, + variant=messages.Variant.INT32) + + +class EnumDescriptor(messages.Message): + """Enum class descriptor. + + Fields: + name: Name of Enum without any qualification. + values: Values defined by Enum class. + """ + + name = messages.StringField(1) + values = messages.MessageField(EnumValueDescriptor, 2, repeated=True) + + +class FieldDescriptor(messages.Message): + """Field definition descriptor. + + Enums: + Variant: Wire format hint sub-types for field. + Label: Values for optional, required and repeated fields. + + Fields: + name: Name of field. + number: Number of field. + variant: Variant of field. + type_name: Type name for message and enum fields. + default_value: String representation of default value. + """ + + Variant = messages.Variant + + class Label(messages.Enum): + """Field label.""" + + OPTIONAL = 1 + REQUIRED = 2 + REPEATED = 3 + + name = messages.StringField(1, required=True) + number = messages.IntegerField(3, + required=True, + variant=messages.Variant.INT32) + label = messages.EnumField(Label, 4, default=Label.OPTIONAL) + variant = messages.EnumField(Variant, 5) + type_name = messages.StringField(6) + + # For numeric types, contains the original text representation of the value. + # For booleans, "true" or "false". + # For strings, contains the default text contents (not escaped in any way). + # For bytes, contains the C escaped value. All bytes < 128 are that are + # traditionally considered unprintable are also escaped. + default_value = messages.StringField(7) + + +class MessageDescriptor(messages.Message): + """Message definition descriptor. + + Fields: + name: Name of Message without any qualification. + fields: Fields defined for message. + message_types: Nested Message classes defined on message. + enum_types: Nested Enum classes defined on message. + """ + + name = messages.StringField(1) + fields = messages.MessageField(FieldDescriptor, 2, repeated=True) + + message_types = messages.MessageField( + 'apitools.base.protorpclite.descriptor.MessageDescriptor', 3, repeated=True) + enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True) + + +class FileDescriptor(messages.Message): + """Description of file containing protobuf definitions. + + Fields: + package: Fully qualified name of package that definitions belong to. + message_types: Message definitions contained in file. + enum_types: Enum definitions contained in file. + """ + + package = messages.StringField(2) + + # TODO(rafek): Add dependency field + + message_types = messages.MessageField(MessageDescriptor, 4, repeated=True) + enum_types = messages.MessageField(EnumDescriptor, 5, repeated=True) + + +class FileSet(messages.Message): + """A collection of FileDescriptors. + + Fields: + files: Files in file-set. + """ + + files = messages.MessageField(FileDescriptor, 1, repeated=True) + + +def describe_enum_value(enum_value): + """Build descriptor for Enum instance. + + Args: + enum_value: Enum value to provide descriptor for. + + Returns: + Initialized EnumValueDescriptor instance describing the Enum instance. + """ + enum_value_descriptor = EnumValueDescriptor() + enum_value_descriptor.name = six.text_type(enum_value.name) + enum_value_descriptor.number = enum_value.number + return enum_value_descriptor + + +def describe_enum(enum_definition): + """Build descriptor for Enum class. + + Args: + enum_definition: Enum class to provide descriptor for. + + Returns: + Initialized EnumDescriptor instance describing the Enum class. + """ + enum_descriptor = EnumDescriptor() + enum_descriptor.name = enum_definition.definition_name().split('.')[-1] + + values = [] + for number in enum_definition.numbers(): + value = enum_definition.lookup_by_number(number) + values.append(describe_enum_value(value)) + + if values: + enum_descriptor.values = values + + return enum_descriptor + + +def describe_field(field_definition): + """Build descriptor for Field instance. + + Args: + field_definition: Field instance to provide descriptor for. + + Returns: + Initialized FieldDescriptor instance describing the Field instance. + """ + field_descriptor = FieldDescriptor() + field_descriptor.name = field_definition.name + field_descriptor.number = field_definition.number + field_descriptor.variant = field_definition.variant + + if isinstance(field_definition, messages.EnumField): + field_descriptor.type_name = field_definition.type.definition_name() + + if isinstance(field_definition, messages.MessageField): + field_descriptor.type_name = field_definition.message_type.definition_name() + + if field_definition.default is not None: + field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[ + type(field_definition)](field_definition.default) + + # Set label. + if field_definition.repeated: + field_descriptor.label = FieldDescriptor.Label.REPEATED + elif field_definition.required: + field_descriptor.label = FieldDescriptor.Label.REQUIRED + else: + field_descriptor.label = FieldDescriptor.Label.OPTIONAL + + return field_descriptor + + +def describe_message(message_definition): + """Build descriptor for Message class. + + Args: + message_definition: Message class to provide descriptor for. + + Returns: + Initialized MessageDescriptor instance describing the Message class. + """ + message_descriptor = MessageDescriptor() + message_descriptor.name = message_definition.definition_name().split('.')[-1] + + fields = sorted(message_definition.all_fields(), + key=lambda v: v.number) + if fields: + message_descriptor.fields = [describe_field(field) for field in fields] + + try: + nested_messages = message_definition.__messages__ + except AttributeError: + pass + else: + message_descriptors = [] + for name in nested_messages: + value = getattr(message_definition, name) + message_descriptors.append(describe_message(value)) + + message_descriptor.message_types = message_descriptors + + try: + nested_enums = message_definition.__enums__ + except AttributeError: + pass + else: + enum_descriptors = [] + for name in nested_enums: + value = getattr(message_definition, name) + enum_descriptors.append(describe_enum(value)) + + message_descriptor.enum_types = enum_descriptors + + return message_descriptor + + +def describe_file(module): + """Build a file from a specified Python module. + + Args: + module: Python module to describe. + + Returns: + Initialized FileDescriptor instance describing the module. + """ + # May not import remote at top of file because remote depends on this + # file + # TODO(rafek): Straighten out this dependency. Possibly move these functions + # from descriptor to their own module. + + descriptor = FileDescriptor() + descriptor.package = util.get_package_for_module(module) + + if not descriptor.package: + descriptor.package = None + + message_descriptors = [] + enum_descriptors = [] + + # Need to iterate over all top level attributes of the module looking for + # message and enum definitions. Each definition must be itself described. + for name in sorted(dir(module)): + value = getattr(module, name) + + if isinstance(value, type): + if issubclass(value, messages.Message): + message_descriptors.append(describe_message(value)) + + elif issubclass(value, messages.Enum): + enum_descriptors.append(describe_enum(value)) + + if message_descriptors: + descriptor.message_types = message_descriptors + + if enum_descriptors: + descriptor.enum_types = enum_descriptors + + return descriptor + + +def describe_file_set(modules): + """Build a file set from a specified Python modules. + + Args: + modules: Iterable of Python module to describe. + + Returns: + Initialized FileSet instance describing the modules. + """ + descriptor = FileSet() + file_descriptors = [] + for module in modules: + file_descriptors.append(describe_file(module)) + + if file_descriptors: + descriptor.files = file_descriptors + + return descriptor + + +def describe(value): + """Describe any value as a descriptor. + + Helper function for describing any object with an appropriate descriptor + object. + + Args: + value: Value to describe as a descriptor. + + Returns: + Descriptor message class if object is describable as a descriptor, else + None. + """ + if isinstance(value, types.ModuleType): + return describe_file(value) + elif isinstance(value, messages.Field): + return describe_field(value) + elif isinstance(value, messages.Enum): + return describe_enum_value(value) + elif isinstance(value, type): + if issubclass(value, messages.Message): + return describe_message(value) + elif issubclass(value, messages.Enum): + return describe_enum(value) + return None + + +@util.positional(1) +def import_descriptor_loader(definition_name, importer=__import__): + """Find objects by importing modules as needed. + + A definition loader is a function that resolves a definition name to a + descriptor. + + The import finder resolves definitions to their names by importing modules + when necessary. + + Args: + definition_name: Name of definition to find. + importer: Import function used for importing new modules. + + Returns: + Appropriate descriptor for any describable type located by name. + + Raises: + DefinitionNotFoundError when a name does not refer to either a definition + or a module. + """ + # Attempt to import descriptor as a module. + if definition_name.startswith('.'): + definition_name = definition_name[1:] + if not definition_name.startswith('.'): + leaf = definition_name.split('.')[-1] + if definition_name: + try: + module = importer(definition_name, '', '', [leaf]) + except ImportError: + pass + else: + return describe(module) + + try: + # Attempt to use messages.find_definition to find item. + return describe(messages.find_definition(definition_name, + importer=__import__)) + except messages.DefinitionNotFoundError as err: + # There are things that find_definition will not find, but if the parent + # is loaded, its children can be searched for a match. + split_name = definition_name.rsplit('.', 1) + if len(split_name) > 1: + parent, child = split_name + try: + parent_definition = import_descriptor_loader(parent, importer=importer) + except messages.DefinitionNotFoundError: + # Fall through to original error. + pass + else: + # Check the parent definition for a matching descriptor. + if isinstance(parent_definition, EnumDescriptor): + search_list = parent_definition.values or [] + elif isinstance(parent_definition, MessageDescriptor): + search_list = parent_definition.fields or [] + else: + search_list = [] + + for definition in search_list: + if definition.name == child: + return definition + + # Still didn't find. Reraise original exception. + raise err + + +class DescriptorLibrary(object): + """A descriptor library is an object that contains known definitions. + + A descriptor library contains a cache of descriptor objects mapped by + definition name. It contains all types of descriptors except for + file sets. + + When a definition name is requested that the library does not know about + it can be provided with a descriptor loader which attempt to resolve the + missing descriptor. + """ + + @util.positional(1) + def __init__(self, + descriptors=None, + descriptor_loader=import_descriptor_loader): + """Constructor. + + Args: + descriptors: A dictionary or dictionary-like object that can be used + to store and cache descriptors by definition name. + definition_loader: A function used for resolving missing descriptors. + The function takes a definition name as its parameter and returns + an appropriate descriptor. It may raise DefinitionNotFoundError. + """ + self.__descriptor_loader = descriptor_loader + self.__descriptors = descriptors or {} + + def lookup_descriptor(self, definition_name): + """Lookup descriptor by name. + + Get descriptor from library by name. If descriptor is not found will + attempt to find via descriptor loader if provided. + + Args: + definition_name: Definition name to find. + + Returns: + Descriptor that describes definition name. + + Raises: + DefinitionNotFoundError if not descriptor exists for definition name. + """ + try: + return self.__descriptors[definition_name] + except KeyError: + pass + + if self.__descriptor_loader: + definition = self.__descriptor_loader(definition_name) + self.__descriptors[definition_name] = definition + return definition + else: + raise messages.DefinitionNotFoundError( + 'Could not find definition for %s' % definition_name) + + def lookup_package(self, definition_name): + """Determines the package name for any definition. + + Determine the package that any definition name belongs to. May check + parent for package name and will resolve missing descriptors if provided + descriptor loader. + + Args: + definition_name: Definition name to find package for. + """ + while True: + descriptor = self.lookup_descriptor(definition_name) + if isinstance(descriptor, FileDescriptor): + return descriptor.package + else: + index = definition_name.rfind('.') + if index < 0: + return None + definition_name = definition_name[:index] diff --git a/apitools/base/protorpclite/descriptor_test.py b/apitools/base/protorpclite/descriptor_test.py new file mode 100644 index 0000000..f55db1a --- /dev/null +++ b/apitools/base/protorpclite/descriptor_test.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for apitools.base.protorpclite.descriptor.""" + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import types +import unittest + +import six + +from apitools.base.protorpclite import descriptor +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import test_util + + +RUSSIA = u'\u0420\u043e\u0441\u0441\u0438\u044f' + + +class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = descriptor + + +class DescribeEnumValueTest(test_util.TestCase): + + def testDescribe(self): + class MyEnum(messages.Enum): + MY_NAME = 10 + + expected = descriptor.EnumValueDescriptor() + expected.name = 'MY_NAME' + expected.number = 10 + + described = descriptor.describe_enum_value(MyEnum.MY_NAME) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeEnumTest(test_util.TestCase): + + def testEmptyEnum(self): + class EmptyEnum(messages.Enum): + pass + + expected = descriptor.EnumDescriptor() + expected.name = 'EmptyEnum' + + described = descriptor.describe_enum(EmptyEnum) + described.check_initialized() + self.assertEquals(expected, described) + + def testNestedEnum(self): + class MyScope(messages.Message): + class NestedEnum(messages.Enum): + pass + + expected = descriptor.EnumDescriptor() + expected.name = 'NestedEnum' + + described = descriptor.describe_enum(MyScope.NestedEnum) + described.check_initialized() + self.assertEquals(expected, described) + + def testEnumWithItems(self): + class EnumWithItems(messages.Enum): + A = 3 + B = 1 + C = 2 + + expected = descriptor.EnumDescriptor() + expected.name = 'EnumWithItems' + + a = descriptor.EnumValueDescriptor() + a.name = 'A' + a.number = 3 + + b = descriptor.EnumValueDescriptor() + b.name = 'B' + b.number = 1 + + c = descriptor.EnumValueDescriptor() + c.name = 'C' + c.number = 2 + + expected.values = [b, c, a] + + described = descriptor.describe_enum(EnumWithItems) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeFieldTest(test_util.TestCase): + + def testLabel(self): + for repeated, required, expected_label in ( + (True, False, descriptor.FieldDescriptor.Label.REPEATED), + (False, True, descriptor.FieldDescriptor.Label.REQUIRED), + (False, False, descriptor.FieldDescriptor.Label.OPTIONAL)): + field = messages.IntegerField(10, required=required, repeated=repeated) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = expected_label + expected.variant = descriptor.FieldDescriptor.Variant.INT64 + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDefault(self): + for field_class, default, expected_default in ( + (messages.IntegerField, 200, '200'), + (messages.FloatField, 1.5, '1.5'), + (messages.FloatField, 1e6, '1000000.0'), + (messages.BooleanField, True, 'true'), + (messages.BooleanField, False, 'false'), + (messages.BytesField, + b''.join([six.int2byte(x) for x in (31, 32, 33)]), + b'\\x1f !'), + (messages.StringField, RUSSIA, RUSSIA), + ): + field = field_class(10, default=default) + field.name = u'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = u'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = field_class.DEFAULT_VARIANT + expected.default_value = expected_default + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDefault_EnumField(self): + class MyEnum(messages.Enum): + + VAL = 1 + + module_name = test_util.get_module_name(MyEnum) + field = messages.EnumField(MyEnum, 10, default=MyEnum.VAL) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.EnumField.DEFAULT_VARIANT + expected.type_name = '%s.MyEnum' % module_name + expected.default_value = '1' + + described = descriptor.describe_field(field) + self.assertEquals(expected, described) + + def testMessageField(self): + field = messages.MessageField(descriptor.FieldDescriptor, 10) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.MessageField.DEFAULT_VARIANT + expected.type_name = ('apitools.base.protorpclite.descriptor.FieldDescriptor') + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDateTimeField(self): + field = message_types.DateTimeField(20) + field.name = 'a_timestamp' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_timestamp' + expected.number = 20 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.MessageField.DEFAULT_VARIANT + expected.type_name = ('apitools.base.protorpclite.message_types.DateTimeMessage') + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeMessageTest(test_util.TestCase): + + def testEmptyDefinition(self): + class MyMessage(messages.Message): + pass + + expected = descriptor.MessageDescriptor() + expected.name = 'MyMessage' + + described = descriptor.describe_message(MyMessage) + described.check_initialized() + self.assertEquals(expected, described) + + def testDefinitionWithFields(self): + class MessageWithFields(messages.Message): + field1 = messages.IntegerField(10) + field2 = messages.StringField(30) + field3 = messages.IntegerField(20) + + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithFields' + + expected.fields = [ + descriptor.describe_field(MessageWithFields.field_by_name('field1')), + descriptor.describe_field(MessageWithFields.field_by_name('field3')), + descriptor.describe_field(MessageWithFields.field_by_name('field2')), + ] + + described = descriptor.describe_message(MessageWithFields) + described.check_initialized() + self.assertEquals(expected, described) + + def testNestedEnum(self): + class MessageWithEnum(messages.Message): + class Mood(messages.Enum): + GOOD = 1 + BAD = 2 + UGLY = 3 + + class Music(messages.Enum): + CLASSIC = 1 + JAZZ = 2 + BLUES = 3 + + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithEnum' + + expected.enum_types = [descriptor.describe_enum(MessageWithEnum.Mood), + descriptor.describe_enum(MessageWithEnum.Music)] + + described = descriptor.describe_message(MessageWithEnum) + described.check_initialized() + self.assertEquals(expected, described) + + def testNestedMessage(self): + class MessageWithMessage(messages.Message): + class Nesty(messages.Message): + pass + + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithMessage' + + expected.message_types = [ + descriptor.describe_message(MessageWithMessage.Nesty)] + + described = descriptor.describe_message(MessageWithMessage) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeFileTest(test_util.TestCase): + """Test describing modules.""" + + def LoadModule(self, module_name, source): + result = {'__name__': module_name, + 'messages': messages, + } + exec(source, result) + + module = types.ModuleType(module_name) + for name, value in result.items(): + setattr(module, name, value) + + return module + + def testEmptyModule(self): + """Test describing an empty file.""" + module = types.ModuleType('my.package.name') + + expected = descriptor.FileDescriptor() + expected.package = 'my.package.name' + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + def testNoPackageName(self): + """Test describing a module with no module name.""" + module = types.ModuleType('') + + expected = descriptor.FileDescriptor() + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + def testPackageName(self): + """Test using the 'package' module attribute.""" + module = types.ModuleType('my.module.name') + module.package = 'my.package.name' + + expected = descriptor.FileDescriptor() + expected.package = 'my.package.name' + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + def testMain(self): + """Test using the 'package' module attribute.""" + module = types.ModuleType('__main__') + module.__file__ = '/blim/blam/bloom/my_package.py' + + expected = descriptor.FileDescriptor() + expected.package = 'my_package' + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + def testMessages(self): + """Test that messages are described.""" + module = self.LoadModule('my.package', + 'class Message1(messages.Message): pass\n' + 'class Message2(messages.Message): pass\n') + + message1 = descriptor.MessageDescriptor() + message1.name = 'Message1' + + message2 = descriptor.MessageDescriptor() + message2.name = 'Message2' + + expected = descriptor.FileDescriptor() + expected.package = 'my.package' + expected.message_types = [message1, message2] + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + def testEnums(self): + """Test that enums are described.""" + module = self.LoadModule('my.package', + 'class Enum1(messages.Enum): pass\n' + 'class Enum2(messages.Enum): pass\n') + + enum1 = descriptor.EnumDescriptor() + enum1.name = 'Enum1' + + enum2 = descriptor.EnumDescriptor() + enum2.name = 'Enum2' + + expected = descriptor.FileDescriptor() + expected.package = 'my.package' + expected.enum_types = [enum1, enum2] + + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeFileSetTest(test_util.TestCase): + """Test describing multiple modules.""" + + def testNoModules(self): + """Test what happens when no modules provided.""" + described = descriptor.describe_file_set([]) + described.check_initialized() + # The described FileSet.files will be None. + self.assertEquals(descriptor.FileSet(), described) + + def testWithModules(self): + """Test what happens when no modules provided.""" + modules = [types.ModuleType('package1'), types.ModuleType('package1')] + + file1 = descriptor.FileDescriptor() + file1.package = 'package1' + file2 = descriptor.FileDescriptor() + file2.package = 'package2' + + expected = descriptor.FileSet() + expected.files = [file1, file1] + + described = descriptor.describe_file_set(modules) + described.check_initialized() + self.assertEquals(expected, described) + + +class DescribeTest(test_util.TestCase): + + def testModule(self): + self.assertEquals(descriptor.describe_file(test_util), + descriptor.describe(test_util)) + + def testField(self): + self.assertEquals( + descriptor.describe_field(test_util.NestedMessage.a_value), + descriptor.describe(test_util.NestedMessage.a_value)) + + def testEnumValue(self): + self.assertEquals( + descriptor.describe_enum_value( + test_util.OptionalMessage.SimpleEnum.VAL1), + descriptor.describe(test_util.OptionalMessage.SimpleEnum.VAL1)) + + def testMessage(self): + self.assertEquals(descriptor.describe_message(test_util.NestedMessage), + descriptor.describe(test_util.NestedMessage)) + + def testEnum(self): + self.assertEquals( + descriptor.describe_enum(test_util.OptionalMessage.SimpleEnum), + descriptor.describe(test_util.OptionalMessage.SimpleEnum)) + + def testUndescribable(self): + class NonService(object): + + def fn(self): + pass + + for value in (NonService, + NonService.fn, + 1, + 'string', + 1.2, + None): + self.assertEquals(None, descriptor.describe(value)) + + +class ModuleFinderTest(test_util.TestCase): + + def testFindMessage(self): + self.assertEquals( + descriptor.describe_message(descriptor.FileSet), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.descriptor.FileSet')) + + def testFindField(self): + self.assertEquals( + descriptor.describe_field(descriptor.FileSet.files), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.descriptor.FileSet.files')) + + def testFindEnumValue(self): + self.assertEquals( + descriptor.describe_enum_value(test_util.OptionalMessage.SimpleEnum.VAL1), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.test_util.OptionalMessage.SimpleEnum.VAL1')) + + +class DescriptorLibraryTest(test_util.TestCase): + + def setUp(self): + self.packageless = descriptor.MessageDescriptor() + self.packageless.name = 'Packageless' + self.library = descriptor.DescriptorLibrary( + descriptors={ + 'not.real.Packageless': self.packageless, + 'Packageless': self.packageless, + }) + + def testLookupPackage(self): + self.assertEquals('csv', self.library.lookup_package('csv')) + self.assertEquals('apitools.base.protorpclite', + self.library.lookup_package('apitools.base.protorpclite')) + + def testLookupNonPackages(self): + for name in ('', 'a', + 'apitools.base.protorpclite.descriptor.DescriptorLibrary'): + self.assertRaisesWithRegexpMatch( + messages.DefinitionNotFoundError, + 'Could not find definition for %s' % name, + self.library.lookup_package, name) + + def testNoPackage(self): + self.assertRaisesWithRegexpMatch( + messages.DefinitionNotFoundError, + 'Could not find definition for not.real', + self.library.lookup_package, 'not.real.Packageless') + + self.assertEquals(None, self.library.lookup_package('Packageless')) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/apitools/base/protorpclite/message_types.py b/apitools/base/protorpclite/message_types.py new file mode 100644 index 0000000..ec34d17 --- /dev/null +++ b/apitools/base/protorpclite/message_types.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Simple protocol message types. + +Includes new message and field types that are outside what is defined by the +protocol buffers standard. +""" + +__author__ = 'rafek@google.com (Rafe Kaplan)' + +import datetime + +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import util + +__all__ = [ + 'DateTimeField', + 'DateTimeMessage', + 'VoidMessage', +] + +class VoidMessage(messages.Message): + """Empty message.""" + + +class DateTimeMessage(messages.Message): + """Message to store/transmit a DateTime. + + Fields: + milliseconds: Milliseconds since Jan 1st 1970 local time. + time_zone_offset: Optional time zone offset, in minutes from UTC. + """ + milliseconds = messages.IntegerField(1, required=True) + time_zone_offset = messages.IntegerField(2) + + +class DateTimeField(messages.MessageField): + """Field definition for datetime values. + + Stores a python datetime object as a field. If time zone information is + included in the datetime object, it will be included in + the encoded data when this is encoded/decoded. + """ + + type = datetime.datetime + + message_type = DateTimeMessage + + @util.positional(3) + def __init__(self, + number, + **kwargs): + super(DateTimeField, self).__init__(self.message_type, + number, + **kwargs) + + def value_from_message(self, message): + """Convert DateTimeMessage to a datetime. + + Args: + A DateTimeMessage instance. + + Returns: + A datetime instance. + """ + message = super(DateTimeField, self).value_from_message(message) + if message.time_zone_offset is None: + return datetime.datetime.utcfromtimestamp(message.milliseconds / 1000.0) + + # Need to subtract the time zone offset, because when we call + # datetime.fromtimestamp, it will add the time zone offset to the + # value we pass. + milliseconds = (message.milliseconds - + 60000 * message.time_zone_offset) + + timezone = util.TimeZoneOffset(message.time_zone_offset) + return datetime.datetime.fromtimestamp(milliseconds / 1000.0, + tz=timezone) + + def value_to_message(self, value): + value = super(DateTimeField, self).value_to_message(value) + # First, determine the delta from the epoch, so we can fill in + # DateTimeMessage's milliseconds field. + if value.tzinfo is None: + time_zone_offset = 0 + local_epoch = datetime.datetime.utcfromtimestamp(0) + else: + time_zone_offset = util.total_seconds(value.tzinfo.utcoffset(value)) + # Determine Jan 1, 1970 local time. + local_epoch = datetime.datetime.fromtimestamp(-time_zone_offset, + tz=value.tzinfo) + delta = value - local_epoch + + # Create and fill in the DateTimeMessage, including time zone if + # one was specified. + message = DateTimeMessage() + message.milliseconds = int(util.total_seconds(delta) * 1000) + if value.tzinfo is not None: + utc_offset = value.tzinfo.utcoffset(value) + if utc_offset is not None: + message.time_zone_offset = int( + util.total_seconds(value.tzinfo.utcoffset(value)) / 60) + + return message diff --git a/apitools/base/protorpclite/message_types_test.py b/apitools/base/protorpclite/message_types_test.py new file mode 100644 index 0000000..33be248 --- /dev/null +++ b/apitools/base/protorpclite/message_types_test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# +# Copyright 2013 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for apitools.base.protorpclite.message_types.""" + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import datetime + +import unittest + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import test_util +from apitools.base.protorpclite import util + + +class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = message_types + + +class DateTimeFieldTest(test_util.TestCase): + + def testValueToMessage(self): + field = message_types.DateTimeField(1) + message = field.value_to_message(datetime.datetime(2033, 2, 4, 11, 22, 10)) + self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000), + message) + + def testValueToMessageBadValue(self): + field = message_types.DateTimeField(1) + self.assertRaisesWithRegexpMatch( + messages.EncodeError, + 'Expected type datetime, got int: 20', + field.value_to_message, 20) + + def testValueToMessageWithTimeZone(self): + time_zone = util.TimeZoneOffset(60 * 10) + field = message_types.DateTimeField(1) + message = field.value_to_message( + datetime.datetime(2033, 2, 4, 11, 22, 10, tzinfo=time_zone)) + self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000, + time_zone_offset=600), + message) + + def testValueFromMessage(self): + message = message_types.DateTimeMessage(milliseconds=1991128000000) + field = message_types.DateTimeField(1) + timestamp = field.value_from_message(message) + self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40), + timestamp) + + def testValueFromMessageBadValue(self): + field = message_types.DateTimeField(1) + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Expected type DateTimeMessage, got VoidMessage: ', + field.value_from_message, message_types.VoidMessage()) + + def testValueFromMessageWithTimeZone(self): + message = message_types.DateTimeMessage(milliseconds=1991128000000, + time_zone_offset=300) + field = message_types.DateTimeField(1) + timestamp = field.value_from_message(message) + time_zone = util.TimeZoneOffset(60 * 5) + self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40, tzinfo=time_zone), + timestamp) + + +if __name__ == '__main__': + unittest.main() diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py new file mode 100644 index 0000000..a78b7df --- /dev/null +++ b/apitools/base/protorpclite/messages.py @@ -0,0 +1,1935 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Stand-alone implementation of in memory protocol messages. + +Public Classes: + Enum: Represents an enumerated type. + Variant: Hint for wire format to determine how to serialize. + Message: Base class for user defined messages. + IntegerField: Field for integer values. + FloatField: Field for float values. + BooleanField: Field for boolean values. + BytesField: Field for binary string values. + StringField: Field for UTF-8 string values. + MessageField: Field for other message type values. + EnumField: Field for enumerated type values. + +Public Exceptions (indentation indications class hierarchy): + EnumDefinitionError: Raised when enumeration is incorrectly defined. + FieldDefinitionError: Raised when field is incorrectly defined. + InvalidVariantError: Raised when variant is not compatible with field type. + InvalidDefaultError: Raised when default is not compatiable with field. + InvalidNumberError: Raised when field number is out of range or reserved. + MessageDefinitionError: Raised when message is incorrectly defined. + DuplicateNumberError: Raised when field has duplicate number with another. + ValidationError: Raised when a message or field is not valid. + DefinitionNotFoundError: Raised when definition not found. +""" +import six + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import types +import weakref + +from apitools.base.protorpclite import util + +__all__ = ['MAX_ENUM_VALUE', + 'MAX_FIELD_NUMBER', + 'FIRST_RESERVED_FIELD_NUMBER', + 'LAST_RESERVED_FIELD_NUMBER', + + 'Enum', + 'Field', + 'FieldList', + 'Variant', + 'Message', + 'IntegerField', + 'FloatField', + 'BooleanField', + 'BytesField', + 'StringField', + 'MessageField', + 'EnumField', + 'find_definition', + + 'Error', + 'DecodeError', + 'EncodeError', + 'EnumDefinitionError', + 'FieldDefinitionError', + 'InvalidVariantError', + 'InvalidDefaultError', + 'InvalidNumberError', + 'MessageDefinitionError', + 'DuplicateNumberError', + 'ValidationError', + 'DefinitionNotFoundError', + ] + + +# TODO(rafek): Add extended module test to ensure all exceptions +# in services extends Error. +Error = util.Error + + +class EnumDefinitionError(Error): + """Enumeration definition error.""" + + +class FieldDefinitionError(Error): + """Field definition error.""" + + +class InvalidVariantError(FieldDefinitionError): + """Invalid variant provided to field.""" + + +class InvalidDefaultError(FieldDefinitionError): + """Invalid default provided to field.""" + + +class InvalidNumberError(FieldDefinitionError): + """Invalid number provided to field.""" + + +class MessageDefinitionError(Error): + """Message definition error.""" + + +class DuplicateNumberError(Error): + """Duplicate number assigned to field.""" + + +class DefinitionNotFoundError(Error): + """Raised when definition is not found.""" + + +class DecodeError(Error): + """Error found decoding message from encoded form.""" + + +class EncodeError(Error): + """Error found when encoding message.""" + + +class ValidationError(Error): + """Invalid value for message error.""" + + def __str__(self): + """Prints string with field name if present on exception.""" + message = Error.__str__(self) + try: + field_name = self.field_name + except AttributeError: + return message + else: + return message + + +# Attributes that are reserved by a class definition that +# may not be used by either Enum or Message class definitions. +_RESERVED_ATTRIBUTE_NAMES = frozenset( + ['__module__', '__doc__', '__qualname__']) + +_POST_INIT_FIELD_ATTRIBUTE_NAMES = frozenset( + ['name', + '_message_definition', + '_MessageField__type', + '_EnumField__type', + '_EnumField__resolved_default']) + +_POST_INIT_ATTRIBUTE_NAMES = frozenset( + ['_message_definition']) + +# Maximum enumeration value as defined by the protocol buffers standard. +# All enum values must be less than or equal to this value. +MAX_ENUM_VALUE = (2 ** 29) - 1 + +# Maximum field number as defined by the protocol buffers standard. +# All field numbers must be less than or equal to this value. +MAX_FIELD_NUMBER = (2 ** 29) - 1 + +# Field numbers between 19000 and 19999 inclusive are reserved by the +# protobuf protocol and may not be used by fields. +FIRST_RESERVED_FIELD_NUMBER = 19000 +LAST_RESERVED_FIELD_NUMBER = 19999 + + +class _DefinitionClass(type): + """Base meta-class used for definition meta-classes. + + The Enum and Message definition classes share some basic functionality. + Both of these classes may be contained by a Message definition. After + initialization, neither class may have attributes changed + except for the protected _message_definition attribute, and that attribute + may change only once. + """ + + __initialized = False + + def __init__(cls, name, bases, dct): + """Constructor.""" + type.__init__(cls, name, bases, dct) + # Base classes may never be initialized. + if cls.__bases__ != (object,): + cls.__initialized = True + + def message_definition(cls): + """Get outer Message definition that contains this definition. + + Returns: + Containing Message definition if definition is contained within one, + else None. + """ + try: + return cls._message_definition() + except AttributeError: + return None + + def __setattr__(cls, name, value): + """Overridden so that cannot set variables on definition classes after init. + + Setting attributes on a class must work during the period of initialization + to set the enumation value class variables and build the name/number maps. + Once __init__ has set the __initialized flag to True prohibits setting any + more values on the class. The class is in effect frozen. + + Args: + name: Name of value to set. + value: Value to set. + """ + if cls.__initialized and name not in _POST_INIT_ATTRIBUTE_NAMES: + raise AttributeError('May not change values: %s' % name) + else: + type.__setattr__(cls, name, value) + + def __delattr__(cls, name): + """Overridden so that cannot delete varaibles on definition classes.""" + raise TypeError('May not delete attributes on definition class') + + def definition_name(cls): + """Helper method for creating definition name. + + Names will be generated to include the classes package name, scope (if the + class is nested in another definition) and class name. + + By default, the package name for a definition is derived from its module + name. However, this value can be overriden by placing a 'package' attribute + in the module that contains the definition class. For example: + + package = 'some.alternate.package' + + class MyMessage(Message): + ... + + >>> MyMessage.definition_name() + some.alternate.package.MyMessage + + Returns: + Dot-separated fully qualified name of definition. + """ + outer_definition_name = cls.outer_definition_name() + if outer_definition_name is None: + return six.text_type(cls.__name__) + else: + return u'%s.%s' % (outer_definition_name, cls.__name__) + + def outer_definition_name(cls): + """Helper method for creating outer definition name. + + Returns: + If definition is nested, will return the outer definitions name, else the + package name. + """ + outer_definition = cls.message_definition() + if not outer_definition: + return util.get_package_for_module(cls.__module__) + else: + return outer_definition.definition_name() + + def definition_package(cls): + """Helper method for creating creating the package of a definition. + + Returns: + Name of package that definition belongs to. + """ + outer_definition = cls.message_definition() + if not outer_definition: + return util.get_package_for_module(cls.__module__) + else: + return outer_definition.definition_package() + + +class _EnumClass(_DefinitionClass): + """Meta-class used for defining the Enum base class. + + Meta-class enables very specific behavior for any defined Enum + class. All attributes defined on an Enum sub-class must be integers. + Each attribute defined on an Enum sub-class is translated + into an instance of that sub-class, with the name of the attribute + as its name, and the number provided as its value. It also ensures + that only one level of Enum class hierarchy is possible. In other + words it is not possible to delcare sub-classes of sub-classes of + Enum. + + This class also defines some functions in order to restrict the + behavior of the Enum class and its sub-classes. It is not possible + to change the behavior of the Enum class in later classes since + any new classes may be defined with only integer values, and no methods. + """ + + def __init__(cls, name, bases, dct): + # Can only define one level of sub-classes below Enum. + if not (bases == (object,) or bases == (Enum,)): + raise EnumDefinitionError('Enum type %s may only inherit from Enum' % + (name,)) + + cls.__by_number = {} + cls.__by_name = {} + + # Enum base class does not need to be initialized or locked. + if bases != (object,): + # Replace integer with number. + for attribute, value in dct.items(): + + # Module will be in every enum class. + if attribute in _RESERVED_ATTRIBUTE_NAMES: + continue + + # Reject anything that is not an int. + if not isinstance(value, six.integer_types): + raise EnumDefinitionError( + 'May only use integers in Enum definitions. Found: %s = %s' % + (attribute, value)) + + # Protocol buffer standard recommends non-negative values. + # Reject negative values. + if value < 0: + raise EnumDefinitionError( + 'Must use non-negative enum values. Found: %s = %d' % + (attribute, value)) + + if value > MAX_ENUM_VALUE: + raise EnumDefinitionError( + 'Must use enum values less than or equal %d. Found: %s = %d' % + (MAX_ENUM_VALUE, attribute, value)) + + if value in cls.__by_number: + raise EnumDefinitionError( + 'Value for %s = %d is already defined: %s' % + (attribute, value, cls.__by_number[value].name)) + + # Create enum instance and list in new Enum type. + instance = object.__new__(cls) + cls.__init__(instance, attribute, value) + cls.__by_name[instance.name] = instance + cls.__by_number[instance.number] = instance + setattr(cls, attribute, instance) + + _DefinitionClass.__init__(cls, name, bases, dct) + + def __iter__(cls): + """Iterate over all values of enum. + + Yields: + Enumeration instances of the Enum class in arbitrary order. + """ + return iter(cls.__by_number.values()) + + def names(cls): + """Get all names for Enum. + + Returns: + An iterator for names of the enumeration in arbitrary order. + """ + return cls.__by_name.keys() + + def numbers(cls): + """Get all numbers for Enum. + + Returns: + An iterator for all numbers of the enumeration in arbitrary order. + """ + return cls.__by_number.keys() + + def lookup_by_name(cls, name): + """Look up Enum by name. + + Args: + name: Name of enum to find. + + Returns: + Enum sub-class instance of that value. + """ + return cls.__by_name[name] + + def lookup_by_number(cls, number): + """Look up Enum by number. + + Args: + number: Number of enum to find. + + Returns: + Enum sub-class instance of that value. + """ + return cls.__by_number[number] + + def __len__(cls): + return len(cls.__by_name) + + +class Enum(six.with_metaclass(_EnumClass, object)): + """Base class for all enumerated types.""" + + __slots__ = set(('name', 'number')) + + def __new__(cls, index): + """Acts as look-up routine after class is initialized. + + The purpose of overriding __new__ is to provide a way to treat + Enum subclasses as casting types, similar to how the int type + functions. A program can pass a string or an integer and this + method with "convert" that value in to an appropriate Enum instance. + + Args: + index: Name or number to look up. During initialization + this is always the name of the new enum value. + + Raises: + TypeError: When an inappropriate index value is passed provided. + """ + # If is enum type of this class, return it. + if isinstance(index, cls): + return index + + # If number, look up by number. + if isinstance(index, six.integer_types): + try: + return cls.lookup_by_number(index) + except KeyError: + pass + + # If name, look up by name. + if isinstance(index, six.string_types): + try: + return cls.lookup_by_name(index) + except KeyError: + pass + + raise TypeError('No such value for %s in Enum %s' % + (index, cls.__name__)) + + def __init__(self, name, number=None): + """Initialize new Enum instance. + + Since this should only be called during class initialization any + calls that happen after the class is frozen raises an exception. + """ + # Immediately return if __init__ was called after _Enum.__init__(). + # It means that casting operator version of the class constructor + # is being used. + if getattr(type(self), '_DefinitionClass__initialized'): + return + object.__setattr__(self, 'name', name) + object.__setattr__(self, 'number', number) + + def __setattr__(self, name, value): + raise TypeError('May not change enum values') + + def __str__(self): + return self.name + + def __int__(self): + return self.number + + def __repr__(self): + return '%s(%s, %d)' % (type(self).__name__, self.name, self.number) + + def __reduce__(self): + """Enable pickling. + + Returns: + A 2-tuple containing the class and __new__ args to be used for restoring + a pickled instance. + """ + return self.__class__, (self.number,) + + def __cmp__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return cmp(self.number, other.number) + return NotImplemented + + def __lt__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number < other.number + return NotImplemented + + def __le__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number <= other.number + return NotImplemented + + def __eq__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number == other.number + return NotImplemented + + def __ne__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number != other.number + return NotImplemented + + def __ge__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number >= other.number + return NotImplemented + + def __gt__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number > other.number + return NotImplemented + + def __hash__(self): + """Hash by number.""" + return hash(self.number) + + @classmethod + def to_dict(cls): + """Make dictionary version of enumerated class. + + Dictionary created this way can be used with def_num. + + Returns: + A dict (name) -> number + """ + return dict((item.name, item.number) for item in iter(cls)) + + @staticmethod + def def_enum(dct, name): + """Define enum class from dictionary. + + Args: + dct: Dictionary of enumerated values for type. + name: Name of enum. + """ + return type(name, (Enum,), dct) + + +# TODO(rafek): Determine to what degree this enumeration should be compatible +# with FieldDescriptor.Type in: +# +# http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto +class Variant(Enum): + """Wire format variant. + + Used by the 'protobuf' wire format to determine how to transmit + a single piece of data. May be used by other formats. + + See: http://code.google.com/apis/protocolbuffers/docs/encoding.html + + Values: + DOUBLE: 64-bit floating point number. + FLOAT: 32-bit floating point number. + INT64: 64-bit signed integer. + UINT64: 64-bit unsigned integer. + INT32: 32-bit signed integer. + BOOL: Boolean value (True or False). + STRING: String of UTF-8 encoded text. + MESSAGE: Embedded message as byte string. + BYTES: String of 8-bit bytes. + UINT32: 32-bit unsigned integer. + ENUM: Enum value as integer. + SINT32: 32-bit signed integer. Uses "zig-zag" encoding. + SINT64: 64-bit signed integer. Uses "zig-zag" encoding. + """ + DOUBLE = 1 + FLOAT = 2 + INT64 = 3 + UINT64 = 4 + INT32 = 5 + BOOL = 8 + STRING = 9 + MESSAGE = 11 + BYTES = 12 + UINT32 = 13 + ENUM = 14 + SINT32 = 17 + SINT64 = 18 + + +class _MessageClass(_DefinitionClass): + """Meta-class used for defining the Message base class. + + For more details about Message classes, see the Message class docstring. + Information contained there may help understanding this class. + + Meta-class enables very specific behavior for any defined Message + class. All attributes defined on an Message sub-class must be field + instances, Enum class definitions or other Message class definitions. Each + field attribute defined on an Message sub-class is added to the set of + field definitions and the attribute is translated in to a slot. It also + ensures that only one level of Message class hierarchy is possible. In other + words it is not possible to declare sub-classes of sub-classes of + Message. + + This class also defines some functions in order to restrict the + behavior of the Message class and its sub-classes. It is not possible + to change the behavior of the Message class in later classes since + any new classes may be defined with only field, Enums and Messages, and + no methods. + """ + + def __new__(cls, name, bases, dct): + """Create new Message class instance. + + The __new__ method of the _MessageClass type is overridden so as to + allow the translation of Field instances to slots. + """ + by_number = {} + by_name = {} + + variant_map = {} + + if bases != (object,): + # Can only define one level of sub-classes below Message. + if bases != (Message,): + raise MessageDefinitionError( + 'Message types may only inherit from Message') + + enums = [] + messages = [] + # Must not use iteritems because this loop will change the state of dct. + for key, field in dct.items(): + + if key in _RESERVED_ATTRIBUTE_NAMES: + continue + + if isinstance(field, type) and issubclass(field, Enum): + enums.append(key) + continue + + if (isinstance(field, type) and + issubclass(field, Message) and + field is not Message): + messages.append(key) + continue + + # Reject anything that is not a field. + if type(field) is Field or not isinstance(field, Field): + raise MessageDefinitionError( + 'May only use fields in message definitions. Found: %s = %s' % + (key, field)) + + if field.number in by_number: + raise DuplicateNumberError( + 'Field with number %d declared more than once in %s' % + (field.number, name)) + + field.name = key + + # Place in name and number maps. + by_name[key] = field + by_number[field.number] = field + + # Add enums if any exist. + if enums: + dct['__enums__'] = sorted(enums) + + # Add messages if any exist. + if messages: + dct['__messages__'] = sorted(messages) + + dct['_Message__by_number'] = by_number + dct['_Message__by_name'] = by_name + + return _DefinitionClass.__new__(cls, name, bases, dct) + + def __init__(cls, name, bases, dct): + """Initializer required to assign references to new class.""" + if bases != (object,): + for value in dct.values(): + if isinstance(value, _DefinitionClass) and not value is Message: + value._message_definition = weakref.ref(cls) + + for field in cls.all_fields(): + field._message_definition = weakref.ref(cls) + + _DefinitionClass.__init__(cls, name, bases, dct) + + +class Message(six.with_metaclass(_MessageClass, object)): + """Base class for user defined message objects. + + Used to define messages for efficient transmission across network or + process space. Messages are defined using the field classes (IntegerField, + FloatField, EnumField, etc.). + + Messages are more restricted than normal classes in that they may only + contain field attributes and other Message and Enum definitions. These + restrictions are in place because the structure of the Message class is + intentended to itself be transmitted across network or process space and + used directly by clients or even other servers. As such methods and + non-field attributes could not be transmitted with the structural information + causing discrepancies between different languages and implementations. + + Initialization and validation: + + A Message object is considered to be initialized if it has all required + fields and any nested messages are also initialized. + + Calling 'check_initialized' will raise a ValidationException if it is not + initialized; 'is_initialized' returns a boolean value indicating if it is + valid. + + Validation automatically occurs when Message objects are created + and populated. Validation that a given value will be compatible with + a field that it is assigned to can be done through the Field instances + validate() method. The validate method used on a message will check that + all values of a message and its sub-messages are valid. Assingning an + invalid value to a field will raise a ValidationException. + + Example: + + # Trade type. + class TradeType(Enum): + BUY = 1 + SELL = 2 + SHORT = 3 + CALL = 4 + + class Lot(Message): + price = IntegerField(1, required=True) + quantity = IntegerField(2, required=True) + + class Order(Message): + symbol = StringField(1, required=True) + total_quantity = IntegerField(2, required=True) + trade_type = EnumField(TradeType, 3, required=True) + lots = MessageField(Lot, 4, repeated=True) + limit = IntegerField(5) + + order = Order(symbol='GOOG', + total_quantity=10, + trade_type=TradeType.BUY) + + lot1 = Lot(price=304, + quantity=7) + + lot2 = Lot(price = 305, + quantity=3) + + order.lots = [lot1, lot2] + + # Now object is initialized! + order.check_initialized() + """ + + def __init__(self, **kwargs): + """Initialize internal messages state. + + Args: + A message can be initialized via the constructor by passing in keyword + arguments corresponding to fields. For example: + + class Date(Message): + day = IntegerField(1) + month = IntegerField(2) + year = IntegerField(3) + + Invoking: + + date = Date(day=6, month=6, year=1911) + + is the same as doing: + + date = Date() + date.day = 6 + date.month = 6 + date.year = 1911 + """ + # Tag being an essential implementation detail must be private. + self.__tags = {} + self.__unrecognized_fields = {} + + assigned = set() + for name, value in kwargs.items(): + setattr(self, name, value) + assigned.add(name) + + # initialize repeated fields. + for field in self.all_fields(): + if field.repeated and field.name not in assigned: + setattr(self, field.name, []) + + + def check_initialized(self): + """Check class for initialization status. + + Check that all required fields are initialized + + Raises: + ValidationError: If message is not initialized. + """ + for name, field in self.__by_name.items(): + value = getattr(self, name) + if value is None: + if field.required: + raise ValidationError("Message %s is missing required field %s" % + (type(self).__name__, name)) + else: + try: + if (isinstance(field, MessageField) and + issubclass(field.message_type, Message)): + if field.repeated: + for item in value: + item_message_value = field.value_to_message(item) + item_message_value.check_initialized() + else: + message_value = field.value_to_message(value) + message_value.check_initialized() + except ValidationError as err: + if not hasattr(err, 'message_name'): + err.message_name = type(self).__name__ + raise + + def is_initialized(self): + """Get initialization status. + + Returns: + True if message is valid, else False. + """ + try: + self.check_initialized() + except ValidationError: + return False + else: + return True + + @classmethod + def all_fields(cls): + """Get all field definition objects. + + Ordering is arbitrary. + + Returns: + Iterator over all values in arbitrary order. + """ + return cls.__by_name.values() + + @classmethod + def field_by_name(cls, name): + """Get field by name. + + Returns: + Field object associated with name. + + Raises: + KeyError if no field found by that name. + """ + return cls.__by_name[name] + + @classmethod + def field_by_number(cls, number): + """Get field by number. + + Returns: + Field object associated with number. + + Raises: + KeyError if no field found by that number. + """ + return cls.__by_number[number] + + def get_assigned_value(self, name): + """Get the assigned value of an attribute. + + Get the underlying value of an attribute. If value has not been set, will + not return the default for the field. + + Args: + name: Name of attribute to get. + + Returns: + Value of attribute, None if it has not been set. + """ + message_type = type(self) + try: + field = message_type.field_by_name(name) + except KeyError: + raise AttributeError('Message %s has no field %s' % ( + message_type.__name__, name)) + return self.__tags.get(field.number) + + def reset(self, name): + """Reset assigned value for field. + + Resetting a field will return it to its default value or None. + + Args: + name: Name of field to reset. + """ + message_type = type(self) + try: + field = message_type.field_by_name(name) + except KeyError: + if name not in message_type.__by_name: + raise AttributeError('Message %s has no field %s' % ( + message_type.__name__, name)) + if field.repeated: + self.__tags[field.number] = FieldList(field, []) + else: + self.__tags.pop(field.number, None) + + def all_unrecognized_fields(self): + """Get the names of all unrecognized fields in this message.""" + return list(self.__unrecognized_fields.keys()) + + def get_unrecognized_field_info(self, key, value_default=None, + variant_default=None): + """Get the value and variant of an unknown field in this message. + + Args: + key: The name or number of the field to retrieve. + value_default: Value to be returned if the key isn't found. + variant_default: Value to be returned as variant if the key isn't + found. + + Returns: + (value, variant), where value and variant are whatever was passed + to set_unrecognized_field. + """ + value, variant = self.__unrecognized_fields.get(key, (value_default, + variant_default)) + return value, variant + + def set_unrecognized_field(self, key, value, variant): + """Set an unrecognized field, used when decoding a message. + + Args: + key: The name or number used to refer to this unknown value. + value: The value of the field. + variant: Type information needed to interpret the value or re-encode it. + + Raises: + TypeError: If the variant is not an instance of messages.Variant. + """ + if not isinstance(variant, Variant): + raise TypeError('Variant type %s is not valid.' % variant) + self.__unrecognized_fields[key] = value, variant + + def __setattr__(self, name, value): + """Change set behavior for messages. + + Messages may only be assigned values that are fields. + + Does not try to validate field when set. + + Args: + name: Name of field to assign to. + value: Value to assign to field. + + Raises: + AttributeError when trying to assign value that is not a field. + """ + if name in self.__by_name or name.startswith('_Message__'): + object.__setattr__(self, name, value) + else: + raise AttributeError("May not assign arbitrary value %s " + "to message %s" % (name, type(self).__name__)) + + def __repr__(self): + """Make string representation of message. + + Example: + + class MyMessage(messages.Message): + integer_value = messages.IntegerField(1) + string_value = messages.StringField(2) + + my_message = MyMessage() + my_message.integer_value = 42 + my_message.string_value = u'A string' + + print my_message + >>> + + Returns: + String representation of message, including the values + of all fields and repr of all sub-messages. + """ + body = ['<', type(self).__name__] + for field in sorted(self.all_fields(), + key=lambda f: f.number): + attribute = field.name + value = self.get_assigned_value(field.name) + if value is not None: + body.append('\n %s: %s' % (attribute, repr(value))) + body.append('>') + return ''.join(body) + + def __eq__(self, other): + """Equality operator. + + Does field by field comparison with other message. For + equality, must be same type and values of all fields must be + equal. + + Messages not required to be initialized for comparison. + + Does not attempt to determine equality for values that have + default values that are not set. In other words: + + class HasDefault(Message): + + attr1 = StringField(1, default='default value') + + message1 = HasDefault() + message2 = HasDefault() + message2.attr1 = 'default value' + + message1 != message2 + + Does not compare unknown values. + + Args: + other: Other message to compare with. + """ + # TODO(rafek): Implement "equivalent" which does comparisons + # taking default values in to consideration. + if self is other: + return True + + if type(self) is not type(other): + return False + + return self.__tags == other.__tags + + def __ne__(self, other): + """Not equals operator. + + Does field by field comparison with other message. For + non-equality, must be different type or any value of a field must be + non-equal to the same field in the other instance. + + Messages not required to be initialized for comparison. + + Args: + other: Other message to compare with. + """ + return not self.__eq__(other) + + +class FieldList(list): + """List implementation that validates field values. + + This list implementation overrides all methods that add values in to a list + in order to validate those new elements. Attempting to add or set list + values that are not of the correct type will raise ValidationError. + """ + + def __init__(self, field_instance, sequence): + """Constructor. + + Args: + field_instance: Instance of field that validates the list. + sequence: List or tuple to construct list from. + """ + if not field_instance.repeated: + raise FieldDefinitionError('FieldList may only accept repeated fields') + self.__field = field_instance + self.__field.validate(sequence) + list.__init__(self, sequence) + + def __getstate__(self): + """Enable pickling. + + The assigned field instance can't be pickled if it belongs to a Message + definition (message_definition uses a weakref), so the Message class and + field number are returned in that case. + + Returns: + A 3-tuple containing: + - The field instance, or None if it belongs to a Message class. + - The Message class that the field instance belongs to, or None. + - The field instance number of the Message class it belongs to, or None. + """ + message_class = self.__field.message_definition() + if message_class is None: + return self.__field, None, None + else: + return None, message_class, self.__field.number + + def __setstate__(self, state): + """Enable unpickling. + + Args: + state: A 3-tuple containing: + - The field instance, or None if it belongs to a Message class. + - The Message class that the field instance belongs to, or None. + - The field instance number of the Message class it belongs to, or None. + """ + field_instance, message_class, number = state + if field_instance is None: + self.__field = message_class.field_by_number(number) + else: + self.__field = field_instance + + @property + def field(self): + """Field that validates list.""" + return self.__field + + def __setslice__(self, i, j, sequence): + """Validate slice assignment to list.""" + self.__field.validate(sequence) + list.__setslice__(self, i, j, sequence) + + def __setitem__(self, index, value): + """Validate item assignment to list.""" + if isinstance(index, slice): + self.__field.validate(value) + else: + self.__field.validate_element(value) + list.__setitem__(self, index, value) + + def append(self, value): + """Validate item appending to list.""" + self.__field.validate_element(value) + return list.append(self, value) + + def extend(self, sequence): + """Validate extension of list.""" + self.__field.validate(sequence) + return list.extend(self, sequence) + + def insert(self, index, value): + """Validate item insertion to list.""" + self.__field.validate_element(value) + return list.insert(self, index, value) + + +class _FieldMeta(type): + + def __init__(cls, name, bases, dct): + getattr(cls, '_Field__variant_to_type').update( + (variant, cls) for variant in dct.get('VARIANTS', [])) + type.__init__(cls, name, bases, dct) + + +# TODO(rafek): Prevent additional field subclasses. +class Field(six.with_metaclass(_FieldMeta, object)): + + __initialized = False + __variant_to_type = {} + + # TODO(craigcitro): Remove this alias. + # + # We add an alias here for backwards compatibility; note that in + # python3, this attribute will silently be ignored. + __metaclass__ = _FieldMeta + + @util.positional(2) + def __init__(self, + number, + required=False, + repeated=False, + variant=None, + default=None): + """Constructor. + + The required and repeated parameters are mutually exclusive. Setting both + to True will raise a FieldDefinitionError. + + Sub-class Attributes: + Each sub-class of Field must define the following: + VARIANTS: Set of variant types accepted by that field. + DEFAULT_VARIANT: Default variant type if not specified in constructor. + + Args: + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive with + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive with + 'required'. + variant: Wire-format variant hint. + default: Default value for field if not found in stream. + + Raises: + InvalidVariantError when invalid variant for field is provided. + InvalidDefaultError when invalid default for field is provided. + FieldDefinitionError when invalid number provided or mutually exclusive + fields are used. + InvalidNumberError when the field number is out of range or reserved. + """ + if not isinstance(number, int) or not 1 <= number <= MAX_FIELD_NUMBER: + raise InvalidNumberError('Invalid number for field: %s\n' + 'Number must be 1 or greater and %d or less' % + (number, MAX_FIELD_NUMBER)) + + if FIRST_RESERVED_FIELD_NUMBER <= number <= LAST_RESERVED_FIELD_NUMBER: + raise InvalidNumberError('Tag number %d is a reserved number.\n' + 'Numbers %d to %d are reserved' % + (number, FIRST_RESERVED_FIELD_NUMBER, + LAST_RESERVED_FIELD_NUMBER)) + + if repeated and required: + raise FieldDefinitionError('Cannot set both repeated and required') + + if variant is None: + variant = self.DEFAULT_VARIANT + + if repeated and default is not None: + raise FieldDefinitionError('Repeated fields may not have defaults') + + if variant not in self.VARIANTS: + raise InvalidVariantError( + 'Invalid variant: %s\nValid variants for %s are %r' % + (variant, type(self).__name__, sorted(self.VARIANTS))) + + self.number = number + self.required = required + self.repeated = repeated + self.variant = variant + + if default is not None: + try: + self.validate_default(default) + except ValidationError as err: + try: + name = self.name + except AttributeError: + # For when raising error before name initialization. + raise InvalidDefaultError('Invalid default value for %s: %r: %s' % + (self.__class__.__name__, default, err)) + else: + raise InvalidDefaultError('Invalid default value for field %s: ' + '%r: %s' % (name, default, err)) + + self.__default = default + self.__initialized = True + + def __setattr__(self, name, value): + """Setter overidden to prevent assignment to fields after creation. + + Args: + name: Name of attribute to set. + value: Value to assign. + """ + # Special case post-init names. They need to be set after constructor. + if name in _POST_INIT_FIELD_ATTRIBUTE_NAMES: + object.__setattr__(self, name, value) + return + + # All other attributes must be set before __initialized. + if not self.__initialized: + # Not initialized yet, allow assignment. + object.__setattr__(self, name, value) + else: + raise AttributeError('Field objects are read-only') + + def __set__(self, message_instance, value): + """Set value on message. + + Args: + message_instance: Message instance to set value on. + value: Value to set on message. + """ + # Reaches in to message instance directly to assign to private tags. + if value is None: + if self.repeated: + raise ValidationError( + 'May not assign None to repeated field %s' % self.name) + else: + message_instance._Message__tags.pop(self.number, None) + else: + if self.repeated: + value = FieldList(self, value) + else: + self.validate(value) + message_instance._Message__tags[self.number] = value + + def __get__(self, message_instance, message_class): + if message_instance is None: + return self + + result = message_instance._Message__tags.get(self.number) + if result is None: + return self.default + else: + return result + + def validate_element(self, value): + """Validate single element of field. + + This is different from validate in that it is used on individual + values of repeated fields. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected type. + """ + if not isinstance(value, self.type): + if value is None: + if self.required: + raise ValidationError('Required field is missing') + else: + try: + name = self.name + except AttributeError: + raise ValidationError('Expected type %s for %s, ' + 'found %s (type %s)' % + (self.type, self.__class__.__name__, + value, type(value))) + else: + raise ValidationError('Expected type %s for field %s, ' + 'found %s (type %s)' % + (self.type, name, value, type(value))) + + def __validate(self, value, validate_element): + """Internal validation function. + + Validate an internal value using a function to validate individual elements. + + Args: + value: Value to validate. + validate_element: Function to use to validate individual elements. + + Raises: + ValidationError if value is not expected type. + """ + if not self.repeated: + validate_element(value) + else: + # Must be a list or tuple, may not be a string. + if isinstance(value, (list, tuple)): + for element in value: + if element is None: + try: + name = self.name + except AttributeError: + raise ValidationError('Repeated values for %s ' + 'may not be None' % self.__class__.__name__) + else: + raise ValidationError('Repeated values for field %s ' + 'may not be None' % name) + validate_element(element) + elif value is not None: + try: + name = self.name + except AttributeError: + raise ValidationError('%s is repeated. Found: %s' % ( + self.__class__.__name__, value)) + else: + raise ValidationError('Field %s is repeated. Found: %s' % (name, + value)) + + def validate(self, value): + """Validate value assigned to field. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.__validate(value, self.validate_element) + + def validate_default_element(self, value): + """Validate value as assigned to field default field. + + Some fields may allow for delayed resolution of default types necessary + in the case of circular definition references. In this case, the default + value might be a place holder that is resolved when needed after all the + message classes are defined. + + Args: + value: Default value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.validate_element(value) + + def validate_default(self, value): + """Validate default value assigned to field. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.__validate(value, self.validate_default_element) + + def message_definition(self): + """Get Message definition that contains this Field definition. + + Returns: + Containing Message definition for Field. Will return None if for + some reason Field is defined outside of a Message class. + """ + try: + return self._message_definition() + except AttributeError: + return None + + @property + def default(self): + """Get default value for field.""" + return self.__default + + @classmethod + def lookup_field_type_by_variant(cls, variant): + return cls.__variant_to_type[variant] + + +class IntegerField(Field): + """Field definition for integer values.""" + + VARIANTS = frozenset([Variant.INT32, + Variant.INT64, + Variant.UINT32, + Variant.UINT64, + Variant.SINT32, + Variant.SINT64, + ]) + + DEFAULT_VARIANT = Variant.INT64 + + type = six.integer_types + + +class FloatField(Field): + """Field definition for float values.""" + + VARIANTS = frozenset([Variant.FLOAT, + Variant.DOUBLE, + ]) + + DEFAULT_VARIANT = Variant.DOUBLE + + type = float + + +class BooleanField(Field): + """Field definition for boolean values.""" + + VARIANTS = frozenset([Variant.BOOL]) + + DEFAULT_VARIANT = Variant.BOOL + + type = bool + + +class BytesField(Field): + """Field definition for byte string values.""" + + VARIANTS = frozenset([Variant.BYTES]) + + DEFAULT_VARIANT = Variant.BYTES + + type = bytes + + +class StringField(Field): + """Field definition for unicode string values.""" + + VARIANTS = frozenset([Variant.STRING]) + + DEFAULT_VARIANT = Variant.STRING + + type = six.text_type + + def validate_element(self, value): + """Validate StringField allowing for str and unicode. + + Raises: + ValidationError if a str value is not 7-bit ascii. + """ + # If value is str is it considered valid. Satisfies "required=True". + if isinstance(value, bytes): + try: + six.text_type(value, 'ascii') + except UnicodeDecodeError as err: + try: + name = self.name + except AttributeError: + validation_error = ValidationError( + 'Field encountered non-ASCII string %r: %s' % (value, + err)) + else: + validation_error = ValidationError( + 'Field %s encountered non-ASCII string %r: %s' % (self.name, + value, + err)) + validation_error.field_name = self.name + raise validation_error + else: + super(StringField, self).validate_element(value) + + +class MessageField(Field): + """Field definition for sub-message values. + + Message fields contain instance of other messages. Instances stored + on messages stored on message fields are considered to be owned by + the containing message instance and should not be shared between + owning instances. + + Message fields must be defined to reference a single type of message. + Normally message field are defined by passing the referenced message + class in to the constructor. + + It is possible to define a message field for a type that does not yet + exist by passing the name of the message in to the constructor instead + of a message class. Resolution of the actual type of the message is + deferred until it is needed, for example, during message verification. + Names provided to the constructor must refer to a class within the same + python module as the class that is using it. Names refer to messages + relative to the containing messages scope. For example, the two fields + of OuterMessage refer to the same message type: + + class Outer(Message): + + inner_relative = MessageField('Inner', 1) + inner_absolute = MessageField('Outer.Inner', 2) + + class Inner(Message): + ... + + When resolving an actual type, MessageField will traverse the entire + scope of nested messages to match a message name. This makes it easy + for siblings to reference siblings: + + class Outer(Message): + + class Inner(Message): + + sibling = MessageField('Sibling', 1) + + class Sibling(Message): + ... + """ + + VARIANTS = frozenset([Variant.MESSAGE]) + + DEFAULT_VARIANT = Variant.MESSAGE + + @util.positional(3) + def __init__(self, + message_type, + number, + required=False, + repeated=False, + variant=None): + """Constructor. + + Args: + message_type: Message type for field. Must be subclass of Message. + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive to + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive to + 'required'. + variant: Wire-format variant hint. + + Raises: + FieldDefinitionError when invalid message_type is provided. + """ + valid_type = (isinstance(message_type, six.string_types) or + (message_type is not Message and + isinstance(message_type, type) and + issubclass(message_type, Message))) + + if not valid_type: + raise FieldDefinitionError('Invalid message class: %s' % message_type) + + if isinstance(message_type, six.string_types): + self.__type_name = message_type + self.__type = None + else: + self.__type = message_type + + super(MessageField, self).__init__(number, + required=required, + repeated=repeated, + variant=variant) + + def __set__(self, message_instance, value): + """Set value on message. + + Args: + message_instance: Message instance to set value on. + value: Value to set on message. + """ + message_type = self.type + if isinstance(message_type, type) and issubclass(message_type, Message): + if self.repeated: + if value and isinstance(value, (list, tuple)): + value = [(message_type(**v) if isinstance(v, dict) else v) + for v in value] + elif isinstance(value, dict): + value = message_type(**value) + super(MessageField, self).__set__(message_instance, value) + + @property + def type(self): + """Message type used for field.""" + if self.__type is None: + message_type = find_definition(self.__type_name, self.message_definition()) + if not (message_type is not Message and + isinstance(message_type, type) and + issubclass(message_type, Message)): + raise FieldDefinitionError('Invalid message class: %s' % message_type) + self.__type = message_type + return self.__type + + @property + def message_type(self): + """Underlying message type used for serialization. + + Will always be a sub-class of Message. This is different from type + which represents the python value that message_type is mapped to for + use by the user. + """ + return self.type + + def value_from_message(self, message): + """Convert a message to a value instance. + + Used by deserializers to convert from underlying messages to + value of expected user type. + + Args: + message: A message instance of type self.message_type. + + Returns: + Value of self.message_type. + """ + if not isinstance(message, self.message_type): + raise DecodeError('Expected type %s, got %s: %r' % + (self.message_type.__name__, + type(message).__name__, + message)) + return message + + def value_to_message(self, value): + """Convert a value instance to a message. + + Used by serializers to convert Python user types to underlying + messages for transmission. + + Args: + value: A value of type self.type. + + Returns: + An instance of type self.message_type. + """ + if not isinstance(value, self.type): + raise EncodeError('Expected type %s, got %s: %r' % + (self.type.__name__, + type(value).__name__, + value)) + return value + + +class EnumField(Field): + """Field definition for enum values. + + Enum fields may have default values that are delayed until the associated enum + type is resolved. This is necessary to support certain circular references. + + For example: + + class Message1(Message): + + class Color(Enum): + + RED = 1 + GREEN = 2 + BLUE = 3 + + # This field default value will be validated when default is accessed. + animal = EnumField('Message2.Animal', 1, default='HORSE') + + class Message2(Message): + + class Animal(Enum): + + DOG = 1 + CAT = 2 + HORSE = 3 + + # This fields default value will be validated right away since Color is + # already fully resolved. + color = EnumField(Message1.Color, 1, default='RED') + """ + + VARIANTS = frozenset([Variant.ENUM]) + + DEFAULT_VARIANT = Variant.ENUM + + def __init__(self, enum_type, number, **kwargs): + """Constructor. + + Args: + enum_type: Enum type for field. Must be subclass of Enum. + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive to + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive to + 'required'. + variant: Wire-format variant hint. + default: Default value for field if not found in stream. + + Raises: + FieldDefinitionError when invalid enum_type is provided. + """ + valid_type = (isinstance(enum_type, six.string_types) or + (enum_type is not Enum and + isinstance(enum_type, type) and + issubclass(enum_type, Enum))) + + if not valid_type: + raise FieldDefinitionError('Invalid enum type: %s' % enum_type) + + if isinstance(enum_type, six.string_types): + self.__type_name = enum_type + self.__type = None + else: + self.__type = enum_type + + super(EnumField, self).__init__(number, **kwargs) + + def validate_default_element(self, value): + """Validate default element of Enum field. + + Enum fields allow for delayed resolution of default values when the type + of the field has not been resolved. The default value of a field may be + a string or an integer. If the Enum type of the field has been resolved, + the default value is validated against that type. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected message type. + """ + if isinstance(value, (six.string_types, six.integer_types)): + # Validation of the value does not happen for delayed resolution + # enumerated types. Ignore if type is not yet resolved. + if self.__type: + self.__type(value) + return + + super(EnumField, self).validate_default_element(value) + + @property + def type(self): + """Enum type used for field.""" + if self.__type is None: + found_type = find_definition(self.__type_name, self.message_definition()) + if not (found_type is not Enum and + isinstance(found_type, type) and + issubclass(found_type, Enum)): + raise FieldDefinitionError('Invalid enum type: %s' % found_type) + + self.__type = found_type + return self.__type + + @property + def default(self): + """Default for enum field. + + Will cause resolution of Enum type and unresolved default value. + """ + try: + return self.__resolved_default + except AttributeError: + resolved_default = super(EnumField, self).default + if isinstance(resolved_default, (six.string_types, six.integer_types)): + resolved_default = self.type(resolved_default) + self.__resolved_default = resolved_default + return self.__resolved_default + + +@util.positional(2) +def find_definition(name, relative_to=None, importer=__import__): + """Find definition by name in module-space. + + The find algorthm will look for definitions by name relative to a message + definition or by fully qualfied name. If no definition is found relative + to the relative_to parameter it will do the same search against the container + of relative_to. If relative_to is a nested Message, it will search its + message_definition(). If that message has no message_definition() it will + search its module. If relative_to is a module, it will attempt to look for + the containing module and search relative to it. If the module is a top-level + module, it will look for the a message using a fully qualified name. If + no message is found then, the search fails and DefinitionNotFoundError is + raised. + + For example, when looking for any definition 'foo.bar.ADefinition' relative to + an actual message definition abc.xyz.SomeMessage: + + find_definition('foo.bar.ADefinition', SomeMessage) + + It is like looking for the following fully qualified names: + + abc.xyz.SomeMessage. foo.bar.ADefinition + abc.xyz. foo.bar.ADefinition + abc. foo.bar.ADefinition + foo.bar.ADefinition + + When resolving the name relative to Message definitions and modules, the + algorithm searches any Messages or sub-modules found in its path. + Non-Message values are not searched. + + A name that begins with '.' is considered to be a fully qualified name. The + name is always searched for from the topmost package. For example, assume + two message types: + + abc.xyz.SomeMessage + xyz.SomeMessage + + Searching for '.xyz.SomeMessage' relative to 'abc' will resolve to + 'xyz.SomeMessage' and not 'abc.xyz.SomeMessage'. For this kind of name, + the relative_to parameter is effectively ignored and always set to None. + + For more information about package name resolution, please see: + + http://code.google.com/apis/protocolbuffers/docs/proto.html#packages + + Args: + name: Name of definition to find. May be fully qualified or relative name. + relative_to: Search for definition relative to message definition or module. + None will cause a fully qualified name search. + importer: Import function to use for resolving modules. + + Returns: + Enum or Message class definition associated with name. + + Raises: + DefinitionNotFoundError if no definition is found in any search path. + """ + # Check parameters. + if not (relative_to is None or + isinstance(relative_to, types.ModuleType) or + isinstance(relative_to, type) and issubclass(relative_to, Message)): + raise TypeError('relative_to must be None, Message definition or module. ' + 'Found: %s' % relative_to) + + name_path = name.split('.') + + # Handle absolute path reference. + if not name_path[0]: + relative_to = None + name_path = name_path[1:] + + def search_path(): + """Performs a single iteration searching the path from relative_to. + + This is the function that searches up the path from a relative object. + + fully.qualified.object . relative.or.nested.Definition + ----------------------------> + ^ + | + this part of search --+ + + Returns: + Message or Enum at the end of name_path, else None. + """ + next = relative_to + for node in name_path: + # Look for attribute first. + attribute = getattr(next, node, None) + + if attribute is not None: + next = attribute + else: + # If module, look for sub-module. + if next is None or isinstance(next, types.ModuleType): + if next is None: + module_name = node + else: + module_name = '%s.%s' % (next.__name__, node) + + try: + fromitem = module_name.split('.')[-1] + next = importer(module_name, '', '', [str(fromitem)]) + except ImportError: + return None + else: + return None + + if (not isinstance(next, types.ModuleType) and + not (isinstance(next, type) and + issubclass(next, (Message, Enum)))): + return None + + return next + + while True: + found = search_path() + if isinstance(found, type) and issubclass(found, (Enum, Message)): + return found + else: + # Find next relative_to to search against. + # + # fully.qualified.object . relative.or.nested.Definition + # <--------------------- + # ^ + # | + # does this part of search + if relative_to is None: + # Fully qualified search was done. Nothing found. Fail. + raise DefinitionNotFoundError('Could not find definition for %s' + % (name,)) + else: + if isinstance(relative_to, types.ModuleType): + # Find parent module. + module_path = relative_to.__name__.split('.')[:-1] + if not module_path: + relative_to = None + else: + # Should not raise ImportError. If it does... weird and + # unexepected. Propagate. + relative_to = importer( + '.'.join(module_path), '', '', [module_path[-1]]) + elif (isinstance(relative_to, type) and + issubclass(relative_to, Message)): + parent = relative_to.message_definition() + if parent is None: + last_module_name = relative_to.__module__.split('.')[-1] + relative_to = importer( + relative_to.__module__, '', '', [last_module_name]) + else: + relative_to = parent diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py new file mode 100644 index 0000000..a3b6dc0 --- /dev/null +++ b/apitools/base/protorpclite/messages_test.py @@ -0,0 +1,2084 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for apitools.base.protorpclite.messages.""" +import six + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import pickle +import re +import sys +import types +import unittest + +from apitools.base.protorpclite import descriptor +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import test_util + + +class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = messages + + +class ValidationErrorTest(test_util.TestCase): + + def testStr_NoFieldName(self): + """Test string version of ValidationError when no name provided.""" + self.assertEquals('Validation error', + str(messages.ValidationError('Validation error'))) + + def testStr_FieldName(self): + """Test string version of ValidationError when no name provided.""" + validation_error = messages.ValidationError('Validation error') + validation_error.field_name = 'a_field' + self.assertEquals('Validation error', str(validation_error)) + + +class EnumTest(test_util.TestCase): + + def setUp(self): + """Set up tests.""" + # Redefine Color class in case so that changes to it (an error) in one test + # does not affect other tests. + global Color + class Color(messages.Enum): + RED = 20 + ORANGE = 2 + YELLOW = 40 + GREEN = 4 + BLUE = 50 + INDIGO = 5 + VIOLET = 80 + + def testNames(self): + """Test that names iterates over enum names.""" + self.assertEquals( + set(['BLUE', 'GREEN', 'INDIGO', 'ORANGE', 'RED', 'VIOLET', 'YELLOW']), + set(Color.names())) + + def testNumbers(self): + """Tests that numbers iterates of enum numbers.""" + self.assertEquals(set([2, 4, 5, 20, 40, 50, 80]), set(Color.numbers())) + + def testIterate(self): + """Test that __iter__ iterates over all enum values.""" + self.assertEquals(set(Color), + set([Color.RED, + Color.ORANGE, + Color.YELLOW, + Color.GREEN, + Color.BLUE, + Color.INDIGO, + Color.VIOLET])) + + def testNaturalOrder(self): + """Test that natural order enumeration is in numeric order.""" + self.assertEquals([Color.ORANGE, + Color.GREEN, + Color.INDIGO, + Color.RED, + Color.YELLOW, + Color.BLUE, + Color.VIOLET], + sorted(Color)) + + def testByName(self): + """Test look-up by name.""" + self.assertEquals(Color.RED, Color.lookup_by_name('RED')) + self.assertRaises(KeyError, Color.lookup_by_name, 20) + self.assertRaises(KeyError, Color.lookup_by_name, Color.RED) + + def testByNumber(self): + """Test look-up by number.""" + self.assertRaises(KeyError, Color.lookup_by_number, 'RED') + self.assertEquals(Color.RED, Color.lookup_by_number(20)) + self.assertRaises(KeyError, Color.lookup_by_number, Color.RED) + + def testConstructor(self): + """Test that constructor look-up by name or number.""" + self.assertEquals(Color.RED, Color('RED')) + self.assertEquals(Color.RED, Color(u'RED')) + self.assertEquals(Color.RED, Color(20)) + if six.PY2: + self.assertEquals(Color.RED, Color(long(20))) + self.assertEquals(Color.RED, Color(Color.RED)) + self.assertRaises(TypeError, Color, 'Not exists') + self.assertRaises(TypeError, Color, 'Red') + self.assertRaises(TypeError, Color, 100) + self.assertRaises(TypeError, Color, 10.0) + + def testLen(self): + """Test that len function works to count enums.""" + self.assertEquals(7, len(Color)) + + def testNoSubclasses(self): + """Test that it is not possible to sub-class enum classes.""" + def declare_subclass(): + class MoreColor(Color): + pass + self.assertRaises(messages.EnumDefinitionError, + declare_subclass) + + def testClassNotMutable(self): + """Test that enum classes themselves are not mutable.""" + self.assertRaises(AttributeError, + setattr, + Color, + 'something_new', + 10) + + def testInstancesMutable(self): + """Test that enum instances are not mutable.""" + self.assertRaises(TypeError, + setattr, + Color.RED, + 'something_new', + 10) + + def testDefEnum(self): + """Test def_enum works by building enum class from dict.""" + WeekDay = messages.Enum.def_enum({'Monday': 1, + 'Tuesday': 2, + 'Wednesday': 3, + 'Thursday': 4, + 'Friday': 6, + 'Saturday': 7, + 'Sunday': 8}, + 'WeekDay') + self.assertEquals('Wednesday', WeekDay(3).name) + self.assertEquals(6, WeekDay('Friday').number) + self.assertEquals(WeekDay.Sunday, WeekDay('Sunday')) + + def testNonInt(self): + """Test that non-integer values rejection by enum def.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': '1'}, + 'BadEnum') + + def testNegativeInt(self): + """Test that negative numbers rejection by enum def.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': -1}, + 'BadEnum') + + def testLowerBound(self): + """Test that zero is accepted by enum def.""" + class NotImportant(messages.Enum): + """Testing for value zero""" + VALUE = 0 + + self.assertEquals(0, int(NotImportant.VALUE)) + + def testTooLargeInt(self): + """Test that numbers too large are rejected.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': (2 ** 29)}, + 'BadEnum') + + def testRepeatedInt(self): + """Test duplicated numbers are forbidden.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Ok': 1, 'Repeated': 1}, + 'BadEnum') + + def testStr(self): + """Test converting to string.""" + self.assertEquals('RED', str(Color.RED)) + self.assertEquals('ORANGE', str(Color.ORANGE)) + + def testInt(self): + """Test converting to int.""" + self.assertEquals(20, int(Color.RED)) + self.assertEquals(2, int(Color.ORANGE)) + + def testRepr(self): + """Test enum representation.""" + self.assertEquals('Color(RED, 20)', repr(Color.RED)) + self.assertEquals('Color(YELLOW, 40)', repr(Color.YELLOW)) + + def testDocstring(self): + """Test that docstring is supported ok.""" + class NotImportant(messages.Enum): + """I have a docstring.""" + + VALUE1 = 1 + + self.assertEquals('I have a docstring.', NotImportant.__doc__) + + def testDeleteEnumValue(self): + """Test that enum values cannot be deleted.""" + self.assertRaises(TypeError, delattr, Color, 'RED') + + def testEnumName(self): + """Test enum name.""" + module_name = test_util.get_module_name(EnumTest) + self.assertEquals('%s.Color' % module_name, Color.definition_name()) + self.assertEquals(module_name, Color.outer_definition_name()) + self.assertEquals(module_name, Color.definition_package()) + + def testDefinitionName_OverrideModule(self): + """Test enum module is overriden by module package name.""" + global package + try: + package = 'my.package' + self.assertEquals('my.package.Color', Color.definition_name()) + self.assertEquals('my.package', Color.outer_definition_name()) + self.assertEquals('my.package', Color.definition_package()) + finally: + del package + + def testDefinitionName_NoModule(self): + """Test what happens when there is no module for enum.""" + class Enum1(messages.Enum): + pass + + original_modules = sys.modules + sys.modules = dict(sys.modules) + try: + del sys.modules[__name__] + self.assertEquals('Enum1', Enum1.definition_name()) + self.assertEquals(None, Enum1.outer_definition_name()) + self.assertEquals(None, Enum1.definition_package()) + self.assertEquals(six.text_type, type(Enum1.definition_name())) + finally: + sys.modules = original_modules + + def testDefinitionName_Nested(self): + """Test nested Enum names.""" + class MyMessage(messages.Message): + + class NestedEnum(messages.Enum): + + pass + + class NestedMessage(messages.Message): + + class NestedEnum(messages.Enum): + + pass + + module_name = test_util.get_module_name(EnumTest) + self.assertEquals('%s.MyMessage.NestedEnum' % module_name, + MyMessage.NestedEnum.definition_name()) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.NestedEnum.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedEnum.definition_package()) + + self.assertEquals('%s.MyMessage.NestedMessage.NestedEnum' % module_name, + MyMessage.NestedMessage.NestedEnum.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedEnum.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedMessage.NestedEnum.definition_package()) + + def testMessageDefinition(self): + """Test that enumeration knows its enclosing message definition.""" + class OuterEnum(messages.Enum): + pass + + self.assertEquals(None, OuterEnum.message_definition()) + + class OuterMessage(messages.Message): + + class InnerEnum(messages.Enum): + pass + + self.assertEquals(OuterMessage, OuterMessage.InnerEnum.message_definition()) + + def testComparison(self): + """Test comparing various enums to different types.""" + class Enum1(messages.Enum): + VAL1 = 1 + VAL2 = 2 + + class Enum2(messages.Enum): + VAL1 = 1 + + self.assertEquals(Enum1.VAL1, Enum1.VAL1) + self.assertNotEquals(Enum1.VAL1, Enum1.VAL2) + self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) + self.assertNotEquals(Enum1.VAL1, 'VAL1') + self.assertNotEquals(Enum1.VAL1, 1) + self.assertNotEquals(Enum1.VAL1, 2) + self.assertNotEquals(Enum1.VAL1, None) + self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) + + self.assertTrue(Enum1.VAL1 < Enum1.VAL2) + self.assertTrue(Enum1.VAL2 > Enum1.VAL1) + + self.assertNotEquals(1, Enum2.VAL1) + + def testPickle(self): + """Testing pickling and unpickling of Enum instances.""" + colors = list(Color) + unpickled = pickle.loads(pickle.dumps(colors)) + self.assertEquals(colors, unpickled) + # Unpickling shouldn't create new enum instances. + for i, color in enumerate(colors): + self.assertTrue(color is unpickled[i]) + + +class FieldListTest(test_util.TestCase): + + def setUp(self): + self.integer_field = messages.IntegerField(1, repeated=True) + + def testConstructor(self): + self.assertEquals([1, 2, 3], + messages.FieldList(self.integer_field, [1, 2, 3])) + self.assertEquals([1, 2, 3], + messages.FieldList(self.integer_field, (1, 2, 3))) + self.assertEquals([], messages.FieldList(self.integer_field, [])) + + def testNone(self): + self.assertRaises(TypeError, messages.FieldList, self.integer_field, None) + + def testDoNotAutoConvertString(self): + string_field = messages.StringField(1, repeated=True) + self.assertRaises(messages.ValidationError, + messages.FieldList, string_field, 'abc') + + def testConstructorCopies(self): + a_list = [1, 3, 6] + field_list = messages.FieldList(self.integer_field, a_list) + self.assertFalse(a_list is field_list) + self.assertFalse(field_list is + messages.FieldList(self.integer_field, field_list)) + + def testNonRepeatedField(self): + self.assertRaisesWithRegexpMatch( + messages.FieldDefinitionError, + 'FieldList may only accept repeated fields', + messages.FieldList, + messages.IntegerField(1), + []) + + def testConstructor_InvalidValues(self): + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 1 (type %r)" + % (six.integer_types, str)), + messages.FieldList, self.integer_field, ["1", "2", "3"]) + + def testConstructor_Scalars(self): + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "IntegerField is repeated. Found: 3", + messages.FieldList, self.integer_field, 3) + + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "IntegerField is repeated. Found: <(list[_]?|sequence)iterator object", + messages.FieldList, self.integer_field, iter([1, 2, 3])) + + def testSetSlice(self): + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + field_list[1:3] = [10, 20] + self.assertEquals([1, 10, 20, 4, 5], field_list) + + def testSetSlice_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + + def setslice(): + field_list[1:3] = ['10', '20'] + + msg_re = re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + msg_re, + setslice) + + def testSetItem(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list[0] = 10 + self.assertEquals([10], field_list) + + def testSetItem_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + + def setitem(): + field_list[0] = '10' + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + setitem) + + def testAppend(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.append(10) + self.assertEquals([2, 10], field_list) + + def testAppend_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.name = 'a_field' + + def append(): + field_list.append('10') + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + append) + + def testExtend(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.extend([10]) + self.assertEquals([2, 10], field_list) + + def testExtend_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + + def extend(): + field_list.extend(['10']) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + extend) + + def testInsert(self): + field_list = messages.FieldList(self.integer_field, [2, 3]) + field_list.insert(1, 10) + self.assertEquals([2, 10, 3], field_list) + + def testInsert_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2, 3]) + + def insert(): + field_list.insert(1, '10') + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + insert) + + def testPickle(self): + """Testing pickling and unpickling of disconnected FieldList instances.""" + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + unpickled = pickle.loads(pickle.dumps(field_list)) + self.assertEquals(field_list, unpickled) + self.assertIsInstance(unpickled.field, messages.IntegerField) + self.assertEquals(1, unpickled.field.number) + self.assertTrue(unpickled.field.repeated) + + +class FieldTest(test_util.TestCase): + + def ActionOnAllFieldClasses(self, action): + """Test all field classes except Message and Enum. + + Message and Enum require separate tests. + + Args: + action: Callable that takes the field class as a parameter. + """ + for field_class in (messages.IntegerField, + messages.FloatField, + messages.BooleanField, + messages.BytesField, + messages.StringField, + ): + action(field_class) + + def testNumberAttribute(self): + """Test setting the number attribute.""" + def action(field_class): + # Check range. + self.assertRaises(messages.InvalidNumberError, + field_class, + 0) + self.assertRaises(messages.InvalidNumberError, + field_class, + -1) + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.MAX_FIELD_NUMBER + 1) + + # Check reserved. + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.FIRST_RESERVED_FIELD_NUMBER) + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.LAST_RESERVED_FIELD_NUMBER) + self.assertRaises(messages.InvalidNumberError, + field_class, + '1') + + # This one should work. + field_class(number=1) + self.ActionOnAllFieldClasses(action) + + def testRequiredAndRepeated(self): + """Test setting the required and repeated fields.""" + def action(field_class): + field_class(1, required=True) + field_class(1, repeated=True) + self.assertRaises(messages.FieldDefinitionError, + field_class, + 1, + required=True, + repeated=True) + self.ActionOnAllFieldClasses(action) + + def testInvalidVariant(self): + """Test field with invalid variants.""" + def action(field_class): + if field_class is not message_types.DateTimeField: + self.assertRaises(messages.InvalidVariantError, + field_class, + 1, + variant=messages.Variant.ENUM) + self.ActionOnAllFieldClasses(action) + + def testDefaultVariant(self): + """Test that default variant is used when not set.""" + def action(field_class): + field = field_class(1) + self.assertEquals(field_class.DEFAULT_VARIANT, field.variant) + + self.ActionOnAllFieldClasses(action) + + def testAlternateVariant(self): + """Test that default variant is used when not set.""" + field = messages.IntegerField(1, variant=messages.Variant.UINT32) + self.assertEquals(messages.Variant.UINT32, field.variant) + + def testDefaultFields_Single(self): + """Test default field is correct type (single).""" + defaults = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } + + def action(field_class): + field_class(1, default=defaults[field_class]) + self.ActionOnAllFieldClasses(action) + + # Run defaults test again checking for str/unicode compatiblity. + defaults[messages.StringField] = 'abc' + self.ActionOnAllFieldClasses(action) + + def testStringField_BadUnicodeInDefault(self): + """Test binary values in string field.""" + self.assertRaisesWithRegexpMatch( + messages.InvalidDefaultError, + r"Invalid default value for StringField:.*: " + r"Field encountered non-ASCII string .*: " + r"'ascii' codec can't decode byte 0x89 in position 0: " + r"ordinal not in range", + messages.StringField, 1, default=b'\x89') + + def testDefaultFields_InvalidSingle(self): + """Test default field is correct type (invalid single).""" + def action(field_class): + self.assertRaises(messages.InvalidDefaultError, + field_class, + 1, + default=object()) + self.ActionOnAllFieldClasses(action) + + def testDefaultFields_InvalidRepeated(self): + """Test default field does not accept defaults.""" + self.assertRaisesWithRegexpMatch( + messages.FieldDefinitionError, + 'Repeated fields may not have defaults', + messages.StringField, 1, repeated=True, default=[1, 2, 3]) + + def testDefaultFields_None(self): + """Test none is always acceptable.""" + def action(field_class): + field_class(1, default=None) + field_class(1, required=True, default=None) + field_class(1, repeated=True, default=None) + self.ActionOnAllFieldClasses(action) + + def testDefaultFields_Enum(self): + """Test the default for enum fields.""" + class Symbol(messages.Enum): + + ALPHA = 1 + BETA = 2 + GAMMA = 3 + + field = messages.EnumField(Symbol, 1, default=Symbol.ALPHA) + + self.assertEquals(Symbol.ALPHA, field.default) + + def testDefaultFields_EnumStringDelayedResolution(self): + """Test that enum fields resolve default strings.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default='OPTIONAL') + + self.assertEquals(descriptor.FieldDescriptor.Label.OPTIONAL, field.default) + + def testDefaultFields_EnumIntDelayedResolution(self): + """Test that enum fields resolve default integers.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=2) + + self.assertEquals(descriptor.FieldDescriptor.Label.REQUIRED, field.default) + + def testDefaultFields_EnumOkIfTypeKnown(self): + """Test that enum fields accept valid default values when type is known.""" + field = messages.EnumField(descriptor.FieldDescriptor.Label, + 1, + default='REPEATED') + + self.assertEquals(descriptor.FieldDescriptor.Label.REPEATED, field.default) + + def testDefaultFields_EnumForceCheckIfTypeKnown(self): + """Test that enum fields validate default values if type is known.""" + self.assertRaisesWithRegexpMatch(TypeError, + 'No such value for NOT_A_LABEL in ' + 'Enum Label', + messages.EnumField, + descriptor.FieldDescriptor.Label, + 1, + default='NOT_A_LABEL') + + def testDefaultFields_EnumInvalidDelayedResolution(self): + """Test that enum fields raise errors upon delayed resolution error.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=200) + + self.assertRaisesWithRegexpMatch(TypeError, + 'No such value for 200 in Enum Label', + getattr, + field, + 'default') + + def testValidate_Valid(self): + """Test validation of valid values.""" + values = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } + def action(field_class): + # Optional. + field = field_class(1) + field.validate(values[field_class]) + + # Required. + field = field_class(1, required=True) + field.validate(values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + field.validate([]) + field.validate(()) + field.validate([values[field_class]]) + field.validate((values[field_class],)) + + # Right value, but not repeated. + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + self.ActionOnAllFieldClasses(action) + + def testValidate_Invalid(self): + """Test validation of valid values.""" + values = {messages.IntegerField: "10", + messages.FloatField: 1, + messages.BooleanField: 0, + messages.BytesField: 10.20, + messages.StringField: 42, + } + def action(field_class): + # Optional. + field = field_class(1) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + # Required. + field = field_class(1, required=True) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + self.assertRaises(messages.ValidationError, + field.validate, + [values[field_class]]) + self.assertRaises(messages.ValidationError, + field.validate, + (values[field_class],)) + self.ActionOnAllFieldClasses(action) + + def testValidate_None(self): + """Test that None is valid for non-required fields.""" + def action(field_class): + # Optional. + field = field_class(1) + field.validate(None) + + # Required. + field = field_class(1, required=True) + self.assertRaisesWithRegexpMatch(messages.ValidationError, + 'Required field is missing', + field.validate, + None) + + # Repeated. + field = field_class(1, repeated=True) + field.validate(None) + self.assertRaisesWithRegexpMatch(messages.ValidationError, + 'Repeated values for %s may ' + 'not be None' % field_class.__name__, + field.validate, + [None]) + self.assertRaises(messages.ValidationError, + field.validate, + (None,)) + self.ActionOnAllFieldClasses(action) + + def testValidateElement(self): + """Test validation of valid values.""" + values = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: 'abc', + messages.StringField: u'abc', + } + def action(field_class): + # Optional. + field = field_class(1) + field.validate_element(values[field_class]) + + # Required. + field = field_class(1, required=True) + field.validate_element(values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + self.assertRaises(message.VAlidationError, + field.validate_element, + []) + self.assertRaises(message.VAlidationError, + field.validate_element, + ()) + field.validate_element(values[field_class]) + field.validate_element(values[field_class]) + + # Right value, but repeated. + self.assertRaises(messages.ValidationError, + field.validate_element, + [values[field_class]]) + self.assertRaises(messages.ValidationError, + field.validate_element, + (values[field_class],)) + + def testReadOnly(self): + """Test that objects are all read-only.""" + def action(field_class): + field = field_class(10) + self.assertRaises(AttributeError, + setattr, + field, + 'number', + 20) + self.assertRaises(AttributeError, + setattr, + field, + 'anything_else', + 'whatever') + self.ActionOnAllFieldClasses(action) + + def testMessageField(self): + """Test the construction of message fields.""" + self.assertRaises(messages.FieldDefinitionError, + messages.MessageField, + str, + 10) + + self.assertRaises(messages.FieldDefinitionError, + messages.MessageField, + messages.Message, + 10) + + class MyMessage(messages.Message): + pass + + field = messages.MessageField(MyMessage, 10) + self.assertEquals(MyMessage, field.type) + + def testMessageField_ForwardReference(self): + """Test the construction of forward reference message fields.""" + global MyMessage + global ForwardMessage + try: + class MyMessage(messages.Message): + + self_reference = messages.MessageField('MyMessage', 1) + forward = messages.MessageField('ForwardMessage', 2) + nested = messages.MessageField('ForwardMessage.NestedMessage', 3) + inner = messages.MessageField('Inner', 4) + + class Inner(messages.Message): + + sibling = messages.MessageField('Sibling', 1) + + class Sibling(messages.Message): + + pass + + class ForwardMessage(messages.Message): + + class NestedMessage(messages.Message): + + pass + + self.assertEquals(MyMessage, + MyMessage.field_by_name('self_reference').type) + + self.assertEquals(ForwardMessage, + MyMessage.field_by_name('forward').type) + + self.assertEquals(ForwardMessage.NestedMessage, + MyMessage.field_by_name('nested').type) + + self.assertEquals(MyMessage.Inner, + MyMessage.field_by_name('inner').type) + + self.assertEquals(MyMessage.Sibling, + MyMessage.Inner.field_by_name('sibling').type) + finally: + try: + del MyMessage + del ForwardMessage + except: + pass + + def testMessageField_WrongType(self): + """Test that forward referencing the wrong type raises an error.""" + global AnEnum + try: + class AnEnum(messages.Enum): + pass + + class AnotherMessage(messages.Message): + + a_field = messages.MessageField('AnEnum', 1) + + self.assertRaises(messages.FieldDefinitionError, + getattr, + AnotherMessage.field_by_name('a_field'), + 'type') + finally: + del AnEnum + + def testMessageFieldValidate(self): + """Test validation on message field.""" + class MyMessage(messages.Message): + pass + + class AnotherMessage(messages.Message): + pass + + field = messages.MessageField(MyMessage, 10) + field.validate(MyMessage()) + + self.assertRaises(messages.ValidationError, + field.validate, + AnotherMessage()) + + def testMessageFieldMessageType(self): + """Test message_type property.""" + class MyMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + self.assertEqual(HasMessage.field.type, HasMessage.field.message_type) + + def testMessageFieldValueFromMessage(self): + class MyMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + instance = MyMessage() + + self.assertTrue(instance is HasMessage.field.value_from_message(instance)) + + def testMessageFieldValueFromMessageWrongType(self): + class MyMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Expected type MyMessage, got int: 10', + HasMessage.field.value_from_message, 10) + + def testMessageFieldValueToMessage(self): + class MyMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + instance = MyMessage() + + self.assertTrue(instance is HasMessage.field.value_to_message(instance)) + + def testMessageFieldValueToMessageWrongType(self): + class MyMessage(messages.Message): + pass + + class MyOtherMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + instance = MyOtherMessage() + + self.assertRaisesWithRegexpMatch( + messages.EncodeError, + 'Expected type MyMessage, got MyOtherMessage: ', + HasMessage.field.value_to_message, instance) + + def testIntegerField_AllowLong(self): + """Test that the integer field allows for longs.""" + if six.PY2: + messages.IntegerField(10, default=long(10)) + + def testMessageFieldValidate_Initialized(self): + """Test validation on message field.""" + class MyMessage(messages.Message): + field1 = messages.IntegerField(1, required=True) + + field = messages.MessageField(MyMessage, 10) + + # Will validate messages where is_initialized() is False. + message = MyMessage() + field.validate(message) + message.field1 = 20 + field.validate(message) + + def testEnumField(self): + """Test the construction of enum fields.""" + self.assertRaises(messages.FieldDefinitionError, + messages.EnumField, + str, + 10) + + self.assertRaises(messages.FieldDefinitionError, + messages.EnumField, + messages.Enum, + 10) + + class Color(messages.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 + + field = messages.EnumField(Color, 10) + self.assertEquals(Color, field.type) + + class Another(messages.Enum): + VALUE = 1 + + self.assertRaises(messages.InvalidDefaultError, + messages.EnumField, + Color, + 10, + default=Another.VALUE) + + def testEnumField_ForwardReference(self): + """Test the construction of forward reference enum fields.""" + global MyMessage + global ForwardEnum + global ForwardMessage + try: + class MyMessage(messages.Message): + + forward = messages.EnumField('ForwardEnum', 1) + nested = messages.EnumField('ForwardMessage.NestedEnum', 2) + inner = messages.EnumField('Inner', 3) + + class Inner(messages.Enum): + pass + + class ForwardEnum(messages.Enum): + pass + + class ForwardMessage(messages.Message): + + class NestedEnum(messages.Enum): + pass + + self.assertEquals(ForwardEnum, + MyMessage.field_by_name('forward').type) + + self.assertEquals(ForwardMessage.NestedEnum, + MyMessage.field_by_name('nested').type) + + self.assertEquals(MyMessage.Inner, + MyMessage.field_by_name('inner').type) + finally: + try: + del MyMessage + del ForwardEnum + del ForwardMessage + except: + pass + + def testEnumField_WrongType(self): + """Test that forward referencing the wrong type raises an error.""" + global AMessage + try: + class AMessage(messages.Message): + pass + + class AnotherMessage(messages.Message): + + a_field = messages.EnumField('AMessage', 1) + + self.assertRaises(messages.FieldDefinitionError, + getattr, + AnotherMessage.field_by_name('a_field'), + 'type') + finally: + del AMessage + + def testMessageDefinition(self): + """Test that message definition is set on fields.""" + class MyMessage(messages.Message): + + my_field = messages.StringField(1) + + self.assertEquals(MyMessage, + MyMessage.field_by_name('my_field').message_definition()) + + def testNoneAssignment(self): + """Test that assigning None does not change comparison.""" + class MyMessage(messages.Message): + + my_field = messages.StringField(1) + + m1 = MyMessage() + m2 = MyMessage() + m2.my_field = None + self.assertEquals(m1, m2) + + def testNonAsciiStr(self): + """Test validation fails for non-ascii StringField values.""" + class Thing(messages.Message): + string_field = messages.StringField(2) + + thing = Thing() + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + 'Field string_field encountered non-ASCII string', + setattr, thing, 'string_field', test_util.BINARY) + + +class MessageTest(test_util.TestCase): + """Tests for message class.""" + + def CreateMessageClass(self): + """Creates a simple message class with 3 fields. + + Fields are defined in alphabetical order but with conflicting numeric + order. + """ + class ComplexMessage(messages.Message): + a3 = messages.IntegerField(3) + b1 = messages.StringField(1) + c2 = messages.StringField(2) + + return ComplexMessage + + def testSameNumbers(self): + """Test that cannot assign two fields with same numbers.""" + + def action(): + class BadMessage(messages.Message): + f1 = messages.IntegerField(1) + f2 = messages.IntegerField(1) + self.assertRaises(messages.DuplicateNumberError, + action) + + def testStrictAssignment(self): + """Tests that cannot assign to unknown or non-reserved attributes.""" + class SimpleMessage(messages.Message): + field = messages.IntegerField(1) + + simple_message = SimpleMessage() + self.assertRaises(AttributeError, + setattr, + simple_message, + 'does_not_exist', + 10) + + def testListAssignmentDoesNotCopy(self): + class SimpleMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + message = SimpleMessage() + original = message.repeated + message.repeated = [] + self.assertFalse(original is message.repeated) + + def testValidate_Optional(self): + """Tests validation of optional fields.""" + class SimpleMessage(messages.Message): + non_required = messages.IntegerField(1) + + simple_message = SimpleMessage() + simple_message.check_initialized() + simple_message.non_required = 10 + simple_message.check_initialized() + + def testValidate_Required(self): + """Tests validation of required fields.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + simple_message = SimpleMessage() + self.assertRaises(messages.ValidationError, + simple_message.check_initialized) + simple_message.required = 10 + simple_message.check_initialized() + + def testValidate_Repeated(self): + """Tests validation of repeated fields.""" + class SimpleMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + simple_message = SimpleMessage() + + # Check valid values. + for valid_value in [], [10], [10, 20], (), (10,), (10, 20): + simple_message.repeated = valid_value + simple_message.check_initialized() + + # Check cleared. + simple_message.repeated = [] + simple_message.check_initialized() + + # Check invalid values. + for invalid_value in 10, ['10', '20'], [None], (None,): + self.assertRaises(messages.ValidationError, + setattr, simple_message, 'repeated', invalid_value) + + def testIsInitialized(self): + """Tests is_initialized.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + simple_message = SimpleMessage() + self.assertFalse(simple_message.is_initialized()) + + simple_message.required = 10 + + self.assertTrue(simple_message.is_initialized()) + + def testIsInitializedNestedField(self): + """Tests is_initialized for nested fields.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + class NestedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1) + + simple_message = SimpleMessage() + self.assertFalse(simple_message.is_initialized()) + nested_message = NestedMessage(simple=simple_message) + self.assertFalse(nested_message.is_initialized()) + + simple_message.required = 10 + + self.assertTrue(simple_message.is_initialized()) + self.assertTrue(nested_message.is_initialized()) + + def testInitializeNestedFieldFromDict(self): + """Tests initializing nested fields from dict.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + class NestedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1) + + class RepeatedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1, repeated=True) + + nested_message1 = NestedMessage(simple={'required': 10}) + self.assertTrue(nested_message1.is_initialized()) + self.assertTrue(nested_message1.simple.is_initialized()) + + nested_message2 = NestedMessage() + nested_message2.simple = {'required': 10} + self.assertTrue(nested_message2.is_initialized()) + self.assertTrue(nested_message2.simple.is_initialized()) + + repeated_values = [{}, {'required': 10}, SimpleMessage(required=20)] + + repeated_message1 = RepeatedMessage(simple=repeated_values) + self.assertEquals(3, len(repeated_message1.simple)) + self.assertFalse(repeated_message1.is_initialized()) + + repeated_message1.simple[0].required = 0 + self.assertTrue(repeated_message1.is_initialized()) + + repeated_message2 = RepeatedMessage() + repeated_message2.simple = repeated_values + self.assertEquals(3, len(repeated_message2.simple)) + self.assertFalse(repeated_message2.is_initialized()) + + repeated_message2.simple[0].required = 0 + self.assertTrue(repeated_message2.is_initialized()) + + def testNestedMethodsNotAllowed(self): + """Test that method definitions on Message classes are not allowed.""" + def action(): + class WithMethods(messages.Message): + def not_allowed(self): + pass + + self.assertRaises(messages.MessageDefinitionError, + action) + + def testNestedAttributesNotAllowed(self): + """Test that attribute assignment on Message classes are not allowed.""" + def int_attribute(): + class WithMethods(messages.Message): + not_allowed = 1 + + def string_attribute(): + class WithMethods(messages.Message): + not_allowed = 'not allowed' + + def enum_attribute(): + class WithMethods(messages.Message): + not_allowed = Color.RED + + for action in (int_attribute, string_attribute, enum_attribute): + self.assertRaises(messages.MessageDefinitionError, + action) + + def testNameIsSetOnFields(self): + """Make sure name is set on fields after Message class init.""" + class HasNamedFields(messages.Message): + field = messages.StringField(1) + + self.assertEquals('field', HasNamedFields.field_by_number(1).name) + + def testSubclassingMessageDisallowed(self): + """Not permitted to create sub-classes of message classes.""" + class SuperClass(messages.Message): + pass + + def action(): + class SubClass(SuperClass): + pass + + self.assertRaises(messages.MessageDefinitionError, + action) + + def testAllFields(self): + """Test all_fields method.""" + ComplexMessage = self.CreateMessageClass() + fields = list(ComplexMessage.all_fields()) + + # Order does not matter, so sort now. + fields = sorted(fields, key=lambda f: f.name) + + self.assertEquals(3, len(fields)) + self.assertEquals('a3', fields[0].name) + self.assertEquals('b1', fields[1].name) + self.assertEquals('c2', fields[2].name) + + def testFieldByName(self): + """Test getting field by name.""" + ComplexMessage = self.CreateMessageClass() + + self.assertEquals(3, ComplexMessage.field_by_name('a3').number) + self.assertEquals(1, ComplexMessage.field_by_name('b1').number) + self.assertEquals(2, ComplexMessage.field_by_name('c2').number) + + self.assertRaises(KeyError, + ComplexMessage.field_by_name, + 'unknown') + + def testFieldByNumber(self): + """Test getting field by number.""" + ComplexMessage = self.CreateMessageClass() + + self.assertEquals('a3', ComplexMessage.field_by_number(3).name) + self.assertEquals('b1', ComplexMessage.field_by_number(1).name) + self.assertEquals('c2', ComplexMessage.field_by_number(2).name) + + self.assertRaises(KeyError, + ComplexMessage.field_by_number, + 4) + + def testGetAssignedValue(self): + """Test getting the assigned value of a field.""" + class SomeMessage(messages.Message): + a_value = messages.StringField(1, default=u'a default') + + message = SomeMessage() + self.assertEquals(None, message.get_assigned_value('a_value')) + + message.a_value = u'a string' + self.assertEquals(u'a string', message.get_assigned_value('a_value')) + + message.a_value = u'a default' + self.assertEquals(u'a default', message.get_assigned_value('a_value')) + + self.assertRaisesWithRegexpMatch( + AttributeError, + 'Message SomeMessage has no field no_such_field', + message.get_assigned_value, + 'no_such_field') + + def testReset(self): + """Test resetting a field value.""" + class SomeMessage(messages.Message): + a_value = messages.StringField(1, default=u'a default') + repeated = messages.IntegerField(2, repeated=True) + + message = SomeMessage() + + self.assertRaises(AttributeError, message.reset, 'unknown') + + self.assertEquals(u'a default', message.a_value) + message.reset('a_value') + self.assertEquals(u'a default', message.a_value) + + message.a_value = u'a new value' + self.assertEquals(u'a new value', message.a_value) + message.reset('a_value') + self.assertEquals(u'a default', message.a_value) + + message.repeated = [1, 2, 3] + self.assertEquals([1, 2, 3], message.repeated) + saved = message.repeated + message.reset('repeated') + self.assertEquals([], message.repeated) + self.assertIsInstance(message.repeated, messages.FieldList) + self.assertEquals([1, 2, 3], saved) + + def testAllowNestedEnums(self): + """Test allowing nested enums in a message definition.""" + class Trade(messages.Message): + class Duration(messages.Enum): + GTC = 1 + DAY = 2 + + class Currency(messages.Enum): + USD = 1 + GBP = 2 + INR = 3 + + # Sorted by name order seems to be the only feasible option. + self.assertEquals(['Currency', 'Duration'], Trade.__enums__) + + # Message definition will now be set on Enumerated objects. + self.assertEquals(Trade, Trade.Duration.message_definition()) + + def testAllowNestedMessages(self): + """Test allowing nested messages in a message definition.""" + class Trade(messages.Message): + class Lot(messages.Message): + pass + + class Agent(messages.Message): + pass + + # Sorted by name order seems to be the only feasible option. + self.assertEquals(['Agent', 'Lot'], Trade.__messages__) + self.assertEquals(Trade, Trade.Agent.message_definition()) + self.assertEquals(Trade, Trade.Lot.message_definition()) + + # But not Message itself. + def action(): + class Trade(messages.Message): + NiceTry = messages.Message + self.assertRaises(messages.MessageDefinitionError, action) + + def testDisallowClassAssignments(self): + """Test setting class attributes may not happen.""" + class MyMessage(messages.Message): + pass + + self.assertRaises(AttributeError, + setattr, + MyMessage, + 'x', + 'do not assign') + + def testEquality(self): + """Test message class equality.""" + # Comparison against enums must work. + class MyEnum(messages.Enum): + val1 = 1 + val2 = 2 + + # Comparisons against nested messages must work. + class AnotherMessage(messages.Message): + string = messages.StringField(1) + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + field2 = messages.EnumField(MyEnum, 2) + field3 = messages.MessageField(AnotherMessage, 3) + + message1 = MyMessage() + + self.assertNotEquals('hi', message1) + self.assertNotEquals(AnotherMessage(), message1) + self.assertEquals(message1, message1) + + message2 = MyMessage() + + self.assertEquals(message1, message2) + + message1.field1 = 10 + self.assertNotEquals(message1, message2) + + message2.field1 = 20 + self.assertNotEquals(message1, message2) + + message2.field1 = 10 + self.assertEquals(message1, message2) + + message1.field2 = MyEnum.val1 + self.assertNotEquals(message1, message2) + + message2.field2 = MyEnum.val2 + self.assertNotEquals(message1, message2) + + message2.field2 = MyEnum.val1 + self.assertEquals(message1, message2) + + message1.field3 = AnotherMessage() + message1.field3.string = 'value1' + self.assertNotEquals(message1, message2) + + message2.field3 = AnotherMessage() + message2.field3.string = 'value2' + self.assertNotEquals(message1, message2) + + message2.field3.string = 'value1' + self.assertEquals(message1, message2) + + def testEqualityWithUnknowns(self): + """Test message class equality with unknown fields.""" + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message1 = MyMessage() + message2 = MyMessage() + self.assertEquals(message1, message2) + message1.set_unrecognized_field('unknown1', 'value1', + messages.Variant.STRING) + self.assertEquals(message1, message2) + + message1.set_unrecognized_field('unknown2', ['asdf', 3], + messages.Variant.STRING) + message1.set_unrecognized_field('unknown3', 4.7, + messages.Variant.DOUBLE) + self.assertEquals(message1, message2) + + def testUnrecognizedFieldInvalidVariant(self): + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message1 = MyMessage() + self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, None) + self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, 123) + + def testRepr(self): + """Test represtation of Message object.""" + class MyMessage(messages.Message): + integer_value = messages.IntegerField(1) + string_value = messages.StringField(2) + unassigned = messages.StringField(3) + unassigned_with_default = messages.StringField(4, default=u'a default') + + my_message = MyMessage() + my_message.integer_value = 42 + my_message.string_value = u'A string' + + pat = re.compile(r"") + self.assertTrue(pat.match(repr(my_message)) is not None) + + def testValidation(self): + """Test validation of message values.""" + # Test optional. + class SubMessage(messages.Message): + pass + + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + message.val = SubMessage() + message_field.validate(message) + self.assertRaises(messages.ValidationError, + setattr, message, 'val', [SubMessage()]) + + # Test required. + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1, required=True) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + message.val = SubMessage() + message_field.validate(message) + self.assertRaises(messages.ValidationError, + setattr, message, 'val', [SubMessage()]) + + # Test repeated. + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1, repeated=True) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "Field val is repeated. Found: ", + setattr, message, 'val', SubMessage()) + message.val = [SubMessage()] + message_field.validate(message) + + def testDefinitionName(self): + """Test message name.""" + class MyMessage(messages.Message): + pass + + module_name = test_util.get_module_name(FieldTest) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.definition_name()) + self.assertEquals(module_name, MyMessage.outer_definition_name()) + self.assertEquals(module_name, MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + self.assertEquals(six.text_type, type(MyMessage.outer_definition_name())) + self.assertEquals(six.text_type, type(MyMessage.definition_package())) + + def testDefinitionName_OverrideModule(self): + """Test message module is overriden by module package name.""" + class MyMessage(messages.Message): + pass + + global package + package = 'my.package' + + try: + self.assertEquals('my.package.MyMessage', MyMessage.definition_name()) + self.assertEquals('my.package', MyMessage.outer_definition_name()) + self.assertEquals('my.package', MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + self.assertEquals(six.text_type, type(MyMessage.outer_definition_name())) + self.assertEquals(six.text_type, type(MyMessage.definition_package())) + finally: + del package + + def testDefinitionName_NoModule(self): + """Test what happens when there is no module for message.""" + class MyMessage(messages.Message): + pass + + original_modules = sys.modules + sys.modules = dict(sys.modules) + try: + del sys.modules[__name__] + self.assertEquals('MyMessage', MyMessage.definition_name()) + self.assertEquals(None, MyMessage.outer_definition_name()) + self.assertEquals(None, MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + finally: + sys.modules = original_modules + + def testDefinitionName_Nested(self): + """Test nested message names.""" + class MyMessage(messages.Message): + + class NestedMessage(messages.Message): + + class NestedMessage(messages.Message): + + pass + + module_name = test_util.get_module_name(MessageTest) + self.assertEquals('%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.definition_name()) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.NestedMessage.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedMessage.definition_package()) + + self.assertEquals('%s.MyMessage.NestedMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedMessage.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedMessage.outer_definition_name()) + self.assertEquals( + module_name, + MyMessage.NestedMessage.NestedMessage.definition_package()) + + + def testMessageDefinition(self): + """Test that enumeration knows its enclosing message definition.""" + class OuterMessage(messages.Message): + + class InnerMessage(messages.Message): + pass + + self.assertEquals(None, OuterMessage.message_definition()) + self.assertEquals(OuterMessage, + OuterMessage.InnerMessage.message_definition()) + + def testConstructorKwargs(self): + """Test kwargs via constructor.""" + class SomeMessage(messages.Message): + name = messages.StringField(1) + number = messages.IntegerField(2) + + expected = SomeMessage() + expected.name = 'my name' + expected.number = 200 + self.assertEquals(expected, SomeMessage(name='my name', number=200)) + + def testConstructorNotAField(self): + """Test kwargs via constructor with wrong names.""" + class SomeMessage(messages.Message): + pass + + self.assertRaisesWithRegexpMatch( + AttributeError, + 'May not assign arbitrary value does_not_exist to message SomeMessage', + SomeMessage, + does_not_exist=10) + + def testGetUnsetRepeatedValue(self): + class SomeMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + instance = SomeMessage() + self.assertEquals([], instance.repeated) + self.assertTrue(isinstance(instance.repeated, messages.FieldList)) + + def testCompareAutoInitializedRepeatedFields(self): + class SomeMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + message1 = SomeMessage(repeated=[]) + message2 = SomeMessage() + self.assertEquals(message1, message2) + + def testUnknownValues(self): + """Test message class equality with unknown fields.""" + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message = MyMessage() + self.assertEquals([], message.all_unrecognized_fields()) + self.assertEquals((None, None), + message.get_unrecognized_field_info('doesntexist')) + self.assertEquals((None, None), + message.get_unrecognized_field_info( + 'doesntexist', None, None)) + self.assertEquals(('defaultvalue', 'defaultwire'), + message.get_unrecognized_field_info( + 'doesntexist', 'defaultvalue', 'defaultwire')) + self.assertEquals((3, None), + message.get_unrecognized_field_info( + 'doesntexist', value_default=3)) + + message.set_unrecognized_field('exists', 9.5, messages.Variant.DOUBLE) + self.assertEquals(1, len(message.all_unrecognized_fields())) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists')) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists', 'type', + 1234)) + self.assertEquals((1234, None), + message.get_unrecognized_field_info('doesntexist', 1234)) + + message.set_unrecognized_field('another', 'value', messages.Variant.STRING) + self.assertEquals(2, len(message.all_unrecognized_fields())) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertTrue('another' in message.all_unrecognized_fields()) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists')) + self.assertEquals(('value', messages.Variant.STRING), + message.get_unrecognized_field_info('another')) + + message.set_unrecognized_field('typetest1', ['list', 0, ('test',)], + messages.Variant.STRING) + self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), + message.get_unrecognized_field_info('typetest1')) + message.set_unrecognized_field('typetest2', '', messages.Variant.STRING) + self.assertEquals(('', messages.Variant.STRING), + message.get_unrecognized_field_info('typetest2')) + + def testPickle(self): + """Testing pickling and unpickling of Message instances.""" + global MyEnum + global AnotherMessage + global MyMessage + + class MyEnum(messages.Enum): + val1 = 1 + val2 = 2 + + class AnotherMessage(messages.Message): + string = messages.StringField(1, repeated=True) + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + field2 = messages.EnumField(MyEnum, 2) + field3 = messages.MessageField(AnotherMessage, 3) + + message = MyMessage(field1=1, field2=MyEnum.val2, + field3=AnotherMessage(string=['a', 'b', 'c'])) + message.set_unrecognized_field('exists', 'value', messages.Variant.STRING) + message.set_unrecognized_field('repeated', ['list', 0, ('test',)], + messages.Variant.STRING) + unpickled = pickle.loads(pickle.dumps(message)) + self.assertEquals(message, unpickled) + self.assertTrue(AnotherMessage.string is unpickled.field3.string.field) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertEquals(('value', messages.Variant.STRING), + message.get_unrecognized_field_info('exists')) + self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), + message.get_unrecognized_field_info('repeated')) + + +class FindDefinitionTest(test_util.TestCase): + """Test finding definitions relative to various definitions and modules.""" + + def setUp(self): + """Set up module-space. Starts off empty.""" + self.modules = {} + + def DefineModule(self, name): + """Define a module and its parents in module space. + + Modules that are already defined in self.modules are not re-created. + + Args: + name: Fully qualified name of modules to create. + + Returns: + Deepest nested module. For example: + + DefineModule('a.b.c') # Returns c. + """ + name_path = name.split('.') + full_path = [] + for node in name_path: + full_path.append(node) + full_name = '.'.join(full_path) + self.modules.setdefault(full_name, types.ModuleType(full_name)) + return self.modules[name] + + def DefineMessage(self, module, name, children={}, add_to_module=True): + """Define a new Message class in the context of a module. + + Used for easily describing complex Message hierarchy. Message is defined + including all child definitions. + + Args: + module: Fully qualified name of module to place Message class in. + name: Name of Message to define within module. + children: Define any level of nesting of children definitions. To define + a message, map the name to another dictionary. The dictionary can + itself contain additional definitions, and so on. To map to an Enum, + define the Enum class separately and map it by name. + add_to_module: If True, new Message class is added to module. If False, + new Message is not added. + """ + # Make sure module exists. + module_instance = self.DefineModule(module) + + # Recursively define all child messages. + for attribute, value in children.items(): + if isinstance(value, dict): + children[attribute] = self.DefineMessage( + module, attribute, value, False) + + # Override default __module__ variable. + children['__module__'] = module + + # Instantiate and possibly add to module. + message_class = type(name, (messages.Message,), dict(children)) + if add_to_module: + setattr(module_instance, name, message_class) + return message_class + + def Importer(self, module, globals='', locals='', fromlist=None): + """Importer function. + + Acts like __import__. Only loads modules from self.modules. Does not + try to load real modules defined elsewhere. Does not try to handle relative + imports. + + Args: + module: Fully qualified name of module to load from self.modules. + """ + if fromlist is None: + module = module.split('.')[0] + try: + return self.modules[module] + except KeyError: + raise ImportError() + + def testNoSuchModule(self): + """Test searching for definitions that do no exist.""" + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'does.not.exist', + importer=self.Importer) + + def testRefersToModule(self): + """Test that referring to a module does not return that module.""" + self.DefineModule('i.am.a.module') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module', + importer=self.Importer) + + def testNoDefinition(self): + """Test not finding a definition in an existing module.""" + self.DefineModule('i.am.a.module') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module.MyMessage', + importer=self.Importer) + + def testNotADefinition(self): + """Test trying to fetch something that is not a definition.""" + module = self.DefineModule('i.am.a.module') + setattr(module, 'A', 'a string') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module.A', + importer=self.Importer) + + def testGlobalFind(self): + """Test finding definitions from fully qualified module names.""" + A = self.DefineMessage('a.b.c', 'A', {}) + self.assertEquals(A, messages.find_definition('a.b.c.A', + importer=self.Importer)) + B = self.DefineMessage('a.b.c', 'B', {'C':{}}) + self.assertEquals(B.C, messages.find_definition('a.b.c.B.C', + importer=self.Importer)) + + def testRelativeToModule(self): + """Test finding definitions relative to modules.""" + # Define modules. + a = self.DefineModule('a') + b = self.DefineModule('a.b') + c = self.DefineModule('a.b.c') + + # Define messages. + A = self.DefineMessage('a', 'A') + B = self.DefineMessage('a.b', 'B') + C = self.DefineMessage('a.b.c', 'C') + D = self.DefineMessage('a.b.d', 'D') + + # Find A, B, C and D relative to a. + self.assertEquals(A, messages.find_definition( + 'A', a, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'b.B', a, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'b.c.C', a, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'b.d.D', a, importer=self.Importer)) + + # Find A, B, C and D relative to b. + self.assertEquals(A, messages.find_definition( + 'A', b, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', b, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'c.C', b, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'd.D', b, importer=self.Importer)) + + # Find A, B, C and D relative to c. Module d is the same case as c. + self.assertEquals(A, messages.find_definition( + 'A', c, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', c, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', c, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'd.D', c, importer=self.Importer)) + + def testRelativeToMessages(self): + """Test finding definitions relative to Message definitions.""" + A = self.DefineMessage('a.b', 'A', {'B': {'C': {}, 'D': {}}}) + B = A.B + C = A.B.C + D = A.B.D + + # Find relative to A. + self.assertEquals(A, messages.find_definition( + 'A', A, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', A, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'B.C', A, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'B.D', A, importer=self.Importer)) + + # Find relative to B. + self.assertEquals(A, messages.find_definition( + 'A', B, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', B, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', B, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'D', B, importer=self.Importer)) + + # Find relative to C. + self.assertEquals(A, messages.find_definition( + 'A', C, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', C, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', C, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'D', C, importer=self.Importer)) + + # Find relative to C searching from c. + self.assertEquals(A, messages.find_definition( + 'b.A', C, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'b.A.B', C, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'b.A.B.C', C, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'b.A.B.D', C, importer=self.Importer)) + + def testAbsoluteReference(self): + """Test finding absolute definition names.""" + # Define modules. + a = self.DefineModule('a') + b = self.DefineModule('a.a') + + # Define messages. + aA = self.DefineMessage('a', 'A') + aaA = self.DefineMessage('a.a', 'A') + + # Always find a.A. + self.assertEquals(aA, messages.find_definition('.a.A', None, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', a, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', aA, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', aaA, + importer=self.Importer)) + + def testFindEnum(self): + """Test that Enums are found.""" + class Color(messages.Enum): + pass + A = self.DefineMessage('a', 'A', {'Color': Color}) + + self.assertEquals( + Color, + messages.find_definition('Color', A, importer=self.Importer)) + + def testFalseScope(self): + """Test that Message definitions nested in strange objects are hidden.""" + global X + class X(object): + class A(messages.Message): + pass + + self.assertRaises(TypeError, messages.find_definition, 'A', X) + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'X.A', sys.modules[__name__]) + + def testSearchAttributeFirst(self): + """Make sure not faked out by module, but continues searching.""" + A = self.DefineMessage('a', 'A') + module_A = self.DefineModule('a.A') + + self.assertEquals(A, messages.find_definition( + 'a.A', None, importer=self.Importer)) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py new file mode 100644 index 0000000..0e9c6bb --- /dev/null +++ b/apitools/base/protorpclite/protojson.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""JSON support for message types. + +Public classes: + MessageJSONEncoder: JSON encoder for message objects. + +Public functions: + encode_message: Encodes a message in to a JSON string. + decode_message: Merge from a JSON string in to a message. +""" +import six + +__author__ = 'rafek@google.com (Rafe Kaplan)' + +import base64 +import binascii +import logging + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import util + +__all__ = [ + 'ALTERNATIVE_CONTENT_TYPES', + 'CONTENT_TYPE', + 'MessageJSONEncoder', + 'encode_message', + 'decode_message', + 'ProtoJson', +] + + +def _load_json_module(): + """Try to load a valid json module. + + There are more than one json modules that might be installed. They are + mostly compatible with one another but some versions may be different. + This function attempts to load various json modules in a preferred order. + It does a basic check to guess if a loaded version of json is compatible. + + Returns: + Compatible json module. + + Raises: + ImportError if there are no json modules or the loaded json module is + not compatible with ProtoRPC. + """ + first_import_error = None + for module_name in ['json', + 'simplejson']: + try: + module = __import__(module_name, {}, {}, 'json') + if not hasattr(module, 'JSONEncoder'): + message = ('json library "%s" is not compatible with ProtoRPC' % + module_name) + logging.warning(message) + raise ImportError(message) + else: + return module + except ImportError as err: + if not first_import_error: + first_import_error = err + + logging.error('Must use valid json library (Python 2.6 json or simplejson)') + raise first_import_error +json = _load_json_module() + + +# TODO: Rename this to MessageJsonEncoder. +class MessageJSONEncoder(json.JSONEncoder): + """Message JSON encoder class. + + Extension of JSONEncoder that can build JSON from a message object. + """ + + def __init__(self, protojson_protocol=None, **kwargs): + """Constructor. + + Args: + protojson_protocol: ProtoJson instance. + """ + super(MessageJSONEncoder, self).__init__(**kwargs) + self.__protojson_protocol = protojson_protocol or ProtoJson.get_default() + + def default(self, value): + """Return dictionary instance from a message object. + + Args: + value: Value to get dictionary for. If not encodable, will + call superclasses default method. + """ + if isinstance(value, messages.Enum): + return str(value) + + if six.PY3 and isinstance(value, bytes): + return value.decode('utf8') + + if isinstance(value, messages.Message): + result = {} + for field in value.all_fields(): + item = value.get_assigned_value(field.name) + if item not in (None, [], ()): + result[field.name] = self.__protojson_protocol.encode_field( + field, item) + # Handle unrecognized fields, so they're included when a message is + # decoded then encoded. + for unknown_key in value.all_unrecognized_fields(): + unrecognized_field, _ = value.get_unrecognized_field_info(unknown_key) + result[unknown_key] = unrecognized_field + return result + else: + return super(MessageJSONEncoder, self).default(value) + + +class ProtoJson(object): + """ProtoRPC JSON implementation class. + + Implementation of JSON based protocol used for serializing and deserializing + message objects. Instances of remote.ProtocolConfig constructor or used with + remote.Protocols.add_protocol. See the remote.py module for more details. + """ + + CONTENT_TYPE = 'application/json' + ALTERNATIVE_CONTENT_TYPES = [ + 'application/x-javascript', + 'text/javascript', + 'text/x-javascript', + 'text/x-json', + 'text/json', + ] + + def encode_field(self, field, value): + """Encode a python field value to a JSON value. + + Args: + field: A ProtoRPC field instance. + value: A python value supported by field. + + Returns: + A JSON serializable value appropriate for field. + """ + if isinstance(field, messages.BytesField): + if field.repeated: + value = [base64.b64encode(byte) for byte in value] + else: + value = base64.b64encode(value) + elif isinstance(field, message_types.DateTimeField): + # DateTimeField stores its data as a RFC 3339 compliant string. + if field.repeated: + value = [i.isoformat() for i in value] + else: + value = value.isoformat() + return value + + def encode_message(self, message): + """Encode Message instance to JSON string. + + Args: + Message instance to encode in to JSON string. + + Returns: + String encoding of Message instance in protocol JSON format. + + Raises: + messages.ValidationError if message is not initialized. + """ + message.check_initialized() + + return json.dumps(message, cls=MessageJSONEncoder, protojson_protocol=self) + + def decode_message(self, message_type, encoded_message): + """Merge JSON structure to Message instance. + + Args: + message_type: Message to decode data to. + encoded_message: JSON encoded version of message. + + Returns: + Decoded instance of message_type. + + Raises: + ValueError: If encoded_message is not valid JSON. + messages.ValidationError if merged message is not initialized. + """ + if not encoded_message.strip(): + return message_type() + + dictionary = json.loads(encoded_message) + message = self.__decode_dictionary(message_type, dictionary) + message.check_initialized() + return message + + def __find_variant(self, value): + """Find the messages.Variant type that describes this value. + + Args: + value: The value whose variant type is being determined. + + Returns: + The messages.Variant value that best describes value's type, or None if + it's a type we don't know how to handle. + """ + if isinstance(value, bool): + return messages.Variant.BOOL + elif isinstance(value, six.integer_types): + return messages.Variant.INT64 + elif isinstance(value, float): + return messages.Variant.DOUBLE + elif isinstance(value, six.string_types): + return messages.Variant.STRING + elif isinstance(value, (list, tuple)): + # Find the most specific variant that covers all elements. + variant_priority = [None, messages.Variant.INT64, messages.Variant.DOUBLE, + messages.Variant.STRING] + chosen_priority = 0 + for v in value: + variant = self.__find_variant(v) + try: + priority = variant_priority.index(variant) + except IndexError: + priority = -1 + if priority > chosen_priority: + chosen_priority = priority + return variant_priority[chosen_priority] + # Unrecognized type. + return None + + def __decode_dictionary(self, message_type, dictionary): + """Merge dictionary in to message. + + Args: + message: Message to merge dictionary in to. + dictionary: Dictionary to extract information from. Dictionary + is as parsed from JSON. Nested objects will also be dictionaries. + """ + message = message_type() + for key, value in six.iteritems(dictionary): + if value is None: + try: + message.reset(key) + except AttributeError: + pass # This is an unrecognized field, skip it. + continue + + try: + field = message.field_by_name(key) + except KeyError: + # Save unknown values. + variant = self.__find_variant(value) + if variant: + if key.isdigit(): + key = int(key) + message.set_unrecognized_field(key, value, variant) + else: + logging.warning('No variant found for unrecognized field: %s', key) + continue + + # Normalize values in to a list. + if isinstance(value, list): + if not value: + continue + else: + value = [value] + + valid_value = [] + for item in value: + valid_value.append(self.decode_field(field, item)) + + if field.repeated: + existing_value = getattr(message, field.name) + setattr(message, field.name, valid_value) + else: + setattr(message, field.name, valid_value[-1]) + return message + + def decode_field(self, field, value): + """Decode a JSON value to a python value. + + Args: + field: A ProtoRPC field instance. + value: A serialized JSON value. + + Return: + A Python value compatible with field. + """ + if isinstance(field, messages.EnumField): + try: + return field.type(value) + except TypeError: + raise messages.DecodeError('Invalid enum value "%s"' % (value or '')) + + elif isinstance(field, messages.BytesField): + try: + return base64.b64decode(value) + except (binascii.Error, TypeError) as err: + raise messages.DecodeError('Base64 decoding error: %s' % err) + + elif isinstance(field, message_types.DateTimeField): + try: + return util.decode_datetime(value) + except ValueError as err: + raise messages.DecodeError(err) + + elif (isinstance(field, messages.MessageField) and + issubclass(field.type, messages.Message)): + return self.__decode_dictionary(field.type, value) + + elif (isinstance(field, messages.FloatField) and + isinstance(value, (six.integer_types, six.string_types))): + try: + return float(value) + except: + pass + + elif (isinstance(field, messages.IntegerField) and + isinstance(value, six.string_types)): + try: + return int(value) + except: + pass + + return value + + @staticmethod + def get_default(): + """Get default instanceof ProtoJson.""" + try: + return ProtoJson.__default + except AttributeError: + ProtoJson.__default = ProtoJson() + return ProtoJson.__default + + @staticmethod + def set_default(protocol): + """Set the default instance of ProtoJson. + + Args: + protocol: A ProtoJson instance. + """ + if not isinstance(protocol, ProtoJson): + raise TypeError('Expected protocol of type ProtoJson') + ProtoJson.__default = protocol + +CONTENT_TYPE = ProtoJson.CONTENT_TYPE + +ALTERNATIVE_CONTENT_TYPES = ProtoJson.ALTERNATIVE_CONTENT_TYPES + +encode_message = ProtoJson.get_default().encode_message + +decode_message = ProtoJson.get_default().decode_message diff --git a/apitools/base/protorpclite/protojson_test.py b/apitools/base/protorpclite/protojson_test.py new file mode 100644 index 0000000..a0349d4 --- /dev/null +++ b/apitools/base/protorpclite/protojson_test.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for apitools.base.protorpclite.protojson.""" + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import datetime +import json +import unittest + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import protojson +from apitools.base.protorpclite import test_util + + +class CustomField(messages.MessageField): + """Custom MessageField class.""" + + type = int + message_type = message_types.VoidMessage + + def __init__(self, number, **kwargs): + super(CustomField, self).__init__(self.message_type, number, **kwargs) + + def value_to_message(self, value): + return self.message_type() + + +class MyMessage(messages.Message): + """Test message containing various types.""" + + class Color(messages.Enum): + + RED = 1 + GREEN = 2 + BLUE = 3 + + class Nested(messages.Message): + + nested_value = messages.StringField(1) + + a_string = messages.StringField(2) + an_integer = messages.IntegerField(3) + a_float = messages.FloatField(4) + a_boolean = messages.BooleanField(5) + an_enum = messages.EnumField(Color, 6) + a_nested = messages.MessageField(Nested, 7) + a_repeated = messages.IntegerField(8, repeated=True) + a_repeated_float = messages.FloatField(9, repeated=True) + a_datetime = message_types.DateTimeField(10) + a_repeated_datetime = message_types.DateTimeField(11, repeated=True) + a_custom = CustomField(12) + a_repeated_custom = CustomField(13, repeated=True) + + +class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = protojson + + +# TODO(rafek): Convert this test to the compliance test in test_util. +class ProtojsonTest(test_util.TestCase, + test_util.ProtoConformanceTestBase): + """Test JSON encoding and decoding.""" + + PROTOLIB = protojson + + def CompareEncoded(self, expected_encoded, actual_encoded): + """JSON encoding will be laundered to remove string differences.""" + self.assertEquals(json.loads(expected_encoded), + json.loads(actual_encoded)) + + encoded_empty_message = '{}' + + encoded_partial = """{ + "double_value": 1.23, + "int64_value": -100000000000, + "int32_value": 1020, + "string_value": "a string", + "enum_value": "VAL2" + } + """ + + encoded_full = """{ + "double_value": 1.23, + "float_value": -2.5, + "int64_value": -100000000000, + "uint64_value": 102020202020, + "int32_value": 1020, + "bool_value": true, + "string_value": "a string\u044f", + "bytes_value": "YSBieXRlc//+", + "enum_value": "VAL2" + } + """ + + encoded_repeated = """{ + "double_value": [1.23, 2.3], + "float_value": [-2.5, 0.5], + "int64_value": [-100000000000, 20], + "uint64_value": [102020202020, 10], + "int32_value": [1020, 718], + "bool_value": [true, false], + "string_value": ["a string\u044f", "another string"], + "bytes_value": ["YSBieXRlc//+", "YW5vdGhlciBieXRlcw=="], + "enum_value": ["VAL2", "VAL1"] + } + """ + + encoded_nested = """{ + "nested": { + "a_value": "a string" + } + } + """ + + encoded_repeated_nested = """{ + "repeated_nested": [{"a_value": "a string"}, + {"a_value": "another string"}] + } + """ + + unexpected_tag_message = '{"unknown": "value"}' + + encoded_default_assigned = '{"a_value": "a default"}' + + encoded_nested_empty = '{"nested": {}}' + + encoded_repeated_nested_empty = '{"repeated_nested": [{}, {}]}' + + encoded_extend_message = '{"int64_value": [400, 50, 6000]}' + + encoded_string_types = '{"string_value": "Latin"}' + + encoded_invalid_enum = '{"enum_value": "undefined"}' + + def testConvertIntegerToFloat(self): + """Test that integers passed in to float fields are converted. + + This is necessary because JSON outputs integers for numbers with 0 decimals. + """ + message = protojson.decode_message(MyMessage, '{"a_float": 10}') + + self.assertTrue(isinstance(message.a_float, float)) + self.assertEquals(10.0, message.a_float) + + def testConvertStringToNumbers(self): + """Test that strings passed to integer fields are converted.""" + message = protojson.decode_message(MyMessage, + """{"an_integer": "10", + "a_float": "3.5", + "a_repeated": ["1", "2"], + "a_repeated_float": ["1.5", "2", 10] + }""") + + self.assertEquals(MyMessage(an_integer=10, + a_float=3.5, + a_repeated=[1, 2], + a_repeated_float=[1.5, 2.0, 10.0]), + message) + + def testWrongTypeAssignment(self): + """Test when wrong type is assigned to a field.""" + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"a_string": 10}') + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"an_integer": 10.2}') + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"an_integer": "10.2"}') + + def testNumericEnumeration(self): + """Test that numbers work for enum values.""" + message = protojson.decode_message(MyMessage, '{"an_enum": 2}') + + expected_message = MyMessage() + expected_message.an_enum = MyMessage.Color.GREEN + + self.assertEquals(expected_message, message) + + def testNumericEnumerationNegativeTest(self): + """Test with an invalid number for the enum value.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value "89"', + protojson.decode_message, + MyMessage, + '{"an_enum": 89}') + + def testAlphaEnumeration(self): + """Test that alpha enum values work.""" + message = protojson.decode_message(MyMessage, '{"an_enum": "RED"}') + + expected_message = MyMessage() + expected_message.an_enum = MyMessage.Color.RED + + self.assertEquals(expected_message, message) + + def testAlphaEnumerationNegativeTest(self): + """The alpha enum value is invalid.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value "IAMINVALID"', + protojson.decode_message, + MyMessage, + '{"an_enum": "IAMINVALID"}') + + def testEnumerationNegativeTestWithEmptyString(self): + """The enum value is an empty string.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value ""', + protojson.decode_message, + MyMessage, + '{"an_enum": ""}') + + def testNullValues(self): + """Test that null values overwrite existing values.""" + self.assertEquals(MyMessage(), + protojson.decode_message(MyMessage, + ('{"an_integer": null,' + ' "a_nested": null,' + ' "an_enum": null' + '}'))) + + def testEmptyList(self): + """Test that empty lists are ignored.""" + self.assertEquals(MyMessage(), + protojson.decode_message(MyMessage, + '{"a_repeated": []}')) + + def testNotJSON(self): + """Test error when string is not valid JSON.""" + self.assertRaises(ValueError, + protojson.decode_message, MyMessage, '{this is not json}') + + def testDoNotEncodeStrangeObjects(self): + """Test trying to encode a strange object. + + The main purpose of this test is to complete coverage. It ensures that + the default behavior of the JSON encoder is preserved when someone tries to + serialized an unexpected type. + """ + class BogusObject(object): + + def check_initialized(self): + pass + + self.assertRaises(TypeError, + protojson.encode_message, + BogusObject()) + + def testMergeEmptyString(self): + """Test merging the empty or space only string.""" + message = protojson.decode_message(test_util.OptionalMessage, '') + self.assertEquals(test_util.OptionalMessage(), message) + + message = protojson.decode_message(test_util.OptionalMessage, ' ') + self.assertEquals(test_util.OptionalMessage(), message) + + def testProtojsonUnrecognizedFieldName(self): + """Test that unrecognized fields are saved and can be accessed.""" + decoded = protojson.decode_message(MyMessage, + ('{"an_integer": 1, "unknown_val": 2}')) + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(1, len(decoded.all_unrecognized_fields())) + self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) + self.assertEquals((2, messages.Variant.INT64), + decoded.get_unrecognized_field_info('unknown_val')) + + def testProtojsonUnrecognizedFieldNumber(self): + """Test that unrecognized fields are saved and can be accessed.""" + decoded = protojson.decode_message( + MyMessage, + '{"an_integer": 1, "1001": "unknown", "-123": "negative", ' + '"456_mixed": 2}') + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(3, len(decoded.all_unrecognized_fields())) + self.assertTrue(1001 in decoded.all_unrecognized_fields()) + self.assertEquals(('unknown', messages.Variant.STRING), + decoded.get_unrecognized_field_info(1001)) + self.assertTrue('-123' in decoded.all_unrecognized_fields()) + self.assertEquals(('negative', messages.Variant.STRING), + decoded.get_unrecognized_field_info('-123')) + self.assertTrue('456_mixed' in decoded.all_unrecognized_fields()) + self.assertEquals((2, messages.Variant.INT64), + decoded.get_unrecognized_field_info('456_mixed')) + + def testProtojsonUnrecognizedNull(self): + """Test that unrecognized fields that are None are skipped.""" + decoded = protojson.decode_message( + MyMessage, + '{"an_integer": 1, "unrecognized_null": null}') + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(decoded.all_unrecognized_fields(), []) + + def testUnrecognizedFieldVariants(self): + """Test that unrecognized fields are mapped to the right variants.""" + for encoded, expected_variant in ( + ('{"an_integer": 1, "unknown_val": 2}', messages.Variant.INT64), + ('{"an_integer": 1, "unknown_val": 2.0}', messages.Variant.DOUBLE), + ('{"an_integer": 1, "unknown_val": "string value"}', + messages.Variant.STRING), + ('{"an_integer": 1, "unknown_val": [1, 2, 3]}', messages.Variant.INT64), + ('{"an_integer": 1, "unknown_val": [1, 2.0, 3]}', + messages.Variant.DOUBLE), + ('{"an_integer": 1, "unknown_val": [1, "foo", 3]}', + messages.Variant.STRING), + ('{"an_integer": 1, "unknown_val": true}', messages.Variant.BOOL)): + decoded = protojson.decode_message(MyMessage, encoded) + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(1, len(decoded.all_unrecognized_fields())) + self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) + _, decoded_variant = decoded.get_unrecognized_field_info('unknown_val') + self.assertEquals(expected_variant, decoded_variant) + + def testDecodeDateTime(self): + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + message = protojson.decode_message( + MyMessage, '{"a_datetime": "%s"}' % datetime_string) + expected_message = MyMessage( + a_datetime=datetime.datetime(*datetime_vals)) + + self.assertEquals(expected_message, message) + + def testDecodeInvalidDateTime(self): + self.assertRaises(messages.DecodeError, protojson.decode_message, + MyMessage, '{"a_datetime": "invalid"}') + + def testEncodeDateTime(self): + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262000', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50.262123', (2012, 9, 30, 15, 31, 50, 262123)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + decoded_message = protojson.encode_message( + MyMessage(a_datetime=datetime.datetime(*datetime_vals))) + expected_decoding = '{"a_datetime": "%s"}' % datetime_string + self.CompareEncoded(expected_decoding, decoded_message) + + def testDecodeRepeatedDateTime(self): + message = protojson.decode_message( + MyMessage, + '{"a_repeated_datetime": ["2012-09-30T15:31:50.262", ' + '"2010-01-21T09:52:00", "2000-01-01T01:00:59.999999"]}') + expected_message = MyMessage( + a_repeated_datetime=[ + datetime.datetime(2012, 9, 30, 15, 31, 50, 262000), + datetime.datetime(2010, 1, 21, 9, 52), + datetime.datetime(2000, 1, 1, 1, 0, 59, 999999)]) + + self.assertEquals(expected_message, message) + + def testDecodeCustom(self): + message = protojson.decode_message(MyMessage, '{"a_custom": 1}') + self.assertEquals(MyMessage(a_custom=1), message) + + def testDecodeInvalidCustom(self): + self.assertRaises(messages.ValidationError, protojson.decode_message, + MyMessage, '{"a_custom": "invalid"}') + + def testEncodeCustom(self): + decoded_message = protojson.encode_message(MyMessage(a_custom=1)) + self.CompareEncoded('{"a_custom": 1}', decoded_message) + + def testDecodeRepeatedCustom(self): + message = protojson.decode_message( + MyMessage, '{"a_repeated_custom": [1, 2, 3]}') + self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message) + + def testDecodeBadBase64BytesField(self): + """Test decoding improperly encoded base64 bytes value.""" + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Base64 decoding error: Incorrect padding', + protojson.decode_message, + test_util.OptionalMessage, + '{"bytes_value": "abcdefghijklmnopq"}') + + +class CustomProtoJson(protojson.ProtoJson): + + def encode_field(self, field, value): + return '{encoded}' + value + + def decode_field(self, field, value): + return '{decoded}' + value + + +class CustomProtoJsonTest(test_util.TestCase): + """Tests for serialization overriding functionality.""" + + def setUp(self): + self.protojson = CustomProtoJson() + + def testEncode(self): + self.assertEqual('{"a_string": "{encoded}xyz"}', + self.protojson.encode_message(MyMessage(a_string='xyz'))) + + def testDecode(self): + self.assertEqual( + MyMessage(a_string='{decoded}xyz'), + self.protojson.decode_message(MyMessage, '{"a_string": "xyz"}')) + + def testDecodeEmptyMessage(self): + self.assertEqual( + MyMessage(a_string='{decoded}'), + self.protojson.decode_message(MyMessage, '{"a_string": ""}')) + + def testDefault(self): + self.assertTrue(protojson.ProtoJson.get_default(), + protojson.ProtoJson.get_default()) + + instance = CustomProtoJson() + protojson.ProtoJson.set_default(instance) + self.assertTrue(instance is protojson.ProtoJson.get_default()) + + +if __name__ == '__main__': + unittest.main() diff --git a/apitools/base/protorpclite/test_util.py b/apitools/base/protorpclite/test_util.py new file mode 100644 index 0000000..b6b3537 --- /dev/null +++ b/apitools/base/protorpclite/test_util.py @@ -0,0 +1,665 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Test utilities for message testing. + +Includes module interface test to ensure that public parts of module are +correctly declared in __all__. + +Includes message types that correspond to those defined in +services_test.proto. + +Includes additional test utilities to make sure encoding/decoding libraries +conform. +""" +__author__ = 'rafek@google.com (Rafe Kaplan)' + +import cgi +import datetime +import inspect +import os +import re +import socket +import types +import unittest2 as unittest + +import six +from six.moves import range + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import util + +# Unicode of the word "Russian" in cyrillic. +RUSSIAN = u'\u0440\u0443\u0441\u0441\u043a\u0438\u0439' + +# All characters binary value interspersed with nulls. +BINARY = b''.join(six.int2byte(value) + b'\0' for value in range(256)) + + +class TestCase(unittest.TestCase): + + def assertRaisesWithRegexpMatch(self, + exception, + regexp, + function, + *params, + **kwargs): + """Check that exception is raised and text matches regular expression. + + Args: + exception: Exception type that is expected. + regexp: String regular expression that is expected in error message. + function: Callable to test. + params: Parameters to forward to function. + kwargs: Keyword arguments to forward to function. + """ + try: + function(*params, **kwargs) + self.fail('Expected exception %s was not raised' % exception.__name__) + except exception as err: + match = bool(re.match(regexp, str(err))) + self.assertTrue(match, 'Expected match "%s", found "%s"' % (regexp, + err)) + + def assertHeaderSame(self, header1, header2): + """Check that two HTTP headers are the same. + + Args: + header1: Header value string 1. + header2: header value string 2. + """ + value1, params1 = cgi.parse_header(header1) + value2, params2 = cgi.parse_header(header2) + self.assertEqual(value1, value2) + self.assertEqual(params1, params2) + + def assertIterEqual(self, iter1, iter2): + """Check that two iterators or iterables are equal independent of order. + + Similar to Python 2.7 assertItemsEqual. Named differently in order to + avoid potential conflict. + + Args: + iter1: An iterator or iterable. + iter2: An iterator or iterable. + """ + list1 = list(iter1) + list2 = list(iter2) + + unmatched1 = list() + + while list1: + item1 = list1[0] + del list1[0] + for index in range(len(list2)): + if item1 == list2[index]: + del list2[index] + break + else: + unmatched1.append(item1) + + error_message = [] + for item in unmatched1: + error_message.append( + ' Item from iter1 not found in iter2: %r' % item) + for item in list2: + error_message.append( + ' Item from iter2 not found in iter1: %r' % item) + if error_message: + self.fail('Collections not equivalent:\n' + '\n'.join(error_message)) + + +class ModuleInterfaceTest(object): + """Test to ensure module interface is carefully constructed. + + A module interface is the set of public objects listed in the module __all__ + attribute. Modules that that are considered public should have this interface + carefully declared. At all times, the __all__ attribute should have objects + intended to be publically used and all other objects in the module should be + considered unused. + + Protected attributes (those beginning with '_') and other imported modules + should not be part of this set of variables. An exception is for variables + that begin and end with '__' which are implicitly part of the interface + (eg. __name__, __file__, __all__ itself, etc.). + + Modules that are imported in to the tested modules are an exception and may + be left out of the __all__ definition. The test is done by checking the value + of what would otherwise be a public name and not allowing it to be exported + if it is an instance of a module. Modules that are explicitly exported are + for the time being not permitted. + + To use this test class a module should define a new class that inherits first + from ModuleInterfaceTest and then from test_util.TestCase. No other tests + should be added to this test case, making the order of inheritance less + important, but if setUp for some reason is overidden, it is important that + ModuleInterfaceTest is first in the list so that its setUp method is + invoked. + + Multiple inheretance is required so that ModuleInterfaceTest is not itself + a test, and is not itself executed as one. + + The test class is expected to have the following class attributes defined: + + MODULE: A reference to the module that is being validated for interface + correctness. + + Example: + Module definition (hello.py): + + import sys + + __all__ = ['hello'] + + def _get_outputter(): + return sys.stdout + + def hello(): + _get_outputter().write('Hello\n') + + Test definition: + + import unittest + from protorpc import test_util + + import hello + + class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = hello + + + class HelloTest(test_util.TestCase): + ... Test 'hello' module ... + + + if __name__ == '__main__': + unittest.main() + """ + + def setUp(self): + """Set up makes sure that MODULE and IMPORTED_MODULES is defined. + + This is a basic configuration test for the test itself so does not + get it's own test case. + """ + if not hasattr(self, 'MODULE'): + self.fail( + "You must define 'MODULE' on ModuleInterfaceTest sub-class %s." % + type(self).__name__) + + def testAllExist(self): + """Test that all attributes defined in __all__ exist.""" + missing_attributes = [] + for attribute in self.MODULE.__all__: + if not hasattr(self.MODULE, attribute): + missing_attributes.append(attribute) + if missing_attributes: + self.fail('%s of __all__ are not defined in module.' % + missing_attributes) + + def testAllExported(self): + """Test that all public attributes not imported are in __all__.""" + missing_attributes = [] + for attribute in dir(self.MODULE): + if not attribute.startswith('_'): + if (attribute not in self.MODULE.__all__ and + not isinstance(getattr(self.MODULE, attribute), + types.ModuleType) and + attribute != 'with_statement'): + missing_attributes.append(attribute) + if missing_attributes: + self.fail('%s are not modules and not defined in __all__.' % + missing_attributes) + + def testNoExportedProtectedVariables(self): + """Test that there are no protected variables listed in __all__.""" + protected_variables = [] + for attribute in self.MODULE.__all__: + if attribute.startswith('_'): + protected_variables.append(attribute) + if protected_variables: + self.fail('%s are protected variables and may not be exported.' % + protected_variables) + + def testNoExportedModules(self): + """Test that no modules exist in __all__.""" + exported_modules = [] + for attribute in self.MODULE.__all__: + try: + value = getattr(self.MODULE, attribute) + except AttributeError: + # This is a different error case tested for in testAllExist. + pass + else: + if isinstance(value, types.ModuleType): + exported_modules.append(attribute) + if exported_modules: + self.fail('%s are modules and may not be exported.' % exported_modules) + + +class NestedMessage(messages.Message): + """Simple message that gets nested in another message.""" + + a_value = messages.StringField(1, required=True) + + +class HasNestedMessage(messages.Message): + """Message that has another message nested in it.""" + + nested = messages.MessageField(NestedMessage, 1) + repeated_nested = messages.MessageField(NestedMessage, 2, repeated=True) + + +class HasDefault(messages.Message): + """Has a default value.""" + + a_value = messages.StringField(1, default=u'a default') + + +class OptionalMessage(messages.Message): + """Contains all message types.""" + + class SimpleEnum(messages.Enum): + """Simple enumeration type.""" + VAL1 = 1 + VAL2 = 2 + + double_value = messages.FloatField(1, variant=messages.Variant.DOUBLE) + float_value = messages.FloatField(2, variant=messages.Variant.FLOAT) + int64_value = messages.IntegerField(3, variant=messages.Variant.INT64) + uint64_value = messages.IntegerField(4, variant=messages.Variant.UINT64) + int32_value = messages.IntegerField(5, variant=messages.Variant.INT32) + bool_value = messages.BooleanField(6, variant=messages.Variant.BOOL) + string_value = messages.StringField(7, variant=messages.Variant.STRING) + bytes_value = messages.BytesField(8, variant=messages.Variant.BYTES) + enum_value = messages.EnumField(SimpleEnum, 10) + + # TODO(rafek): Add support for these variants. + # uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) + # sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) + # sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) + + +class RepeatedMessage(messages.Message): + """Contains all message types as repeated fields.""" + + class SimpleEnum(messages.Enum): + """Simple enumeration type.""" + VAL1 = 1 + VAL2 = 2 + + double_value = messages.FloatField(1, + variant=messages.Variant.DOUBLE, + repeated=True) + float_value = messages.FloatField(2, + variant=messages.Variant.FLOAT, + repeated=True) + int64_value = messages.IntegerField(3, + variant=messages.Variant.INT64, + repeated=True) + uint64_value = messages.IntegerField(4, + variant=messages.Variant.UINT64, + repeated=True) + int32_value = messages.IntegerField(5, + variant=messages.Variant.INT32, + repeated=True) + bool_value = messages.BooleanField(6, + variant=messages.Variant.BOOL, + repeated=True) + string_value = messages.StringField(7, + variant=messages.Variant.STRING, + repeated=True) + bytes_value = messages.BytesField(8, + variant=messages.Variant.BYTES, + repeated=True) + #uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) + enum_value = messages.EnumField(SimpleEnum, + 10, + repeated=True) + #sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) + #sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) + + +class HasOptionalNestedMessage(messages.Message): + + nested = messages.MessageField(OptionalMessage, 1) + repeated_nested = messages.MessageField(OptionalMessage, 2, repeated=True) + + +class ProtoConformanceTestBase(object): + """Protocol conformance test base class. + + Each supported protocol should implement two methods that support encoding + and decoding of Message objects in that format: + + encode_message(message) - Serialize to encoding. + encode_message(message, encoded_message) - Deserialize from encoding. + + Tests for the modules where these functions are implemented should extend + this class in order to support basic behavioral expectations. This ensures + that protocols correctly encode and decode message transparently to the + caller. + + In order to support these test, the base class should also extend the TestCase + class and implement the following class attributes which define the encoded + version of certain protocol buffers: + + encoded_partial: + + + encoded_full: + + + encoded_repeated: + + + encoded_nested: + + > + + encoded_repeated_nested: + , + + ] + > + + unexpected_tag_message: + An encoded message that has an undefined tag or number in the stream. + + encoded_default_assigned: + + + encoded_nested_empty: + + > + + encoded_invalid_enum: + + """ + + encoded_empty_message = '' + + def testEncodeInvalidMessage(self): + message = NestedMessage() + self.assertRaises(messages.ValidationError, + self.PROTOLIB.encode_message, message) + + def CompareEncoded(self, expected_encoded, actual_encoded): + """Compare two encoded protocol values. + + Can be overridden by sub-classes to special case comparison. + For example, to eliminate white space from output that is not + relevant to encoding. + + Args: + expected_encoded: Expected string encoded value. + actual_encoded: Actual string encoded value. + """ + self.assertEquals(expected_encoded, actual_encoded) + + def EncodeDecode(self, encoded, expected_message): + message = self.PROTOLIB.decode_message(type(expected_message), encoded) + self.assertEquals(expected_message, message) + self.CompareEncoded(encoded, self.PROTOLIB.encode_message(message)) + + def testEmptyMessage(self): + self.EncodeDecode(self.encoded_empty_message, OptionalMessage()) + + def testPartial(self): + """Test message with a few values set.""" + message = OptionalMessage() + message.double_value = 1.23 + message.int64_value = -100000000000 + message.int32_value = 1020 + message.string_value = u'a string' + message.enum_value = OptionalMessage.SimpleEnum.VAL2 + + self.EncodeDecode(self.encoded_partial, message) + + def testFull(self): + """Test all types.""" + message = OptionalMessage() + message.double_value = 1.23 + message.float_value = -2.5 + message.int64_value = -100000000000 + message.uint64_value = 102020202020 + message.int32_value = 1020 + message.bool_value = True + message.string_value = u'a string\u044f' + message.bytes_value = b'a bytes\xff\xfe' + message.enum_value = OptionalMessage.SimpleEnum.VAL2 + + self.EncodeDecode(self.encoded_full, message) + + def testRepeated(self): + """Test repeated fields.""" + message = RepeatedMessage() + message.double_value = [1.23, 2.3] + message.float_value = [-2.5, 0.5] + message.int64_value = [-100000000000, 20] + message.uint64_value = [102020202020, 10] + message.int32_value = [1020, 718] + message.bool_value = [True, False] + message.string_value = [u'a string\u044f', u'another string'] + message.bytes_value = [b'a bytes\xff\xfe', b'another bytes'] + message.enum_value = [RepeatedMessage.SimpleEnum.VAL2, + RepeatedMessage.SimpleEnum.VAL1] + + self.EncodeDecode(self.encoded_repeated, message) + + def testNested(self): + """Test nested messages.""" + nested_message = NestedMessage() + nested_message.a_value = u'a string' + + message = HasNestedMessage() + message.nested = nested_message + + self.EncodeDecode(self.encoded_nested, message) + + def testRepeatedNested(self): + """Test repeated nested messages.""" + nested_message1 = NestedMessage() + nested_message1.a_value = u'a string' + nested_message2 = NestedMessage() + nested_message2.a_value = u'another string' + + message = HasNestedMessage() + message.repeated_nested = [nested_message1, nested_message2] + + self.EncodeDecode(self.encoded_repeated_nested, message) + + def testStringTypes(self): + """Test that encoding str on StringField works.""" + message = OptionalMessage() + message.string_value = 'Latin' + self.EncodeDecode(self.encoded_string_types, message) + + def testEncodeUninitialized(self): + """Test that cannot encode uninitialized message.""" + required = NestedMessage() + self.assertRaisesWithRegexpMatch(messages.ValidationError, + "Message NestedMessage is missing " + "required field a_value", + self.PROTOLIB.encode_message, + required) + + def testUnexpectedField(self): + """Test decoding and encoding unexpected fields.""" + loaded_message = self.PROTOLIB.decode_message(OptionalMessage, + self.unexpected_tag_message) + # Message should be equal to an empty message, since unknown values aren't + # included in equality. + self.assertEquals(OptionalMessage(), loaded_message) + # Verify that the encoded message matches the source, including the + # unknown value. + self.assertEquals(self.unexpected_tag_message, + self.PROTOLIB.encode_message(loaded_message)) + + def testDoNotSendDefault(self): + """Test that default is not sent when nothing is assigned.""" + self.EncodeDecode(self.encoded_empty_message, HasDefault()) + + def testSendDefaultExplicitlyAssigned(self): + """Test that default is sent when explcitly assigned.""" + message = HasDefault() + + message.a_value = HasDefault.a_value.default + + self.EncodeDecode(self.encoded_default_assigned, message) + + def testEncodingNestedEmptyMessage(self): + """Test encoding a nested empty message.""" + message = HasOptionalNestedMessage() + message.nested = OptionalMessage() + + self.EncodeDecode(self.encoded_nested_empty, message) + + def testEncodingRepeatedNestedEmptyMessage(self): + """Test encoding a nested empty message.""" + message = HasOptionalNestedMessage() + message.repeated_nested = [OptionalMessage(), OptionalMessage()] + + self.EncodeDecode(self.encoded_repeated_nested_empty, message) + + def testContentType(self): + self.assertTrue(isinstance(self.PROTOLIB.CONTENT_TYPE, str)) + + def testDecodeInvalidEnumType(self): + self.assertRaisesWithRegexpMatch(messages.DecodeError, + 'Invalid enum value ', + self.PROTOLIB.decode_message, + OptionalMessage, + self.encoded_invalid_enum) + + def testDateTimeNoTimeZone(self): + """Test that DateTimeFields are encoded/decoded correctly.""" + + class MyMessage(messages.Message): + value = message_types.DateTimeField(1) + + value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000) + message = MyMessage(value=value) + decoded = self.PROTOLIB.decode_message( + MyMessage, self.PROTOLIB.encode_message(message)) + self.assertEquals(decoded.value, value) + + def testDateTimeWithTimeZone(self): + """Test DateTimeFields with time zones.""" + + class MyMessage(messages.Message): + value = message_types.DateTimeField(1) + + value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000, + util.TimeZoneOffset(8 * 60)) + message = MyMessage(value=value) + decoded = self.PROTOLIB.decode_message( + MyMessage, self.PROTOLIB.encode_message(message)) + self.assertEquals(decoded.value, value) + + +def do_with(context, function, *args, **kwargs): + """Simulate a with statement. + + Avoids need to import with from future. + + Does not support simulation of 'as'. + + Args: + context: Context object normally used with 'with'. + function: Callable to evoke. Replaces with-block. + """ + context.__enter__() + try: + function(*args, **kwargs) + except: + context.__exit__(*sys.exc_info()) + finally: + context.__exit__(None, None, None) + + +def pick_unused_port(): + """Find an unused port to use in tests. + + Derived from Damon Kohlers example: + + http://code.activestate.com/recipes/531822-pick-unused-port + """ + temp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + temp.bind(('localhost', 0)) + port = temp.getsockname()[1] + finally: + temp.close() + return port + + +def get_module_name(module_attribute): + """Get the module name. + + Args: + module_attribute: An attribute of the module. + + Returns: + The fully qualified module name or simple module name where + 'module_attribute' is defined if the module name is "__main__". + """ + if module_attribute.__module__ == '__main__': + module_file = inspect.getfile(module_attribute) + default = os.path.basename(module_file).split('.')[0] + return default + else: + return module_attribute.__module__ diff --git a/apitools/base/protorpclite/util.py b/apitools/base/protorpclite/util.py new file mode 100644 index 0000000..4881889 --- /dev/null +++ b/apitools/base/protorpclite/util.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Common utility library.""" + +from __future__ import with_statement +import six + +__author__ = ['rafek@google.com (Rafe Kaplan)', + 'guido@google.com (Guido van Rossum)', +] + +import cgi +import datetime +import functools +import inspect +import os +import re +import sys + +__all__ = ['Error', + 'decode_datetime', + 'get_package_for_module', + 'positional', + 'TimeZoneOffset', + 'total_seconds', +] + + +class Error(Exception): + """Base class for protorpc exceptions.""" + + +_TIME_ZONE_RE_STRING = r""" + # Examples: + # +01:00 + # -05:30 + # Z12:00 + ((?PZ) | (?P[-+]) + (?P\d\d) : + (?P\d\d))$ +""" +_TIME_ZONE_RE = re.compile(_TIME_ZONE_RE_STRING, re.IGNORECASE | re.VERBOSE) + + +def positional(max_positional_args): + """A decorator to declare that only the first N arguments may be positional. + + This decorator makes it easy to support Python 3 style keyword-only + parameters. For example, in Python 3 it is possible to write: + + def fn(pos1, *, kwonly1=None, kwonly1=None): + ... + + All named parameters after * must be a keyword: + + fn(10, 'kw1', 'kw2') # Raises exception. + fn(10, kwonly1='kw1') # Ok. + + Example: + To define a function like above, do: + + @positional(1) + def fn(pos1, kwonly1=None, kwonly2=None): + ... + + If no default value is provided to a keyword argument, it becomes a required + keyword argument: + + @positional(0) + def fn(required_kw): + ... + + This must be called with the keyword parameter: + + fn() # Raises exception. + fn(10) # Raises exception. + fn(required_kw=10) # Ok. + + When defining instance or class methods always remember to account for + 'self' and 'cls': + + class MyClass(object): + + @positional(2) + def my_method(self, pos1, kwonly1=None): + ... + + @classmethod + @positional(2) + def my_method(cls, pos1, kwonly1=None): + ... + + One can omit the argument to 'positional' altogether, and then no + arguments with default values may be passed positionally. This + would be equivalent to placing a '*' before the first argument + with a default value in Python 3. If there are no arguments with + default values, and no argument is given to 'positional', an error + is raised. + + @positional + def fn(arg1, arg2, required_kw1=None, required_kw2=0): + ... + + fn(1, 3, 5) # Raises exception. + fn(1, 3) # Ok. + fn(1, 3, required_kw1=5) # Ok. + + Args: + max_positional_arguments: Maximum number of positional arguments. All + parameters after the this index must be keyword only. + + Returns: + A decorator that prevents using arguments after max_positional_args from + being used as positional parameters. + + Raises: + TypeError if a keyword-only argument is provided as a positional parameter. + ValueError if no maximum number of arguments is provided and the function + has no arguments with default values. + """ + def positional_decorator(wrapped): + @functools.wraps(wrapped) + def positional_wrapper(*args, **kwargs): + if len(args) > max_positional_args: + plural_s = '' + if max_positional_args != 1: + plural_s = 's' + raise TypeError('%s() takes at most %d positional argument%s ' + '(%d given)' % (wrapped.__name__, + max_positional_args, + plural_s, len(args))) + return wrapped(*args, **kwargs) + return positional_wrapper + + if isinstance(max_positional_args, six.integer_types): + return positional_decorator + else: + args, _, _, defaults = inspect.getargspec(max_positional_args) + if defaults is None: + raise ValueError( + 'Functions with no keyword arguments must specify ' + 'max_positional_args') + return positional(len(args) - len(defaults))(max_positional_args) + + +@positional(1) +def get_package_for_module(module): + """Get package name for a module. + + Helper calculates the package name of a module. + + Args: + module: Module to get name for. If module is a string, try to find + module in sys.modules. + + Returns: + If module contains 'package' attribute, uses that as package name. + Else, if module is not the '__main__' module, the module __name__. + Else, the base name of the module file name. Else None. + """ + if isinstance(module, six.string_types): + try: + module = sys.modules[module] + except KeyError: + return None + + try: + return six.text_type(module.package) + except AttributeError: + if module.__name__ == '__main__': + try: + file_name = module.__file__ + except AttributeError: + pass + else: + base_name = os.path.basename(file_name) + split_name = os.path.splitext(base_name) + if len(split_name) == 1: + return six.text_type(base_name) + else: + return u'.'.join(split_name[:-1]) + + return six.text_type(module.__name__) + + +def total_seconds(offset): + """Backport of offset.total_seconds() from python 2.7+.""" + seconds = offset.days * 24 * 60 * 60 + offset.seconds + microseconds = seconds * 10**6 + offset.microseconds + return microseconds / (10**6 * 1.0) + + +class TimeZoneOffset(datetime.tzinfo): + """Time zone information as encoded/decoded for DateTimeFields.""" + + def __init__(self, offset): + """Initialize a time zone offset. + + Args: + offset: Integer or timedelta time zone offset, in minutes from UTC. This + can be negative. + """ + super(TimeZoneOffset, self).__init__() + if isinstance(offset, datetime.timedelta): + offset = total_seconds(offset) / 60 + self.__offset = offset + + def utcoffset(self, dt): + """Get the a timedelta with the time zone's offset from UTC. + + Returns: + The time zone offset from UTC, as a timedelta. + """ + return datetime.timedelta(minutes=self.__offset) + + def dst(self, dt): + """Get the daylight savings time offset. + + The formats that ProtoRPC uses to encode/decode time zone information don't + contain any information about daylight savings time. So this always + returns a timedelta of 0. + + Returns: + A timedelta of 0. + """ + return datetime.timedelta(0) + + +def decode_datetime(encoded_datetime): + """Decode a DateTimeField parameter from a string to a python datetime. + + Args: + encoded_datetime: A string in RFC 3339 format. + + Returns: + A datetime object with the date and time specified in encoded_datetime. + + Raises: + ValueError: If the string is not in a recognized format. + """ + # Check if the string includes a time zone offset. Break out the + # part that doesn't include time zone info. Convert to uppercase + # because all our comparisons should be case-insensitive. + time_zone_match = _TIME_ZONE_RE.search(encoded_datetime) + if time_zone_match: + time_string = encoded_datetime[:time_zone_match.start(1)].upper() + else: + time_string = encoded_datetime.upper() + + if '.' in time_string: + format_string = '%Y-%m-%dT%H:%M:%S.%f' + else: + format_string = '%Y-%m-%dT%H:%M:%S' + + decoded_datetime = datetime.datetime.strptime(time_string, format_string) + + if not time_zone_match: + return decoded_datetime + + # Time zone info was included in the parameter. Add a tzinfo + # object to the datetime. Datetimes can't be changed after they're + # created, so we'll need to create a new one. + if time_zone_match.group('z'): + offset_minutes = 0 + else: + sign = time_zone_match.group('sign') + hours, minutes = [int(value) for value in + time_zone_match.group('hours', 'minutes')] + offset_minutes = hours * 60 + minutes + if sign == '-': + offset_minutes *= -1 + + return datetime.datetime(decoded_datetime.year, + decoded_datetime.month, + decoded_datetime.day, + decoded_datetime.hour, + decoded_datetime.minute, + decoded_datetime.second, + decoded_datetime.microsecond, + TimeZoneOffset(offset_minutes)) diff --git a/apitools/base/protorpclite/util_test.py b/apitools/base/protorpclite/util_test.py new file mode 100644 index 0000000..a61a94e --- /dev/null +++ b/apitools/base/protorpclite/util_test.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +# +# Copyright 2010 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Tests for apitools.base.protorpclite.util.""" +import six + +__author__ = 'rafek@google.com (Rafe Kaplan)' + + +import datetime +import random +import sys +import types +import unittest + +from apitools.base.protorpclite import test_util +from apitools.base.protorpclite import util + + +class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + + MODULE = util + + +class UtilTest(test_util.TestCase): + + def testDecoratedFunction_LengthZero(self): + @util.positional(0) + def fn(kwonly=1): + return [kwonly] + self.assertEquals([1], fn()) + self.assertEquals([2], fn(kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 0 positional ' + r'arguments \(1 given\)', + fn, 1) + + def testDecoratedFunction_LengthOne(self): + @util.positional(1) + def fn(pos, kwonly=1): + return [pos, kwonly] + self.assertEquals([1, 1], fn(1)) + self.assertEquals([2, 2], fn(2, kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 1 positional ' + r'argument \(2 given\)', + fn, 2, 3) + + def testDecoratedFunction_LengthTwoWithDefault(self): + @util.positional(2) + def fn(pos1, pos2=1, kwonly=1): + return [pos1, pos2, kwonly] + self.assertEquals([1, 1, 1], fn(1)) + self.assertEquals([2, 2, 1], fn(2, 2)) + self.assertEquals([2, 3, 4], fn(2, 3, kwonly=4)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + fn, 2, 3, 4) + + def testDecoratedMethod(self): + class MyClass(object): + @util.positional(2) + def meth(self, pos1, kwonly=1): + return [pos1, kwonly] + self.assertEquals([1, 1], MyClass().meth(1)) + self.assertEquals([2, 2], MyClass().meth(2, kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'meth\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + MyClass().meth, 2, 3) + + def testDefaultDecoration(self): + @util.positional + def fn(a, b, c=None): + return a, b, c + self.assertEquals((1, 2, 3), fn(1, 2, c=3)) + self.assertEquals((3, 4, None), fn(3, b=4)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + fn, 2, 3, 4) + + def testDefaultDecorationNoKwdsFails(self): + def fn(a): + return a + self.assertRaisesRegexp( + ValueError, + 'Functions with no keyword arguments must specify max_positional_args', + util.positional, fn) + + def testDecoratedFunctionDocstring(self): + @util.positional(0) + def fn(kwonly=1): + """fn docstring.""" + return [kwonly] + self.assertEquals('fn docstring.', fn.__doc__) + + +class GetPackageForModuleTest(test_util.TestCase): + + def setUp(self): + self.original_modules = dict(sys.modules) + + def tearDown(self): + sys.modules.clear() + sys.modules.update(self.original_modules) + + def CreateModule(self, name, file_name=None): + if file_name is None: + file_name = '%s.py' % name + module = types.ModuleType(name) + sys.modules[name] = module + return module + + def assertPackageEquals(self, expected, actual): + self.assertEquals(expected, actual) + if actual is not None: + self.assertTrue(isinstance(actual, six.text_type)) + + def testByString(self): + module = self.CreateModule('service_module') + module.package = 'my_package' + self.assertPackageEquals('my_package', + util.get_package_for_module('service_module')) + + def testModuleNameNotInSys(self): + self.assertPackageEquals(None, + util.get_package_for_module('service_module')) + + def testHasPackage(self): + module = self.CreateModule('service_module') + module.package = 'my_package' + self.assertPackageEquals('my_package', util.get_package_for_module(module)) + + def testHasModuleName(self): + module = self.CreateModule('service_module') + self.assertPackageEquals('service_module', + util.get_package_for_module(module)) + + def testIsMain(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file.py' + self.assertPackageEquals('my_file', util.get_package_for_module(module)) + + def testIsMainCompiled(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file.pyc' + self.assertPackageEquals('my_file', util.get_package_for_module(module)) + + def testNoExtension(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file' + self.assertPackageEquals('my_file', util.get_package_for_module(module)) + + def testNoPackageAtAll(self): + module = self.CreateModule('__main__') + self.assertPackageEquals('__main__', util.get_package_for_module(module)) + + +class DateTimeTests(test_util.TestCase): + + def testDecodeDateTime(self): + """Test that a RFC 3339 datetime string is decoded properly.""" + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + decoded = util.decode_datetime(datetime_string) + expected = datetime.datetime(*datetime_vals) + self.assertEquals(expected, decoded) + + def testDateTimeTimeZones(self): + """Test that a datetime string with a timezone is decoded correctly.""" + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262-06:00', + (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(-360))), + ('2012-09-30T15:31:50.262+01:30', + (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(90))), + ('2012-09-30T15:31:50+00:05', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(5))), + ('2012-09-30T15:31:50+00:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30t15:31:50-00:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30t15:31:50z', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30T15:31:50-23:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(-1380)))): + decoded = util.decode_datetime(datetime_string) + expected = datetime.datetime(*datetime_vals) + self.assertEquals(expected, decoded) + + def testDecodeDateTimeInvalid(self): + """Test that decoding malformed datetime strings raises execptions.""" + for datetime_string in ('invalid', + '2012-09-30T15:31:50.', + '-08:00 2012-09-30T15:31:50.262', + '2012-09-30T15:31', + '2012-09-30T15:31Z', + '2012-09-30T15:31:50ZZ', + '2012-09-30T15:31:50.262 blah blah -08:00', + '1000-99-99T25:99:99.999-99:99'): + self.assertRaises(ValueError, util.decode_datetime, datetime_string) + + def testTimeZoneOffsetDelta(self): + """Test that delta works with TimeZoneOffset.""" + time_zone = util.TimeZoneOffset(datetime.timedelta(minutes=3)) + epoch = time_zone.utcoffset(datetime.datetime.utcfromtimestamp(0)) + self.assertEqual(180, util.total_seconds(epoch)) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 97bbb6f..5623c61 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -8,13 +8,13 @@ import logging import pprint -from protorpc import message_types -from protorpc import messages import six from six.moves import http_client from six.moves import urllib +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages from apitools.base.py import credentials_lib from apitools.base.py import encoding from apitools.base.py import exceptions diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 7758c43..2b03185 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -2,12 +2,12 @@ import base64 import datetime import sys -from protorpc import message_types -from protorpc import messages import six from six.moves import urllib_parse import unittest2 +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages from apitools.base.py import base_api from apitools.base.py import encoding from apitools.base.py import http_wrapper diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 972a95f..abac330 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -9,11 +9,11 @@ import logging import os import sys -from protorpc import message_types -from protorpc import messages -from protorpc import protojson import six +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import protojson from apitools.base.py import exceptions __all__ = [ @@ -195,7 +195,7 @@ def MessageToRepr(msg, multiline=False, **kwargs): def __repr__(self): s = 'TimeZoneOffset(' + repr(self.offset) + ')' if not kwargs.get('no_modules'): - s = 'protorpc.util.' + s + s = 'apitools.base.protorpclite.util.' + s return s msg = datetime.datetime( diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 0389af1..7f0005b 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -3,11 +3,11 @@ import datetime import json import sys -from protorpc import message_types -from protorpc import messages -from protorpc import util import unittest2 +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import util from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import extra_types @@ -371,7 +371,7 @@ class EncodingTest(unittest2.TestCase): encoding.MessageToRepr(msg, multiline=True), ('%s.TimeMessage(\n ' 'timefield=datetime.datetime(2014, 7, 2, 23, 33, 25, 541000, ' - 'tzinfo=protorpc.util.TimeZoneOffset(' + 'tzinfo=apitools.base.protorpclite.util.TimeZoneOffset(' 'datetime.timedelta(0))),\n)') % __name__) self.assertEqual( encoding.MessageToRepr(msg, multiline=True, no_modules=True), diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 403ceda..0649e57 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -10,11 +10,11 @@ import datetime import json import numbers -from protorpc import message_types -from protorpc import messages -from protorpc import protojson import six +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages +from apitools.base.protorpclite import protojson from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import util diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index d2fd5b0..6106463 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -2,9 +2,9 @@ import datetime import json import math -from protorpc import messages import unittest2 +from apitools.base.protorpclite import messages from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import extra_types diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index 75a8b07..be09616 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -9,9 +9,9 @@ as it's all done within the context of a mock. import difflib -from protorpc import messages import six +from apitools.base.protorpclite import messages import apitools.base.py as apitools_base diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py index 4a6c57c..cff0d01 100644 --- a/apitools/base/py/testing/mock_test.py +++ b/apitools/base/py/testing/mock_test.py @@ -1,8 +1,9 @@ """Tests for apitools.base.py.testing.mock.""" -from protorpc import messages import unittest2 +from apitools.base.protorpclite import messages + import apitools.base.py as apitools_base from apitools.base.py.testing import mock from apitools.base.py.testing import testclient as fusiontables diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py index 68284b5..12cd27e 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py @@ -4,7 +4,7 @@ API for working with Fusion Tables data. """ # NOTE: This file is autogenerated and should not be edited by hand. -from protorpc import messages as _messages +from apitools.base.protorpclite import messages as _messages package = 'fusiontables' diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index b92c9f8..06e01a2 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -5,13 +5,13 @@ import collections import os import random -from protorpc import messages import six from six.moves import http_client import six.moves.urllib.error as urllib_error import six.moves.urllib.parse as urllib_parse import six.moves.urllib.request as urllib_request +from apitools.base.protorpclite import messages from apitools.base.py import encoding from apitools.base.py import exceptions diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 6cb551d..4deda8f 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -1,7 +1,7 @@ """Tests for util.py.""" -from protorpc import messages import unittest2 +from apitools.base.protorpclite import messages from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import util diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index c9e94be..9c6ac7b 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -4,9 +4,8 @@ import logging import textwrap -from protorpc import descriptor -from protorpc import messages - +from apitools.base.protorpclite import descriptor +from apitools.base.protorpclite import messages from apitools.gen import extended_descriptor # This is a code generator; we're purposely verbose. @@ -216,7 +215,7 @@ class CommandRegistry(object): type_name = '' if field.variant in (messages.Variant.MESSAGE, messages.Variant.ENUM): - if field.type_name.startswith('protorpc.'): + if field.type_name.startswith('apitools.base.protorpclite.'): type_name = field.type_name else: field_message = self.__LookupMessage(extended_message, field) @@ -462,9 +461,8 @@ class CommandRegistry(object): printer('import platform') printer('import sys') printer() - printer('import protorpc') - printer('from protorpc import message_types') - printer('from protorpc import messages') + printer('from apitools.base.protorpclite import message_types') + printer('from apitools.base.protorpclite import messages') printer() appcommands_import = 'from google.apputils import appcommands' printer(appcommands_import) diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index 5274100..a59bb86 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -16,11 +16,11 @@ import abc import operator import textwrap -from protorpc import descriptor as protorpc_descriptor -from protorpc import message_types -from protorpc import messages import six +from apitools.base.protorpclite import descriptor as protorpc_descriptor +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages import apitools.base.py as apitools_base diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index d467e9b..a8e0322 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -5,10 +5,10 @@ import collections import contextlib import json -from protorpc import descriptor -from protorpc import messages import six +from apitools.base.protorpclite import descriptor +from apitools.base.protorpclite import messages from apitools.gen import extended_descriptor from apitools.gen import util @@ -57,7 +57,7 @@ class MessageRegistry(object): 'date': TypeInfo(type_name='extra_types.DateField', variant=messages.Variant.STRING), 'date-time': TypeInfo( - type_name='protorpc.message_types.DateTimeMessage', + type_name='apitools.base.protorpclite.message_types.DateTimeMessage', variant=messages.Variant.MESSAGE), } @@ -73,7 +73,7 @@ class MessageRegistry(object): package=self.__package, description=self.__description) # Add required imports self.__file_descriptor.additional_imports = [ - 'from protorpc import messages as _messages', + 'from apitools.base.protorpclite import messages as _messages', ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() @@ -392,10 +392,12 @@ class MessageRegistry(object): return self.PRIMITIVE_TYPE_INFO_MAP[type_name] raise ValueError('Unknown type/format "%s"/"%s"' % ( attrs['format'], type_name)) - if (type_info.type_name.startswith('protorpc.message_types.') or - type_info.type_name.startswith('message_types.')): + if type_info.type_name.startswith(( + 'apitools.base.protorpclite.message_types.', + 'message_types.')): self.__AddImport( - 'from protorpc import message_types as _message_types') + 'from apitools.base.protorpclite import message_types ' + 'as _message_types') if type_info.type_name.startswith('extra_types.'): self.__AddImport( 'from %s import extra_types' % self.__base_files_package) diff --git a/setup.py b/setup.py index 0b04356..89fa5d0 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ except ImportError: REQUIRED_PACKAGES = [ 'httplib2>=0.8', 'oauth2client>=1.4.8', - 'protorpc>=0.11.0', 'six>=1.9.0', ] -- GitLab From 09c5d735369892578157eed356be19bdb305a3ad Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 11:25:50 -0700 Subject: [PATCH 171/295] Fix protorpclite indentation with autopep8. --- apitools/base/protorpclite/descriptor.py | 716 +-- apitools/base/protorpclite/descriptor_test.py | 726 +-- apitools/base/protorpclite/message_types.py | 148 +- .../base/protorpclite/message_types_test.py | 93 +- apitools/base/protorpclite/messages.py | 3062 ++++++------- apitools/base/protorpclite/messages_test.py | 3906 +++++++++-------- apitools/base/protorpclite/protojson.py | 588 +-- apitools/base/protorpclite/protojson_test.py | 619 +-- apitools/base/protorpclite/test_util.py | 1057 ++--- apitools/base/protorpclite/util.py | 392 +- apitools/base/protorpclite/util_test.py | 363 +- 11 files changed, 5867 insertions(+), 5803 deletions(-) diff --git a/apitools/base/protorpclite/descriptor.py b/apitools/base/protorpclite/descriptor.py index cf5fe12..de7a17f 100644 --- a/apitools/base/protorpclite/descriptor.py +++ b/apitools/base/protorpclite/descriptor.py @@ -120,7 +120,7 @@ __all__ = ['EnumDescriptor', 'describe_file_set', 'describe', 'import_descriptor_loader', - ] + ] # NOTE: MessageField is missing because message fields cannot have @@ -150,464 +150,466 @@ _DEFAULT_FROM_STRING_MAP = { class EnumValueDescriptor(messages.Message): - """Enum value descriptor. + """Enum value descriptor. - Fields: - name: Name of enumeration value. - number: Number of enumeration value. - """ + Fields: + name: Name of enumeration value. + number: Number of enumeration value. + """ - # TODO(rafek): Why are these listed as optional in descriptor.proto. - # Harmonize? - name = messages.StringField(1, required=True) - number = messages.IntegerField(2, - required=True, - variant=messages.Variant.INT32) + # TODO(rafek): Why are these listed as optional in descriptor.proto. + # Harmonize? + name = messages.StringField(1, required=True) + number = messages.IntegerField(2, + required=True, + variant=messages.Variant.INT32) class EnumDescriptor(messages.Message): - """Enum class descriptor. + """Enum class descriptor. - Fields: - name: Name of Enum without any qualification. - values: Values defined by Enum class. - """ + Fields: + name: Name of Enum without any qualification. + values: Values defined by Enum class. + """ - name = messages.StringField(1) - values = messages.MessageField(EnumValueDescriptor, 2, repeated=True) + name = messages.StringField(1) + values = messages.MessageField(EnumValueDescriptor, 2, repeated=True) class FieldDescriptor(messages.Message): - """Field definition descriptor. - - Enums: - Variant: Wire format hint sub-types for field. - Label: Values for optional, required and repeated fields. - - Fields: - name: Name of field. - number: Number of field. - variant: Variant of field. - type_name: Type name for message and enum fields. - default_value: String representation of default value. - """ + """Field definition descriptor. + + Enums: + Variant: Wire format hint sub-types for field. + Label: Values for optional, required and repeated fields. + + Fields: + name: Name of field. + number: Number of field. + variant: Variant of field. + type_name: Type name for message and enum fields. + default_value: String representation of default value. + """ - Variant = messages.Variant + Variant = messages.Variant - class Label(messages.Enum): - """Field label.""" + class Label(messages.Enum): + """Field label.""" - OPTIONAL = 1 - REQUIRED = 2 - REPEATED = 3 + OPTIONAL = 1 + REQUIRED = 2 + REPEATED = 3 - name = messages.StringField(1, required=True) - number = messages.IntegerField(3, - required=True, - variant=messages.Variant.INT32) - label = messages.EnumField(Label, 4, default=Label.OPTIONAL) - variant = messages.EnumField(Variant, 5) - type_name = messages.StringField(6) + name = messages.StringField(1, required=True) + number = messages.IntegerField(3, + required=True, + variant=messages.Variant.INT32) + label = messages.EnumField(Label, 4, default=Label.OPTIONAL) + variant = messages.EnumField(Variant, 5) + type_name = messages.StringField(6) - # For numeric types, contains the original text representation of the value. - # For booleans, "true" or "false". - # For strings, contains the default text contents (not escaped in any way). - # For bytes, contains the C escaped value. All bytes < 128 are that are - # traditionally considered unprintable are also escaped. - default_value = messages.StringField(7) + # For numeric types, contains the original text representation of the value. + # For booleans, "true" or "false". + # For strings, contains the default text contents (not escaped in any way). + # For bytes, contains the C escaped value. All bytes < 128 are that are + # traditionally considered unprintable are also escaped. + default_value = messages.StringField(7) class MessageDescriptor(messages.Message): - """Message definition descriptor. + """Message definition descriptor. - Fields: - name: Name of Message without any qualification. - fields: Fields defined for message. - message_types: Nested Message classes defined on message. - enum_types: Nested Enum classes defined on message. - """ + Fields: + name: Name of Message without any qualification. + fields: Fields defined for message. + message_types: Nested Message classes defined on message. + enum_types: Nested Enum classes defined on message. + """ - name = messages.StringField(1) - fields = messages.MessageField(FieldDescriptor, 2, repeated=True) + name = messages.StringField(1) + fields = messages.MessageField(FieldDescriptor, 2, repeated=True) - message_types = messages.MessageField( - 'apitools.base.protorpclite.descriptor.MessageDescriptor', 3, repeated=True) - enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True) + message_types = messages.MessageField( + 'apitools.base.protorpclite.descriptor.MessageDescriptor', 3, repeated=True) + enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True) class FileDescriptor(messages.Message): - """Description of file containing protobuf definitions. + """Description of file containing protobuf definitions. - Fields: - package: Fully qualified name of package that definitions belong to. - message_types: Message definitions contained in file. - enum_types: Enum definitions contained in file. - """ + Fields: + package: Fully qualified name of package that definitions belong to. + message_types: Message definitions contained in file. + enum_types: Enum definitions contained in file. + """ - package = messages.StringField(2) + package = messages.StringField(2) - # TODO(rafek): Add dependency field + # TODO(rafek): Add dependency field - message_types = messages.MessageField(MessageDescriptor, 4, repeated=True) - enum_types = messages.MessageField(EnumDescriptor, 5, repeated=True) + message_types = messages.MessageField(MessageDescriptor, 4, repeated=True) + enum_types = messages.MessageField(EnumDescriptor, 5, repeated=True) class FileSet(messages.Message): - """A collection of FileDescriptors. + """A collection of FileDescriptors. - Fields: - files: Files in file-set. - """ + Fields: + files: Files in file-set. + """ - files = messages.MessageField(FileDescriptor, 1, repeated=True) + files = messages.MessageField(FileDescriptor, 1, repeated=True) def describe_enum_value(enum_value): - """Build descriptor for Enum instance. + """Build descriptor for Enum instance. - Args: - enum_value: Enum value to provide descriptor for. + Args: + enum_value: Enum value to provide descriptor for. - Returns: - Initialized EnumValueDescriptor instance describing the Enum instance. - """ - enum_value_descriptor = EnumValueDescriptor() - enum_value_descriptor.name = six.text_type(enum_value.name) - enum_value_descriptor.number = enum_value.number - return enum_value_descriptor + Returns: + Initialized EnumValueDescriptor instance describing the Enum instance. + """ + enum_value_descriptor = EnumValueDescriptor() + enum_value_descriptor.name = six.text_type(enum_value.name) + enum_value_descriptor.number = enum_value.number + return enum_value_descriptor def describe_enum(enum_definition): - """Build descriptor for Enum class. + """Build descriptor for Enum class. - Args: - enum_definition: Enum class to provide descriptor for. + Args: + enum_definition: Enum class to provide descriptor for. - Returns: - Initialized EnumDescriptor instance describing the Enum class. - """ - enum_descriptor = EnumDescriptor() - enum_descriptor.name = enum_definition.definition_name().split('.')[-1] + Returns: + Initialized EnumDescriptor instance describing the Enum class. + """ + enum_descriptor = EnumDescriptor() + enum_descriptor.name = enum_definition.definition_name().split('.')[-1] - values = [] - for number in enum_definition.numbers(): - value = enum_definition.lookup_by_number(number) - values.append(describe_enum_value(value)) + values = [] + for number in enum_definition.numbers(): + value = enum_definition.lookup_by_number(number) + values.append(describe_enum_value(value)) - if values: - enum_descriptor.values = values + if values: + enum_descriptor.values = values - return enum_descriptor + return enum_descriptor def describe_field(field_definition): - """Build descriptor for Field instance. + """Build descriptor for Field instance. - Args: - field_definition: Field instance to provide descriptor for. + Args: + field_definition: Field instance to provide descriptor for. - Returns: - Initialized FieldDescriptor instance describing the Field instance. - """ - field_descriptor = FieldDescriptor() - field_descriptor.name = field_definition.name - field_descriptor.number = field_definition.number - field_descriptor.variant = field_definition.variant + Returns: + Initialized FieldDescriptor instance describing the Field instance. + """ + field_descriptor = FieldDescriptor() + field_descriptor.name = field_definition.name + field_descriptor.number = field_definition.number + field_descriptor.variant = field_definition.variant + + if isinstance(field_definition, messages.EnumField): + field_descriptor.type_name = field_definition.type.definition_name() + + if isinstance(field_definition, messages.MessageField): + field_descriptor.type_name = field_definition.message_type.definition_name() + + if field_definition.default is not None: + field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[ + type(field_definition)](field_definition.default) + + # Set label. + if field_definition.repeated: + field_descriptor.label = FieldDescriptor.Label.REPEATED + elif field_definition.required: + field_descriptor.label = FieldDescriptor.Label.REQUIRED + else: + field_descriptor.label = FieldDescriptor.Label.OPTIONAL - if isinstance(field_definition, messages.EnumField): - field_descriptor.type_name = field_definition.type.definition_name() + return field_descriptor - if isinstance(field_definition, messages.MessageField): - field_descriptor.type_name = field_definition.message_type.definition_name() - if field_definition.default is not None: - field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[ - type(field_definition)](field_definition.default) +def describe_message(message_definition): + """Build descriptor for Message class. - # Set label. - if field_definition.repeated: - field_descriptor.label = FieldDescriptor.Label.REPEATED - elif field_definition.required: - field_descriptor.label = FieldDescriptor.Label.REQUIRED - else: - field_descriptor.label = FieldDescriptor.Label.OPTIONAL + Args: + message_definition: Message class to provide descriptor for. - return field_descriptor + Returns: + Initialized MessageDescriptor instance describing the Message class. + """ + message_descriptor = MessageDescriptor() + message_descriptor.name = message_definition.definition_name().split( + '.')[-1] + fields = sorted(message_definition.all_fields(), + key=lambda v: v.number) + if fields: + message_descriptor.fields = [describe_field(field) for field in fields] -def describe_message(message_definition): - """Build descriptor for Message class. - - Args: - message_definition: Message class to provide descriptor for. - - Returns: - Initialized MessageDescriptor instance describing the Message class. - """ - message_descriptor = MessageDescriptor() - message_descriptor.name = message_definition.definition_name().split('.')[-1] - - fields = sorted(message_definition.all_fields(), - key=lambda v: v.number) - if fields: - message_descriptor.fields = [describe_field(field) for field in fields] - - try: - nested_messages = message_definition.__messages__ - except AttributeError: - pass - else: - message_descriptors = [] - for name in nested_messages: - value = getattr(message_definition, name) - message_descriptors.append(describe_message(value)) + try: + nested_messages = message_definition.__messages__ + except AttributeError: + pass + else: + message_descriptors = [] + for name in nested_messages: + value = getattr(message_definition, name) + message_descriptors.append(describe_message(value)) - message_descriptor.message_types = message_descriptors + message_descriptor.message_types = message_descriptors - try: - nested_enums = message_definition.__enums__ - except AttributeError: - pass - else: - enum_descriptors = [] - for name in nested_enums: - value = getattr(message_definition, name) - enum_descriptors.append(describe_enum(value)) + try: + nested_enums = message_definition.__enums__ + except AttributeError: + pass + else: + enum_descriptors = [] + for name in nested_enums: + value = getattr(message_definition, name) + enum_descriptors.append(describe_enum(value)) - message_descriptor.enum_types = enum_descriptors + message_descriptor.enum_types = enum_descriptors - return message_descriptor + return message_descriptor def describe_file(module): - """Build a file from a specified Python module. + """Build a file from a specified Python module. - Args: - module: Python module to describe. + Args: + module: Python module to describe. - Returns: - Initialized FileDescriptor instance describing the module. - """ - # May not import remote at top of file because remote depends on this - # file - # TODO(rafek): Straighten out this dependency. Possibly move these functions - # from descriptor to their own module. + Returns: + Initialized FileDescriptor instance describing the module. + """ + # May not import remote at top of file because remote depends on this + # file + # TODO(rafek): Straighten out this dependency. Possibly move these functions + # from descriptor to their own module. - descriptor = FileDescriptor() - descriptor.package = util.get_package_for_module(module) + descriptor = FileDescriptor() + descriptor.package = util.get_package_for_module(module) - if not descriptor.package: - descriptor.package = None + if not descriptor.package: + descriptor.package = None - message_descriptors = [] - enum_descriptors = [] + message_descriptors = [] + enum_descriptors = [] - # Need to iterate over all top level attributes of the module looking for - # message and enum definitions. Each definition must be itself described. - for name in sorted(dir(module)): - value = getattr(module, name) + # Need to iterate over all top level attributes of the module looking for + # message and enum definitions. Each definition must be itself described. + for name in sorted(dir(module)): + value = getattr(module, name) - if isinstance(value, type): - if issubclass(value, messages.Message): - message_descriptors.append(describe_message(value)) + if isinstance(value, type): + if issubclass(value, messages.Message): + message_descriptors.append(describe_message(value)) - elif issubclass(value, messages.Enum): - enum_descriptors.append(describe_enum(value)) + elif issubclass(value, messages.Enum): + enum_descriptors.append(describe_enum(value)) - if message_descriptors: - descriptor.message_types = message_descriptors + if message_descriptors: + descriptor.message_types = message_descriptors - if enum_descriptors: - descriptor.enum_types = enum_descriptors + if enum_descriptors: + descriptor.enum_types = enum_descriptors - return descriptor + return descriptor def describe_file_set(modules): - """Build a file set from a specified Python modules. + """Build a file set from a specified Python modules. - Args: - modules: Iterable of Python module to describe. + Args: + modules: Iterable of Python module to describe. - Returns: - Initialized FileSet instance describing the modules. - """ - descriptor = FileSet() - file_descriptors = [] - for module in modules: - file_descriptors.append(describe_file(module)) + Returns: + Initialized FileSet instance describing the modules. + """ + descriptor = FileSet() + file_descriptors = [] + for module in modules: + file_descriptors.append(describe_file(module)) - if file_descriptors: - descriptor.files = file_descriptors + if file_descriptors: + descriptor.files = file_descriptors - return descriptor + return descriptor def describe(value): - """Describe any value as a descriptor. - - Helper function for describing any object with an appropriate descriptor - object. - - Args: - value: Value to describe as a descriptor. - - Returns: - Descriptor message class if object is describable as a descriptor, else - None. - """ - if isinstance(value, types.ModuleType): - return describe_file(value) - elif isinstance(value, messages.Field): - return describe_field(value) - elif isinstance(value, messages.Enum): - return describe_enum_value(value) - elif isinstance(value, type): - if issubclass(value, messages.Message): - return describe_message(value) - elif issubclass(value, messages.Enum): - return describe_enum(value) - return None + """Describe any value as a descriptor. + Helper function for describing any object with an appropriate descriptor + object. -@util.positional(1) -def import_descriptor_loader(definition_name, importer=__import__): - """Find objects by importing modules as needed. - - A definition loader is a function that resolves a definition name to a - descriptor. - - The import finder resolves definitions to their names by importing modules - when necessary. - - Args: - definition_name: Name of definition to find. - importer: Import function used for importing new modules. - - Returns: - Appropriate descriptor for any describable type located by name. - - Raises: - DefinitionNotFoundError when a name does not refer to either a definition - or a module. - """ - # Attempt to import descriptor as a module. - if definition_name.startswith('.'): - definition_name = definition_name[1:] - if not definition_name.startswith('.'): - leaf = definition_name.split('.')[-1] - if definition_name: - try: - module = importer(definition_name, '', '', [leaf]) - except ImportError: - pass - else: - return describe(module) - - try: - # Attempt to use messages.find_definition to find item. - return describe(messages.find_definition(definition_name, - importer=__import__)) - except messages.DefinitionNotFoundError as err: - # There are things that find_definition will not find, but if the parent - # is loaded, its children can be searched for a match. - split_name = definition_name.rsplit('.', 1) - if len(split_name) > 1: - parent, child = split_name - try: - parent_definition = import_descriptor_loader(parent, importer=importer) - except messages.DefinitionNotFoundError: - # Fall through to original error. - pass - else: - # Check the parent definition for a matching descriptor. - if isinstance(parent_definition, EnumDescriptor): - search_list = parent_definition.values or [] - elif isinstance(parent_definition, MessageDescriptor): - search_list = parent_definition.fields or [] - else: - search_list = [] - - for definition in search_list: - if definition.name == child: - return definition - - # Still didn't find. Reraise original exception. - raise err - - -class DescriptorLibrary(object): - """A descriptor library is an object that contains known definitions. - - A descriptor library contains a cache of descriptor objects mapped by - definition name. It contains all types of descriptors except for - file sets. + Args: + value: Value to describe as a descriptor. - When a definition name is requested that the library does not know about - it can be provided with a descriptor loader which attempt to resolve the - missing descriptor. - """ + Returns: + Descriptor message class if object is describable as a descriptor, else + None. + """ + if isinstance(value, types.ModuleType): + return describe_file(value) + elif isinstance(value, messages.Field): + return describe_field(value) + elif isinstance(value, messages.Enum): + return describe_enum_value(value) + elif isinstance(value, type): + if issubclass(value, messages.Message): + return describe_message(value) + elif issubclass(value, messages.Enum): + return describe_enum(value) + return None - @util.positional(1) - def __init__(self, - descriptors=None, - descriptor_loader=import_descriptor_loader): - """Constructor. - Args: - descriptors: A dictionary or dictionary-like object that can be used - to store and cache descriptors by definition name. - definition_loader: A function used for resolving missing descriptors. - The function takes a definition name as its parameter and returns - an appropriate descriptor. It may raise DefinitionNotFoundError. - """ - self.__descriptor_loader = descriptor_loader - self.__descriptors = descriptors or {} +@util.positional(1) +def import_descriptor_loader(definition_name, importer=__import__): + """Find objects by importing modules as needed. - def lookup_descriptor(self, definition_name): - """Lookup descriptor by name. + A definition loader is a function that resolves a definition name to a + descriptor. - Get descriptor from library by name. If descriptor is not found will - attempt to find via descriptor loader if provided. + The import finder resolves definitions to their names by importing modules + when necessary. Args: - definition_name: Definition name to find. + definition_name: Name of definition to find. + importer: Import function used for importing new modules. Returns: - Descriptor that describes definition name. + Appropriate descriptor for any describable type located by name. Raises: - DefinitionNotFoundError if not descriptor exists for definition name. + DefinitionNotFoundError when a name does not refer to either a definition + or a module. """ + # Attempt to import descriptor as a module. + if definition_name.startswith('.'): + definition_name = definition_name[1:] + if not definition_name.startswith('.'): + leaf = definition_name.split('.')[-1] + if definition_name: + try: + module = importer(definition_name, '', '', [leaf]) + except ImportError: + pass + else: + return describe(module) + try: - return self.__descriptors[definition_name] - except KeyError: - pass - - if self.__descriptor_loader: - definition = self.__descriptor_loader(definition_name) - self.__descriptors[definition_name] = definition - return definition - else: - raise messages.DefinitionNotFoundError( - 'Could not find definition for %s' % definition_name) + # Attempt to use messages.find_definition to find item. + return describe(messages.find_definition(definition_name, + importer=__import__)) + except messages.DefinitionNotFoundError as err: + # There are things that find_definition will not find, but if the parent + # is loaded, its children can be searched for a match. + split_name = definition_name.rsplit('.', 1) + if len(split_name) > 1: + parent, child = split_name + try: + parent_definition = import_descriptor_loader( + parent, importer=importer) + except messages.DefinitionNotFoundError: + # Fall through to original error. + pass + else: + # Check the parent definition for a matching descriptor. + if isinstance(parent_definition, EnumDescriptor): + search_list = parent_definition.values or [] + elif isinstance(parent_definition, MessageDescriptor): + search_list = parent_definition.fields or [] + else: + search_list = [] + + for definition in search_list: + if definition.name == child: + return definition + + # Still didn't find. Reraise original exception. + raise err - def lookup_package(self, definition_name): - """Determines the package name for any definition. - Determine the package that any definition name belongs to. May check - parent for package name and will resolve missing descriptors if provided - descriptor loader. +class DescriptorLibrary(object): + """A descriptor library is an object that contains known definitions. - Args: - definition_name: Definition name to find package for. + A descriptor library contains a cache of descriptor objects mapped by + definition name. It contains all types of descriptors except for + file sets. + + When a definition name is requested that the library does not know about + it can be provided with a descriptor loader which attempt to resolve the + missing descriptor. """ - while True: - descriptor = self.lookup_descriptor(definition_name) - if isinstance(descriptor, FileDescriptor): - return descriptor.package - else: - index = definition_name.rfind('.') - if index < 0: - return None - definition_name = definition_name[:index] + + @util.positional(1) + def __init__(self, + descriptors=None, + descriptor_loader=import_descriptor_loader): + """Constructor. + + Args: + descriptors: A dictionary or dictionary-like object that can be used + to store and cache descriptors by definition name. + definition_loader: A function used for resolving missing descriptors. + The function takes a definition name as its parameter and returns + an appropriate descriptor. It may raise DefinitionNotFoundError. + """ + self.__descriptor_loader = descriptor_loader + self.__descriptors = descriptors or {} + + def lookup_descriptor(self, definition_name): + """Lookup descriptor by name. + + Get descriptor from library by name. If descriptor is not found will + attempt to find via descriptor loader if provided. + + Args: + definition_name: Definition name to find. + + Returns: + Descriptor that describes definition name. + + Raises: + DefinitionNotFoundError if not descriptor exists for definition name. + """ + try: + return self.__descriptors[definition_name] + except KeyError: + pass + + if self.__descriptor_loader: + definition = self.__descriptor_loader(definition_name) + self.__descriptors[definition_name] = definition + return definition + else: + raise messages.DefinitionNotFoundError( + 'Could not find definition for %s' % definition_name) + + def lookup_package(self, definition_name): + """Determines the package name for any definition. + + Determine the package that any definition name belongs to. May check + parent for package name and will resolve missing descriptors if provided + descriptor loader. + + Args: + definition_name: Definition name to find package for. + """ + while True: + descriptor = self.lookup_descriptor(definition_name) + if isinstance(descriptor, FileDescriptor): + return descriptor.package + else: + index = definition_name.rfind('.') + if index < 0: + return None + definition_name = definition_name[:index] diff --git a/apitools/base/protorpclite/descriptor_test.py b/apitools/base/protorpclite/descriptor_test.py index f55db1a..c9943b8 100644 --- a/apitools/base/protorpclite/descriptor_test.py +++ b/apitools/base/protorpclite/descriptor_test.py @@ -37,470 +37,480 @@ RUSSIA = u'\u0420\u043e\u0441\u0441\u0438\u044f' class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): - MODULE = descriptor + MODULE = descriptor class DescribeEnumValueTest(test_util.TestCase): - def testDescribe(self): - class MyEnum(messages.Enum): - MY_NAME = 10 + def testDescribe(self): + class MyEnum(messages.Enum): + MY_NAME = 10 - expected = descriptor.EnumValueDescriptor() - expected.name = 'MY_NAME' - expected.number = 10 + expected = descriptor.EnumValueDescriptor() + expected.name = 'MY_NAME' + expected.number = 10 - described = descriptor.describe_enum_value(MyEnum.MY_NAME) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_enum_value(MyEnum.MY_NAME) + described.check_initialized() + self.assertEquals(expected, described) class DescribeEnumTest(test_util.TestCase): - def testEmptyEnum(self): - class EmptyEnum(messages.Enum): - pass + def testEmptyEnum(self): + class EmptyEnum(messages.Enum): + pass - expected = descriptor.EnumDescriptor() - expected.name = 'EmptyEnum' + expected = descriptor.EnumDescriptor() + expected.name = 'EmptyEnum' - described = descriptor.describe_enum(EmptyEnum) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_enum(EmptyEnum) + described.check_initialized() + self.assertEquals(expected, described) - def testNestedEnum(self): - class MyScope(messages.Message): - class NestedEnum(messages.Enum): - pass + def testNestedEnum(self): + class MyScope(messages.Message): - expected = descriptor.EnumDescriptor() - expected.name = 'NestedEnum' + class NestedEnum(messages.Enum): + pass - described = descriptor.describe_enum(MyScope.NestedEnum) - described.check_initialized() - self.assertEquals(expected, described) + expected = descriptor.EnumDescriptor() + expected.name = 'NestedEnum' - def testEnumWithItems(self): - class EnumWithItems(messages.Enum): - A = 3 - B = 1 - C = 2 + described = descriptor.describe_enum(MyScope.NestedEnum) + described.check_initialized() + self.assertEquals(expected, described) - expected = descriptor.EnumDescriptor() - expected.name = 'EnumWithItems' + def testEnumWithItems(self): + class EnumWithItems(messages.Enum): + A = 3 + B = 1 + C = 2 - a = descriptor.EnumValueDescriptor() - a.name = 'A' - a.number = 3 + expected = descriptor.EnumDescriptor() + expected.name = 'EnumWithItems' - b = descriptor.EnumValueDescriptor() - b.name = 'B' - b.number = 1 + a = descriptor.EnumValueDescriptor() + a.name = 'A' + a.number = 3 - c = descriptor.EnumValueDescriptor() - c.name = 'C' - c.number = 2 + b = descriptor.EnumValueDescriptor() + b.name = 'B' + b.number = 1 - expected.values = [b, c, a] + c = descriptor.EnumValueDescriptor() + c.name = 'C' + c.number = 2 - described = descriptor.describe_enum(EnumWithItems) - described.check_initialized() - self.assertEquals(expected, described) + expected.values = [b, c, a] + + described = descriptor.describe_enum(EnumWithItems) + described.check_initialized() + self.assertEquals(expected, described) class DescribeFieldTest(test_util.TestCase): - def testLabel(self): - for repeated, required, expected_label in ( - (True, False, descriptor.FieldDescriptor.Label.REPEATED), - (False, True, descriptor.FieldDescriptor.Label.REQUIRED), - (False, False, descriptor.FieldDescriptor.Label.OPTIONAL)): - field = messages.IntegerField(10, required=required, repeated=repeated) - field.name = 'a_field' - - expected = descriptor.FieldDescriptor() - expected.name = 'a_field' - expected.number = 10 - expected.label = expected_label - expected.variant = descriptor.FieldDescriptor.Variant.INT64 - - described = descriptor.describe_field(field) - described.check_initialized() - self.assertEquals(expected, described) - - def testDefault(self): - for field_class, default, expected_default in ( - (messages.IntegerField, 200, '200'), - (messages.FloatField, 1.5, '1.5'), - (messages.FloatField, 1e6, '1000000.0'), - (messages.BooleanField, True, 'true'), - (messages.BooleanField, False, 'false'), - (messages.BytesField, - b''.join([six.int2byte(x) for x in (31, 32, 33)]), - b'\\x1f !'), - (messages.StringField, RUSSIA, RUSSIA), + def testLabel(self): + for repeated, required, expected_label in ( + (True, False, descriptor.FieldDescriptor.Label.REPEATED), + (False, True, descriptor.FieldDescriptor.Label.REQUIRED), + (False, False, descriptor.FieldDescriptor.Label.OPTIONAL)): + field = messages.IntegerField( + 10, required=required, repeated=repeated) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = expected_label + expected.variant = descriptor.FieldDescriptor.Variant.INT64 + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDefault(self): + for field_class, default, expected_default in ( + (messages.IntegerField, 200, '200'), + (messages.FloatField, 1.5, '1.5'), + (messages.FloatField, 1e6, '1000000.0'), + (messages.BooleanField, True, 'true'), + (messages.BooleanField, False, 'false'), + (messages.BytesField, + b''.join([six.int2byte(x) for x in (31, 32, 33)]), + b'\\x1f !'), + (messages.StringField, RUSSIA, RUSSIA), ): - field = field_class(10, default=default) - field.name = u'a_field' - - expected = descriptor.FieldDescriptor() - expected.name = u'a_field' - expected.number = 10 - expected.label = descriptor.FieldDescriptor.Label.OPTIONAL - expected.variant = field_class.DEFAULT_VARIANT - expected.default_value = expected_default - - described = descriptor.describe_field(field) - described.check_initialized() - self.assertEquals(expected, described) - - def testDefault_EnumField(self): - class MyEnum(messages.Enum): - - VAL = 1 - - module_name = test_util.get_module_name(MyEnum) - field = messages.EnumField(MyEnum, 10, default=MyEnum.VAL) - field.name = 'a_field' - - expected = descriptor.FieldDescriptor() - expected.name = 'a_field' - expected.number = 10 - expected.label = descriptor.FieldDescriptor.Label.OPTIONAL - expected.variant = messages.EnumField.DEFAULT_VARIANT - expected.type_name = '%s.MyEnum' % module_name - expected.default_value = '1' - - described = descriptor.describe_field(field) - self.assertEquals(expected, described) - - def testMessageField(self): - field = messages.MessageField(descriptor.FieldDescriptor, 10) - field.name = 'a_field' - - expected = descriptor.FieldDescriptor() - expected.name = 'a_field' - expected.number = 10 - expected.label = descriptor.FieldDescriptor.Label.OPTIONAL - expected.variant = messages.MessageField.DEFAULT_VARIANT - expected.type_name = ('apitools.base.protorpclite.descriptor.FieldDescriptor') - - described = descriptor.describe_field(field) - described.check_initialized() - self.assertEquals(expected, described) - - def testDateTimeField(self): - field = message_types.DateTimeField(20) - field.name = 'a_timestamp' - - expected = descriptor.FieldDescriptor() - expected.name = 'a_timestamp' - expected.number = 20 - expected.label = descriptor.FieldDescriptor.Label.OPTIONAL - expected.variant = messages.MessageField.DEFAULT_VARIANT - expected.type_name = ('apitools.base.protorpclite.message_types.DateTimeMessage') - - described = descriptor.describe_field(field) - described.check_initialized() - self.assertEquals(expected, described) + field = field_class(10, default=default) + field.name = u'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = u'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = field_class.DEFAULT_VARIANT + expected.default_value = expected_default + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDefault_EnumField(self): + class MyEnum(messages.Enum): + + VAL = 1 + + module_name = test_util.get_module_name(MyEnum) + field = messages.EnumField(MyEnum, 10, default=MyEnum.VAL) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.EnumField.DEFAULT_VARIANT + expected.type_name = '%s.MyEnum' % module_name + expected.default_value = '1' + + described = descriptor.describe_field(field) + self.assertEquals(expected, described) + + def testMessageField(self): + field = messages.MessageField(descriptor.FieldDescriptor, 10) + field.name = 'a_field' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_field' + expected.number = 10 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.MessageField.DEFAULT_VARIANT + expected.type_name = ( + 'apitools.base.protorpclite.descriptor.FieldDescriptor') + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) + + def testDateTimeField(self): + field = message_types.DateTimeField(20) + field.name = 'a_timestamp' + + expected = descriptor.FieldDescriptor() + expected.name = 'a_timestamp' + expected.number = 20 + expected.label = descriptor.FieldDescriptor.Label.OPTIONAL + expected.variant = messages.MessageField.DEFAULT_VARIANT + expected.type_name = ( + 'apitools.base.protorpclite.message_types.DateTimeMessage') + + described = descriptor.describe_field(field) + described.check_initialized() + self.assertEquals(expected, described) class DescribeMessageTest(test_util.TestCase): - def testEmptyDefinition(self): - class MyMessage(messages.Message): - pass + def testEmptyDefinition(self): + class MyMessage(messages.Message): + pass + + expected = descriptor.MessageDescriptor() + expected.name = 'MyMessage' + + described = descriptor.describe_message(MyMessage) + described.check_initialized() + self.assertEquals(expected, described) - expected = descriptor.MessageDescriptor() - expected.name = 'MyMessage' + def testDefinitionWithFields(self): + class MessageWithFields(messages.Message): + field1 = messages.IntegerField(10) + field2 = messages.StringField(30) + field3 = messages.IntegerField(20) - described = descriptor.describe_message(MyMessage) - described.check_initialized() - self.assertEquals(expected, described) + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithFields' - def testDefinitionWithFields(self): - class MessageWithFields(messages.Message): - field1 = messages.IntegerField(10) - field2 = messages.StringField(30) - field3 = messages.IntegerField(20) + expected.fields = [ + descriptor.describe_field( + MessageWithFields.field_by_name('field1')), + descriptor.describe_field( + MessageWithFields.field_by_name('field3')), + descriptor.describe_field( + MessageWithFields.field_by_name('field2')), + ] - expected = descriptor.MessageDescriptor() - expected.name = 'MessageWithFields' + described = descriptor.describe_message(MessageWithFields) + described.check_initialized() + self.assertEquals(expected, described) - expected.fields = [ - descriptor.describe_field(MessageWithFields.field_by_name('field1')), - descriptor.describe_field(MessageWithFields.field_by_name('field3')), - descriptor.describe_field(MessageWithFields.field_by_name('field2')), - ] + def testNestedEnum(self): + class MessageWithEnum(messages.Message): - described = descriptor.describe_message(MessageWithFields) - described.check_initialized() - self.assertEquals(expected, described) + class Mood(messages.Enum): + GOOD = 1 + BAD = 2 + UGLY = 3 - def testNestedEnum(self): - class MessageWithEnum(messages.Message): - class Mood(messages.Enum): - GOOD = 1 - BAD = 2 - UGLY = 3 + class Music(messages.Enum): + CLASSIC = 1 + JAZZ = 2 + BLUES = 3 - class Music(messages.Enum): - CLASSIC = 1 - JAZZ = 2 - BLUES = 3 + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithEnum' - expected = descriptor.MessageDescriptor() - expected.name = 'MessageWithEnum' + expected.enum_types = [descriptor.describe_enum(MessageWithEnum.Mood), + descriptor.describe_enum(MessageWithEnum.Music)] - expected.enum_types = [descriptor.describe_enum(MessageWithEnum.Mood), - descriptor.describe_enum(MessageWithEnum.Music)] + described = descriptor.describe_message(MessageWithEnum) + described.check_initialized() + self.assertEquals(expected, described) - described = descriptor.describe_message(MessageWithEnum) - described.check_initialized() - self.assertEquals(expected, described) + def testNestedMessage(self): + class MessageWithMessage(messages.Message): - def testNestedMessage(self): - class MessageWithMessage(messages.Message): - class Nesty(messages.Message): - pass + class Nesty(messages.Message): + pass - expected = descriptor.MessageDescriptor() - expected.name = 'MessageWithMessage' + expected = descriptor.MessageDescriptor() + expected.name = 'MessageWithMessage' - expected.message_types = [ - descriptor.describe_message(MessageWithMessage.Nesty)] + expected.message_types = [ + descriptor.describe_message(MessageWithMessage.Nesty)] - described = descriptor.describe_message(MessageWithMessage) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_message(MessageWithMessage) + described.check_initialized() + self.assertEquals(expected, described) class DescribeFileTest(test_util.TestCase): - """Test describing modules.""" + """Test describing modules.""" - def LoadModule(self, module_name, source): - result = {'__name__': module_name, - 'messages': messages, - } - exec(source, result) + def LoadModule(self, module_name, source): + result = {'__name__': module_name, + 'messages': messages, + } + exec(source, result) - module = types.ModuleType(module_name) - for name, value in result.items(): - setattr(module, name, value) + module = types.ModuleType(module_name) + for name, value in result.items(): + setattr(module, name, value) - return module + return module - def testEmptyModule(self): - """Test describing an empty file.""" - module = types.ModuleType('my.package.name') + def testEmptyModule(self): + """Test describing an empty file.""" + module = types.ModuleType('my.package.name') - expected = descriptor.FileDescriptor() - expected.package = 'my.package.name' + expected = descriptor.FileDescriptor() + expected.package = 'my.package.name' - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) - def testNoPackageName(self): - """Test describing a module with no module name.""" - module = types.ModuleType('') + def testNoPackageName(self): + """Test describing a module with no module name.""" + module = types.ModuleType('') - expected = descriptor.FileDescriptor() + expected = descriptor.FileDescriptor() - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) - def testPackageName(self): - """Test using the 'package' module attribute.""" - module = types.ModuleType('my.module.name') - module.package = 'my.package.name' + def testPackageName(self): + """Test using the 'package' module attribute.""" + module = types.ModuleType('my.module.name') + module.package = 'my.package.name' - expected = descriptor.FileDescriptor() - expected.package = 'my.package.name' + expected = descriptor.FileDescriptor() + expected.package = 'my.package.name' - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) - def testMain(self): - """Test using the 'package' module attribute.""" - module = types.ModuleType('__main__') - module.__file__ = '/blim/blam/bloom/my_package.py' + def testMain(self): + """Test using the 'package' module attribute.""" + module = types.ModuleType('__main__') + module.__file__ = '/blim/blam/bloom/my_package.py' - expected = descriptor.FileDescriptor() - expected.package = 'my_package' + expected = descriptor.FileDescriptor() + expected.package = 'my_package' - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) - def testMessages(self): - """Test that messages are described.""" - module = self.LoadModule('my.package', - 'class Message1(messages.Message): pass\n' - 'class Message2(messages.Message): pass\n') + def testMessages(self): + """Test that messages are described.""" + module = self.LoadModule('my.package', + 'class Message1(messages.Message): pass\n' + 'class Message2(messages.Message): pass\n') - message1 = descriptor.MessageDescriptor() - message1.name = 'Message1' + message1 = descriptor.MessageDescriptor() + message1.name = 'Message1' - message2 = descriptor.MessageDescriptor() - message2.name = 'Message2' + message2 = descriptor.MessageDescriptor() + message2.name = 'Message2' - expected = descriptor.FileDescriptor() - expected.package = 'my.package' - expected.message_types = [message1, message2] + expected = descriptor.FileDescriptor() + expected.package = 'my.package' + expected.message_types = [message1, message2] - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) - def testEnums(self): - """Test that enums are described.""" - module = self.LoadModule('my.package', - 'class Enum1(messages.Enum): pass\n' - 'class Enum2(messages.Enum): pass\n') + def testEnums(self): + """Test that enums are described.""" + module = self.LoadModule('my.package', + 'class Enum1(messages.Enum): pass\n' + 'class Enum2(messages.Enum): pass\n') - enum1 = descriptor.EnumDescriptor() - enum1.name = 'Enum1' + enum1 = descriptor.EnumDescriptor() + enum1.name = 'Enum1' - enum2 = descriptor.EnumDescriptor() - enum2.name = 'Enum2' + enum2 = descriptor.EnumDescriptor() + enum2.name = 'Enum2' - expected = descriptor.FileDescriptor() - expected.package = 'my.package' - expected.enum_types = [enum1, enum2] + expected = descriptor.FileDescriptor() + expected.package = 'my.package' + expected.enum_types = [enum1, enum2] - described = descriptor.describe_file(module) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file(module) + described.check_initialized() + self.assertEquals(expected, described) class DescribeFileSetTest(test_util.TestCase): - """Test describing multiple modules.""" + """Test describing multiple modules.""" - def testNoModules(self): - """Test what happens when no modules provided.""" - described = descriptor.describe_file_set([]) - described.check_initialized() - # The described FileSet.files will be None. - self.assertEquals(descriptor.FileSet(), described) + def testNoModules(self): + """Test what happens when no modules provided.""" + described = descriptor.describe_file_set([]) + described.check_initialized() + # The described FileSet.files will be None. + self.assertEquals(descriptor.FileSet(), described) - def testWithModules(self): - """Test what happens when no modules provided.""" - modules = [types.ModuleType('package1'), types.ModuleType('package1')] + def testWithModules(self): + """Test what happens when no modules provided.""" + modules = [types.ModuleType('package1'), types.ModuleType('package1')] - file1 = descriptor.FileDescriptor() - file1.package = 'package1' - file2 = descriptor.FileDescriptor() - file2.package = 'package2' + file1 = descriptor.FileDescriptor() + file1.package = 'package1' + file2 = descriptor.FileDescriptor() + file2.package = 'package2' - expected = descriptor.FileSet() - expected.files = [file1, file1] + expected = descriptor.FileSet() + expected.files = [file1, file1] - described = descriptor.describe_file_set(modules) - described.check_initialized() - self.assertEquals(expected, described) + described = descriptor.describe_file_set(modules) + described.check_initialized() + self.assertEquals(expected, described) class DescribeTest(test_util.TestCase): - def testModule(self): - self.assertEquals(descriptor.describe_file(test_util), - descriptor.describe(test_util)) + def testModule(self): + self.assertEquals(descriptor.describe_file(test_util), + descriptor.describe(test_util)) - def testField(self): - self.assertEquals( - descriptor.describe_field(test_util.NestedMessage.a_value), - descriptor.describe(test_util.NestedMessage.a_value)) + def testField(self): + self.assertEquals( + descriptor.describe_field(test_util.NestedMessage.a_value), + descriptor.describe(test_util.NestedMessage.a_value)) - def testEnumValue(self): - self.assertEquals( - descriptor.describe_enum_value( - test_util.OptionalMessage.SimpleEnum.VAL1), - descriptor.describe(test_util.OptionalMessage.SimpleEnum.VAL1)) + def testEnumValue(self): + self.assertEquals( + descriptor.describe_enum_value( + test_util.OptionalMessage.SimpleEnum.VAL1), + descriptor.describe(test_util.OptionalMessage.SimpleEnum.VAL1)) - def testMessage(self): - self.assertEquals(descriptor.describe_message(test_util.NestedMessage), - descriptor.describe(test_util.NestedMessage)) + def testMessage(self): + self.assertEquals(descriptor.describe_message(test_util.NestedMessage), + descriptor.describe(test_util.NestedMessage)) - def testEnum(self): - self.assertEquals( - descriptor.describe_enum(test_util.OptionalMessage.SimpleEnum), - descriptor.describe(test_util.OptionalMessage.SimpleEnum)) + def testEnum(self): + self.assertEquals( + descriptor.describe_enum(test_util.OptionalMessage.SimpleEnum), + descriptor.describe(test_util.OptionalMessage.SimpleEnum)) - def testUndescribable(self): - class NonService(object): + def testUndescribable(self): + class NonService(object): - def fn(self): - pass + def fn(self): + pass - for value in (NonService, - NonService.fn, - 1, - 'string', - 1.2, - None): - self.assertEquals(None, descriptor.describe(value)) + for value in (NonService, + NonService.fn, + 1, + 'string', + 1.2, + None): + self.assertEquals(None, descriptor.describe(value)) class ModuleFinderTest(test_util.TestCase): - def testFindMessage(self): - self.assertEquals( - descriptor.describe_message(descriptor.FileSet), - descriptor.import_descriptor_loader( - 'apitools.base.protorpclite.descriptor.FileSet')) + def testFindMessage(self): + self.assertEquals( + descriptor.describe_message(descriptor.FileSet), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.descriptor.FileSet')) - def testFindField(self): - self.assertEquals( - descriptor.describe_field(descriptor.FileSet.files), - descriptor.import_descriptor_loader( - 'apitools.base.protorpclite.descriptor.FileSet.files')) + def testFindField(self): + self.assertEquals( + descriptor.describe_field(descriptor.FileSet.files), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.descriptor.FileSet.files')) - def testFindEnumValue(self): - self.assertEquals( - descriptor.describe_enum_value(test_util.OptionalMessage.SimpleEnum.VAL1), - descriptor.import_descriptor_loader( - 'apitools.base.protorpclite.test_util.OptionalMessage.SimpleEnum.VAL1')) + def testFindEnumValue(self): + self.assertEquals( + descriptor.describe_enum_value( + test_util.OptionalMessage.SimpleEnum.VAL1), + descriptor.import_descriptor_loader( + 'apitools.base.protorpclite.test_util.OptionalMessage.SimpleEnum.VAL1')) class DescriptorLibraryTest(test_util.TestCase): - def setUp(self): - self.packageless = descriptor.MessageDescriptor() - self.packageless.name = 'Packageless' - self.library = descriptor.DescriptorLibrary( - descriptors={ - 'not.real.Packageless': self.packageless, - 'Packageless': self.packageless, - }) - - def testLookupPackage(self): - self.assertEquals('csv', self.library.lookup_package('csv')) - self.assertEquals('apitools.base.protorpclite', - self.library.lookup_package('apitools.base.protorpclite')) - - def testLookupNonPackages(self): - for name in ('', 'a', - 'apitools.base.protorpclite.descriptor.DescriptorLibrary'): - self.assertRaisesWithRegexpMatch( - messages.DefinitionNotFoundError, - 'Could not find definition for %s' % name, - self.library.lookup_package, name) - - def testNoPackage(self): - self.assertRaisesWithRegexpMatch( - messages.DefinitionNotFoundError, - 'Could not find definition for not.real', - self.library.lookup_package, 'not.real.Packageless') - - self.assertEquals(None, self.library.lookup_package('Packageless')) + def setUp(self): + self.packageless = descriptor.MessageDescriptor() + self.packageless.name = 'Packageless' + self.library = descriptor.DescriptorLibrary( + descriptors={ + 'not.real.Packageless': self.packageless, + 'Packageless': self.packageless, + }) + + def testLookupPackage(self): + self.assertEquals('csv', self.library.lookup_package('csv')) + self.assertEquals('apitools.base.protorpclite', + self.library.lookup_package('apitools.base.protorpclite')) + + def testLookupNonPackages(self): + for name in ('', 'a', + 'apitools.base.protorpclite.descriptor.DescriptorLibrary'): + self.assertRaisesWithRegexpMatch( + messages.DefinitionNotFoundError, + 'Could not find definition for %s' % name, + self.library.lookup_package, name) + + def testNoPackage(self): + self.assertRaisesWithRegexpMatch( + messages.DefinitionNotFoundError, + 'Could not find definition for not.real', + self.library.lookup_package, 'not.real.Packageless') + + self.assertEquals(None, self.library.lookup_package('Packageless')) def main(): - unittest.main() + unittest.main() if __name__ == '__main__': - main() + main() diff --git a/apitools/base/protorpclite/message_types.py b/apitools/base/protorpclite/message_types.py index ec34d17..fb55ddc 100644 --- a/apitools/base/protorpclite/message_types.py +++ b/apitools/base/protorpclite/message_types.py @@ -34,86 +34,88 @@ __all__ = [ 'VoidMessage', ] + class VoidMessage(messages.Message): - """Empty message.""" + """Empty message.""" class DateTimeMessage(messages.Message): - """Message to store/transmit a DateTime. + """Message to store/transmit a DateTime. - Fields: - milliseconds: Milliseconds since Jan 1st 1970 local time. - time_zone_offset: Optional time zone offset, in minutes from UTC. - """ - milliseconds = messages.IntegerField(1, required=True) - time_zone_offset = messages.IntegerField(2) + Fields: + milliseconds: Milliseconds since Jan 1st 1970 local time. + time_zone_offset: Optional time zone offset, in minutes from UTC. + """ + milliseconds = messages.IntegerField(1, required=True) + time_zone_offset = messages.IntegerField(2) class DateTimeField(messages.MessageField): - """Field definition for datetime values. - - Stores a python datetime object as a field. If time zone information is - included in the datetime object, it will be included in - the encoded data when this is encoded/decoded. - """ - - type = datetime.datetime - - message_type = DateTimeMessage + """Field definition for datetime values. - @util.positional(3) - def __init__(self, - number, - **kwargs): - super(DateTimeField, self).__init__(self.message_type, - number, - **kwargs) - - def value_from_message(self, message): - """Convert DateTimeMessage to a datetime. - - Args: - A DateTimeMessage instance. - - Returns: - A datetime instance. + Stores a python datetime object as a field. If time zone information is + included in the datetime object, it will be included in + the encoded data when this is encoded/decoded. """ - message = super(DateTimeField, self).value_from_message(message) - if message.time_zone_offset is None: - return datetime.datetime.utcfromtimestamp(message.milliseconds / 1000.0) - - # Need to subtract the time zone offset, because when we call - # datetime.fromtimestamp, it will add the time zone offset to the - # value we pass. - milliseconds = (message.milliseconds - - 60000 * message.time_zone_offset) - - timezone = util.TimeZoneOffset(message.time_zone_offset) - return datetime.datetime.fromtimestamp(milliseconds / 1000.0, - tz=timezone) - - def value_to_message(self, value): - value = super(DateTimeField, self).value_to_message(value) - # First, determine the delta from the epoch, so we can fill in - # DateTimeMessage's milliseconds field. - if value.tzinfo is None: - time_zone_offset = 0 - local_epoch = datetime.datetime.utcfromtimestamp(0) - else: - time_zone_offset = util.total_seconds(value.tzinfo.utcoffset(value)) - # Determine Jan 1, 1970 local time. - local_epoch = datetime.datetime.fromtimestamp(-time_zone_offset, - tz=value.tzinfo) - delta = value - local_epoch - - # Create and fill in the DateTimeMessage, including time zone if - # one was specified. - message = DateTimeMessage() - message.milliseconds = int(util.total_seconds(delta) * 1000) - if value.tzinfo is not None: - utc_offset = value.tzinfo.utcoffset(value) - if utc_offset is not None: - message.time_zone_offset = int( - util.total_seconds(value.tzinfo.utcoffset(value)) / 60) - - return message + + type = datetime.datetime + + message_type = DateTimeMessage + + @util.positional(3) + def __init__(self, + number, + **kwargs): + super(DateTimeField, self).__init__(self.message_type, + number, + **kwargs) + + def value_from_message(self, message): + """Convert DateTimeMessage to a datetime. + + Args: + A DateTimeMessage instance. + + Returns: + A datetime instance. + """ + message = super(DateTimeField, self).value_from_message(message) + if message.time_zone_offset is None: + return datetime.datetime.utcfromtimestamp(message.milliseconds / 1000.0) + + # Need to subtract the time zone offset, because when we call + # datetime.fromtimestamp, it will add the time zone offset to the + # value we pass. + milliseconds = (message.milliseconds - + 60000 * message.time_zone_offset) + + timezone = util.TimeZoneOffset(message.time_zone_offset) + return datetime.datetime.fromtimestamp(milliseconds / 1000.0, + tz=timezone) + + def value_to_message(self, value): + value = super(DateTimeField, self).value_to_message(value) + # First, determine the delta from the epoch, so we can fill in + # DateTimeMessage's milliseconds field. + if value.tzinfo is None: + time_zone_offset = 0 + local_epoch = datetime.datetime.utcfromtimestamp(0) + else: + time_zone_offset = util.total_seconds( + value.tzinfo.utcoffset(value)) + # Determine Jan 1, 1970 local time. + local_epoch = datetime.datetime.fromtimestamp(-time_zone_offset, + tz=value.tzinfo) + delta = value - local_epoch + + # Create and fill in the DateTimeMessage, including time zone if + # one was specified. + message = DateTimeMessage() + message.milliseconds = int(util.total_seconds(delta) * 1000) + if value.tzinfo is not None: + utc_offset = value.tzinfo.utcoffset(value) + if utc_offset is not None: + message.time_zone_offset = int( + util.total_seconds(value.tzinfo.utcoffset(value)) / 60) + + return message diff --git a/apitools/base/protorpclite/message_types_test.py b/apitools/base/protorpclite/message_types_test.py index 33be248..39f1f4a 100644 --- a/apitools/base/protorpclite/message_types_test.py +++ b/apitools/base/protorpclite/message_types_test.py @@ -33,56 +33,57 @@ from apitools.base.protorpclite import util class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): - MODULE = message_types + MODULE = message_types class DateTimeFieldTest(test_util.TestCase): - def testValueToMessage(self): - field = message_types.DateTimeField(1) - message = field.value_to_message(datetime.datetime(2033, 2, 4, 11, 22, 10)) - self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000), - message) - - def testValueToMessageBadValue(self): - field = message_types.DateTimeField(1) - self.assertRaisesWithRegexpMatch( - messages.EncodeError, - 'Expected type datetime, got int: 20', - field.value_to_message, 20) - - def testValueToMessageWithTimeZone(self): - time_zone = util.TimeZoneOffset(60 * 10) - field = message_types.DateTimeField(1) - message = field.value_to_message( - datetime.datetime(2033, 2, 4, 11, 22, 10, tzinfo=time_zone)) - self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000, - time_zone_offset=600), - message) - - def testValueFromMessage(self): - message = message_types.DateTimeMessage(milliseconds=1991128000000) - field = message_types.DateTimeField(1) - timestamp = field.value_from_message(message) - self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40), - timestamp) - - def testValueFromMessageBadValue(self): - field = message_types.DateTimeField(1) - self.assertRaisesWithRegexpMatch( - messages.DecodeError, - 'Expected type DateTimeMessage, got VoidMessage: ', - field.value_from_message, message_types.VoidMessage()) - - def testValueFromMessageWithTimeZone(self): - message = message_types.DateTimeMessage(milliseconds=1991128000000, - time_zone_offset=300) - field = message_types.DateTimeField(1) - timestamp = field.value_from_message(message) - time_zone = util.TimeZoneOffset(60 * 5) - self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40, tzinfo=time_zone), - timestamp) + def testValueToMessage(self): + field = message_types.DateTimeField(1) + message = field.value_to_message( + datetime.datetime(2033, 2, 4, 11, 22, 10)) + self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000), + message) + + def testValueToMessageBadValue(self): + field = message_types.DateTimeField(1) + self.assertRaisesWithRegexpMatch( + messages.EncodeError, + 'Expected type datetime, got int: 20', + field.value_to_message, 20) + + def testValueToMessageWithTimeZone(self): + time_zone = util.TimeZoneOffset(60 * 10) + field = message_types.DateTimeField(1) + message = field.value_to_message( + datetime.datetime(2033, 2, 4, 11, 22, 10, tzinfo=time_zone)) + self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000, + time_zone_offset=600), + message) + + def testValueFromMessage(self): + message = message_types.DateTimeMessage(milliseconds=1991128000000) + field = message_types.DateTimeField(1) + timestamp = field.value_from_message(message) + self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40), + timestamp) + + def testValueFromMessageBadValue(self): + field = message_types.DateTimeField(1) + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Expected type DateTimeMessage, got VoidMessage: ', + field.value_from_message, message_types.VoidMessage()) + + def testValueFromMessageWithTimeZone(self): + message = message_types.DateTimeMessage(milliseconds=1991128000000, + time_zone_offset=300) + field = message_types.DateTimeField(1) + timestamp = field.value_from_message(message) + time_zone = util.TimeZoneOffset(60 * 5) + self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40, tzinfo=time_zone), + timestamp) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index a78b7df..196a640 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -81,7 +81,7 @@ __all__ = ['MAX_ENUM_VALUE', 'DuplicateNumberError', 'ValidationError', 'DefinitionNotFoundError', - ] + ] # TODO(rafek): Add extended module test to ensure all exceptions @@ -90,57 +90,57 @@ Error = util.Error class EnumDefinitionError(Error): - """Enumeration definition error.""" + """Enumeration definition error.""" class FieldDefinitionError(Error): - """Field definition error.""" + """Field definition error.""" class InvalidVariantError(FieldDefinitionError): - """Invalid variant provided to field.""" + """Invalid variant provided to field.""" class InvalidDefaultError(FieldDefinitionError): - """Invalid default provided to field.""" + """Invalid default provided to field.""" class InvalidNumberError(FieldDefinitionError): - """Invalid number provided to field.""" + """Invalid number provided to field.""" class MessageDefinitionError(Error): - """Message definition error.""" + """Message definition error.""" class DuplicateNumberError(Error): - """Duplicate number assigned to field.""" + """Duplicate number assigned to field.""" class DefinitionNotFoundError(Error): - """Raised when definition is not found.""" + """Raised when definition is not found.""" class DecodeError(Error): - """Error found decoding message from encoded form.""" + """Error found decoding message from encoded form.""" class EncodeError(Error): - """Error found when encoding message.""" + """Error found when encoding message.""" class ValidationError(Error): - """Invalid value for message error.""" + """Invalid value for message error.""" - def __str__(self): - """Prints string with field name if present on exception.""" - message = Error.__str__(self) - try: - field_name = self.field_name - except AttributeError: - return message - else: - return message + def __str__(self): + """Prints string with field name if present on exception.""" + message = Error.__str__(self) + try: + field_name = self.field_name + except AttributeError: + return message + else: + return message # Attributes that are reserved by a class definition that @@ -173,370 +173,370 @@ LAST_RESERVED_FIELD_NUMBER = 19999 class _DefinitionClass(type): - """Base meta-class used for definition meta-classes. + """Base meta-class used for definition meta-classes. - The Enum and Message definition classes share some basic functionality. - Both of these classes may be contained by a Message definition. After - initialization, neither class may have attributes changed - except for the protected _message_definition attribute, and that attribute - may change only once. - """ + The Enum and Message definition classes share some basic functionality. + Both of these classes may be contained by a Message definition. After + initialization, neither class may have attributes changed + except for the protected _message_definition attribute, and that attribute + may change only once. + """ - __initialized = False + __initialized = False - def __init__(cls, name, bases, dct): - """Constructor.""" - type.__init__(cls, name, bases, dct) - # Base classes may never be initialized. - if cls.__bases__ != (object,): - cls.__initialized = True + def __init__(cls, name, bases, dct): + """Constructor.""" + type.__init__(cls, name, bases, dct) + # Base classes may never be initialized. + if cls.__bases__ != (object,): + cls.__initialized = True - def message_definition(cls): - """Get outer Message definition that contains this definition. + def message_definition(cls): + """Get outer Message definition that contains this definition. - Returns: - Containing Message definition if definition is contained within one, - else None. - """ - try: - return cls._message_definition() - except AttributeError: - return None + Returns: + Containing Message definition if definition is contained within one, + else None. + """ + try: + return cls._message_definition() + except AttributeError: + return None - def __setattr__(cls, name, value): - """Overridden so that cannot set variables on definition classes after init. + def __setattr__(cls, name, value): + """Overridden so that cannot set variables on definition classes after init. - Setting attributes on a class must work during the period of initialization - to set the enumation value class variables and build the name/number maps. - Once __init__ has set the __initialized flag to True prohibits setting any - more values on the class. The class is in effect frozen. + Setting attributes on a class must work during the period of initialization + to set the enumation value class variables and build the name/number maps. + Once __init__ has set the __initialized flag to True prohibits setting any + more values on the class. The class is in effect frozen. - Args: - name: Name of value to set. - value: Value to set. - """ - if cls.__initialized and name not in _POST_INIT_ATTRIBUTE_NAMES: - raise AttributeError('May not change values: %s' % name) - else: - type.__setattr__(cls, name, value) + Args: + name: Name of value to set. + value: Value to set. + """ + if cls.__initialized and name not in _POST_INIT_ATTRIBUTE_NAMES: + raise AttributeError('May not change values: %s' % name) + else: + type.__setattr__(cls, name, value) - def __delattr__(cls, name): - """Overridden so that cannot delete varaibles on definition classes.""" - raise TypeError('May not delete attributes on definition class') + def __delattr__(cls, name): + """Overridden so that cannot delete varaibles on definition classes.""" + raise TypeError('May not delete attributes on definition class') - def definition_name(cls): - """Helper method for creating definition name. + def definition_name(cls): + """Helper method for creating definition name. - Names will be generated to include the classes package name, scope (if the - class is nested in another definition) and class name. + Names will be generated to include the classes package name, scope (if the + class is nested in another definition) and class name. - By default, the package name for a definition is derived from its module - name. However, this value can be overriden by placing a 'package' attribute - in the module that contains the definition class. For example: + By default, the package name for a definition is derived from its module + name. However, this value can be overriden by placing a 'package' attribute + in the module that contains the definition class. For example: - package = 'some.alternate.package' + package = 'some.alternate.package' - class MyMessage(Message): - ... + class MyMessage(Message): + ... - >>> MyMessage.definition_name() - some.alternate.package.MyMessage + >>> MyMessage.definition_name() + some.alternate.package.MyMessage - Returns: - Dot-separated fully qualified name of definition. - """ - outer_definition_name = cls.outer_definition_name() - if outer_definition_name is None: - return six.text_type(cls.__name__) - else: - return u'%s.%s' % (outer_definition_name, cls.__name__) + Returns: + Dot-separated fully qualified name of definition. + """ + outer_definition_name = cls.outer_definition_name() + if outer_definition_name is None: + return six.text_type(cls.__name__) + else: + return u'%s.%s' % (outer_definition_name, cls.__name__) + + def outer_definition_name(cls): + """Helper method for creating outer definition name. + + Returns: + If definition is nested, will return the outer definitions name, else the + package name. + """ + outer_definition = cls.message_definition() + if not outer_definition: + return util.get_package_for_module(cls.__module__) + else: + return outer_definition.definition_name() - def outer_definition_name(cls): - """Helper method for creating outer definition name. + def definition_package(cls): + """Helper method for creating creating the package of a definition. - Returns: - If definition is nested, will return the outer definitions name, else the - package name. - """ - outer_definition = cls.message_definition() - if not outer_definition: - return util.get_package_for_module(cls.__module__) - else: - return outer_definition.definition_name() + Returns: + Name of package that definition belongs to. + """ + outer_definition = cls.message_definition() + if not outer_definition: + return util.get_package_for_module(cls.__module__) + else: + return outer_definition.definition_package() - def definition_package(cls): - """Helper method for creating creating the package of a definition. - Returns: - Name of package that definition belongs to. +class _EnumClass(_DefinitionClass): + """Meta-class used for defining the Enum base class. + + Meta-class enables very specific behavior for any defined Enum + class. All attributes defined on an Enum sub-class must be integers. + Each attribute defined on an Enum sub-class is translated + into an instance of that sub-class, with the name of the attribute + as its name, and the number provided as its value. It also ensures + that only one level of Enum class hierarchy is possible. In other + words it is not possible to delcare sub-classes of sub-classes of + Enum. + + This class also defines some functions in order to restrict the + behavior of the Enum class and its sub-classes. It is not possible + to change the behavior of the Enum class in later classes since + any new classes may be defined with only integer values, and no methods. """ - outer_definition = cls.message_definition() - if not outer_definition: - return util.get_package_for_module(cls.__module__) - else: - return outer_definition.definition_package() + def __init__(cls, name, bases, dct): + # Can only define one level of sub-classes below Enum. + if not (bases == (object,) or bases == (Enum,)): + raise EnumDefinitionError('Enum type %s may only inherit from Enum' % + (name,)) -class _EnumClass(_DefinitionClass): - """Meta-class used for defining the Enum base class. - - Meta-class enables very specific behavior for any defined Enum - class. All attributes defined on an Enum sub-class must be integers. - Each attribute defined on an Enum sub-class is translated - into an instance of that sub-class, with the name of the attribute - as its name, and the number provided as its value. It also ensures - that only one level of Enum class hierarchy is possible. In other - words it is not possible to delcare sub-classes of sub-classes of - Enum. - - This class also defines some functions in order to restrict the - behavior of the Enum class and its sub-classes. It is not possible - to change the behavior of the Enum class in later classes since - any new classes may be defined with only integer values, and no methods. - """ - - def __init__(cls, name, bases, dct): - # Can only define one level of sub-classes below Enum. - if not (bases == (object,) or bases == (Enum,)): - raise EnumDefinitionError('Enum type %s may only inherit from Enum' % - (name,)) - - cls.__by_number = {} - cls.__by_name = {} - - # Enum base class does not need to be initialized or locked. - if bases != (object,): - # Replace integer with number. - for attribute, value in dct.items(): - - # Module will be in every enum class. - if attribute in _RESERVED_ATTRIBUTE_NAMES: - continue - - # Reject anything that is not an int. - if not isinstance(value, six.integer_types): - raise EnumDefinitionError( - 'May only use integers in Enum definitions. Found: %s = %s' % - (attribute, value)) - - # Protocol buffer standard recommends non-negative values. - # Reject negative values. - if value < 0: - raise EnumDefinitionError( - 'Must use non-negative enum values. Found: %s = %d' % - (attribute, value)) - - if value > MAX_ENUM_VALUE: - raise EnumDefinitionError( - 'Must use enum values less than or equal %d. Found: %s = %d' % - (MAX_ENUM_VALUE, attribute, value)) - - if value in cls.__by_number: - raise EnumDefinitionError( - 'Value for %s = %d is already defined: %s' % - (attribute, value, cls.__by_number[value].name)) - - # Create enum instance and list in new Enum type. - instance = object.__new__(cls) - cls.__init__(instance, attribute, value) - cls.__by_name[instance.name] = instance - cls.__by_number[instance.number] = instance - setattr(cls, attribute, instance) - - _DefinitionClass.__init__(cls, name, bases, dct) - - def __iter__(cls): - """Iterate over all values of enum. - - Yields: - Enumeration instances of the Enum class in arbitrary order. - """ - return iter(cls.__by_number.values()) + cls.__by_number = {} + cls.__by_name = {} - def names(cls): - """Get all names for Enum. + # Enum base class does not need to be initialized or locked. + if bases != (object,): + # Replace integer with number. + for attribute, value in dct.items(): - Returns: - An iterator for names of the enumeration in arbitrary order. - """ - return cls.__by_name.keys() + # Module will be in every enum class. + if attribute in _RESERVED_ATTRIBUTE_NAMES: + continue - def numbers(cls): - """Get all numbers for Enum. + # Reject anything that is not an int. + if not isinstance(value, six.integer_types): + raise EnumDefinitionError( + 'May only use integers in Enum definitions. Found: %s = %s' % + (attribute, value)) - Returns: - An iterator for all numbers of the enumeration in arbitrary order. - """ - return cls.__by_number.keys() + # Protocol buffer standard recommends non-negative values. + # Reject negative values. + if value < 0: + raise EnumDefinitionError( + 'Must use non-negative enum values. Found: %s = %d' % + (attribute, value)) - def lookup_by_name(cls, name): - """Look up Enum by name. + if value > MAX_ENUM_VALUE: + raise EnumDefinitionError( + 'Must use enum values less than or equal %d. Found: %s = %d' % + (MAX_ENUM_VALUE, attribute, value)) - Args: - name: Name of enum to find. + if value in cls.__by_number: + raise EnumDefinitionError( + 'Value for %s = %d is already defined: %s' % + (attribute, value, cls.__by_number[value].name)) - Returns: - Enum sub-class instance of that value. - """ - return cls.__by_name[name] + # Create enum instance and list in new Enum type. + instance = object.__new__(cls) + cls.__init__(instance, attribute, value) + cls.__by_name[instance.name] = instance + cls.__by_number[instance.number] = instance + setattr(cls, attribute, instance) - def lookup_by_number(cls, number): - """Look up Enum by number. + _DefinitionClass.__init__(cls, name, bases, dct) - Args: - number: Number of enum to find. + def __iter__(cls): + """Iterate over all values of enum. - Returns: - Enum sub-class instance of that value. - """ - return cls.__by_number[number] + Yields: + Enumeration instances of the Enum class in arbitrary order. + """ + return iter(cls.__by_number.values()) - def __len__(cls): - return len(cls.__by_name) + def names(cls): + """Get all names for Enum. + Returns: + An iterator for names of the enumeration in arbitrary order. + """ + return cls.__by_name.keys() -class Enum(six.with_metaclass(_EnumClass, object)): - """Base class for all enumerated types.""" + def numbers(cls): + """Get all numbers for Enum. - __slots__ = set(('name', 'number')) + Returns: + An iterator for all numbers of the enumeration in arbitrary order. + """ + return cls.__by_number.keys() - def __new__(cls, index): - """Acts as look-up routine after class is initialized. + def lookup_by_name(cls, name): + """Look up Enum by name. - The purpose of overriding __new__ is to provide a way to treat - Enum subclasses as casting types, similar to how the int type - functions. A program can pass a string or an integer and this - method with "convert" that value in to an appropriate Enum instance. + Args: + name: Name of enum to find. - Args: - index: Name or number to look up. During initialization - this is always the name of the new enum value. + Returns: + Enum sub-class instance of that value. + """ + return cls.__by_name[name] - Raises: - TypeError: When an inappropriate index value is passed provided. - """ - # If is enum type of this class, return it. - if isinstance(index, cls): - return index - - # If number, look up by number. - if isinstance(index, six.integer_types): - try: - return cls.lookup_by_number(index) - except KeyError: - pass - - # If name, look up by name. - if isinstance(index, six.string_types): - try: - return cls.lookup_by_name(index) - except KeyError: - pass - - raise TypeError('No such value for %s in Enum %s' % - (index, cls.__name__)) - - def __init__(self, name, number=None): - """Initialize new Enum instance. - - Since this should only be called during class initialization any - calls that happen after the class is frozen raises an exception. - """ - # Immediately return if __init__ was called after _Enum.__init__(). - # It means that casting operator version of the class constructor - # is being used. - if getattr(type(self), '_DefinitionClass__initialized'): - return - object.__setattr__(self, 'name', name) - object.__setattr__(self, 'number', number) + def lookup_by_number(cls, number): + """Look up Enum by number. - def __setattr__(self, name, value): - raise TypeError('May not change enum values') + Args: + number: Number of enum to find. - def __str__(self): - return self.name + Returns: + Enum sub-class instance of that value. + """ + return cls.__by_number[number] - def __int__(self): - return self.number + def __len__(cls): + return len(cls.__by_name) - def __repr__(self): - return '%s(%s, %d)' % (type(self).__name__, self.name, self.number) - def __reduce__(self): - """Enable pickling. +class Enum(six.with_metaclass(_EnumClass, object)): + """Base class for all enumerated types.""" - Returns: - A 2-tuple containing the class and __new__ args to be used for restoring - a pickled instance. - """ - return self.__class__, (self.number,) - - def __cmp__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return cmp(self.number, other.number) - return NotImplemented - - def __lt__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number < other.number - return NotImplemented - - def __le__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number <= other.number - return NotImplemented - - def __eq__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number == other.number - return NotImplemented - - def __ne__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number != other.number - return NotImplemented - - def __ge__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number >= other.number - return NotImplemented - - def __gt__(self, other): - """Order is by number.""" - if isinstance(other, type(self)): - return self.number > other.number - return NotImplemented - - def __hash__(self): - """Hash by number.""" - return hash(self.number) - - @classmethod - def to_dict(cls): - """Make dictionary version of enumerated class. - - Dictionary created this way can be used with def_num. + __slots__ = set(('name', 'number')) - Returns: - A dict (name) -> number - """ - return dict((item.name, item.number) for item in iter(cls)) + def __new__(cls, index): + """Acts as look-up routine after class is initialized. - @staticmethod - def def_enum(dct, name): - """Define enum class from dictionary. + The purpose of overriding __new__ is to provide a way to treat + Enum subclasses as casting types, similar to how the int type + functions. A program can pass a string or an integer and this + method with "convert" that value in to an appropriate Enum instance. - Args: - dct: Dictionary of enumerated values for type. - name: Name of enum. - """ - return type(name, (Enum,), dct) + Args: + index: Name or number to look up. During initialization + this is always the name of the new enum value. + + Raises: + TypeError: When an inappropriate index value is passed provided. + """ + # If is enum type of this class, return it. + if isinstance(index, cls): + return index + + # If number, look up by number. + if isinstance(index, six.integer_types): + try: + return cls.lookup_by_number(index) + except KeyError: + pass + + # If name, look up by name. + if isinstance(index, six.string_types): + try: + return cls.lookup_by_name(index) + except KeyError: + pass + + raise TypeError('No such value for %s in Enum %s' % + (index, cls.__name__)) + + def __init__(self, name, number=None): + """Initialize new Enum instance. + + Since this should only be called during class initialization any + calls that happen after the class is frozen raises an exception. + """ + # Immediately return if __init__ was called after _Enum.__init__(). + # It means that casting operator version of the class constructor + # is being used. + if getattr(type(self), '_DefinitionClass__initialized'): + return + object.__setattr__(self, 'name', name) + object.__setattr__(self, 'number', number) + + def __setattr__(self, name, value): + raise TypeError('May not change enum values') + + def __str__(self): + return self.name + + def __int__(self): + return self.number + + def __repr__(self): + return '%s(%s, %d)' % (type(self).__name__, self.name, self.number) + + def __reduce__(self): + """Enable pickling. + + Returns: + A 2-tuple containing the class and __new__ args to be used for restoring + a pickled instance. + """ + return self.__class__, (self.number,) + + def __cmp__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return cmp(self.number, other.number) + return NotImplemented + + def __lt__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number < other.number + return NotImplemented + + def __le__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number <= other.number + return NotImplemented + + def __eq__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number == other.number + return NotImplemented + + def __ne__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number != other.number + return NotImplemented + + def __ge__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number >= other.number + return NotImplemented + + def __gt__(self, other): + """Order is by number.""" + if isinstance(other, type(self)): + return self.number > other.number + return NotImplemented + + def __hash__(self): + """Hash by number.""" + return hash(self.number) + + @classmethod + def to_dict(cls): + """Make dictionary version of enumerated class. + + Dictionary created this way can be used with def_num. + + Returns: + A dict (name) -> number + """ + return dict((item.name, item.number) for item in iter(cls)) + + @staticmethod + def def_enum(dct, name): + """Define enum class from dictionary. + + Args: + dct: Dictionary of enumerated values for type. + name: Name of enum. + """ + return type(name, (Enum,), dct) # TODO(rafek): Determine to what degree this enumeration should be compatible @@ -544,1392 +544,1400 @@ class Enum(six.with_metaclass(_EnumClass, object)): # # http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto class Variant(Enum): - """Wire format variant. - - Used by the 'protobuf' wire format to determine how to transmit - a single piece of data. May be used by other formats. - - See: http://code.google.com/apis/protocolbuffers/docs/encoding.html - - Values: - DOUBLE: 64-bit floating point number. - FLOAT: 32-bit floating point number. - INT64: 64-bit signed integer. - UINT64: 64-bit unsigned integer. - INT32: 32-bit signed integer. - BOOL: Boolean value (True or False). - STRING: String of UTF-8 encoded text. - MESSAGE: Embedded message as byte string. - BYTES: String of 8-bit bytes. - UINT32: 32-bit unsigned integer. - ENUM: Enum value as integer. - SINT32: 32-bit signed integer. Uses "zig-zag" encoding. - SINT64: 64-bit signed integer. Uses "zig-zag" encoding. - """ - DOUBLE = 1 - FLOAT = 2 - INT64 = 3 - UINT64 = 4 - INT32 = 5 - BOOL = 8 - STRING = 9 - MESSAGE = 11 - BYTES = 12 - UINT32 = 13 - ENUM = 14 - SINT32 = 17 - SINT64 = 18 + """Wire format variant. + + Used by the 'protobuf' wire format to determine how to transmit + a single piece of data. May be used by other formats. + + See: http://code.google.com/apis/protocolbuffers/docs/encoding.html + + Values: + DOUBLE: 64-bit floating point number. + FLOAT: 32-bit floating point number. + INT64: 64-bit signed integer. + UINT64: 64-bit unsigned integer. + INT32: 32-bit signed integer. + BOOL: Boolean value (True or False). + STRING: String of UTF-8 encoded text. + MESSAGE: Embedded message as byte string. + BYTES: String of 8-bit bytes. + UINT32: 32-bit unsigned integer. + ENUM: Enum value as integer. + SINT32: 32-bit signed integer. Uses "zig-zag" encoding. + SINT64: 64-bit signed integer. Uses "zig-zag" encoding. + """ + DOUBLE = 1 + FLOAT = 2 + INT64 = 3 + UINT64 = 4 + INT32 = 5 + BOOL = 8 + STRING = 9 + MESSAGE = 11 + BYTES = 12 + UINT32 = 13 + ENUM = 14 + SINT32 = 17 + SINT64 = 18 class _MessageClass(_DefinitionClass): - """Meta-class used for defining the Message base class. - - For more details about Message classes, see the Message class docstring. - Information contained there may help understanding this class. - - Meta-class enables very specific behavior for any defined Message - class. All attributes defined on an Message sub-class must be field - instances, Enum class definitions or other Message class definitions. Each - field attribute defined on an Message sub-class is added to the set of - field definitions and the attribute is translated in to a slot. It also - ensures that only one level of Message class hierarchy is possible. In other - words it is not possible to declare sub-classes of sub-classes of - Message. - - This class also defines some functions in order to restrict the - behavior of the Message class and its sub-classes. It is not possible - to change the behavior of the Message class in later classes since - any new classes may be defined with only field, Enums and Messages, and - no methods. - """ - - def __new__(cls, name, bases, dct): - """Create new Message class instance. - - The __new__ method of the _MessageClass type is overridden so as to - allow the translation of Field instances to slots. + """Meta-class used for defining the Message base class. + + For more details about Message classes, see the Message class docstring. + Information contained there may help understanding this class. + + Meta-class enables very specific behavior for any defined Message + class. All attributes defined on an Message sub-class must be field + instances, Enum class definitions or other Message class definitions. Each + field attribute defined on an Message sub-class is added to the set of + field definitions and the attribute is translated in to a slot. It also + ensures that only one level of Message class hierarchy is possible. In other + words it is not possible to declare sub-classes of sub-classes of + Message. + + This class also defines some functions in order to restrict the + behavior of the Message class and its sub-classes. It is not possible + to change the behavior of the Message class in later classes since + any new classes may be defined with only field, Enums and Messages, and + no methods. """ - by_number = {} - by_name = {} - - variant_map = {} - - if bases != (object,): - # Can only define one level of sub-classes below Message. - if bases != (Message,): - raise MessageDefinitionError( - 'Message types may only inherit from Message') - enums = [] - messages = [] - # Must not use iteritems because this loop will change the state of dct. - for key, field in dct.items(): + def __new__(cls, name, bases, dct): + """Create new Message class instance. - if key in _RESERVED_ATTRIBUTE_NAMES: - continue + The __new__ method of the _MessageClass type is overridden so as to + allow the translation of Field instances to slots. + """ + by_number = {} + by_name = {} - if isinstance(field, type) and issubclass(field, Enum): - enums.append(key) - continue + variant_map = {} - if (isinstance(field, type) and - issubclass(field, Message) and - field is not Message): - messages.append(key) - continue + if bases != (object,): + # Can only define one level of sub-classes below Message. + if bases != (Message,): + raise MessageDefinitionError( + 'Message types may only inherit from Message') - # Reject anything that is not a field. - if type(field) is Field or not isinstance(field, Field): - raise MessageDefinitionError( - 'May only use fields in message definitions. Found: %s = %s' % - (key, field)) + enums = [] + messages = [] + # Must not use iteritems because this loop will change the state of + # dct. + for key, field in dct.items(): - if field.number in by_number: - raise DuplicateNumberError( - 'Field with number %d declared more than once in %s' % - (field.number, name)) + if key in _RESERVED_ATTRIBUTE_NAMES: + continue - field.name = key + if isinstance(field, type) and issubclass(field, Enum): + enums.append(key) + continue - # Place in name and number maps. - by_name[key] = field - by_number[field.number] = field + if (isinstance(field, type) and + issubclass(field, Message) and + field is not Message): + messages.append(key) + continue - # Add enums if any exist. - if enums: - dct['__enums__'] = sorted(enums) + # Reject anything that is not a field. + if type(field) is Field or not isinstance(field, Field): + raise MessageDefinitionError( + 'May only use fields in message definitions. Found: %s = %s' % + (key, field)) - # Add messages if any exist. - if messages: - dct['__messages__'] = sorted(messages) + if field.number in by_number: + raise DuplicateNumberError( + 'Field with number %d declared more than once in %s' % + (field.number, name)) - dct['_Message__by_number'] = by_number - dct['_Message__by_name'] = by_name + field.name = key - return _DefinitionClass.__new__(cls, name, bases, dct) + # Place in name and number maps. + by_name[key] = field + by_number[field.number] = field - def __init__(cls, name, bases, dct): - """Initializer required to assign references to new class.""" - if bases != (object,): - for value in dct.values(): - if isinstance(value, _DefinitionClass) and not value is Message: - value._message_definition = weakref.ref(cls) + # Add enums if any exist. + if enums: + dct['__enums__'] = sorted(enums) - for field in cls.all_fields(): - field._message_definition = weakref.ref(cls) - - _DefinitionClass.__init__(cls, name, bases, dct) - - -class Message(six.with_metaclass(_MessageClass, object)): - """Base class for user defined message objects. + # Add messages if any exist. + if messages: + dct['__messages__'] = sorted(messages) - Used to define messages for efficient transmission across network or - process space. Messages are defined using the field classes (IntegerField, - FloatField, EnumField, etc.). + dct['_Message__by_number'] = by_number + dct['_Message__by_name'] = by_name - Messages are more restricted than normal classes in that they may only - contain field attributes and other Message and Enum definitions. These - restrictions are in place because the structure of the Message class is - intentended to itself be transmitted across network or process space and - used directly by clients or even other servers. As such methods and - non-field attributes could not be transmitted with the structural information - causing discrepancies between different languages and implementations. + return _DefinitionClass.__new__(cls, name, bases, dct) - Initialization and validation: + def __init__(cls, name, bases, dct): + """Initializer required to assign references to new class.""" + if bases != (object,): + for value in dct.values(): + if isinstance(value, _DefinitionClass) and not value is Message: + value._message_definition = weakref.ref(cls) - A Message object is considered to be initialized if it has all required - fields and any nested messages are also initialized. + for field in cls.all_fields(): + field._message_definition = weakref.ref(cls) - Calling 'check_initialized' will raise a ValidationException if it is not - initialized; 'is_initialized' returns a boolean value indicating if it is - valid. + _DefinitionClass.__init__(cls, name, bases, dct) - Validation automatically occurs when Message objects are created - and populated. Validation that a given value will be compatible with - a field that it is assigned to can be done through the Field instances - validate() method. The validate method used on a message will check that - all values of a message and its sub-messages are valid. Assingning an - invalid value to a field will raise a ValidationException. - Example: +class Message(six.with_metaclass(_MessageClass, object)): + """Base class for user defined message objects. - # Trade type. - class TradeType(Enum): - BUY = 1 - SELL = 2 - SHORT = 3 - CALL = 4 + Used to define messages for efficient transmission across network or + process space. Messages are defined using the field classes (IntegerField, + FloatField, EnumField, etc.). - class Lot(Message): - price = IntegerField(1, required=True) - quantity = IntegerField(2, required=True) + Messages are more restricted than normal classes in that they may only + contain field attributes and other Message and Enum definitions. These + restrictions are in place because the structure of the Message class is + intentended to itself be transmitted across network or process space and + used directly by clients or even other servers. As such methods and + non-field attributes could not be transmitted with the structural information + causing discrepancies between different languages and implementations. - class Order(Message): - symbol = StringField(1, required=True) - total_quantity = IntegerField(2, required=True) - trade_type = EnumField(TradeType, 3, required=True) - lots = MessageField(Lot, 4, repeated=True) - limit = IntegerField(5) + Initialization and validation: - order = Order(symbol='GOOG', - total_quantity=10, - trade_type=TradeType.BUY) + A Message object is considered to be initialized if it has all required + fields and any nested messages are also initialized. - lot1 = Lot(price=304, - quantity=7) + Calling 'check_initialized' will raise a ValidationException if it is not + initialized; 'is_initialized' returns a boolean value indicating if it is + valid. - lot2 = Lot(price = 305, - quantity=3) + Validation automatically occurs when Message objects are created + and populated. Validation that a given value will be compatible with + a field that it is assigned to can be done through the Field instances + validate() method. The validate method used on a message will check that + all values of a message and its sub-messages are valid. Assingning an + invalid value to a field will raise a ValidationException. - order.lots = [lot1, lot2] + Example: - # Now object is initialized! - order.check_initialized() - """ + # Trade type. + class TradeType(Enum): + BUY = 1 + SELL = 2 + SHORT = 3 + CALL = 4 - def __init__(self, **kwargs): - """Initialize internal messages state. + class Lot(Message): + price = IntegerField(1, required=True) + quantity = IntegerField(2, required=True) - Args: - A message can be initialized via the constructor by passing in keyword - arguments corresponding to fields. For example: + class Order(Message): + symbol = StringField(1, required=True) + total_quantity = IntegerField(2, required=True) + trade_type = EnumField(TradeType, 3, required=True) + lots = MessageField(Lot, 4, repeated=True) + limit = IntegerField(5) - class Date(Message): - day = IntegerField(1) - month = IntegerField(2) - year = IntegerField(3) + order = Order(symbol='GOOG', + total_quantity=10, + trade_type=TradeType.BUY) - Invoking: + lot1 = Lot(price=304, + quantity=7) - date = Date(day=6, month=6, year=1911) + lot2 = Lot(price = 305, + quantity=3) - is the same as doing: + order.lots = [lot1, lot2] - date = Date() - date.day = 6 - date.month = 6 - date.year = 1911 + # Now object is initialized! + order.check_initialized() """ - # Tag being an essential implementation detail must be private. - self.__tags = {} - self.__unrecognized_fields = {} - - assigned = set() - for name, value in kwargs.items(): - setattr(self, name, value) - assigned.add(name) - - # initialize repeated fields. - for field in self.all_fields(): - if field.repeated and field.name not in assigned: - setattr(self, field.name, []) - - - def check_initialized(self): - """Check class for initialization status. - - Check that all required fields are initialized - Raises: - ValidationError: If message is not initialized. - """ - for name, field in self.__by_name.items(): - value = getattr(self, name) - if value is None: - if field.required: - raise ValidationError("Message %s is missing required field %s" % - (type(self).__name__, name)) - else: - try: - if (isinstance(field, MessageField) and - issubclass(field.message_type, Message)): - if field.repeated: - for item in value: - item_message_value = field.value_to_message(item) - item_message_value.check_initialized() + def __init__(self, **kwargs): + """Initialize internal messages state. + + Args: + A message can be initialized via the constructor by passing in keyword + arguments corresponding to fields. For example: + + class Date(Message): + day = IntegerField(1) + month = IntegerField(2) + year = IntegerField(3) + + Invoking: + + date = Date(day=6, month=6, year=1911) + + is the same as doing: + + date = Date() + date.day = 6 + date.month = 6 + date.year = 1911 + """ + # Tag being an essential implementation detail must be private. + self.__tags = {} + self.__unrecognized_fields = {} + + assigned = set() + for name, value in kwargs.items(): + setattr(self, name, value) + assigned.add(name) + + # initialize repeated fields. + for field in self.all_fields(): + if field.repeated and field.name not in assigned: + setattr(self, field.name, []) + + def check_initialized(self): + """Check class for initialization status. + + Check that all required fields are initialized + + Raises: + ValidationError: If message is not initialized. + """ + for name, field in self.__by_name.items(): + value = getattr(self, name) + if value is None: + if field.required: + raise ValidationError("Message %s is missing required field %s" % + (type(self).__name__, name)) else: - message_value = field.value_to_message(value) - message_value.check_initialized() - except ValidationError as err: - if not hasattr(err, 'message_name'): - err.message_name = type(self).__name__ - raise - - def is_initialized(self): - """Get initialization status. - - Returns: - True if message is valid, else False. - """ - try: - self.check_initialized() - except ValidationError: - return False - else: - return True - - @classmethod - def all_fields(cls): - """Get all field definition objects. - - Ordering is arbitrary. - - Returns: - Iterator over all values in arbitrary order. - """ - return cls.__by_name.values() - - @classmethod - def field_by_name(cls, name): - """Get field by name. - - Returns: - Field object associated with name. - - Raises: - KeyError if no field found by that name. - """ - return cls.__by_name[name] - - @classmethod - def field_by_number(cls, number): - """Get field by number. + try: + if (isinstance(field, MessageField) and + issubclass(field.message_type, Message)): + if field.repeated: + for item in value: + item_message_value = field.value_to_message( + item) + item_message_value.check_initialized() + else: + message_value = field.value_to_message(value) + message_value.check_initialized() + except ValidationError as err: + if not hasattr(err, 'message_name'): + err.message_name = type(self).__name__ + raise + + def is_initialized(self): + """Get initialization status. + + Returns: + True if message is valid, else False. + """ + try: + self.check_initialized() + except ValidationError: + return False + else: + return True - Returns: - Field object associated with number. + @classmethod + def all_fields(cls): + """Get all field definition objects. - Raises: - KeyError if no field found by that number. - """ - return cls.__by_number[number] + Ordering is arbitrary. - def get_assigned_value(self, name): - """Get the assigned value of an attribute. + Returns: + Iterator over all values in arbitrary order. + """ + return cls.__by_name.values() - Get the underlying value of an attribute. If value has not been set, will - not return the default for the field. + @classmethod + def field_by_name(cls, name): + """Get field by name. - Args: - name: Name of attribute to get. + Returns: + Field object associated with name. - Returns: - Value of attribute, None if it has not been set. - """ - message_type = type(self) - try: - field = message_type.field_by_name(name) - except KeyError: - raise AttributeError('Message %s has no field %s' % ( - message_type.__name__, name)) - return self.__tags.get(field.number) + Raises: + KeyError if no field found by that name. + """ + return cls.__by_name[name] - def reset(self, name): - """Reset assigned value for field. + @classmethod + def field_by_number(cls, number): + """Get field by number. - Resetting a field will return it to its default value or None. + Returns: + Field object associated with number. - Args: - name: Name of field to reset. - """ - message_type = type(self) - try: - field = message_type.field_by_name(name) - except KeyError: - if name not in message_type.__by_name: - raise AttributeError('Message %s has no field %s' % ( - message_type.__name__, name)) - if field.repeated: - self.__tags[field.number] = FieldList(field, []) - else: - self.__tags.pop(field.number, None) - - def all_unrecognized_fields(self): - """Get the names of all unrecognized fields in this message.""" - return list(self.__unrecognized_fields.keys()) - - def get_unrecognized_field_info(self, key, value_default=None, - variant_default=None): - """Get the value and variant of an unknown field in this message. + Raises: + KeyError if no field found by that number. + """ + return cls.__by_number[number] - Args: - key: The name or number of the field to retrieve. - value_default: Value to be returned if the key isn't found. - variant_default: Value to be returned as variant if the key isn't - found. + def get_assigned_value(self, name): + """Get the assigned value of an attribute. - Returns: - (value, variant), where value and variant are whatever was passed - to set_unrecognized_field. - """ - value, variant = self.__unrecognized_fields.get(key, (value_default, - variant_default)) - return value, variant + Get the underlying value of an attribute. If value has not been set, will + not return the default for the field. - def set_unrecognized_field(self, key, value, variant): - """Set an unrecognized field, used when decoding a message. + Args: + name: Name of attribute to get. - Args: - key: The name or number used to refer to this unknown value. - value: The value of the field. - variant: Type information needed to interpret the value or re-encode it. - - Raises: - TypeError: If the variant is not an instance of messages.Variant. - """ - if not isinstance(variant, Variant): - raise TypeError('Variant type %s is not valid.' % variant) - self.__unrecognized_fields[key] = value, variant - - def __setattr__(self, name, value): - """Change set behavior for messages. - - Messages may only be assigned values that are fields. + Returns: + Value of attribute, None if it has not been set. + """ + message_type = type(self) + try: + field = message_type.field_by_name(name) + except KeyError: + raise AttributeError('Message %s has no field %s' % ( + message_type.__name__, name)) + return self.__tags.get(field.number) - Does not try to validate field when set. + def reset(self, name): + """Reset assigned value for field. - Args: - name: Name of field to assign to. - value: Value to assign to field. + Resetting a field will return it to its default value or None. - Raises: - AttributeError when trying to assign value that is not a field. - """ - if name in self.__by_name or name.startswith('_Message__'): - object.__setattr__(self, name, value) - else: - raise AttributeError("May not assign arbitrary value %s " - "to message %s" % (name, type(self).__name__)) + Args: + name: Name of field to reset. + """ + message_type = type(self) + try: + field = message_type.field_by_name(name) + except KeyError: + if name not in message_type.__by_name: + raise AttributeError('Message %s has no field %s' % ( + message_type.__name__, name)) + if field.repeated: + self.__tags[field.number] = FieldList(field, []) + else: + self.__tags.pop(field.number, None) + + def all_unrecognized_fields(self): + """Get the names of all unrecognized fields in this message.""" + return list(self.__unrecognized_fields.keys()) + + def get_unrecognized_field_info(self, key, value_default=None, + variant_default=None): + """Get the value and variant of an unknown field in this message. + + Args: + key: The name or number of the field to retrieve. + value_default: Value to be returned if the key isn't found. + variant_default: Value to be returned as variant if the key isn't + found. + + Returns: + (value, variant), where value and variant are whatever was passed + to set_unrecognized_field. + """ + value, variant = self.__unrecognized_fields.get(key, (value_default, + variant_default)) + return value, variant + + def set_unrecognized_field(self, key, value, variant): + """Set an unrecognized field, used when decoding a message. + + Args: + key: The name or number used to refer to this unknown value. + value: The value of the field. + variant: Type information needed to interpret the value or re-encode it. + + Raises: + TypeError: If the variant is not an instance of messages.Variant. + """ + if not isinstance(variant, Variant): + raise TypeError('Variant type %s is not valid.' % variant) + self.__unrecognized_fields[key] = value, variant + + def __setattr__(self, name, value): + """Change set behavior for messages. + + Messages may only be assigned values that are fields. + + Does not try to validate field when set. + + Args: + name: Name of field to assign to. + value: Value to assign to field. + + Raises: + AttributeError when trying to assign value that is not a field. + """ + if name in self.__by_name or name.startswith('_Message__'): + object.__setattr__(self, name, value) + else: + raise AttributeError("May not assign arbitrary value %s " + "to message %s" % (name, type(self).__name__)) - def __repr__(self): - """Make string representation of message. + def __repr__(self): + """Make string representation of message. - Example: + Example: - class MyMessage(messages.Message): - integer_value = messages.IntegerField(1) - string_value = messages.StringField(2) + class MyMessage(messages.Message): + integer_value = messages.IntegerField(1) + string_value = messages.StringField(2) - my_message = MyMessage() - my_message.integer_value = 42 - my_message.string_value = u'A string' + my_message = MyMessage() + my_message.integer_value = 42 + my_message.string_value = u'A string' - print my_message - >>> + print my_message + >>> - Returns: - String representation of message, including the values - of all fields and repr of all sub-messages. - """ - body = ['<', type(self).__name__] - for field in sorted(self.all_fields(), - key=lambda f: f.number): - attribute = field.name - value = self.get_assigned_value(field.name) - if value is not None: - body.append('\n %s: %s' % (attribute, repr(value))) - body.append('>') - return ''.join(body) + Returns: + String representation of message, including the values + of all fields and repr of all sub-messages. + """ + body = ['<', type(self).__name__] + for field in sorted(self.all_fields(), + key=lambda f: f.number): + attribute = field.name + value = self.get_assigned_value(field.name) + if value is not None: + body.append('\n %s: %s' % (attribute, repr(value))) + body.append('>') + return ''.join(body) - def __eq__(self, other): - """Equality operator. + def __eq__(self, other): + """Equality operator. - Does field by field comparison with other message. For - equality, must be same type and values of all fields must be - equal. + Does field by field comparison with other message. For + equality, must be same type and values of all fields must be + equal. - Messages not required to be initialized for comparison. + Messages not required to be initialized for comparison. - Does not attempt to determine equality for values that have - default values that are not set. In other words: + Does not attempt to determine equality for values that have + default values that are not set. In other words: - class HasDefault(Message): + class HasDefault(Message): - attr1 = StringField(1, default='default value') + attr1 = StringField(1, default='default value') - message1 = HasDefault() - message2 = HasDefault() - message2.attr1 = 'default value' + message1 = HasDefault() + message2 = HasDefault() + message2.attr1 = 'default value' - message1 != message2 + message1 != message2 - Does not compare unknown values. + Does not compare unknown values. - Args: - other: Other message to compare with. - """ - # TODO(rafek): Implement "equivalent" which does comparisons - # taking default values in to consideration. - if self is other: - return True + Args: + other: Other message to compare with. + """ + # TODO(rafek): Implement "equivalent" which does comparisons + # taking default values in to consideration. + if self is other: + return True - if type(self) is not type(other): - return False + if type(self) is not type(other): + return False - return self.__tags == other.__tags + return self.__tags == other.__tags - def __ne__(self, other): - """Not equals operator. + def __ne__(self, other): + """Not equals operator. - Does field by field comparison with other message. For - non-equality, must be different type or any value of a field must be - non-equal to the same field in the other instance. + Does field by field comparison with other message. For + non-equality, must be different type or any value of a field must be + non-equal to the same field in the other instance. - Messages not required to be initialized for comparison. + Messages not required to be initialized for comparison. - Args: - other: Other message to compare with. - """ - return not self.__eq__(other) + Args: + other: Other message to compare with. + """ + return not self.__eq__(other) class FieldList(list): - """List implementation that validates field values. - - This list implementation overrides all methods that add values in to a list - in order to validate those new elements. Attempting to add or set list - values that are not of the correct type will raise ValidationError. - """ - - def __init__(self, field_instance, sequence): - """Constructor. - - Args: - field_instance: Instance of field that validates the list. - sequence: List or tuple to construct list from. - """ - if not field_instance.repeated: - raise FieldDefinitionError('FieldList may only accept repeated fields') - self.__field = field_instance - self.__field.validate(sequence) - list.__init__(self, sequence) + """List implementation that validates field values. - def __getstate__(self): - """Enable pickling. - - The assigned field instance can't be pickled if it belongs to a Message - definition (message_definition uses a weakref), so the Message class and - field number are returned in that case. - - Returns: - A 3-tuple containing: - - The field instance, or None if it belongs to a Message class. - - The Message class that the field instance belongs to, or None. - - The field instance number of the Message class it belongs to, or None. + This list implementation overrides all methods that add values in to a list + in order to validate those new elements. Attempting to add or set list + values that are not of the correct type will raise ValidationError. """ - message_class = self.__field.message_definition() - if message_class is None: - return self.__field, None, None - else: - return None, message_class, self.__field.number - def __setstate__(self, state): - """Enable unpickling. + def __init__(self, field_instance, sequence): + """Constructor. + + Args: + field_instance: Instance of field that validates the list. + sequence: List or tuple to construct list from. + """ + if not field_instance.repeated: + raise FieldDefinitionError( + 'FieldList may only accept repeated fields') + self.__field = field_instance + self.__field.validate(sequence) + list.__init__(self, sequence) + + def __getstate__(self): + """Enable pickling. + + The assigned field instance can't be pickled if it belongs to a Message + definition (message_definition uses a weakref), so the Message class and + field number are returned in that case. + + Returns: + A 3-tuple containing: + - The field instance, or None if it belongs to a Message class. + - The Message class that the field instance belongs to, or None. + - The field instance number of the Message class it belongs to, or None. + """ + message_class = self.__field.message_definition() + if message_class is None: + return self.__field, None, None + else: + return None, message_class, self.__field.number + + def __setstate__(self, state): + """Enable unpickling. + + Args: + state: A 3-tuple containing: + - The field instance, or None if it belongs to a Message class. + - The Message class that the field instance belongs to, or None. + - The field instance number of the Message class it belongs to, or None. + """ + field_instance, message_class, number = state + if field_instance is None: + self.__field = message_class.field_by_number(number) + else: + self.__field = field_instance + + @property + def field(self): + """Field that validates list.""" + return self.__field + + def __setslice__(self, i, j, sequence): + """Validate slice assignment to list.""" + self.__field.validate(sequence) + list.__setslice__(self, i, j, sequence) + + def __setitem__(self, index, value): + """Validate item assignment to list.""" + if isinstance(index, slice): + self.__field.validate(value) + else: + self.__field.validate_element(value) + list.__setitem__(self, index, value) - Args: - state: A 3-tuple containing: - - The field instance, or None if it belongs to a Message class. - - The Message class that the field instance belongs to, or None. - - The field instance number of the Message class it belongs to, or None. - """ - field_instance, message_class, number = state - if field_instance is None: - self.__field = message_class.field_by_number(number) - else: - self.__field = field_instance - - @property - def field(self): - """Field that validates list.""" - return self.__field - - def __setslice__(self, i, j, sequence): - """Validate slice assignment to list.""" - self.__field.validate(sequence) - list.__setslice__(self, i, j, sequence) - - def __setitem__(self, index, value): - """Validate item assignment to list.""" - if isinstance(index, slice): - self.__field.validate(value) - else: + def append(self, value): + """Validate item appending to list.""" self.__field.validate_element(value) - list.__setitem__(self, index, value) + return list.append(self, value) - def append(self, value): - """Validate item appending to list.""" - self.__field.validate_element(value) - return list.append(self, value) + def extend(self, sequence): + """Validate extension of list.""" + self.__field.validate(sequence) + return list.extend(self, sequence) - def extend(self, sequence): - """Validate extension of list.""" - self.__field.validate(sequence) - return list.extend(self, sequence) - - def insert(self, index, value): - """Validate item insertion to list.""" - self.__field.validate_element(value) - return list.insert(self, index, value) + def insert(self, index, value): + """Validate item insertion to list.""" + self.__field.validate_element(value) + return list.insert(self, index, value) class _FieldMeta(type): - def __init__(cls, name, bases, dct): - getattr(cls, '_Field__variant_to_type').update( - (variant, cls) for variant in dct.get('VARIANTS', [])) - type.__init__(cls, name, bases, dct) + def __init__(cls, name, bases, dct): + getattr(cls, '_Field__variant_to_type').update( + (variant, cls) for variant in dct.get('VARIANTS', [])) + type.__init__(cls, name, bases, dct) # TODO(rafek): Prevent additional field subclasses. class Field(six.with_metaclass(_FieldMeta, object)): - __initialized = False - __variant_to_type = {} - - # TODO(craigcitro): Remove this alias. - # - # We add an alias here for backwards compatibility; note that in - # python3, this attribute will silently be ignored. - __metaclass__ = _FieldMeta - - @util.positional(2) - def __init__(self, - number, - required=False, - repeated=False, - variant=None, - default=None): - """Constructor. - - The required and repeated parameters are mutually exclusive. Setting both - to True will raise a FieldDefinitionError. - - Sub-class Attributes: - Each sub-class of Field must define the following: - VARIANTS: Set of variant types accepted by that field. - DEFAULT_VARIANT: Default variant type if not specified in constructor. - - Args: - number: Number of field. Must be unique per message class. - required: Whether or not field is required. Mutually exclusive with - 'repeated'. - repeated: Whether or not field is repeated. Mutually exclusive with - 'required'. - variant: Wire-format variant hint. - default: Default value for field if not found in stream. - - Raises: - InvalidVariantError when invalid variant for field is provided. - InvalidDefaultError when invalid default for field is provided. - FieldDefinitionError when invalid number provided or mutually exclusive - fields are used. - InvalidNumberError when the field number is out of range or reserved. - """ - if not isinstance(number, int) or not 1 <= number <= MAX_FIELD_NUMBER: - raise InvalidNumberError('Invalid number for field: %s\n' - 'Number must be 1 or greater and %d or less' % - (number, MAX_FIELD_NUMBER)) - - if FIRST_RESERVED_FIELD_NUMBER <= number <= LAST_RESERVED_FIELD_NUMBER: - raise InvalidNumberError('Tag number %d is a reserved number.\n' - 'Numbers %d to %d are reserved' % - (number, FIRST_RESERVED_FIELD_NUMBER, - LAST_RESERVED_FIELD_NUMBER)) - - if repeated and required: - raise FieldDefinitionError('Cannot set both repeated and required') - - if variant is None: - variant = self.DEFAULT_VARIANT - - if repeated and default is not None: - raise FieldDefinitionError('Repeated fields may not have defaults') - - if variant not in self.VARIANTS: - raise InvalidVariantError( - 'Invalid variant: %s\nValid variants for %s are %r' % - (variant, type(self).__name__, sorted(self.VARIANTS))) - - self.number = number - self.required = required - self.repeated = repeated - self.variant = variant - - if default is not None: - try: - self.validate_default(default) - except ValidationError as err: - try: - name = self.name - except AttributeError: - # For when raising error before name initialization. - raise InvalidDefaultError('Invalid default value for %s: %r: %s' % - (self.__class__.__name__, default, err)) + __initialized = False + __variant_to_type = {} + + # TODO(craigcitro): Remove this alias. + # + # We add an alias here for backwards compatibility; note that in + # python3, this attribute will silently be ignored. + __metaclass__ = _FieldMeta + + @util.positional(2) + def __init__(self, + number, + required=False, + repeated=False, + variant=None, + default=None): + """Constructor. + + The required and repeated parameters are mutually exclusive. Setting both + to True will raise a FieldDefinitionError. + + Sub-class Attributes: + Each sub-class of Field must define the following: + VARIANTS: Set of variant types accepted by that field. + DEFAULT_VARIANT: Default variant type if not specified in constructor. + + Args: + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive with + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive with + 'required'. + variant: Wire-format variant hint. + default: Default value for field if not found in stream. + + Raises: + InvalidVariantError when invalid variant for field is provided. + InvalidDefaultError when invalid default for field is provided. + FieldDefinitionError when invalid number provided or mutually exclusive + fields are used. + InvalidNumberError when the field number is out of range or reserved. + """ + if not isinstance(number, int) or not 1 <= number <= MAX_FIELD_NUMBER: + raise InvalidNumberError('Invalid number for field: %s\n' + 'Number must be 1 or greater and %d or less' % + (number, MAX_FIELD_NUMBER)) + + if FIRST_RESERVED_FIELD_NUMBER <= number <= LAST_RESERVED_FIELD_NUMBER: + raise InvalidNumberError('Tag number %d is a reserved number.\n' + 'Numbers %d to %d are reserved' % + (number, FIRST_RESERVED_FIELD_NUMBER, + LAST_RESERVED_FIELD_NUMBER)) + + if repeated and required: + raise FieldDefinitionError('Cannot set both repeated and required') + + if variant is None: + variant = self.DEFAULT_VARIANT + + if repeated and default is not None: + raise FieldDefinitionError('Repeated fields may not have defaults') + + if variant not in self.VARIANTS: + raise InvalidVariantError( + 'Invalid variant: %s\nValid variants for %s are %r' % + (variant, type(self).__name__, sorted(self.VARIANTS))) + + self.number = number + self.required = required + self.repeated = repeated + self.variant = variant + + if default is not None: + try: + self.validate_default(default) + except ValidationError as err: + try: + name = self.name + except AttributeError: + # For when raising error before name initialization. + raise InvalidDefaultError('Invalid default value for %s: %r: %s' % + (self.__class__.__name__, default, err)) + else: + raise InvalidDefaultError('Invalid default value for field %s: ' + '%r: %s' % (name, default, err)) + + self.__default = default + self.__initialized = True + + def __setattr__(self, name, value): + """Setter overidden to prevent assignment to fields after creation. + + Args: + name: Name of attribute to set. + value: Value to assign. + """ + # Special case post-init names. They need to be set after constructor. + if name in _POST_INIT_FIELD_ATTRIBUTE_NAMES: + object.__setattr__(self, name, value) + return + + # All other attributes must be set before __initialized. + if not self.__initialized: + # Not initialized yet, allow assignment. + object.__setattr__(self, name, value) else: - raise InvalidDefaultError('Invalid default value for field %s: ' - '%r: %s' % (name, default, err)) - - self.__default = default - self.__initialized = True - - def __setattr__(self, name, value): - """Setter overidden to prevent assignment to fields after creation. - - Args: - name: Name of attribute to set. - value: Value to assign. - """ - # Special case post-init names. They need to be set after constructor. - if name in _POST_INIT_FIELD_ATTRIBUTE_NAMES: - object.__setattr__(self, name, value) - return - - # All other attributes must be set before __initialized. - if not self.__initialized: - # Not initialized yet, allow assignment. - object.__setattr__(self, name, value) - else: - raise AttributeError('Field objects are read-only') - - def __set__(self, message_instance, value): - """Set value on message. - - Args: - message_instance: Message instance to set value on. - value: Value to set on message. - """ - # Reaches in to message instance directly to assign to private tags. - if value is None: - if self.repeated: - raise ValidationError( - 'May not assign None to repeated field %s' % self.name) - else: - message_instance._Message__tags.pop(self.number, None) - else: - if self.repeated: - value = FieldList(self, value) - else: - self.validate(value) - message_instance._Message__tags[self.number] = value - - def __get__(self, message_instance, message_class): - if message_instance is None: - return self - - result = message_instance._Message__tags.get(self.number) - if result is None: - return self.default - else: - return result - - def validate_element(self, value): - """Validate single element of field. - - This is different from validate in that it is used on individual - values of repeated fields. + raise AttributeError('Field objects are read-only') + + def __set__(self, message_instance, value): + """Set value on message. + + Args: + message_instance: Message instance to set value on. + value: Value to set on message. + """ + # Reaches in to message instance directly to assign to private tags. + if value is None: + if self.repeated: + raise ValidationError( + 'May not assign None to repeated field %s' % self.name) + else: + message_instance._Message__tags.pop(self.number, None) + else: + if self.repeated: + value = FieldList(self, value) + else: + self.validate(value) + message_instance._Message__tags[self.number] = value - Args: - value: Value to validate. + def __get__(self, message_instance, message_class): + if message_instance is None: + return self - Raises: - ValidationError if value is not expected type. - """ - if not isinstance(value, self.type): - if value is None: - if self.required: - raise ValidationError('Required field is missing') - else: - try: - name = self.name - except AttributeError: - raise ValidationError('Expected type %s for %s, ' - 'found %s (type %s)' % - (self.type, self.__class__.__name__, - value, type(value))) + result = message_instance._Message__tags.get(self.number) + if result is None: + return self.default else: - raise ValidationError('Expected type %s for field %s, ' - 'found %s (type %s)' % - (self.type, name, value, type(value))) + return result - def __validate(self, value, validate_element): - """Internal validation function. + def validate_element(self, value): + """Validate single element of field. - Validate an internal value using a function to validate individual elements. + This is different from validate in that it is used on individual + values of repeated fields. - Args: - value: Value to validate. - validate_element: Function to use to validate individual elements. + Args: + value: Value to validate. - Raises: - ValidationError if value is not expected type. - """ - if not self.repeated: - validate_element(value) - else: - # Must be a list or tuple, may not be a string. - if isinstance(value, (list, tuple)): - for element in value: - if element is None: - try: - name = self.name - except AttributeError: - raise ValidationError('Repeated values for %s ' - 'may not be None' % self.__class__.__name__) + Raises: + ValidationError if value is not expected type. + """ + if not isinstance(value, self.type): + if value is None: + if self.required: + raise ValidationError('Required field is missing') else: - raise ValidationError('Repeated values for field %s ' - 'may not be None' % name) - validate_element(element) - elif value is not None: + try: + name = self.name + except AttributeError: + raise ValidationError('Expected type %s for %s, ' + 'found %s (type %s)' % + (self.type, self.__class__.__name__, + value, type(value))) + else: + raise ValidationError('Expected type %s for field %s, ' + 'found %s (type %s)' % + (self.type, name, value, type(value))) + + def __validate(self, value, validate_element): + """Internal validation function. + + Validate an internal value using a function to validate individual elements. + + Args: + value: Value to validate. + validate_element: Function to use to validate individual elements. + + Raises: + ValidationError if value is not expected type. + """ + if not self.repeated: + validate_element(value) + else: + # Must be a list or tuple, may not be a string. + if isinstance(value, (list, tuple)): + for element in value: + if element is None: + try: + name = self.name + except AttributeError: + raise ValidationError('Repeated values for %s ' + 'may not be None' % self.__class__.__name__) + else: + raise ValidationError('Repeated values for field %s ' + 'may not be None' % name) + validate_element(element) + elif value is not None: + try: + name = self.name + except AttributeError: + raise ValidationError('%s is repeated. Found: %s' % ( + self.__class__.__name__, value)) + else: + raise ValidationError('Field %s is repeated. Found: %s' % (name, + value)) + + def validate(self, value): + """Validate value assigned to field. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.__validate(value, self.validate_element) + + def validate_default_element(self, value): + """Validate value as assigned to field default field. + + Some fields may allow for delayed resolution of default types necessary + in the case of circular definition references. In this case, the default + value might be a place holder that is resolved when needed after all the + message classes are defined. + + Args: + value: Default value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.validate_element(value) + + def validate_default(self, value): + """Validate default value assigned to field. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected type. + """ + self.__validate(value, self.validate_default_element) + + def message_definition(self): + """Get Message definition that contains this Field definition. + + Returns: + Containing Message definition for Field. Will return None if for + some reason Field is defined outside of a Message class. + """ try: - name = self.name + return self._message_definition() except AttributeError: - raise ValidationError('%s is repeated. Found: %s' % ( - self.__class__.__name__, value)) - else: - raise ValidationError('Field %s is repeated. Found: %s' % (name, - value)) - - def validate(self, value): - """Validate value assigned to field. - - Args: - value: Value to validate. - - Raises: - ValidationError if value is not expected type. - """ - self.__validate(value, self.validate_element) - - def validate_default_element(self, value): - """Validate value as assigned to field default field. - - Some fields may allow for delayed resolution of default types necessary - in the case of circular definition references. In this case, the default - value might be a place holder that is resolved when needed after all the - message classes are defined. - - Args: - value: Default value to validate. - - Raises: - ValidationError if value is not expected type. - """ - self.validate_element(value) - - def validate_default(self, value): - """Validate default value assigned to field. - - Args: - value: Value to validate. - - Raises: - ValidationError if value is not expected type. - """ - self.__validate(value, self.validate_default_element) - - def message_definition(self): - """Get Message definition that contains this Field definition. - - Returns: - Containing Message definition for Field. Will return None if for - some reason Field is defined outside of a Message class. - """ - try: - return self._message_definition() - except AttributeError: - return None + return None - @property - def default(self): - """Get default value for field.""" - return self.__default + @property + def default(self): + """Get default value for field.""" + return self.__default - @classmethod - def lookup_field_type_by_variant(cls, variant): - return cls.__variant_to_type[variant] + @classmethod + def lookup_field_type_by_variant(cls, variant): + return cls.__variant_to_type[variant] class IntegerField(Field): - """Field definition for integer values.""" + """Field definition for integer values.""" - VARIANTS = frozenset([Variant.INT32, - Variant.INT64, - Variant.UINT32, - Variant.UINT64, - Variant.SINT32, - Variant.SINT64, - ]) + VARIANTS = frozenset([Variant.INT32, + Variant.INT64, + Variant.UINT32, + Variant.UINT64, + Variant.SINT32, + Variant.SINT64, + ]) - DEFAULT_VARIANT = Variant.INT64 + DEFAULT_VARIANT = Variant.INT64 - type = six.integer_types + type = six.integer_types class FloatField(Field): - """Field definition for float values.""" + """Field definition for float values.""" - VARIANTS = frozenset([Variant.FLOAT, - Variant.DOUBLE, - ]) + VARIANTS = frozenset([Variant.FLOAT, + Variant.DOUBLE, + ]) - DEFAULT_VARIANT = Variant.DOUBLE + DEFAULT_VARIANT = Variant.DOUBLE - type = float + type = float class BooleanField(Field): - """Field definition for boolean values.""" + """Field definition for boolean values.""" - VARIANTS = frozenset([Variant.BOOL]) + VARIANTS = frozenset([Variant.BOOL]) - DEFAULT_VARIANT = Variant.BOOL + DEFAULT_VARIANT = Variant.BOOL - type = bool + type = bool class BytesField(Field): - """Field definition for byte string values.""" + """Field definition for byte string values.""" - VARIANTS = frozenset([Variant.BYTES]) + VARIANTS = frozenset([Variant.BYTES]) - DEFAULT_VARIANT = Variant.BYTES + DEFAULT_VARIANT = Variant.BYTES - type = bytes + type = bytes class StringField(Field): - """Field definition for unicode string values.""" + """Field definition for unicode string values.""" - VARIANTS = frozenset([Variant.STRING]) + VARIANTS = frozenset([Variant.STRING]) - DEFAULT_VARIANT = Variant.STRING + DEFAULT_VARIANT = Variant.STRING - type = six.text_type + type = six.text_type - def validate_element(self, value): - """Validate StringField allowing for str and unicode. + def validate_element(self, value): + """Validate StringField allowing for str and unicode. - Raises: - ValidationError if a str value is not 7-bit ascii. - """ - # If value is str is it considered valid. Satisfies "required=True". - if isinstance(value, bytes): - try: - six.text_type(value, 'ascii') - except UnicodeDecodeError as err: - try: - name = self.name - except AttributeError: - validation_error = ValidationError( - 'Field encountered non-ASCII string %r: %s' % (value, - err)) + Raises: + ValidationError if a str value is not 7-bit ascii. + """ + # If value is str is it considered valid. Satisfies "required=True". + if isinstance(value, bytes): + try: + six.text_type(value, 'ascii') + except UnicodeDecodeError as err: + try: + name = self.name + except AttributeError: + validation_error = ValidationError( + 'Field encountered non-ASCII string %r: %s' % (value, + err)) + else: + validation_error = ValidationError( + 'Field %s encountered non-ASCII string %r: %s' % (self.name, + value, + err)) + validation_error.field_name = self.name + raise validation_error else: - validation_error = ValidationError( - 'Field %s encountered non-ASCII string %r: %s' % (self.name, - value, - err)) - validation_error.field_name = self.name - raise validation_error - else: - super(StringField, self).validate_element(value) + super(StringField, self).validate_element(value) class MessageField(Field): - """Field definition for sub-message values. - - Message fields contain instance of other messages. Instances stored - on messages stored on message fields are considered to be owned by - the containing message instance and should not be shared between - owning instances. - - Message fields must be defined to reference a single type of message. - Normally message field are defined by passing the referenced message - class in to the constructor. - - It is possible to define a message field for a type that does not yet - exist by passing the name of the message in to the constructor instead - of a message class. Resolution of the actual type of the message is - deferred until it is needed, for example, during message verification. - Names provided to the constructor must refer to a class within the same - python module as the class that is using it. Names refer to messages - relative to the containing messages scope. For example, the two fields - of OuterMessage refer to the same message type: - - class Outer(Message): - - inner_relative = MessageField('Inner', 1) - inner_absolute = MessageField('Outer.Inner', 2) - - class Inner(Message): - ... - - When resolving an actual type, MessageField will traverse the entire - scope of nested messages to match a message name. This makes it easy - for siblings to reference siblings: - - class Outer(Message): - - class Inner(Message): - - sibling = MessageField('Sibling', 1) - - class Sibling(Message): - ... - """ - - VARIANTS = frozenset([Variant.MESSAGE]) - - DEFAULT_VARIANT = Variant.MESSAGE - - @util.positional(3) - def __init__(self, - message_type, - number, - required=False, - repeated=False, - variant=None): - """Constructor. + """Field definition for sub-message values. - Args: - message_type: Message type for field. Must be subclass of Message. - number: Number of field. Must be unique per message class. - required: Whether or not field is required. Mutually exclusive to - 'repeated'. - repeated: Whether or not field is repeated. Mutually exclusive to - 'required'. - variant: Wire-format variant hint. + Message fields contain instance of other messages. Instances stored + on messages stored on message fields are considered to be owned by + the containing message instance and should not be shared between + owning instances. - Raises: - FieldDefinitionError when invalid message_type is provided. - """ - valid_type = (isinstance(message_type, six.string_types) or - (message_type is not Message and - isinstance(message_type, type) and - issubclass(message_type, Message))) + Message fields must be defined to reference a single type of message. + Normally message field are defined by passing the referenced message + class in to the constructor. - if not valid_type: - raise FieldDefinitionError('Invalid message class: %s' % message_type) + It is possible to define a message field for a type that does not yet + exist by passing the name of the message in to the constructor instead + of a message class. Resolution of the actual type of the message is + deferred until it is needed, for example, during message verification. + Names provided to the constructor must refer to a class within the same + python module as the class that is using it. Names refer to messages + relative to the containing messages scope. For example, the two fields + of OuterMessage refer to the same message type: - if isinstance(message_type, six.string_types): - self.__type_name = message_type - self.__type = None - else: - self.__type = message_type + class Outer(Message): - super(MessageField, self).__init__(number, - required=required, - repeated=repeated, - variant=variant) + inner_relative = MessageField('Inner', 1) + inner_absolute = MessageField('Outer.Inner', 2) - def __set__(self, message_instance, value): - """Set value on message. + class Inner(Message): + ... - Args: - message_instance: Message instance to set value on. - value: Value to set on message. - """ - message_type = self.type - if isinstance(message_type, type) and issubclass(message_type, Message): - if self.repeated: - if value and isinstance(value, (list, tuple)): - value = [(message_type(**v) if isinstance(v, dict) else v) - for v in value] - elif isinstance(value, dict): - value = message_type(**value) - super(MessageField, self).__set__(message_instance, value) - - @property - def type(self): - """Message type used for field.""" - if self.__type is None: - message_type = find_definition(self.__type_name, self.message_definition()) - if not (message_type is not Message and - isinstance(message_type, type) and - issubclass(message_type, Message)): - raise FieldDefinitionError('Invalid message class: %s' % message_type) - self.__type = message_type - return self.__type - - @property - def message_type(self): - """Underlying message type used for serialization. - - Will always be a sub-class of Message. This is different from type - which represents the python value that message_type is mapped to for - use by the user. - """ - return self.type + When resolving an actual type, MessageField will traverse the entire + scope of nested messages to match a message name. This makes it easy + for siblings to reference siblings: - def value_from_message(self, message): - """Convert a message to a value instance. + class Outer(Message): - Used by deserializers to convert from underlying messages to - value of expected user type. + class Inner(Message): - Args: - message: A message instance of type self.message_type. + sibling = MessageField('Sibling', 1) - Returns: - Value of self.message_type. + class Sibling(Message): + ... """ - if not isinstance(message, self.message_type): - raise DecodeError('Expected type %s, got %s: %r' % - (self.message_type.__name__, - type(message).__name__, - message)) - return message - def value_to_message(self, value): - """Convert a value instance to a message. - - Used by serializers to convert Python user types to underlying - messages for transmission. - - Args: - value: A value of type self.type. - - Returns: - An instance of type self.message_type. - """ - if not isinstance(value, self.type): - raise EncodeError('Expected type %s, got %s: %r' % - (self.type.__name__, - type(value).__name__, - value)) - return value + VARIANTS = frozenset([Variant.MESSAGE]) + + DEFAULT_VARIANT = Variant.MESSAGE + + @util.positional(3) + def __init__(self, + message_type, + number, + required=False, + repeated=False, + variant=None): + """Constructor. + + Args: + message_type: Message type for field. Must be subclass of Message. + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive to + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive to + 'required'. + variant: Wire-format variant hint. + + Raises: + FieldDefinitionError when invalid message_type is provided. + """ + valid_type = (isinstance(message_type, six.string_types) or + (message_type is not Message and + isinstance(message_type, type) and + issubclass(message_type, Message))) + + if not valid_type: + raise FieldDefinitionError( + 'Invalid message class: %s' % message_type) + + if isinstance(message_type, six.string_types): + self.__type_name = message_type + self.__type = None + else: + self.__type = message_type + + super(MessageField, self).__init__(number, + required=required, + repeated=repeated, + variant=variant) + + def __set__(self, message_instance, value): + """Set value on message. + + Args: + message_instance: Message instance to set value on. + value: Value to set on message. + """ + message_type = self.type + if isinstance(message_type, type) and issubclass(message_type, Message): + if self.repeated: + if value and isinstance(value, (list, tuple)): + value = [(message_type(**v) if isinstance(v, dict) else v) + for v in value] + elif isinstance(value, dict): + value = message_type(**value) + super(MessageField, self).__set__(message_instance, value) + + @property + def type(self): + """Message type used for field.""" + if self.__type is None: + message_type = find_definition( + self.__type_name, self.message_definition()) + if not (message_type is not Message and + isinstance(message_type, type) and + issubclass(message_type, Message)): + raise FieldDefinitionError( + 'Invalid message class: %s' % message_type) + self.__type = message_type + return self.__type + + @property + def message_type(self): + """Underlying message type used for serialization. + + Will always be a sub-class of Message. This is different from type + which represents the python value that message_type is mapped to for + use by the user. + """ + return self.type + + def value_from_message(self, message): + """Convert a message to a value instance. + + Used by deserializers to convert from underlying messages to + value of expected user type. + + Args: + message: A message instance of type self.message_type. + + Returns: + Value of self.message_type. + """ + if not isinstance(message, self.message_type): + raise DecodeError('Expected type %s, got %s: %r' % + (self.message_type.__name__, + type(message).__name__, + message)) + return message + + def value_to_message(self, value): + """Convert a value instance to a message. + + Used by serializers to convert Python user types to underlying + messages for transmission. + + Args: + value: A value of type self.type. + + Returns: + An instance of type self.message_type. + """ + if not isinstance(value, self.type): + raise EncodeError('Expected type %s, got %s: %r' % + (self.type.__name__, + type(value).__name__, + value)) + return value class EnumField(Field): - """Field definition for enum values. - - Enum fields may have default values that are delayed until the associated enum - type is resolved. This is necessary to support certain circular references. + """Field definition for enum values. - For example: + Enum fields may have default values that are delayed until the associated enum + type is resolved. This is necessary to support certain circular references. - class Message1(Message): + For example: - class Color(Enum): + class Message1(Message): - RED = 1 - GREEN = 2 - BLUE = 3 + class Color(Enum): - # This field default value will be validated when default is accessed. - animal = EnumField('Message2.Animal', 1, default='HORSE') + RED = 1 + GREEN = 2 + BLUE = 3 - class Message2(Message): + # This field default value will be validated when default is accessed. + animal = EnumField('Message2.Animal', 1, default='HORSE') - class Animal(Enum): + class Message2(Message): - DOG = 1 - CAT = 2 - HORSE = 3 + class Animal(Enum): - # This fields default value will be validated right away since Color is - # already fully resolved. - color = EnumField(Message1.Color, 1, default='RED') - """ + DOG = 1 + CAT = 2 + HORSE = 3 - VARIANTS = frozenset([Variant.ENUM]) - - DEFAULT_VARIANT = Variant.ENUM - - def __init__(self, enum_type, number, **kwargs): - """Constructor. - - Args: - enum_type: Enum type for field. Must be subclass of Enum. - number: Number of field. Must be unique per message class. - required: Whether or not field is required. Mutually exclusive to - 'repeated'. - repeated: Whether or not field is repeated. Mutually exclusive to - 'required'. - variant: Wire-format variant hint. - default: Default value for field if not found in stream. - - Raises: - FieldDefinitionError when invalid enum_type is provided. + # This fields default value will be validated right away since Color is + # already fully resolved. + color = EnumField(Message1.Color, 1, default='RED') """ - valid_type = (isinstance(enum_type, six.string_types) or - (enum_type is not Enum and - isinstance(enum_type, type) and - issubclass(enum_type, Enum))) - - if not valid_type: - raise FieldDefinitionError('Invalid enum type: %s' % enum_type) - - if isinstance(enum_type, six.string_types): - self.__type_name = enum_type - self.__type = None - else: - self.__type = enum_type - super(EnumField, self).__init__(number, **kwargs) - - def validate_default_element(self, value): - """Validate default element of Enum field. - - Enum fields allow for delayed resolution of default values when the type - of the field has not been resolved. The default value of a field may be - a string or an integer. If the Enum type of the field has been resolved, - the default value is validated against that type. - - Args: - value: Value to validate. - - Raises: - ValidationError if value is not expected message type. - """ - if isinstance(value, (six.string_types, six.integer_types)): - # Validation of the value does not happen for delayed resolution - # enumerated types. Ignore if type is not yet resolved. - if self.__type: - self.__type(value) - return - - super(EnumField, self).validate_default_element(value) - - @property - def type(self): - """Enum type used for field.""" - if self.__type is None: - found_type = find_definition(self.__type_name, self.message_definition()) - if not (found_type is not Enum and - isinstance(found_type, type) and - issubclass(found_type, Enum)): - raise FieldDefinitionError('Invalid enum type: %s' % found_type) - - self.__type = found_type - return self.__type - - @property - def default(self): - """Default for enum field. - - Will cause resolution of Enum type and unresolved default value. - """ - try: - return self.__resolved_default - except AttributeError: - resolved_default = super(EnumField, self).default - if isinstance(resolved_default, (six.string_types, six.integer_types)): - resolved_default = self.type(resolved_default) - self.__resolved_default = resolved_default - return self.__resolved_default + VARIANTS = frozenset([Variant.ENUM]) + + DEFAULT_VARIANT = Variant.ENUM + + def __init__(self, enum_type, number, **kwargs): + """Constructor. + + Args: + enum_type: Enum type for field. Must be subclass of Enum. + number: Number of field. Must be unique per message class. + required: Whether or not field is required. Mutually exclusive to + 'repeated'. + repeated: Whether or not field is repeated. Mutually exclusive to + 'required'. + variant: Wire-format variant hint. + default: Default value for field if not found in stream. + + Raises: + FieldDefinitionError when invalid enum_type is provided. + """ + valid_type = (isinstance(enum_type, six.string_types) or + (enum_type is not Enum and + isinstance(enum_type, type) and + issubclass(enum_type, Enum))) + + if not valid_type: + raise FieldDefinitionError('Invalid enum type: %s' % enum_type) + + if isinstance(enum_type, six.string_types): + self.__type_name = enum_type + self.__type = None + else: + self.__type = enum_type + + super(EnumField, self).__init__(number, **kwargs) + + def validate_default_element(self, value): + """Validate default element of Enum field. + + Enum fields allow for delayed resolution of default values when the type + of the field has not been resolved. The default value of a field may be + a string or an integer. If the Enum type of the field has been resolved, + the default value is validated against that type. + + Args: + value: Value to validate. + + Raises: + ValidationError if value is not expected message type. + """ + if isinstance(value, (six.string_types, six.integer_types)): + # Validation of the value does not happen for delayed resolution + # enumerated types. Ignore if type is not yet resolved. + if self.__type: + self.__type(value) + return + + super(EnumField, self).validate_default_element(value) + + @property + def type(self): + """Enum type used for field.""" + if self.__type is None: + found_type = find_definition( + self.__type_name, self.message_definition()) + if not (found_type is not Enum and + isinstance(found_type, type) and + issubclass(found_type, Enum)): + raise FieldDefinitionError( + 'Invalid enum type: %s' % found_type) + + self.__type = found_type + return self.__type + + @property + def default(self): + """Default for enum field. + + Will cause resolution of Enum type and unresolved default value. + """ + try: + return self.__resolved_default + except AttributeError: + resolved_default = super(EnumField, self).default + if isinstance(resolved_default, (six.string_types, six.integer_types)): + resolved_default = self.type(resolved_default) + self.__resolved_default = resolved_default + return self.__resolved_default @util.positional(2) def find_definition(name, relative_to=None, importer=__import__): - """Find definition by name in module-space. - - The find algorthm will look for definitions by name relative to a message - definition or by fully qualfied name. If no definition is found relative - to the relative_to parameter it will do the same search against the container - of relative_to. If relative_to is a nested Message, it will search its - message_definition(). If that message has no message_definition() it will - search its module. If relative_to is a module, it will attempt to look for - the containing module and search relative to it. If the module is a top-level - module, it will look for the a message using a fully qualified name. If - no message is found then, the search fails and DefinitionNotFoundError is - raised. - - For example, when looking for any definition 'foo.bar.ADefinition' relative to - an actual message definition abc.xyz.SomeMessage: - - find_definition('foo.bar.ADefinition', SomeMessage) - - It is like looking for the following fully qualified names: - - abc.xyz.SomeMessage. foo.bar.ADefinition - abc.xyz. foo.bar.ADefinition - abc. foo.bar.ADefinition - foo.bar.ADefinition + """Find definition by name in module-space. - When resolving the name relative to Message definitions and modules, the - algorithm searches any Messages or sub-modules found in its path. - Non-Message values are not searched. + The find algorthm will look for definitions by name relative to a message + definition or by fully qualfied name. If no definition is found relative + to the relative_to parameter it will do the same search against the container + of relative_to. If relative_to is a nested Message, it will search its + message_definition(). If that message has no message_definition() it will + search its module. If relative_to is a module, it will attempt to look for + the containing module and search relative to it. If the module is a top-level + module, it will look for the a message using a fully qualified name. If + no message is found then, the search fails and DefinitionNotFoundError is + raised. - A name that begins with '.' is considered to be a fully qualified name. The - name is always searched for from the topmost package. For example, assume - two message types: + For example, when looking for any definition 'foo.bar.ADefinition' relative to + an actual message definition abc.xyz.SomeMessage: - abc.xyz.SomeMessage - xyz.SomeMessage + find_definition('foo.bar.ADefinition', SomeMessage) - Searching for '.xyz.SomeMessage' relative to 'abc' will resolve to - 'xyz.SomeMessage' and not 'abc.xyz.SomeMessage'. For this kind of name, - the relative_to parameter is effectively ignored and always set to None. + It is like looking for the following fully qualified names: - For more information about package name resolution, please see: + abc.xyz.SomeMessage. foo.bar.ADefinition + abc.xyz. foo.bar.ADefinition + abc. foo.bar.ADefinition + foo.bar.ADefinition - http://code.google.com/apis/protocolbuffers/docs/proto.html#packages + When resolving the name relative to Message definitions and modules, the + algorithm searches any Messages or sub-modules found in its path. + Non-Message values are not searched. - Args: - name: Name of definition to find. May be fully qualified or relative name. - relative_to: Search for definition relative to message definition or module. - None will cause a fully qualified name search. - importer: Import function to use for resolving modules. + A name that begins with '.' is considered to be a fully qualified name. The + name is always searched for from the topmost package. For example, assume + two message types: - Returns: - Enum or Message class definition associated with name. + abc.xyz.SomeMessage + xyz.SomeMessage - Raises: - DefinitionNotFoundError if no definition is found in any search path. - """ - # Check parameters. - if not (relative_to is None or - isinstance(relative_to, types.ModuleType) or - isinstance(relative_to, type) and issubclass(relative_to, Message)): - raise TypeError('relative_to must be None, Message definition or module. ' - 'Found: %s' % relative_to) + Searching for '.xyz.SomeMessage' relative to 'abc' will resolve to + 'xyz.SomeMessage' and not 'abc.xyz.SomeMessage'. For this kind of name, + the relative_to parameter is effectively ignored and always set to None. - name_path = name.split('.') + For more information about package name resolution, please see: - # Handle absolute path reference. - if not name_path[0]: - relative_to = None - name_path = name_path[1:] + http://code.google.com/apis/protocolbuffers/docs/proto.html#packages - def search_path(): - """Performs a single iteration searching the path from relative_to. - - This is the function that searches up the path from a relative object. - - fully.qualified.object . relative.or.nested.Definition - ----------------------------> - ^ - | - this part of search --+ + Args: + name: Name of definition to find. May be fully qualified or relative name. + relative_to: Search for definition relative to message definition or module. + None will cause a fully qualified name search. + importer: Import function to use for resolving modules. Returns: - Message or Enum at the end of name_path, else None. + Enum or Message class definition associated with name. + + Raises: + DefinitionNotFoundError if no definition is found in any search path. """ - next = relative_to - for node in name_path: - # Look for attribute first. - attribute = getattr(next, node, None) - - if attribute is not None: - next = attribute - else: - # If module, look for sub-module. - if next is None or isinstance(next, types.ModuleType): - if next is None: - module_name = node - else: - module_name = '%s.%s' % (next.__name__, node) - - try: - fromitem = module_name.split('.')[-1] - next = importer(module_name, '', '', [str(fromitem)]) - except ImportError: - return None + # Check parameters. + if not (relative_to is None or + isinstance(relative_to, types.ModuleType) or + isinstance(relative_to, type) and issubclass(relative_to, Message)): + raise TypeError('relative_to must be None, Message definition or module. ' + 'Found: %s' % relative_to) + + name_path = name.split('.') + + # Handle absolute path reference. + if not name_path[0]: + relative_to = None + name_path = name_path[1:] + + def search_path(): + """Performs a single iteration searching the path from relative_to. + + This is the function that searches up the path from a relative object. + + fully.qualified.object . relative.or.nested.Definition + ----------------------------> + ^ + | + this part of search --+ + + Returns: + Message or Enum at the end of name_path, else None. + """ + next = relative_to + for node in name_path: + # Look for attribute first. + attribute = getattr(next, node, None) + + if attribute is not None: + next = attribute + else: + # If module, look for sub-module. + if next is None or isinstance(next, types.ModuleType): + if next is None: + module_name = node + else: + module_name = '%s.%s' % (next.__name__, node) + + try: + fromitem = module_name.split('.')[-1] + next = importer(module_name, '', '', [str(fromitem)]) + except ImportError: + return None + else: + return None + + if (not isinstance(next, types.ModuleType) and + not (isinstance(next, type) and + issubclass(next, (Message, Enum)))): + return None + + return next + + while True: + found = search_path() + if isinstance(found, type) and issubclass(found, (Enum, Message)): + return found else: - return None - - if (not isinstance(next, types.ModuleType) and - not (isinstance(next, type) and - issubclass(next, (Message, Enum)))): - return None - - return next - - while True: - found = search_path() - if isinstance(found, type) and issubclass(found, (Enum, Message)): - return found - else: - # Find next relative_to to search against. - # - # fully.qualified.object . relative.or.nested.Definition - # <--------------------- - # ^ - # | - # does this part of search - if relative_to is None: - # Fully qualified search was done. Nothing found. Fail. - raise DefinitionNotFoundError('Could not find definition for %s' - % (name,)) - else: - if isinstance(relative_to, types.ModuleType): - # Find parent module. - module_path = relative_to.__name__.split('.')[:-1] - if not module_path: - relative_to = None - else: - # Should not raise ImportError. If it does... weird and - # unexepected. Propagate. - relative_to = importer( - '.'.join(module_path), '', '', [module_path[-1]]) - elif (isinstance(relative_to, type) and - issubclass(relative_to, Message)): - parent = relative_to.message_definition() - if parent is None: - last_module_name = relative_to.__module__.split('.')[-1] - relative_to = importer( - relative_to.__module__, '', '', [last_module_name]) - else: - relative_to = parent + # Find next relative_to to search against. + # + # fully.qualified.object . relative.or.nested.Definition + # <--------------------- + # ^ + # | + # does this part of search + if relative_to is None: + # Fully qualified search was done. Nothing found. Fail. + raise DefinitionNotFoundError('Could not find definition for %s' + % (name,)) + else: + if isinstance(relative_to, types.ModuleType): + # Find parent module. + module_path = relative_to.__name__.split('.')[:-1] + if not module_path: + relative_to = None + else: + # Should not raise ImportError. If it does... weird and + # unexepected. Propagate. + relative_to = importer( + '.'.join(module_path), '', '', [module_path[-1]]) + elif (isinstance(relative_to, type) and + issubclass(relative_to, Message)): + parent = relative_to.message_definition() + if parent is None: + last_module_name = relative_to.__module__.split( + '.')[-1] + relative_to = importer( + relative_to.__module__, '', '', [last_module_name]) + else: + relative_to = parent diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index a3b6dc0..31001ab 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -36,2049 +36,2073 @@ from apitools.base.protorpclite import test_util class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): - MODULE = messages + MODULE = messages class ValidationErrorTest(test_util.TestCase): - def testStr_NoFieldName(self): - """Test string version of ValidationError when no name provided.""" - self.assertEquals('Validation error', - str(messages.ValidationError('Validation error'))) + def testStr_NoFieldName(self): + """Test string version of ValidationError when no name provided.""" + self.assertEquals('Validation error', + str(messages.ValidationError('Validation error'))) - def testStr_FieldName(self): - """Test string version of ValidationError when no name provided.""" - validation_error = messages.ValidationError('Validation error') - validation_error.field_name = 'a_field' - self.assertEquals('Validation error', str(validation_error)) + def testStr_FieldName(self): + """Test string version of ValidationError when no name provided.""" + validation_error = messages.ValidationError('Validation error') + validation_error.field_name = 'a_field' + self.assertEquals('Validation error', str(validation_error)) class EnumTest(test_util.TestCase): - def setUp(self): - """Set up tests.""" - # Redefine Color class in case so that changes to it (an error) in one test - # does not affect other tests. - global Color - class Color(messages.Enum): - RED = 20 - ORANGE = 2 - YELLOW = 40 - GREEN = 4 - BLUE = 50 - INDIGO = 5 - VIOLET = 80 - - def testNames(self): - """Test that names iterates over enum names.""" - self.assertEquals( - set(['BLUE', 'GREEN', 'INDIGO', 'ORANGE', 'RED', 'VIOLET', 'YELLOW']), - set(Color.names())) - - def testNumbers(self): - """Tests that numbers iterates of enum numbers.""" - self.assertEquals(set([2, 4, 5, 20, 40, 50, 80]), set(Color.numbers())) - - def testIterate(self): - """Test that __iter__ iterates over all enum values.""" - self.assertEquals(set(Color), - set([Color.RED, - Color.ORANGE, - Color.YELLOW, + def setUp(self): + """Set up tests.""" + # Redefine Color class in case so that changes to it (an error) in one test + # does not affect other tests. + global Color + + class Color(messages.Enum): + RED = 20 + ORANGE = 2 + YELLOW = 40 + GREEN = 4 + BLUE = 50 + INDIGO = 5 + VIOLET = 80 + + def testNames(self): + """Test that names iterates over enum names.""" + self.assertEquals( + set(['BLUE', 'GREEN', 'INDIGO', 'ORANGE', 'RED', 'VIOLET', 'YELLOW']), + set(Color.names())) + + def testNumbers(self): + """Tests that numbers iterates of enum numbers.""" + self.assertEquals(set([2, 4, 5, 20, 40, 50, 80]), set(Color.numbers())) + + def testIterate(self): + """Test that __iter__ iterates over all enum values.""" + self.assertEquals(set(Color), + set([Color.RED, + Color.ORANGE, + Color.YELLOW, + Color.GREEN, + Color.BLUE, + Color.INDIGO, + Color.VIOLET])) + + def testNaturalOrder(self): + """Test that natural order enumeration is in numeric order.""" + self.assertEquals([Color.ORANGE, Color.GREEN, - Color.BLUE, Color.INDIGO, - Color.VIOLET])) - - def testNaturalOrder(self): - """Test that natural order enumeration is in numeric order.""" - self.assertEquals([Color.ORANGE, - Color.GREEN, - Color.INDIGO, - Color.RED, - Color.YELLOW, - Color.BLUE, - Color.VIOLET], - sorted(Color)) - - def testByName(self): - """Test look-up by name.""" - self.assertEquals(Color.RED, Color.lookup_by_name('RED')) - self.assertRaises(KeyError, Color.lookup_by_name, 20) - self.assertRaises(KeyError, Color.lookup_by_name, Color.RED) - - def testByNumber(self): - """Test look-up by number.""" - self.assertRaises(KeyError, Color.lookup_by_number, 'RED') - self.assertEquals(Color.RED, Color.lookup_by_number(20)) - self.assertRaises(KeyError, Color.lookup_by_number, Color.RED) - - def testConstructor(self): - """Test that constructor look-up by name or number.""" - self.assertEquals(Color.RED, Color('RED')) - self.assertEquals(Color.RED, Color(u'RED')) - self.assertEquals(Color.RED, Color(20)) - if six.PY2: - self.assertEquals(Color.RED, Color(long(20))) - self.assertEquals(Color.RED, Color(Color.RED)) - self.assertRaises(TypeError, Color, 'Not exists') - self.assertRaises(TypeError, Color, 'Red') - self.assertRaises(TypeError, Color, 100) - self.assertRaises(TypeError, Color, 10.0) - - def testLen(self): - """Test that len function works to count enums.""" - self.assertEquals(7, len(Color)) - - def testNoSubclasses(self): - """Test that it is not possible to sub-class enum classes.""" - def declare_subclass(): - class MoreColor(Color): - pass - self.assertRaises(messages.EnumDefinitionError, - declare_subclass) - - def testClassNotMutable(self): - """Test that enum classes themselves are not mutable.""" - self.assertRaises(AttributeError, - setattr, - Color, - 'something_new', - 10) - - def testInstancesMutable(self): - """Test that enum instances are not mutable.""" - self.assertRaises(TypeError, - setattr, - Color.RED, - 'something_new', - 10) - - def testDefEnum(self): - """Test def_enum works by building enum class from dict.""" - WeekDay = messages.Enum.def_enum({'Monday': 1, - 'Tuesday': 2, - 'Wednesday': 3, - 'Thursday': 4, - 'Friday': 6, - 'Saturday': 7, - 'Sunday': 8}, - 'WeekDay') - self.assertEquals('Wednesday', WeekDay(3).name) - self.assertEquals(6, WeekDay('Friday').number) - self.assertEquals(WeekDay.Sunday, WeekDay('Sunday')) - - def testNonInt(self): - """Test that non-integer values rejection by enum def.""" - self.assertRaises(messages.EnumDefinitionError, - messages.Enum.def_enum, - {'Bad': '1'}, - 'BadEnum') - - def testNegativeInt(self): - """Test that negative numbers rejection by enum def.""" - self.assertRaises(messages.EnumDefinitionError, - messages.Enum.def_enum, - {'Bad': -1}, - 'BadEnum') - - def testLowerBound(self): - """Test that zero is accepted by enum def.""" - class NotImportant(messages.Enum): - """Testing for value zero""" - VALUE = 0 - - self.assertEquals(0, int(NotImportant.VALUE)) - - def testTooLargeInt(self): - """Test that numbers too large are rejected.""" - self.assertRaises(messages.EnumDefinitionError, - messages.Enum.def_enum, - {'Bad': (2 ** 29)}, - 'BadEnum') - - def testRepeatedInt(self): - """Test duplicated numbers are forbidden.""" - self.assertRaises(messages.EnumDefinitionError, - messages.Enum.def_enum, - {'Ok': 1, 'Repeated': 1}, - 'BadEnum') - - def testStr(self): - """Test converting to string.""" - self.assertEquals('RED', str(Color.RED)) - self.assertEquals('ORANGE', str(Color.ORANGE)) - - def testInt(self): - """Test converting to int.""" - self.assertEquals(20, int(Color.RED)) - self.assertEquals(2, int(Color.ORANGE)) - - def testRepr(self): - """Test enum representation.""" - self.assertEquals('Color(RED, 20)', repr(Color.RED)) - self.assertEquals('Color(YELLOW, 40)', repr(Color.YELLOW)) - - def testDocstring(self): - """Test that docstring is supported ok.""" - class NotImportant(messages.Enum): - """I have a docstring.""" - - VALUE1 = 1 - - self.assertEquals('I have a docstring.', NotImportant.__doc__) - - def testDeleteEnumValue(self): - """Test that enum values cannot be deleted.""" - self.assertRaises(TypeError, delattr, Color, 'RED') - - def testEnumName(self): - """Test enum name.""" - module_name = test_util.get_module_name(EnumTest) - self.assertEquals('%s.Color' % module_name, Color.definition_name()) - self.assertEquals(module_name, Color.outer_definition_name()) - self.assertEquals(module_name, Color.definition_package()) - - def testDefinitionName_OverrideModule(self): - """Test enum module is overriden by module package name.""" - global package - try: - package = 'my.package' - self.assertEquals('my.package.Color', Color.definition_name()) - self.assertEquals('my.package', Color.outer_definition_name()) - self.assertEquals('my.package', Color.definition_package()) - finally: - del package - - def testDefinitionName_NoModule(self): - """Test what happens when there is no module for enum.""" - class Enum1(messages.Enum): - pass - - original_modules = sys.modules - sys.modules = dict(sys.modules) - try: - del sys.modules[__name__] - self.assertEquals('Enum1', Enum1.definition_name()) - self.assertEquals(None, Enum1.outer_definition_name()) - self.assertEquals(None, Enum1.definition_package()) - self.assertEquals(six.text_type, type(Enum1.definition_name())) - finally: - sys.modules = original_modules - - def testDefinitionName_Nested(self): - """Test nested Enum names.""" - class MyMessage(messages.Message): - - class NestedEnum(messages.Enum): - - pass - - class NestedMessage(messages.Message): - - class NestedEnum(messages.Enum): - - pass - - module_name = test_util.get_module_name(EnumTest) - self.assertEquals('%s.MyMessage.NestedEnum' % module_name, - MyMessage.NestedEnum.definition_name()) - self.assertEquals('%s.MyMessage' % module_name, - MyMessage.NestedEnum.outer_definition_name()) - self.assertEquals(module_name, - MyMessage.NestedEnum.definition_package()) - - self.assertEquals('%s.MyMessage.NestedMessage.NestedEnum' % module_name, - MyMessage.NestedMessage.NestedEnum.definition_name()) - self.assertEquals( - '%s.MyMessage.NestedMessage' % module_name, - MyMessage.NestedMessage.NestedEnum.outer_definition_name()) - self.assertEquals(module_name, - MyMessage.NestedMessage.NestedEnum.definition_package()) - - def testMessageDefinition(self): - """Test that enumeration knows its enclosing message definition.""" - class OuterEnum(messages.Enum): - pass - - self.assertEquals(None, OuterEnum.message_definition()) - - class OuterMessage(messages.Message): - - class InnerEnum(messages.Enum): - pass - - self.assertEquals(OuterMessage, OuterMessage.InnerEnum.message_definition()) - - def testComparison(self): - """Test comparing various enums to different types.""" - class Enum1(messages.Enum): - VAL1 = 1 - VAL2 = 2 - - class Enum2(messages.Enum): - VAL1 = 1 - - self.assertEquals(Enum1.VAL1, Enum1.VAL1) - self.assertNotEquals(Enum1.VAL1, Enum1.VAL2) - self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) - self.assertNotEquals(Enum1.VAL1, 'VAL1') - self.assertNotEquals(Enum1.VAL1, 1) - self.assertNotEquals(Enum1.VAL1, 2) - self.assertNotEquals(Enum1.VAL1, None) - self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) - - self.assertTrue(Enum1.VAL1 < Enum1.VAL2) - self.assertTrue(Enum1.VAL2 > Enum1.VAL1) - - self.assertNotEquals(1, Enum2.VAL1) - - def testPickle(self): - """Testing pickling and unpickling of Enum instances.""" - colors = list(Color) - unpickled = pickle.loads(pickle.dumps(colors)) - self.assertEquals(colors, unpickled) - # Unpickling shouldn't create new enum instances. - for i, color in enumerate(colors): - self.assertTrue(color is unpickled[i]) + Color.RED, + Color.YELLOW, + Color.BLUE, + Color.VIOLET], + sorted(Color)) + + def testByName(self): + """Test look-up by name.""" + self.assertEquals(Color.RED, Color.lookup_by_name('RED')) + self.assertRaises(KeyError, Color.lookup_by_name, 20) + self.assertRaises(KeyError, Color.lookup_by_name, Color.RED) + + def testByNumber(self): + """Test look-up by number.""" + self.assertRaises(KeyError, Color.lookup_by_number, 'RED') + self.assertEquals(Color.RED, Color.lookup_by_number(20)) + self.assertRaises(KeyError, Color.lookup_by_number, Color.RED) + + def testConstructor(self): + """Test that constructor look-up by name or number.""" + self.assertEquals(Color.RED, Color('RED')) + self.assertEquals(Color.RED, Color(u'RED')) + self.assertEquals(Color.RED, Color(20)) + if six.PY2: + self.assertEquals(Color.RED, Color(long(20))) + self.assertEquals(Color.RED, Color(Color.RED)) + self.assertRaises(TypeError, Color, 'Not exists') + self.assertRaises(TypeError, Color, 'Red') + self.assertRaises(TypeError, Color, 100) + self.assertRaises(TypeError, Color, 10.0) + + def testLen(self): + """Test that len function works to count enums.""" + self.assertEquals(7, len(Color)) + + def testNoSubclasses(self): + """Test that it is not possible to sub-class enum classes.""" + def declare_subclass(): + class MoreColor(Color): + pass + self.assertRaises(messages.EnumDefinitionError, + declare_subclass) + + def testClassNotMutable(self): + """Test that enum classes themselves are not mutable.""" + self.assertRaises(AttributeError, + setattr, + Color, + 'something_new', + 10) + + def testInstancesMutable(self): + """Test that enum instances are not mutable.""" + self.assertRaises(TypeError, + setattr, + Color.RED, + 'something_new', + 10) + + def testDefEnum(self): + """Test def_enum works by building enum class from dict.""" + WeekDay = messages.Enum.def_enum({'Monday': 1, + 'Tuesday': 2, + 'Wednesday': 3, + 'Thursday': 4, + 'Friday': 6, + 'Saturday': 7, + 'Sunday': 8}, + 'WeekDay') + self.assertEquals('Wednesday', WeekDay(3).name) + self.assertEquals(6, WeekDay('Friday').number) + self.assertEquals(WeekDay.Sunday, WeekDay('Sunday')) + + def testNonInt(self): + """Test that non-integer values rejection by enum def.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': '1'}, + 'BadEnum') + + def testNegativeInt(self): + """Test that negative numbers rejection by enum def.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': -1}, + 'BadEnum') + + def testLowerBound(self): + """Test that zero is accepted by enum def.""" + class NotImportant(messages.Enum): + """Testing for value zero""" + VALUE = 0 + + self.assertEquals(0, int(NotImportant.VALUE)) + + def testTooLargeInt(self): + """Test that numbers too large are rejected.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Bad': (2 ** 29)}, + 'BadEnum') + + def testRepeatedInt(self): + """Test duplicated numbers are forbidden.""" + self.assertRaises(messages.EnumDefinitionError, + messages.Enum.def_enum, + {'Ok': 1, 'Repeated': 1}, + 'BadEnum') + + def testStr(self): + """Test converting to string.""" + self.assertEquals('RED', str(Color.RED)) + self.assertEquals('ORANGE', str(Color.ORANGE)) + + def testInt(self): + """Test converting to int.""" + self.assertEquals(20, int(Color.RED)) + self.assertEquals(2, int(Color.ORANGE)) + + def testRepr(self): + """Test enum representation.""" + self.assertEquals('Color(RED, 20)', repr(Color.RED)) + self.assertEquals('Color(YELLOW, 40)', repr(Color.YELLOW)) + + def testDocstring(self): + """Test that docstring is supported ok.""" + class NotImportant(messages.Enum): + """I have a docstring.""" + + VALUE1 = 1 + + self.assertEquals('I have a docstring.', NotImportant.__doc__) + + def testDeleteEnumValue(self): + """Test that enum values cannot be deleted.""" + self.assertRaises(TypeError, delattr, Color, 'RED') + + def testEnumName(self): + """Test enum name.""" + module_name = test_util.get_module_name(EnumTest) + self.assertEquals('%s.Color' % module_name, Color.definition_name()) + self.assertEquals(module_name, Color.outer_definition_name()) + self.assertEquals(module_name, Color.definition_package()) + + def testDefinitionName_OverrideModule(self): + """Test enum module is overriden by module package name.""" + global package + try: + package = 'my.package' + self.assertEquals('my.package.Color', Color.definition_name()) + self.assertEquals('my.package', Color.outer_definition_name()) + self.assertEquals('my.package', Color.definition_package()) + finally: + del package + + def testDefinitionName_NoModule(self): + """Test what happens when there is no module for enum.""" + class Enum1(messages.Enum): + pass + + original_modules = sys.modules + sys.modules = dict(sys.modules) + try: + del sys.modules[__name__] + self.assertEquals('Enum1', Enum1.definition_name()) + self.assertEquals(None, Enum1.outer_definition_name()) + self.assertEquals(None, Enum1.definition_package()) + self.assertEquals(six.text_type, type(Enum1.definition_name())) + finally: + sys.modules = original_modules + + def testDefinitionName_Nested(self): + """Test nested Enum names.""" + class MyMessage(messages.Message): + + class NestedEnum(messages.Enum): + + pass + + class NestedMessage(messages.Message): + + class NestedEnum(messages.Enum): + + pass + + module_name = test_util.get_module_name(EnumTest) + self.assertEquals('%s.MyMessage.NestedEnum' % module_name, + MyMessage.NestedEnum.definition_name()) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.NestedEnum.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedEnum.definition_package()) + + self.assertEquals('%s.MyMessage.NestedMessage.NestedEnum' % module_name, + MyMessage.NestedMessage.NestedEnum.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedEnum.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedMessage.NestedEnum.definition_package()) + + def testMessageDefinition(self): + """Test that enumeration knows its enclosing message definition.""" + class OuterEnum(messages.Enum): + pass + + self.assertEquals(None, OuterEnum.message_definition()) + + class OuterMessage(messages.Message): + + class InnerEnum(messages.Enum): + pass + + self.assertEquals( + OuterMessage, OuterMessage.InnerEnum.message_definition()) + + def testComparison(self): + """Test comparing various enums to different types.""" + class Enum1(messages.Enum): + VAL1 = 1 + VAL2 = 2 + + class Enum2(messages.Enum): + VAL1 = 1 + + self.assertEquals(Enum1.VAL1, Enum1.VAL1) + self.assertNotEquals(Enum1.VAL1, Enum1.VAL2) + self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) + self.assertNotEquals(Enum1.VAL1, 'VAL1') + self.assertNotEquals(Enum1.VAL1, 1) + self.assertNotEquals(Enum1.VAL1, 2) + self.assertNotEquals(Enum1.VAL1, None) + self.assertNotEquals(Enum1.VAL1, Enum2.VAL1) + + self.assertTrue(Enum1.VAL1 < Enum1.VAL2) + self.assertTrue(Enum1.VAL2 > Enum1.VAL1) + + self.assertNotEquals(1, Enum2.VAL1) + + def testPickle(self): + """Testing pickling and unpickling of Enum instances.""" + colors = list(Color) + unpickled = pickle.loads(pickle.dumps(colors)) + self.assertEquals(colors, unpickled) + # Unpickling shouldn't create new enum instances. + for i, color in enumerate(colors): + self.assertTrue(color is unpickled[i]) class FieldListTest(test_util.TestCase): - def setUp(self): - self.integer_field = messages.IntegerField(1, repeated=True) - - def testConstructor(self): - self.assertEquals([1, 2, 3], - messages.FieldList(self.integer_field, [1, 2, 3])) - self.assertEquals([1, 2, 3], - messages.FieldList(self.integer_field, (1, 2, 3))) - self.assertEquals([], messages.FieldList(self.integer_field, [])) - - def testNone(self): - self.assertRaises(TypeError, messages.FieldList, self.integer_field, None) - - def testDoNotAutoConvertString(self): - string_field = messages.StringField(1, repeated=True) - self.assertRaises(messages.ValidationError, - messages.FieldList, string_field, 'abc') - - def testConstructorCopies(self): - a_list = [1, 3, 6] - field_list = messages.FieldList(self.integer_field, a_list) - self.assertFalse(a_list is field_list) - self.assertFalse(field_list is - messages.FieldList(self.integer_field, field_list)) - - def testNonRepeatedField(self): - self.assertRaisesWithRegexpMatch( - messages.FieldDefinitionError, - 'FieldList may only accept repeated fields', - messages.FieldList, - messages.IntegerField(1), - []) - - def testConstructor_InvalidValues(self): - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - re.escape("Expected type %r " - "for IntegerField, found 1 (type %r)" - % (six.integer_types, str)), - messages.FieldList, self.integer_field, ["1", "2", "3"]) - - def testConstructor_Scalars(self): - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - "IntegerField is repeated. Found: 3", - messages.FieldList, self.integer_field, 3) - - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - "IntegerField is repeated. Found: <(list[_]?|sequence)iterator object", - messages.FieldList, self.integer_field, iter([1, 2, 3])) - - def testSetSlice(self): - field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) - field_list[1:3] = [10, 20] - self.assertEquals([1, 10, 20, 4, 5], field_list) - - def testSetSlice_InvalidValues(self): - field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) - - def setslice(): - field_list[1:3] = ['10', '20'] - - msg_re = re.escape("Expected type %r " - "for IntegerField, found 10 (type %r)" - % (six.integer_types, str)) - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - msg_re, - setslice) - - def testSetItem(self): - field_list = messages.FieldList(self.integer_field, [2]) - field_list[0] = 10 - self.assertEquals([10], field_list) - - def testSetItem_InvalidValues(self): - field_list = messages.FieldList(self.integer_field, [2]) - - def setitem(): - field_list[0] = '10' - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - re.escape("Expected type %r " - "for IntegerField, found 10 (type %r)" - % (six.integer_types, str)), - setitem) - - def testAppend(self): - field_list = messages.FieldList(self.integer_field, [2]) - field_list.append(10) - self.assertEquals([2, 10], field_list) - - def testAppend_InvalidValues(self): - field_list = messages.FieldList(self.integer_field, [2]) - field_list.name = 'a_field' - - def append(): - field_list.append('10') - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - re.escape("Expected type %r " - "for IntegerField, found 10 (type %r)" - % (six.integer_types, str)), - append) - - def testExtend(self): - field_list = messages.FieldList(self.integer_field, [2]) - field_list.extend([10]) - self.assertEquals([2, 10], field_list) - - def testExtend_InvalidValues(self): - field_list = messages.FieldList(self.integer_field, [2]) - - def extend(): - field_list.extend(['10']) - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - re.escape("Expected type %r " - "for IntegerField, found 10 (type %r)" - % (six.integer_types, str)), - extend) - - def testInsert(self): - field_list = messages.FieldList(self.integer_field, [2, 3]) - field_list.insert(1, 10) - self.assertEquals([2, 10, 3], field_list) - - def testInsert_InvalidValues(self): - field_list = messages.FieldList(self.integer_field, [2, 3]) - - def insert(): - field_list.insert(1, '10') - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - re.escape("Expected type %r " - "for IntegerField, found 10 (type %r)" - % (six.integer_types, str)), - insert) - - def testPickle(self): - """Testing pickling and unpickling of disconnected FieldList instances.""" - field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) - unpickled = pickle.loads(pickle.dumps(field_list)) - self.assertEquals(field_list, unpickled) - self.assertIsInstance(unpickled.field, messages.IntegerField) - self.assertEquals(1, unpickled.field.number) - self.assertTrue(unpickled.field.repeated) + def setUp(self): + self.integer_field = messages.IntegerField(1, repeated=True) + + def testConstructor(self): + self.assertEquals([1, 2, 3], + messages.FieldList(self.integer_field, [1, 2, 3])) + self.assertEquals([1, 2, 3], + messages.FieldList(self.integer_field, (1, 2, 3))) + self.assertEquals([], messages.FieldList(self.integer_field, [])) + + def testNone(self): + self.assertRaises(TypeError, messages.FieldList, + self.integer_field, None) + + def testDoNotAutoConvertString(self): + string_field = messages.StringField(1, repeated=True) + self.assertRaises(messages.ValidationError, + messages.FieldList, string_field, 'abc') + + def testConstructorCopies(self): + a_list = [1, 3, 6] + field_list = messages.FieldList(self.integer_field, a_list) + self.assertFalse(a_list is field_list) + self.assertFalse(field_list is + messages.FieldList(self.integer_field, field_list)) + + def testNonRepeatedField(self): + self.assertRaisesWithRegexpMatch( + messages.FieldDefinitionError, + 'FieldList may only accept repeated fields', + messages.FieldList, + messages.IntegerField(1), + []) + + def testConstructor_InvalidValues(self): + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 1 (type %r)" + % (six.integer_types, str)), + messages.FieldList, self.integer_field, ["1", "2", "3"]) + + def testConstructor_Scalars(self): + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "IntegerField is repeated. Found: 3", + messages.FieldList, self.integer_field, 3) + + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "IntegerField is repeated. Found: <(list[_]?|sequence)iterator object", + messages.FieldList, self.integer_field, iter([1, 2, 3])) + + def testSetSlice(self): + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + field_list[1:3] = [10, 20] + self.assertEquals([1, 10, 20, 4, 5], field_list) + + def testSetSlice_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + + def setslice(): + field_list[1:3] = ['10', '20'] + + msg_re = re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + msg_re, + setslice) + + def testSetItem(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list[0] = 10 + self.assertEquals([10], field_list) + + def testSetItem_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + + def setitem(): + field_list[0] = '10' + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + setitem) + + def testAppend(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.append(10) + self.assertEquals([2, 10], field_list) + + def testAppend_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.name = 'a_field' + + def append(): + field_list.append('10') + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + append) + + def testExtend(self): + field_list = messages.FieldList(self.integer_field, [2]) + field_list.extend([10]) + self.assertEquals([2, 10], field_list) + + def testExtend_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2]) + + def extend(): + field_list.extend(['10']) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + extend) + + def testInsert(self): + field_list = messages.FieldList(self.integer_field, [2, 3]) + field_list.insert(1, 10) + self.assertEquals([2, 10, 3], field_list) + + def testInsert_InvalidValues(self): + field_list = messages.FieldList(self.integer_field, [2, 3]) + + def insert(): + field_list.insert(1, '10') + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + re.escape("Expected type %r " + "for IntegerField, found 10 (type %r)" + % (six.integer_types, str)), + insert) + + def testPickle(self): + """Testing pickling and unpickling of disconnected FieldList instances.""" + field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) + unpickled = pickle.loads(pickle.dumps(field_list)) + self.assertEquals(field_list, unpickled) + self.assertIsInstance(unpickled.field, messages.IntegerField) + self.assertEquals(1, unpickled.field.number) + self.assertTrue(unpickled.field.repeated) class FieldTest(test_util.TestCase): - def ActionOnAllFieldClasses(self, action): - """Test all field classes except Message and Enum. - - Message and Enum require separate tests. - - Args: - action: Callable that takes the field class as a parameter. - """ - for field_class in (messages.IntegerField, - messages.FloatField, - messages.BooleanField, - messages.BytesField, - messages.StringField, - ): - action(field_class) - - def testNumberAttribute(self): - """Test setting the number attribute.""" - def action(field_class): - # Check range. - self.assertRaises(messages.InvalidNumberError, - field_class, - 0) - self.assertRaises(messages.InvalidNumberError, - field_class, - -1) - self.assertRaises(messages.InvalidNumberError, - field_class, - messages.MAX_FIELD_NUMBER + 1) - - # Check reserved. - self.assertRaises(messages.InvalidNumberError, - field_class, - messages.FIRST_RESERVED_FIELD_NUMBER) - self.assertRaises(messages.InvalidNumberError, - field_class, - messages.LAST_RESERVED_FIELD_NUMBER) - self.assertRaises(messages.InvalidNumberError, - field_class, - '1') - - # This one should work. - field_class(number=1) - self.ActionOnAllFieldClasses(action) - - def testRequiredAndRepeated(self): - """Test setting the required and repeated fields.""" - def action(field_class): - field_class(1, required=True) - field_class(1, repeated=True) - self.assertRaises(messages.FieldDefinitionError, - field_class, - 1, - required=True, - repeated=True) - self.ActionOnAllFieldClasses(action) - - def testInvalidVariant(self): - """Test field with invalid variants.""" - def action(field_class): - if field_class is not message_types.DateTimeField: - self.assertRaises(messages.InvalidVariantError, - field_class, - 1, - variant=messages.Variant.ENUM) - self.ActionOnAllFieldClasses(action) - - def testDefaultVariant(self): - """Test that default variant is used when not set.""" - def action(field_class): - field = field_class(1) - self.assertEquals(field_class.DEFAULT_VARIANT, field.variant) - - self.ActionOnAllFieldClasses(action) - - def testAlternateVariant(self): - """Test that default variant is used when not set.""" - field = messages.IntegerField(1, variant=messages.Variant.UINT32) - self.assertEquals(messages.Variant.UINT32, field.variant) - - def testDefaultFields_Single(self): - """Test default field is correct type (single).""" - defaults = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: b'abc', - messages.StringField: u'abc', - } - - def action(field_class): - field_class(1, default=defaults[field_class]) - self.ActionOnAllFieldClasses(action) - - # Run defaults test again checking for str/unicode compatiblity. - defaults[messages.StringField] = 'abc' - self.ActionOnAllFieldClasses(action) - - def testStringField_BadUnicodeInDefault(self): - """Test binary values in string field.""" - self.assertRaisesWithRegexpMatch( - messages.InvalidDefaultError, - r"Invalid default value for StringField:.*: " - r"Field encountered non-ASCII string .*: " - r"'ascii' codec can't decode byte 0x89 in position 0: " - r"ordinal not in range", - messages.StringField, 1, default=b'\x89') - - def testDefaultFields_InvalidSingle(self): - """Test default field is correct type (invalid single).""" - def action(field_class): - self.assertRaises(messages.InvalidDefaultError, - field_class, - 1, - default=object()) - self.ActionOnAllFieldClasses(action) - - def testDefaultFields_InvalidRepeated(self): - """Test default field does not accept defaults.""" - self.assertRaisesWithRegexpMatch( - messages.FieldDefinitionError, - 'Repeated fields may not have defaults', - messages.StringField, 1, repeated=True, default=[1, 2, 3]) - - def testDefaultFields_None(self): - """Test none is always acceptable.""" - def action(field_class): - field_class(1, default=None) - field_class(1, required=True, default=None) - field_class(1, repeated=True, default=None) - self.ActionOnAllFieldClasses(action) - - def testDefaultFields_Enum(self): - """Test the default for enum fields.""" - class Symbol(messages.Enum): - - ALPHA = 1 - BETA = 2 - GAMMA = 3 - - field = messages.EnumField(Symbol, 1, default=Symbol.ALPHA) - - self.assertEquals(Symbol.ALPHA, field.default) - - def testDefaultFields_EnumStringDelayedResolution(self): - """Test that enum fields resolve default strings.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default='OPTIONAL') - - self.assertEquals(descriptor.FieldDescriptor.Label.OPTIONAL, field.default) - - def testDefaultFields_EnumIntDelayedResolution(self): - """Test that enum fields resolve default integers.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default=2) - - self.assertEquals(descriptor.FieldDescriptor.Label.REQUIRED, field.default) - - def testDefaultFields_EnumOkIfTypeKnown(self): - """Test that enum fields accept valid default values when type is known.""" - field = messages.EnumField(descriptor.FieldDescriptor.Label, - 1, - default='REPEATED') - - self.assertEquals(descriptor.FieldDescriptor.Label.REPEATED, field.default) - - def testDefaultFields_EnumForceCheckIfTypeKnown(self): - """Test that enum fields validate default values if type is known.""" - self.assertRaisesWithRegexpMatch(TypeError, - 'No such value for NOT_A_LABEL in ' - 'Enum Label', - messages.EnumField, - descriptor.FieldDescriptor.Label, - 1, - default='NOT_A_LABEL') - - def testDefaultFields_EnumInvalidDelayedResolution(self): - """Test that enum fields raise errors upon delayed resolution error.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default=200) - - self.assertRaisesWithRegexpMatch(TypeError, - 'No such value for 200 in Enum Label', - getattr, - field, - 'default') - - def testValidate_Valid(self): - """Test validation of valid values.""" - values = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: b'abc', - messages.StringField: u'abc', - } - def action(field_class): - # Optional. - field = field_class(1) - field.validate(values[field_class]) - - # Required. - field = field_class(1, required=True) - field.validate(values[field_class]) - - # Repeated. - field = field_class(1, repeated=True) - field.validate([]) - field.validate(()) - field.validate([values[field_class]]) - field.validate((values[field_class],)) - - # Right value, but not repeated. - self.assertRaises(messages.ValidationError, - field.validate, - values[field_class]) - self.assertRaises(messages.ValidationError, - field.validate, - values[field_class]) - - self.ActionOnAllFieldClasses(action) - - def testValidate_Invalid(self): - """Test validation of valid values.""" - values = {messages.IntegerField: "10", - messages.FloatField: 1, - messages.BooleanField: 0, - messages.BytesField: 10.20, - messages.StringField: 42, - } - def action(field_class): - # Optional. - field = field_class(1) - self.assertRaises(messages.ValidationError, - field.validate, - values[field_class]) - - # Required. - field = field_class(1, required=True) - self.assertRaises(messages.ValidationError, - field.validate, - values[field_class]) - - # Repeated. - field = field_class(1, repeated=True) - self.assertRaises(messages.ValidationError, - field.validate, - [values[field_class]]) - self.assertRaises(messages.ValidationError, - field.validate, - (values[field_class],)) - self.ActionOnAllFieldClasses(action) - - def testValidate_None(self): - """Test that None is valid for non-required fields.""" - def action(field_class): - # Optional. - field = field_class(1) - field.validate(None) - - # Required. - field = field_class(1, required=True) - self.assertRaisesWithRegexpMatch(messages.ValidationError, - 'Required field is missing', - field.validate, - None) - - # Repeated. - field = field_class(1, repeated=True) - field.validate(None) - self.assertRaisesWithRegexpMatch(messages.ValidationError, - 'Repeated values for %s may ' - 'not be None' % field_class.__name__, - field.validate, - [None]) - self.assertRaises(messages.ValidationError, - field.validate, - (None,)) - self.ActionOnAllFieldClasses(action) - - def testValidateElement(self): - """Test validation of valid values.""" - values = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: 'abc', - messages.StringField: u'abc', - } - def action(field_class): - # Optional. - field = field_class(1) - field.validate_element(values[field_class]) - - # Required. - field = field_class(1, required=True) - field.validate_element(values[field_class]) - - # Repeated. - field = field_class(1, repeated=True) - self.assertRaises(message.VAlidationError, - field.validate_element, - []) - self.assertRaises(message.VAlidationError, - field.validate_element, - ()) - field.validate_element(values[field_class]) - field.validate_element(values[field_class]) - - # Right value, but repeated. - self.assertRaises(messages.ValidationError, - field.validate_element, - [values[field_class]]) - self.assertRaises(messages.ValidationError, - field.validate_element, - (values[field_class],)) - - def testReadOnly(self): - """Test that objects are all read-only.""" - def action(field_class): - field = field_class(10) - self.assertRaises(AttributeError, - setattr, - field, - 'number', - 20) - self.assertRaises(AttributeError, - setattr, - field, - 'anything_else', - 'whatever') - self.ActionOnAllFieldClasses(action) - - def testMessageField(self): - """Test the construction of message fields.""" - self.assertRaises(messages.FieldDefinitionError, - messages.MessageField, - str, - 10) - - self.assertRaises(messages.FieldDefinitionError, - messages.MessageField, - messages.Message, - 10) - - class MyMessage(messages.Message): - pass - - field = messages.MessageField(MyMessage, 10) - self.assertEquals(MyMessage, field.type) - - def testMessageField_ForwardReference(self): - """Test the construction of forward reference message fields.""" - global MyMessage - global ForwardMessage - try: - class MyMessage(messages.Message): - - self_reference = messages.MessageField('MyMessage', 1) - forward = messages.MessageField('ForwardMessage', 2) - nested = messages.MessageField('ForwardMessage.NestedMessage', 3) - inner = messages.MessageField('Inner', 4) - - class Inner(messages.Message): - - sibling = messages.MessageField('Sibling', 1) - - class Sibling(messages.Message): - - pass - - class ForwardMessage(messages.Message): + def ActionOnAllFieldClasses(self, action): + """Test all field classes except Message and Enum. + + Message and Enum require separate tests. + + Args: + action: Callable that takes the field class as a parameter. + """ + for field_class in (messages.IntegerField, + messages.FloatField, + messages.BooleanField, + messages.BytesField, + messages.StringField, + ): + action(field_class) + + def testNumberAttribute(self): + """Test setting the number attribute.""" + def action(field_class): + # Check range. + self.assertRaises(messages.InvalidNumberError, + field_class, + 0) + self.assertRaises(messages.InvalidNumberError, + field_class, + -1) + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.MAX_FIELD_NUMBER + 1) + + # Check reserved. + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.FIRST_RESERVED_FIELD_NUMBER) + self.assertRaises(messages.InvalidNumberError, + field_class, + messages.LAST_RESERVED_FIELD_NUMBER) + self.assertRaises(messages.InvalidNumberError, + field_class, + '1') + + # This one should work. + field_class(number=1) + self.ActionOnAllFieldClasses(action) + + def testRequiredAndRepeated(self): + """Test setting the required and repeated fields.""" + def action(field_class): + field_class(1, required=True) + field_class(1, repeated=True) + self.assertRaises(messages.FieldDefinitionError, + field_class, + 1, + required=True, + repeated=True) + self.ActionOnAllFieldClasses(action) + + def testInvalidVariant(self): + """Test field with invalid variants.""" + def action(field_class): + if field_class is not message_types.DateTimeField: + self.assertRaises(messages.InvalidVariantError, + field_class, + 1, + variant=messages.Variant.ENUM) + self.ActionOnAllFieldClasses(action) + + def testDefaultVariant(self): + """Test that default variant is used when not set.""" + def action(field_class): + field = field_class(1) + self.assertEquals(field_class.DEFAULT_VARIANT, field.variant) + + self.ActionOnAllFieldClasses(action) + + def testAlternateVariant(self): + """Test that default variant is used when not set.""" + field = messages.IntegerField(1, variant=messages.Variant.UINT32) + self.assertEquals(messages.Variant.UINT32, field.variant) + + def testDefaultFields_Single(self): + """Test default field is correct type (single).""" + defaults = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } + + def action(field_class): + field_class(1, default=defaults[field_class]) + self.ActionOnAllFieldClasses(action) + + # Run defaults test again checking for str/unicode compatiblity. + defaults[messages.StringField] = 'abc' + self.ActionOnAllFieldClasses(action) + + def testStringField_BadUnicodeInDefault(self): + """Test binary values in string field.""" + self.assertRaisesWithRegexpMatch( + messages.InvalidDefaultError, + r"Invalid default value for StringField:.*: " + r"Field encountered non-ASCII string .*: " + r"'ascii' codec can't decode byte 0x89 in position 0: " + r"ordinal not in range", + messages.StringField, 1, default=b'\x89') + + def testDefaultFields_InvalidSingle(self): + """Test default field is correct type (invalid single).""" + def action(field_class): + self.assertRaises(messages.InvalidDefaultError, + field_class, + 1, + default=object()) + self.ActionOnAllFieldClasses(action) + + def testDefaultFields_InvalidRepeated(self): + """Test default field does not accept defaults.""" + self.assertRaisesWithRegexpMatch( + messages.FieldDefinitionError, + 'Repeated fields may not have defaults', + messages.StringField, 1, repeated=True, default=[1, 2, 3]) + + def testDefaultFields_None(self): + """Test none is always acceptable.""" + def action(field_class): + field_class(1, default=None) + field_class(1, required=True, default=None) + field_class(1, repeated=True, default=None) + self.ActionOnAllFieldClasses(action) + + def testDefaultFields_Enum(self): + """Test the default for enum fields.""" + class Symbol(messages.Enum): + + ALPHA = 1 + BETA = 2 + GAMMA = 3 + + field = messages.EnumField(Symbol, 1, default=Symbol.ALPHA) + + self.assertEquals(Symbol.ALPHA, field.default) + + def testDefaultFields_EnumStringDelayedResolution(self): + """Test that enum fields resolve default strings.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default='OPTIONAL') + + self.assertEquals( + descriptor.FieldDescriptor.Label.OPTIONAL, field.default) + + def testDefaultFields_EnumIntDelayedResolution(self): + """Test that enum fields resolve default integers.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=2) + + self.assertEquals( + descriptor.FieldDescriptor.Label.REQUIRED, field.default) + + def testDefaultFields_EnumOkIfTypeKnown(self): + """Test that enum fields accept valid default values when type is known.""" + field = messages.EnumField(descriptor.FieldDescriptor.Label, + 1, + default='REPEATED') + + self.assertEquals( + descriptor.FieldDescriptor.Label.REPEATED, field.default) + + def testDefaultFields_EnumForceCheckIfTypeKnown(self): + """Test that enum fields validate default values if type is known.""" + self.assertRaisesWithRegexpMatch(TypeError, + 'No such value for NOT_A_LABEL in ' + 'Enum Label', + messages.EnumField, + descriptor.FieldDescriptor.Label, + 1, + default='NOT_A_LABEL') + + def testDefaultFields_EnumInvalidDelayedResolution(self): + """Test that enum fields raise errors upon delayed resolution error.""" + field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=200) + + self.assertRaisesWithRegexpMatch(TypeError, + 'No such value for 200 in Enum Label', + getattr, + field, + 'default') + + def testValidate_Valid(self): + """Test validation of valid values.""" + values = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } + + def action(field_class): + # Optional. + field = field_class(1) + field.validate(values[field_class]) + + # Required. + field = field_class(1, required=True) + field.validate(values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + field.validate([]) + field.validate(()) + field.validate([values[field_class]]) + field.validate((values[field_class],)) + + # Right value, but not repeated. + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + self.ActionOnAllFieldClasses(action) + + def testValidate_Invalid(self): + """Test validation of valid values.""" + values = {messages.IntegerField: "10", + messages.FloatField: 1, + messages.BooleanField: 0, + messages.BytesField: 10.20, + messages.StringField: 42, + } + + def action(field_class): + # Optional. + field = field_class(1) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + # Required. + field = field_class(1, required=True) + self.assertRaises(messages.ValidationError, + field.validate, + values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + self.assertRaises(messages.ValidationError, + field.validate, + [values[field_class]]) + self.assertRaises(messages.ValidationError, + field.validate, + (values[field_class],)) + self.ActionOnAllFieldClasses(action) + + def testValidate_None(self): + """Test that None is valid for non-required fields.""" + def action(field_class): + # Optional. + field = field_class(1) + field.validate(None) + + # Required. + field = field_class(1, required=True) + self.assertRaisesWithRegexpMatch(messages.ValidationError, + 'Required field is missing', + field.validate, + None) + + # Repeated. + field = field_class(1, repeated=True) + field.validate(None) + self.assertRaisesWithRegexpMatch(messages.ValidationError, + 'Repeated values for %s may ' + 'not be None' % field_class.__name__, + field.validate, + [None]) + self.assertRaises(messages.ValidationError, + field.validate, + (None,)) + self.ActionOnAllFieldClasses(action) + + def testValidateElement(self): + """Test validation of valid values.""" + values = {messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: 'abc', + messages.StringField: u'abc', + } + + def action(field_class): + # Optional. + field = field_class(1) + field.validate_element(values[field_class]) + + # Required. + field = field_class(1, required=True) + field.validate_element(values[field_class]) + + # Repeated. + field = field_class(1, repeated=True) + self.assertRaises(message.VAlidationError, + field.validate_element, + []) + self.assertRaises(message.VAlidationError, + field.validate_element, + ()) + field.validate_element(values[field_class]) + field.validate_element(values[field_class]) + + # Right value, but repeated. + self.assertRaises(messages.ValidationError, + field.validate_element, + [values[field_class]]) + self.assertRaises(messages.ValidationError, + field.validate_element, + (values[field_class],)) + + def testReadOnly(self): + """Test that objects are all read-only.""" + def action(field_class): + field = field_class(10) + self.assertRaises(AttributeError, + setattr, + field, + 'number', + 20) + self.assertRaises(AttributeError, + setattr, + field, + 'anything_else', + 'whatever') + self.ActionOnAllFieldClasses(action) + + def testMessageField(self): + """Test the construction of message fields.""" + self.assertRaises(messages.FieldDefinitionError, + messages.MessageField, + str, + 10) + + self.assertRaises(messages.FieldDefinitionError, + messages.MessageField, + messages.Message, + 10) + + class MyMessage(messages.Message): + pass + + field = messages.MessageField(MyMessage, 10) + self.assertEquals(MyMessage, field.type) + + def testMessageField_ForwardReference(self): + """Test the construction of forward reference message fields.""" + global MyMessage + global ForwardMessage + try: + class MyMessage(messages.Message): + + self_reference = messages.MessageField('MyMessage', 1) + forward = messages.MessageField('ForwardMessage', 2) + nested = messages.MessageField( + 'ForwardMessage.NestedMessage', 3) + inner = messages.MessageField('Inner', 4) + + class Inner(messages.Message): + + sibling = messages.MessageField('Sibling', 1) + + class Sibling(messages.Message): + + pass + + class ForwardMessage(messages.Message): + + class NestedMessage(messages.Message): + + pass + + self.assertEquals(MyMessage, + MyMessage.field_by_name('self_reference').type) + + self.assertEquals(ForwardMessage, + MyMessage.field_by_name('forward').type) + + self.assertEquals(ForwardMessage.NestedMessage, + MyMessage.field_by_name('nested').type) + + self.assertEquals(MyMessage.Inner, + MyMessage.field_by_name('inner').type) + + self.assertEquals(MyMessage.Sibling, + MyMessage.Inner.field_by_name('sibling').type) + finally: + try: + del MyMessage + del ForwardMessage + except: + pass + + def testMessageField_WrongType(self): + """Test that forward referencing the wrong type raises an error.""" + global AnEnum + try: + class AnEnum(messages.Enum): + pass - class NestedMessage(messages.Message): + class AnotherMessage(messages.Message): + + a_field = messages.MessageField('AnEnum', 1) + + self.assertRaises(messages.FieldDefinitionError, + getattr, + AnotherMessage.field_by_name('a_field'), + 'type') + finally: + del AnEnum - pass + def testMessageFieldValidate(self): + """Test validation on message field.""" + class MyMessage(messages.Message): + pass - self.assertEquals(MyMessage, - MyMessage.field_by_name('self_reference').type) + class AnotherMessage(messages.Message): + pass - self.assertEquals(ForwardMessage, - MyMessage.field_by_name('forward').type) + field = messages.MessageField(MyMessage, 10) + field.validate(MyMessage()) - self.assertEquals(ForwardMessage.NestedMessage, - MyMessage.field_by_name('nested').type) + self.assertRaises(messages.ValidationError, + field.validate, + AnotherMessage()) - self.assertEquals(MyMessage.Inner, - MyMessage.field_by_name('inner').type) + def testMessageFieldMessageType(self): + """Test message_type property.""" + class MyMessage(messages.Message): + pass - self.assertEquals(MyMessage.Sibling, - MyMessage.Inner.field_by_name('sibling').type) - finally: - try: - del MyMessage - del ForwardMessage - except: - pass + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) - def testMessageField_WrongType(self): - """Test that forward referencing the wrong type raises an error.""" - global AnEnum - try: - class AnEnum(messages.Enum): - pass + self.assertEqual(HasMessage.field.type, HasMessage.field.message_type) - class AnotherMessage(messages.Message): + def testMessageFieldValueFromMessage(self): + class MyMessage(messages.Message): + pass - a_field = messages.MessageField('AnEnum', 1) + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) - self.assertRaises(messages.FieldDefinitionError, - getattr, - AnotherMessage.field_by_name('a_field'), - 'type') - finally: - del AnEnum + instance = MyMessage() - def testMessageFieldValidate(self): - """Test validation on message field.""" - class MyMessage(messages.Message): - pass + self.assertTrue( + instance is HasMessage.field.value_from_message(instance)) - class AnotherMessage(messages.Message): - pass + def testMessageFieldValueFromMessageWrongType(self): + class MyMessage(messages.Message): + pass - field = messages.MessageField(MyMessage, 10) - field.validate(MyMessage()) + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) - self.assertRaises(messages.ValidationError, - field.validate, - AnotherMessage()) + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Expected type MyMessage, got int: 10', + HasMessage.field.value_from_message, 10) - def testMessageFieldMessageType(self): - """Test message_type property.""" - class MyMessage(messages.Message): - pass + def testMessageFieldValueToMessage(self): + class MyMessage(messages.Message): + pass - class HasMessage(messages.Message): - field = messages.MessageField(MyMessage, 1) + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) - self.assertEqual(HasMessage.field.type, HasMessage.field.message_type) + instance = MyMessage() - def testMessageFieldValueFromMessage(self): - class MyMessage(messages.Message): - pass + self.assertTrue( + instance is HasMessage.field.value_to_message(instance)) - class HasMessage(messages.Message): - field = messages.MessageField(MyMessage, 1) + def testMessageFieldValueToMessageWrongType(self): + class MyMessage(messages.Message): + pass - instance = MyMessage() + class MyOtherMessage(messages.Message): + pass + + class HasMessage(messages.Message): + field = messages.MessageField(MyMessage, 1) + + instance = MyOtherMessage() - self.assertTrue(instance is HasMessage.field.value_from_message(instance)) + self.assertRaisesWithRegexpMatch( + messages.EncodeError, + 'Expected type MyMessage, got MyOtherMessage: ', + HasMessage.field.value_to_message, instance) - def testMessageFieldValueFromMessageWrongType(self): - class MyMessage(messages.Message): - pass + def testIntegerField_AllowLong(self): + """Test that the integer field allows for longs.""" + if six.PY2: + messages.IntegerField(10, default=long(10)) - class HasMessage(messages.Message): - field = messages.MessageField(MyMessage, 1) + def testMessageFieldValidate_Initialized(self): + """Test validation on message field.""" + class MyMessage(messages.Message): + field1 = messages.IntegerField(1, required=True) - self.assertRaisesWithRegexpMatch( - messages.DecodeError, - 'Expected type MyMessage, got int: 10', - HasMessage.field.value_from_message, 10) + field = messages.MessageField(MyMessage, 10) - def testMessageFieldValueToMessage(self): - class MyMessage(messages.Message): - pass + # Will validate messages where is_initialized() is False. + message = MyMessage() + field.validate(message) + message.field1 = 20 + field.validate(message) - class HasMessage(messages.Message): - field = messages.MessageField(MyMessage, 1) + def testEnumField(self): + """Test the construction of enum fields.""" + self.assertRaises(messages.FieldDefinitionError, + messages.EnumField, + str, + 10) - instance = MyMessage() + self.assertRaises(messages.FieldDefinitionError, + messages.EnumField, + messages.Enum, + 10) - self.assertTrue(instance is HasMessage.field.value_to_message(instance)) + class Color(messages.Enum): + RED = 1 + GREEN = 2 + BLUE = 3 - def testMessageFieldValueToMessageWrongType(self): - class MyMessage(messages.Message): - pass + field = messages.EnumField(Color, 10) + self.assertEquals(Color, field.type) - class MyOtherMessage(messages.Message): - pass + class Another(messages.Enum): + VALUE = 1 - class HasMessage(messages.Message): - field = messages.MessageField(MyMessage, 1) + self.assertRaises(messages.InvalidDefaultError, + messages.EnumField, + Color, + 10, + default=Another.VALUE) - instance = MyOtherMessage() + def testEnumField_ForwardReference(self): + """Test the construction of forward reference enum fields.""" + global MyMessage + global ForwardEnum + global ForwardMessage + try: + class MyMessage(messages.Message): - self.assertRaisesWithRegexpMatch( - messages.EncodeError, - 'Expected type MyMessage, got MyOtherMessage: ', - HasMessage.field.value_to_message, instance) + forward = messages.EnumField('ForwardEnum', 1) + nested = messages.EnumField('ForwardMessage.NestedEnum', 2) + inner = messages.EnumField('Inner', 3) - def testIntegerField_AllowLong(self): - """Test that the integer field allows for longs.""" - if six.PY2: - messages.IntegerField(10, default=long(10)) + class Inner(messages.Enum): + pass - def testMessageFieldValidate_Initialized(self): - """Test validation on message field.""" - class MyMessage(messages.Message): - field1 = messages.IntegerField(1, required=True) + class ForwardEnum(messages.Enum): + pass - field = messages.MessageField(MyMessage, 10) + class ForwardMessage(messages.Message): - # Will validate messages where is_initialized() is False. - message = MyMessage() - field.validate(message) - message.field1 = 20 - field.validate(message) + class NestedEnum(messages.Enum): + pass - def testEnumField(self): - """Test the construction of enum fields.""" - self.assertRaises(messages.FieldDefinitionError, - messages.EnumField, - str, - 10) + self.assertEquals(ForwardEnum, + MyMessage.field_by_name('forward').type) - self.assertRaises(messages.FieldDefinitionError, - messages.EnumField, - messages.Enum, - 10) + self.assertEquals(ForwardMessage.NestedEnum, + MyMessage.field_by_name('nested').type) - class Color(messages.Enum): - RED = 1 - GREEN = 2 - BLUE = 3 + self.assertEquals(MyMessage.Inner, + MyMessage.field_by_name('inner').type) + finally: + try: + del MyMessage + del ForwardEnum + del ForwardMessage + except: + pass - field = messages.EnumField(Color, 10) - self.assertEquals(Color, field.type) + def testEnumField_WrongType(self): + """Test that forward referencing the wrong type raises an error.""" + global AMessage + try: + class AMessage(messages.Message): + pass + + class AnotherMessage(messages.Message): + + a_field = messages.EnumField('AMessage', 1) + + self.assertRaises(messages.FieldDefinitionError, + getattr, + AnotherMessage.field_by_name('a_field'), + 'type') + finally: + del AMessage + + def testMessageDefinition(self): + """Test that message definition is set on fields.""" + class MyMessage(messages.Message): + + my_field = messages.StringField(1) - class Another(messages.Enum): - VALUE = 1 + self.assertEquals(MyMessage, + MyMessage.field_by_name('my_field').message_definition()) + + def testNoneAssignment(self): + """Test that assigning None does not change comparison.""" + class MyMessage(messages.Message): + + my_field = messages.StringField(1) + + m1 = MyMessage() + m2 = MyMessage() + m2.my_field = None + self.assertEquals(m1, m2) - self.assertRaises(messages.InvalidDefaultError, - messages.EnumField, - Color, - 10, - default=Another.VALUE) + def testNonAsciiStr(self): + """Test validation fails for non-ascii StringField values.""" + class Thing(messages.Message): + string_field = messages.StringField(2) - def testEnumField_ForwardReference(self): - """Test the construction of forward reference enum fields.""" - global MyMessage - global ForwardEnum - global ForwardMessage - try: - class MyMessage(messages.Message): + thing = Thing() + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + 'Field string_field encountered non-ASCII string', + setattr, thing, 'string_field', test_util.BINARY) - forward = messages.EnumField('ForwardEnum', 1) - nested = messages.EnumField('ForwardMessage.NestedEnum', 2) - inner = messages.EnumField('Inner', 3) - class Inner(messages.Enum): - pass +class MessageTest(test_util.TestCase): + """Tests for message class.""" + + def CreateMessageClass(self): + """Creates a simple message class with 3 fields. + + Fields are defined in alphabetical order but with conflicting numeric + order. + """ + class ComplexMessage(messages.Message): + a3 = messages.IntegerField(3) + b1 = messages.StringField(1) + c2 = messages.StringField(2) + + return ComplexMessage + + def testSameNumbers(self): + """Test that cannot assign two fields with same numbers.""" + + def action(): + class BadMessage(messages.Message): + f1 = messages.IntegerField(1) + f2 = messages.IntegerField(1) + self.assertRaises(messages.DuplicateNumberError, + action) + + def testStrictAssignment(self): + """Tests that cannot assign to unknown or non-reserved attributes.""" + class SimpleMessage(messages.Message): + field = messages.IntegerField(1) + + simple_message = SimpleMessage() + self.assertRaises(AttributeError, + setattr, + simple_message, + 'does_not_exist', + 10) + + def testListAssignmentDoesNotCopy(self): + class SimpleMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + message = SimpleMessage() + original = message.repeated + message.repeated = [] + self.assertFalse(original is message.repeated) + + def testValidate_Optional(self): + """Tests validation of optional fields.""" + class SimpleMessage(messages.Message): + non_required = messages.IntegerField(1) + + simple_message = SimpleMessage() + simple_message.check_initialized() + simple_message.non_required = 10 + simple_message.check_initialized() + + def testValidate_Required(self): + """Tests validation of required fields.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + simple_message = SimpleMessage() + self.assertRaises(messages.ValidationError, + simple_message.check_initialized) + simple_message.required = 10 + simple_message.check_initialized() + + def testValidate_Repeated(self): + """Tests validation of repeated fields.""" + class SimpleMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + simple_message = SimpleMessage() + + # Check valid values. + for valid_value in [], [10], [10, 20], (), (10,), (10, 20): + simple_message.repeated = valid_value + simple_message.check_initialized() + + # Check cleared. + simple_message.repeated = [] + simple_message.check_initialized() + + # Check invalid values. + for invalid_value in 10, ['10', '20'], [None], (None,): + self.assertRaises(messages.ValidationError, + setattr, simple_message, 'repeated', invalid_value) + + def testIsInitialized(self): + """Tests is_initialized.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) + + simple_message = SimpleMessage() + self.assertFalse(simple_message.is_initialized()) + + simple_message.required = 10 + + self.assertTrue(simple_message.is_initialized()) + + def testIsInitializedNestedField(self): + """Tests is_initialized for nested fields.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) - class ForwardEnum(messages.Enum): - pass + class NestedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1) - class ForwardMessage(messages.Message): + simple_message = SimpleMessage() + self.assertFalse(simple_message.is_initialized()) + nested_message = NestedMessage(simple=simple_message) + self.assertFalse(nested_message.is_initialized()) - class NestedEnum(messages.Enum): - pass + simple_message.required = 10 - self.assertEquals(ForwardEnum, - MyMessage.field_by_name('forward').type) + self.assertTrue(simple_message.is_initialized()) + self.assertTrue(nested_message.is_initialized()) - self.assertEquals(ForwardMessage.NestedEnum, - MyMessage.field_by_name('nested').type) + def testInitializeNestedFieldFromDict(self): + """Tests initializing nested fields from dict.""" + class SimpleMessage(messages.Message): + required = messages.IntegerField(1, required=True) - self.assertEquals(MyMessage.Inner, - MyMessage.field_by_name('inner').type) - finally: - try: - del MyMessage - del ForwardEnum - del ForwardMessage - except: - pass + class NestedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1) - def testEnumField_WrongType(self): - """Test that forward referencing the wrong type raises an error.""" - global AMessage - try: - class AMessage(messages.Message): - pass - - class AnotherMessage(messages.Message): - - a_field = messages.EnumField('AMessage', 1) - - self.assertRaises(messages.FieldDefinitionError, - getattr, - AnotherMessage.field_by_name('a_field'), - 'type') - finally: - del AMessage - - def testMessageDefinition(self): - """Test that message definition is set on fields.""" - class MyMessage(messages.Message): - - my_field = messages.StringField(1) + class RepeatedMessage(messages.Message): + simple = messages.MessageField(SimpleMessage, 1, repeated=True) - self.assertEquals(MyMessage, - MyMessage.field_by_name('my_field').message_definition()) - - def testNoneAssignment(self): - """Test that assigning None does not change comparison.""" - class MyMessage(messages.Message): - - my_field = messages.StringField(1) - - m1 = MyMessage() - m2 = MyMessage() - m2.my_field = None - self.assertEquals(m1, m2) + nested_message1 = NestedMessage(simple={'required': 10}) + self.assertTrue(nested_message1.is_initialized()) + self.assertTrue(nested_message1.simple.is_initialized()) - def testNonAsciiStr(self): - """Test validation fails for non-ascii StringField values.""" - class Thing(messages.Message): - string_field = messages.StringField(2) + nested_message2 = NestedMessage() + nested_message2.simple = {'required': 10} + self.assertTrue(nested_message2.is_initialized()) + self.assertTrue(nested_message2.simple.is_initialized()) - thing = Thing() - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - 'Field string_field encountered non-ASCII string', - setattr, thing, 'string_field', test_util.BINARY) + repeated_values = [{}, {'required': 10}, SimpleMessage(required=20)] + repeated_message1 = RepeatedMessage(simple=repeated_values) + self.assertEquals(3, len(repeated_message1.simple)) + self.assertFalse(repeated_message1.is_initialized()) -class MessageTest(test_util.TestCase): - """Tests for message class.""" - - def CreateMessageClass(self): - """Creates a simple message class with 3 fields. - - Fields are defined in alphabetical order but with conflicting numeric - order. - """ - class ComplexMessage(messages.Message): - a3 = messages.IntegerField(3) - b1 = messages.StringField(1) - c2 = messages.StringField(2) - - return ComplexMessage - - def testSameNumbers(self): - """Test that cannot assign two fields with same numbers.""" - - def action(): - class BadMessage(messages.Message): - f1 = messages.IntegerField(1) - f2 = messages.IntegerField(1) - self.assertRaises(messages.DuplicateNumberError, - action) - - def testStrictAssignment(self): - """Tests that cannot assign to unknown or non-reserved attributes.""" - class SimpleMessage(messages.Message): - field = messages.IntegerField(1) - - simple_message = SimpleMessage() - self.assertRaises(AttributeError, - setattr, - simple_message, - 'does_not_exist', - 10) - - def testListAssignmentDoesNotCopy(self): - class SimpleMessage(messages.Message): - repeated = messages.IntegerField(1, repeated=True) - - message = SimpleMessage() - original = message.repeated - message.repeated = [] - self.assertFalse(original is message.repeated) - - def testValidate_Optional(self): - """Tests validation of optional fields.""" - class SimpleMessage(messages.Message): - non_required = messages.IntegerField(1) - - simple_message = SimpleMessage() - simple_message.check_initialized() - simple_message.non_required = 10 - simple_message.check_initialized() - - def testValidate_Required(self): - """Tests validation of required fields.""" - class SimpleMessage(messages.Message): - required = messages.IntegerField(1, required=True) - - simple_message = SimpleMessage() - self.assertRaises(messages.ValidationError, - simple_message.check_initialized) - simple_message.required = 10 - simple_message.check_initialized() - - def testValidate_Repeated(self): - """Tests validation of repeated fields.""" - class SimpleMessage(messages.Message): - repeated = messages.IntegerField(1, repeated=True) - - simple_message = SimpleMessage() - - # Check valid values. - for valid_value in [], [10], [10, 20], (), (10,), (10, 20): - simple_message.repeated = valid_value - simple_message.check_initialized() - - # Check cleared. - simple_message.repeated = [] - simple_message.check_initialized() - - # Check invalid values. - for invalid_value in 10, ['10', '20'], [None], (None,): - self.assertRaises(messages.ValidationError, - setattr, simple_message, 'repeated', invalid_value) - - def testIsInitialized(self): - """Tests is_initialized.""" - class SimpleMessage(messages.Message): - required = messages.IntegerField(1, required=True) - - simple_message = SimpleMessage() - self.assertFalse(simple_message.is_initialized()) - - simple_message.required = 10 - - self.assertTrue(simple_message.is_initialized()) - - def testIsInitializedNestedField(self): - """Tests is_initialized for nested fields.""" - class SimpleMessage(messages.Message): - required = messages.IntegerField(1, required=True) - - class NestedMessage(messages.Message): - simple = messages.MessageField(SimpleMessage, 1) - - simple_message = SimpleMessage() - self.assertFalse(simple_message.is_initialized()) - nested_message = NestedMessage(simple=simple_message) - self.assertFalse(nested_message.is_initialized()) - - simple_message.required = 10 - - self.assertTrue(simple_message.is_initialized()) - self.assertTrue(nested_message.is_initialized()) - - def testInitializeNestedFieldFromDict(self): - """Tests initializing nested fields from dict.""" - class SimpleMessage(messages.Message): - required = messages.IntegerField(1, required=True) - - class NestedMessage(messages.Message): - simple = messages.MessageField(SimpleMessage, 1) - - class RepeatedMessage(messages.Message): - simple = messages.MessageField(SimpleMessage, 1, repeated=True) - - nested_message1 = NestedMessage(simple={'required': 10}) - self.assertTrue(nested_message1.is_initialized()) - self.assertTrue(nested_message1.simple.is_initialized()) - - nested_message2 = NestedMessage() - nested_message2.simple = {'required': 10} - self.assertTrue(nested_message2.is_initialized()) - self.assertTrue(nested_message2.simple.is_initialized()) - - repeated_values = [{}, {'required': 10}, SimpleMessage(required=20)] + repeated_message1.simple[0].required = 0 + self.assertTrue(repeated_message1.is_initialized()) - repeated_message1 = RepeatedMessage(simple=repeated_values) - self.assertEquals(3, len(repeated_message1.simple)) - self.assertFalse(repeated_message1.is_initialized()) + repeated_message2 = RepeatedMessage() + repeated_message2.simple = repeated_values + self.assertEquals(3, len(repeated_message2.simple)) + self.assertFalse(repeated_message2.is_initialized()) - repeated_message1.simple[0].required = 0 - self.assertTrue(repeated_message1.is_initialized()) + repeated_message2.simple[0].required = 0 + self.assertTrue(repeated_message2.is_initialized()) - repeated_message2 = RepeatedMessage() - repeated_message2.simple = repeated_values - self.assertEquals(3, len(repeated_message2.simple)) - self.assertFalse(repeated_message2.is_initialized()) + def testNestedMethodsNotAllowed(self): + """Test that method definitions on Message classes are not allowed.""" + def action(): + class WithMethods(messages.Message): - repeated_message2.simple[0].required = 0 - self.assertTrue(repeated_message2.is_initialized()) + def not_allowed(self): + pass - def testNestedMethodsNotAllowed(self): - """Test that method definitions on Message classes are not allowed.""" - def action(): - class WithMethods(messages.Message): - def not_allowed(self): - pass + self.assertRaises(messages.MessageDefinitionError, + action) - self.assertRaises(messages.MessageDefinitionError, - action) + def testNestedAttributesNotAllowed(self): + """Test that attribute assignment on Message classes are not allowed.""" + def int_attribute(): + class WithMethods(messages.Message): + not_allowed = 1 - def testNestedAttributesNotAllowed(self): - """Test that attribute assignment on Message classes are not allowed.""" - def int_attribute(): - class WithMethods(messages.Message): - not_allowed = 1 + def string_attribute(): + class WithMethods(messages.Message): + not_allowed = 'not allowed' - def string_attribute(): - class WithMethods(messages.Message): - not_allowed = 'not allowed' + def enum_attribute(): + class WithMethods(messages.Message): + not_allowed = Color.RED - def enum_attribute(): - class WithMethods(messages.Message): - not_allowed = Color.RED + for action in (int_attribute, string_attribute, enum_attribute): + self.assertRaises(messages.MessageDefinitionError, + action) - for action in (int_attribute, string_attribute, enum_attribute): - self.assertRaises(messages.MessageDefinitionError, - action) + def testNameIsSetOnFields(self): + """Make sure name is set on fields after Message class init.""" + class HasNamedFields(messages.Message): + field = messages.StringField(1) - def testNameIsSetOnFields(self): - """Make sure name is set on fields after Message class init.""" - class HasNamedFields(messages.Message): - field = messages.StringField(1) + self.assertEquals('field', HasNamedFields.field_by_number(1).name) - self.assertEquals('field', HasNamedFields.field_by_number(1).name) + def testSubclassingMessageDisallowed(self): + """Not permitted to create sub-classes of message classes.""" + class SuperClass(messages.Message): + pass - def testSubclassingMessageDisallowed(self): - """Not permitted to create sub-classes of message classes.""" - class SuperClass(messages.Message): - pass + def action(): + class SubClass(SuperClass): + pass - def action(): - class SubClass(SuperClass): - pass - - self.assertRaises(messages.MessageDefinitionError, - action) - - def testAllFields(self): - """Test all_fields method.""" - ComplexMessage = self.CreateMessageClass() - fields = list(ComplexMessage.all_fields()) - - # Order does not matter, so sort now. - fields = sorted(fields, key=lambda f: f.name) - - self.assertEquals(3, len(fields)) - self.assertEquals('a3', fields[0].name) - self.assertEquals('b1', fields[1].name) - self.assertEquals('c2', fields[2].name) - - def testFieldByName(self): - """Test getting field by name.""" - ComplexMessage = self.CreateMessageClass() - - self.assertEquals(3, ComplexMessage.field_by_name('a3').number) - self.assertEquals(1, ComplexMessage.field_by_name('b1').number) - self.assertEquals(2, ComplexMessage.field_by_name('c2').number) - - self.assertRaises(KeyError, - ComplexMessage.field_by_name, - 'unknown') - - def testFieldByNumber(self): - """Test getting field by number.""" - ComplexMessage = self.CreateMessageClass() - - self.assertEquals('a3', ComplexMessage.field_by_number(3).name) - self.assertEquals('b1', ComplexMessage.field_by_number(1).name) - self.assertEquals('c2', ComplexMessage.field_by_number(2).name) - - self.assertRaises(KeyError, - ComplexMessage.field_by_number, - 4) - - def testGetAssignedValue(self): - """Test getting the assigned value of a field.""" - class SomeMessage(messages.Message): - a_value = messages.StringField(1, default=u'a default') - - message = SomeMessage() - self.assertEquals(None, message.get_assigned_value('a_value')) - - message.a_value = u'a string' - self.assertEquals(u'a string', message.get_assigned_value('a_value')) - - message.a_value = u'a default' - self.assertEquals(u'a default', message.get_assigned_value('a_value')) - - self.assertRaisesWithRegexpMatch( - AttributeError, - 'Message SomeMessage has no field no_such_field', - message.get_assigned_value, - 'no_such_field') - - def testReset(self): - """Test resetting a field value.""" - class SomeMessage(messages.Message): - a_value = messages.StringField(1, default=u'a default') - repeated = messages.IntegerField(2, repeated=True) - - message = SomeMessage() - - self.assertRaises(AttributeError, message.reset, 'unknown') - - self.assertEquals(u'a default', message.a_value) - message.reset('a_value') - self.assertEquals(u'a default', message.a_value) - - message.a_value = u'a new value' - self.assertEquals(u'a new value', message.a_value) - message.reset('a_value') - self.assertEquals(u'a default', message.a_value) - - message.repeated = [1, 2, 3] - self.assertEquals([1, 2, 3], message.repeated) - saved = message.repeated - message.reset('repeated') - self.assertEquals([], message.repeated) - self.assertIsInstance(message.repeated, messages.FieldList) - self.assertEquals([1, 2, 3], saved) - - def testAllowNestedEnums(self): - """Test allowing nested enums in a message definition.""" - class Trade(messages.Message): - class Duration(messages.Enum): - GTC = 1 - DAY = 2 + self.assertRaises(messages.MessageDefinitionError, + action) - class Currency(messages.Enum): - USD = 1 - GBP = 2 - INR = 3 + def testAllFields(self): + """Test all_fields method.""" + ComplexMessage = self.CreateMessageClass() + fields = list(ComplexMessage.all_fields()) - # Sorted by name order seems to be the only feasible option. - self.assertEquals(['Currency', 'Duration'], Trade.__enums__) - - # Message definition will now be set on Enumerated objects. - self.assertEquals(Trade, Trade.Duration.message_definition()) - - def testAllowNestedMessages(self): - """Test allowing nested messages in a message definition.""" - class Trade(messages.Message): - class Lot(messages.Message): - pass - - class Agent(messages.Message): - pass - - # Sorted by name order seems to be the only feasible option. - self.assertEquals(['Agent', 'Lot'], Trade.__messages__) - self.assertEquals(Trade, Trade.Agent.message_definition()) - self.assertEquals(Trade, Trade.Lot.message_definition()) - - # But not Message itself. - def action(): - class Trade(messages.Message): - NiceTry = messages.Message - self.assertRaises(messages.MessageDefinitionError, action) - - def testDisallowClassAssignments(self): - """Test setting class attributes may not happen.""" - class MyMessage(messages.Message): - pass - - self.assertRaises(AttributeError, - setattr, - MyMessage, - 'x', - 'do not assign') - - def testEquality(self): - """Test message class equality.""" - # Comparison against enums must work. - class MyEnum(messages.Enum): - val1 = 1 - val2 = 2 - - # Comparisons against nested messages must work. - class AnotherMessage(messages.Message): - string = messages.StringField(1) - - class MyMessage(messages.Message): - field1 = messages.IntegerField(1) - field2 = messages.EnumField(MyEnum, 2) - field3 = messages.MessageField(AnotherMessage, 3) - - message1 = MyMessage() - - self.assertNotEquals('hi', message1) - self.assertNotEquals(AnotherMessage(), message1) - self.assertEquals(message1, message1) - - message2 = MyMessage() - - self.assertEquals(message1, message2) - - message1.field1 = 10 - self.assertNotEquals(message1, message2) - - message2.field1 = 20 - self.assertNotEquals(message1, message2) - - message2.field1 = 10 - self.assertEquals(message1, message2) - - message1.field2 = MyEnum.val1 - self.assertNotEquals(message1, message2) - - message2.field2 = MyEnum.val2 - self.assertNotEquals(message1, message2) - - message2.field2 = MyEnum.val1 - self.assertEquals(message1, message2) - - message1.field3 = AnotherMessage() - message1.field3.string = 'value1' - self.assertNotEquals(message1, message2) - - message2.field3 = AnotherMessage() - message2.field3.string = 'value2' - self.assertNotEquals(message1, message2) - - message2.field3.string = 'value1' - self.assertEquals(message1, message2) - - def testEqualityWithUnknowns(self): - """Test message class equality with unknown fields.""" - - class MyMessage(messages.Message): - field1 = messages.IntegerField(1) - - message1 = MyMessage() - message2 = MyMessage() - self.assertEquals(message1, message2) - message1.set_unrecognized_field('unknown1', 'value1', - messages.Variant.STRING) - self.assertEquals(message1, message2) - - message1.set_unrecognized_field('unknown2', ['asdf', 3], - messages.Variant.STRING) - message1.set_unrecognized_field('unknown3', 4.7, - messages.Variant.DOUBLE) - self.assertEquals(message1, message2) - - def testUnrecognizedFieldInvalidVariant(self): - class MyMessage(messages.Message): - field1 = messages.IntegerField(1) - - message1 = MyMessage() - self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', - {'unhandled': 'type'}, None) - self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', - {'unhandled': 'type'}, 123) - - def testRepr(self): - """Test represtation of Message object.""" - class MyMessage(messages.Message): - integer_value = messages.IntegerField(1) - string_value = messages.StringField(2) - unassigned = messages.StringField(3) - unassigned_with_default = messages.StringField(4, default=u'a default') - - my_message = MyMessage() - my_message.integer_value = 42 - my_message.string_value = u'A string' - - pat = re.compile(r"") - self.assertTrue(pat.match(repr(my_message)) is not None) - - def testValidation(self): - """Test validation of message values.""" - # Test optional. - class SubMessage(messages.Message): - pass - - class Message(messages.Message): - val = messages.MessageField(SubMessage, 1) - - message = Message() - - message_field = messages.MessageField(Message, 1) - message_field.validate(message) - message.val = SubMessage() - message_field.validate(message) - self.assertRaises(messages.ValidationError, - setattr, message, 'val', [SubMessage()]) - - # Test required. - class Message(messages.Message): - val = messages.MessageField(SubMessage, 1, required=True) - - message = Message() - - message_field = messages.MessageField(Message, 1) - message_field.validate(message) - message.val = SubMessage() - message_field.validate(message) - self.assertRaises(messages.ValidationError, - setattr, message, 'val', [SubMessage()]) - - # Test repeated. - class Message(messages.Message): - val = messages.MessageField(SubMessage, 1, repeated=True) - - message = Message() - - message_field = messages.MessageField(Message, 1) - message_field.validate(message) - self.assertRaisesWithRegexpMatch( - messages.ValidationError, - "Field val is repeated. Found: ", - setattr, message, 'val', SubMessage()) - message.val = [SubMessage()] - message_field.validate(message) - - def testDefinitionName(self): - """Test message name.""" - class MyMessage(messages.Message): - pass - - module_name = test_util.get_module_name(FieldTest) - self.assertEquals('%s.MyMessage' % module_name, - MyMessage.definition_name()) - self.assertEquals(module_name, MyMessage.outer_definition_name()) - self.assertEquals(module_name, MyMessage.definition_package()) - - self.assertEquals(six.text_type, type(MyMessage.definition_name())) - self.assertEquals(six.text_type, type(MyMessage.outer_definition_name())) - self.assertEquals(six.text_type, type(MyMessage.definition_package())) - - def testDefinitionName_OverrideModule(self): - """Test message module is overriden by module package name.""" - class MyMessage(messages.Message): - pass - - global package - package = 'my.package' - - try: - self.assertEquals('my.package.MyMessage', MyMessage.definition_name()) - self.assertEquals('my.package', MyMessage.outer_definition_name()) - self.assertEquals('my.package', MyMessage.definition_package()) - - self.assertEquals(six.text_type, type(MyMessage.definition_name())) - self.assertEquals(six.text_type, type(MyMessage.outer_definition_name())) - self.assertEquals(six.text_type, type(MyMessage.definition_package())) - finally: - del package - - def testDefinitionName_NoModule(self): - """Test what happens when there is no module for message.""" - class MyMessage(messages.Message): - pass - - original_modules = sys.modules - sys.modules = dict(sys.modules) - try: - del sys.modules[__name__] - self.assertEquals('MyMessage', MyMessage.definition_name()) - self.assertEquals(None, MyMessage.outer_definition_name()) - self.assertEquals(None, MyMessage.definition_package()) - - self.assertEquals(six.text_type, type(MyMessage.definition_name())) - finally: - sys.modules = original_modules - - def testDefinitionName_Nested(self): - """Test nested message names.""" - class MyMessage(messages.Message): - - class NestedMessage(messages.Message): + # Order does not matter, so sort now. + fields = sorted(fields, key=lambda f: f.name) - class NestedMessage(messages.Message): + self.assertEquals(3, len(fields)) + self.assertEquals('a3', fields[0].name) + self.assertEquals('b1', fields[1].name) + self.assertEquals('c2', fields[2].name) + + def testFieldByName(self): + """Test getting field by name.""" + ComplexMessage = self.CreateMessageClass() + + self.assertEquals(3, ComplexMessage.field_by_name('a3').number) + self.assertEquals(1, ComplexMessage.field_by_name('b1').number) + self.assertEquals(2, ComplexMessage.field_by_name('c2').number) + + self.assertRaises(KeyError, + ComplexMessage.field_by_name, + 'unknown') + + def testFieldByNumber(self): + """Test getting field by number.""" + ComplexMessage = self.CreateMessageClass() + + self.assertEquals('a3', ComplexMessage.field_by_number(3).name) + self.assertEquals('b1', ComplexMessage.field_by_number(1).name) + self.assertEquals('c2', ComplexMessage.field_by_number(2).name) + + self.assertRaises(KeyError, + ComplexMessage.field_by_number, + 4) + + def testGetAssignedValue(self): + """Test getting the assigned value of a field.""" + class SomeMessage(messages.Message): + a_value = messages.StringField(1, default=u'a default') + + message = SomeMessage() + self.assertEquals(None, message.get_assigned_value('a_value')) + + message.a_value = u'a string' + self.assertEquals(u'a string', message.get_assigned_value('a_value')) + + message.a_value = u'a default' + self.assertEquals(u'a default', message.get_assigned_value('a_value')) + + self.assertRaisesWithRegexpMatch( + AttributeError, + 'Message SomeMessage has no field no_such_field', + message.get_assigned_value, + 'no_such_field') + + def testReset(self): + """Test resetting a field value.""" + class SomeMessage(messages.Message): + a_value = messages.StringField(1, default=u'a default') + repeated = messages.IntegerField(2, repeated=True) + + message = SomeMessage() + + self.assertRaises(AttributeError, message.reset, 'unknown') + + self.assertEquals(u'a default', message.a_value) + message.reset('a_value') + self.assertEquals(u'a default', message.a_value) + + message.a_value = u'a new value' + self.assertEquals(u'a new value', message.a_value) + message.reset('a_value') + self.assertEquals(u'a default', message.a_value) + + message.repeated = [1, 2, 3] + self.assertEquals([1, 2, 3], message.repeated) + saved = message.repeated + message.reset('repeated') + self.assertEquals([], message.repeated) + self.assertIsInstance(message.repeated, messages.FieldList) + self.assertEquals([1, 2, 3], saved) + + def testAllowNestedEnums(self): + """Test allowing nested enums in a message definition.""" + class Trade(messages.Message): + + class Duration(messages.Enum): + GTC = 1 + DAY = 2 + + class Currency(messages.Enum): + USD = 1 + GBP = 2 + INR = 3 + + # Sorted by name order seems to be the only feasible option. + self.assertEquals(['Currency', 'Duration'], Trade.__enums__) + + # Message definition will now be set on Enumerated objects. + self.assertEquals(Trade, Trade.Duration.message_definition()) + + def testAllowNestedMessages(self): + """Test allowing nested messages in a message definition.""" + class Trade(messages.Message): + + class Lot(messages.Message): + pass + + class Agent(messages.Message): + pass - pass - - module_name = test_util.get_module_name(MessageTest) - self.assertEquals('%s.MyMessage.NestedMessage' % module_name, - MyMessage.NestedMessage.definition_name()) - self.assertEquals('%s.MyMessage' % module_name, - MyMessage.NestedMessage.outer_definition_name()) - self.assertEquals(module_name, - MyMessage.NestedMessage.definition_package()) - - self.assertEquals('%s.MyMessage.NestedMessage.NestedMessage' % module_name, - MyMessage.NestedMessage.NestedMessage.definition_name()) - self.assertEquals( - '%s.MyMessage.NestedMessage' % module_name, - MyMessage.NestedMessage.NestedMessage.outer_definition_name()) - self.assertEquals( - module_name, - MyMessage.NestedMessage.NestedMessage.definition_package()) - - - def testMessageDefinition(self): - """Test that enumeration knows its enclosing message definition.""" - class OuterMessage(messages.Message): - - class InnerMessage(messages.Message): - pass - - self.assertEquals(None, OuterMessage.message_definition()) - self.assertEquals(OuterMessage, - OuterMessage.InnerMessage.message_definition()) - - def testConstructorKwargs(self): - """Test kwargs via constructor.""" - class SomeMessage(messages.Message): - name = messages.StringField(1) - number = messages.IntegerField(2) - - expected = SomeMessage() - expected.name = 'my name' - expected.number = 200 - self.assertEquals(expected, SomeMessage(name='my name', number=200)) - - def testConstructorNotAField(self): - """Test kwargs via constructor with wrong names.""" - class SomeMessage(messages.Message): - pass - - self.assertRaisesWithRegexpMatch( - AttributeError, - 'May not assign arbitrary value does_not_exist to message SomeMessage', - SomeMessage, - does_not_exist=10) - - def testGetUnsetRepeatedValue(self): - class SomeMessage(messages.Message): - repeated = messages.IntegerField(1, repeated=True) - - instance = SomeMessage() - self.assertEquals([], instance.repeated) - self.assertTrue(isinstance(instance.repeated, messages.FieldList)) - - def testCompareAutoInitializedRepeatedFields(self): - class SomeMessage(messages.Message): - repeated = messages.IntegerField(1, repeated=True) - - message1 = SomeMessage(repeated=[]) - message2 = SomeMessage() - self.assertEquals(message1, message2) - - def testUnknownValues(self): - """Test message class equality with unknown fields.""" - class MyMessage(messages.Message): - field1 = messages.IntegerField(1) - - message = MyMessage() - self.assertEquals([], message.all_unrecognized_fields()) - self.assertEquals((None, None), - message.get_unrecognized_field_info('doesntexist')) - self.assertEquals((None, None), - message.get_unrecognized_field_info( - 'doesntexist', None, None)) - self.assertEquals(('defaultvalue', 'defaultwire'), - message.get_unrecognized_field_info( - 'doesntexist', 'defaultvalue', 'defaultwire')) - self.assertEquals((3, None), - message.get_unrecognized_field_info( - 'doesntexist', value_default=3)) - - message.set_unrecognized_field('exists', 9.5, messages.Variant.DOUBLE) - self.assertEquals(1, len(message.all_unrecognized_fields())) - self.assertTrue('exists' in message.all_unrecognized_fields()) - self.assertEquals((9.5, messages.Variant.DOUBLE), - message.get_unrecognized_field_info('exists')) - self.assertEquals((9.5, messages.Variant.DOUBLE), - message.get_unrecognized_field_info('exists', 'type', - 1234)) - self.assertEquals((1234, None), - message.get_unrecognized_field_info('doesntexist', 1234)) - - message.set_unrecognized_field('another', 'value', messages.Variant.STRING) - self.assertEquals(2, len(message.all_unrecognized_fields())) - self.assertTrue('exists' in message.all_unrecognized_fields()) - self.assertTrue('another' in message.all_unrecognized_fields()) - self.assertEquals((9.5, messages.Variant.DOUBLE), - message.get_unrecognized_field_info('exists')) - self.assertEquals(('value', messages.Variant.STRING), - message.get_unrecognized_field_info('another')) - - message.set_unrecognized_field('typetest1', ['list', 0, ('test',)], - messages.Variant.STRING) - self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), - message.get_unrecognized_field_info('typetest1')) - message.set_unrecognized_field('typetest2', '', messages.Variant.STRING) - self.assertEquals(('', messages.Variant.STRING), - message.get_unrecognized_field_info('typetest2')) - - def testPickle(self): - """Testing pickling and unpickling of Message instances.""" - global MyEnum - global AnotherMessage - global MyMessage - - class MyEnum(messages.Enum): - val1 = 1 - val2 = 2 - - class AnotherMessage(messages.Message): - string = messages.StringField(1, repeated=True) - - class MyMessage(messages.Message): - field1 = messages.IntegerField(1) - field2 = messages.EnumField(MyEnum, 2) - field3 = messages.MessageField(AnotherMessage, 3) - - message = MyMessage(field1=1, field2=MyEnum.val2, - field3=AnotherMessage(string=['a', 'b', 'c'])) - message.set_unrecognized_field('exists', 'value', messages.Variant.STRING) - message.set_unrecognized_field('repeated', ['list', 0, ('test',)], - messages.Variant.STRING) - unpickled = pickle.loads(pickle.dumps(message)) - self.assertEquals(message, unpickled) - self.assertTrue(AnotherMessage.string is unpickled.field3.string.field) - self.assertTrue('exists' in message.all_unrecognized_fields()) - self.assertEquals(('value', messages.Variant.STRING), - message.get_unrecognized_field_info('exists')) - self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), - message.get_unrecognized_field_info('repeated')) + # Sorted by name order seems to be the only feasible option. + self.assertEquals(['Agent', 'Lot'], Trade.__messages__) + self.assertEquals(Trade, Trade.Agent.message_definition()) + self.assertEquals(Trade, Trade.Lot.message_definition()) + + # But not Message itself. + def action(): + class Trade(messages.Message): + NiceTry = messages.Message + self.assertRaises(messages.MessageDefinitionError, action) + + def testDisallowClassAssignments(self): + """Test setting class attributes may not happen.""" + class MyMessage(messages.Message): + pass + + self.assertRaises(AttributeError, + setattr, + MyMessage, + 'x', + 'do not assign') + + def testEquality(self): + """Test message class equality.""" + # Comparison against enums must work. + class MyEnum(messages.Enum): + val1 = 1 + val2 = 2 + + # Comparisons against nested messages must work. + class AnotherMessage(messages.Message): + string = messages.StringField(1) + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + field2 = messages.EnumField(MyEnum, 2) + field3 = messages.MessageField(AnotherMessage, 3) + + message1 = MyMessage() + + self.assertNotEquals('hi', message1) + self.assertNotEquals(AnotherMessage(), message1) + self.assertEquals(message1, message1) + + message2 = MyMessage() + + self.assertEquals(message1, message2) + + message1.field1 = 10 + self.assertNotEquals(message1, message2) + + message2.field1 = 20 + self.assertNotEquals(message1, message2) + + message2.field1 = 10 + self.assertEquals(message1, message2) + + message1.field2 = MyEnum.val1 + self.assertNotEquals(message1, message2) + + message2.field2 = MyEnum.val2 + self.assertNotEquals(message1, message2) + + message2.field2 = MyEnum.val1 + self.assertEquals(message1, message2) + + message1.field3 = AnotherMessage() + message1.field3.string = 'value1' + self.assertNotEquals(message1, message2) + + message2.field3 = AnotherMessage() + message2.field3.string = 'value2' + self.assertNotEquals(message1, message2) + + message2.field3.string = 'value1' + self.assertEquals(message1, message2) + + def testEqualityWithUnknowns(self): + """Test message class equality with unknown fields.""" + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message1 = MyMessage() + message2 = MyMessage() + self.assertEquals(message1, message2) + message1.set_unrecognized_field('unknown1', 'value1', + messages.Variant.STRING) + self.assertEquals(message1, message2) + + message1.set_unrecognized_field('unknown2', ['asdf', 3], + messages.Variant.STRING) + message1.set_unrecognized_field('unknown3', 4.7, + messages.Variant.DOUBLE) + self.assertEquals(message1, message2) + + def testUnrecognizedFieldInvalidVariant(self): + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message1 = MyMessage() + self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, None) + self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, 123) + + def testRepr(self): + """Test represtation of Message object.""" + class MyMessage(messages.Message): + integer_value = messages.IntegerField(1) + string_value = messages.StringField(2) + unassigned = messages.StringField(3) + unassigned_with_default = messages.StringField( + 4, default=u'a default') + + my_message = MyMessage() + my_message.integer_value = 42 + my_message.string_value = u'A string' + + pat = re.compile(r"") + self.assertTrue(pat.match(repr(my_message)) is not None) + + def testValidation(self): + """Test validation of message values.""" + # Test optional. + class SubMessage(messages.Message): + pass + + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + message.val = SubMessage() + message_field.validate(message) + self.assertRaises(messages.ValidationError, + setattr, message, 'val', [SubMessage()]) + + # Test required. + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1, required=True) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + message.val = SubMessage() + message_field.validate(message) + self.assertRaises(messages.ValidationError, + setattr, message, 'val', [SubMessage()]) + + # Test repeated. + class Message(messages.Message): + val = messages.MessageField(SubMessage, 1, repeated=True) + + message = Message() + + message_field = messages.MessageField(Message, 1) + message_field.validate(message) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + "Field val is repeated. Found: ", + setattr, message, 'val', SubMessage()) + message.val = [SubMessage()] + message_field.validate(message) + + def testDefinitionName(self): + """Test message name.""" + class MyMessage(messages.Message): + pass + + module_name = test_util.get_module_name(FieldTest) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.definition_name()) + self.assertEquals(module_name, MyMessage.outer_definition_name()) + self.assertEquals(module_name, MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + self.assertEquals(six.text_type, type( + MyMessage.outer_definition_name())) + self.assertEquals(six.text_type, type(MyMessage.definition_package())) + + def testDefinitionName_OverrideModule(self): + """Test message module is overriden by module package name.""" + class MyMessage(messages.Message): + pass + + global package + package = 'my.package' + + try: + self.assertEquals('my.package.MyMessage', + MyMessage.definition_name()) + self.assertEquals('my.package', MyMessage.outer_definition_name()) + self.assertEquals('my.package', MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + self.assertEquals(six.text_type, type( + MyMessage.outer_definition_name())) + self.assertEquals(six.text_type, type( + MyMessage.definition_package())) + finally: + del package + + def testDefinitionName_NoModule(self): + """Test what happens when there is no module for message.""" + class MyMessage(messages.Message): + pass + + original_modules = sys.modules + sys.modules = dict(sys.modules) + try: + del sys.modules[__name__] + self.assertEquals('MyMessage', MyMessage.definition_name()) + self.assertEquals(None, MyMessage.outer_definition_name()) + self.assertEquals(None, MyMessage.definition_package()) + + self.assertEquals(six.text_type, type(MyMessage.definition_name())) + finally: + sys.modules = original_modules + + def testDefinitionName_Nested(self): + """Test nested message names.""" + class MyMessage(messages.Message): + + class NestedMessage(messages.Message): + + class NestedMessage(messages.Message): + + pass + + module_name = test_util.get_module_name(MessageTest) + self.assertEquals('%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.definition_name()) + self.assertEquals('%s.MyMessage' % module_name, + MyMessage.NestedMessage.outer_definition_name()) + self.assertEquals(module_name, + MyMessage.NestedMessage.definition_package()) + + self.assertEquals('%s.MyMessage.NestedMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedMessage.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedMessage.outer_definition_name()) + self.assertEquals( + module_name, + MyMessage.NestedMessage.NestedMessage.definition_package()) + + def testMessageDefinition(self): + """Test that enumeration knows its enclosing message definition.""" + class OuterMessage(messages.Message): + + class InnerMessage(messages.Message): + pass + + self.assertEquals(None, OuterMessage.message_definition()) + self.assertEquals(OuterMessage, + OuterMessage.InnerMessage.message_definition()) + + def testConstructorKwargs(self): + """Test kwargs via constructor.""" + class SomeMessage(messages.Message): + name = messages.StringField(1) + number = messages.IntegerField(2) + + expected = SomeMessage() + expected.name = 'my name' + expected.number = 200 + self.assertEquals(expected, SomeMessage(name='my name', number=200)) + + def testConstructorNotAField(self): + """Test kwargs via constructor with wrong names.""" + class SomeMessage(messages.Message): + pass + + self.assertRaisesWithRegexpMatch( + AttributeError, + 'May not assign arbitrary value does_not_exist to message SomeMessage', + SomeMessage, + does_not_exist=10) + + def testGetUnsetRepeatedValue(self): + class SomeMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + instance = SomeMessage() + self.assertEquals([], instance.repeated) + self.assertTrue(isinstance(instance.repeated, messages.FieldList)) + + def testCompareAutoInitializedRepeatedFields(self): + class SomeMessage(messages.Message): + repeated = messages.IntegerField(1, repeated=True) + + message1 = SomeMessage(repeated=[]) + message2 = SomeMessage() + self.assertEquals(message1, message2) + + def testUnknownValues(self): + """Test message class equality with unknown fields.""" + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + + message = MyMessage() + self.assertEquals([], message.all_unrecognized_fields()) + self.assertEquals((None, None), + message.get_unrecognized_field_info('doesntexist')) + self.assertEquals((None, None), + message.get_unrecognized_field_info( + 'doesntexist', None, None)) + self.assertEquals(('defaultvalue', 'defaultwire'), + message.get_unrecognized_field_info( + 'doesntexist', 'defaultvalue', 'defaultwire')) + self.assertEquals((3, None), + message.get_unrecognized_field_info( + 'doesntexist', value_default=3)) + + message.set_unrecognized_field('exists', 9.5, messages.Variant.DOUBLE) + self.assertEquals(1, len(message.all_unrecognized_fields())) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists')) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists', 'type', + 1234)) + self.assertEquals((1234, None), + message.get_unrecognized_field_info('doesntexist', 1234)) + + message.set_unrecognized_field( + 'another', 'value', messages.Variant.STRING) + self.assertEquals(2, len(message.all_unrecognized_fields())) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertTrue('another' in message.all_unrecognized_fields()) + self.assertEquals((9.5, messages.Variant.DOUBLE), + message.get_unrecognized_field_info('exists')) + self.assertEquals(('value', messages.Variant.STRING), + message.get_unrecognized_field_info('another')) + + message.set_unrecognized_field('typetest1', ['list', 0, ('test',)], + messages.Variant.STRING) + self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), + message.get_unrecognized_field_info('typetest1')) + message.set_unrecognized_field( + 'typetest2', '', messages.Variant.STRING) + self.assertEquals(('', messages.Variant.STRING), + message.get_unrecognized_field_info('typetest2')) + + def testPickle(self): + """Testing pickling and unpickling of Message instances.""" + global MyEnum + global AnotherMessage + global MyMessage + + class MyEnum(messages.Enum): + val1 = 1 + val2 = 2 + + class AnotherMessage(messages.Message): + string = messages.StringField(1, repeated=True) + + class MyMessage(messages.Message): + field1 = messages.IntegerField(1) + field2 = messages.EnumField(MyEnum, 2) + field3 = messages.MessageField(AnotherMessage, 3) + + message = MyMessage(field1=1, field2=MyEnum.val2, + field3=AnotherMessage(string=['a', 'b', 'c'])) + message.set_unrecognized_field( + 'exists', 'value', messages.Variant.STRING) + message.set_unrecognized_field('repeated', ['list', 0, ('test',)], + messages.Variant.STRING) + unpickled = pickle.loads(pickle.dumps(message)) + self.assertEquals(message, unpickled) + self.assertTrue(AnotherMessage.string is unpickled.field3.string.field) + self.assertTrue('exists' in message.all_unrecognized_fields()) + self.assertEquals(('value', messages.Variant.STRING), + message.get_unrecognized_field_info('exists')) + self.assertEquals((['list', 0, ('test',)], messages.Variant.STRING), + message.get_unrecognized_field_info('repeated')) class FindDefinitionTest(test_util.TestCase): - """Test finding definitions relative to various definitions and modules.""" - - def setUp(self): - """Set up module-space. Starts off empty.""" - self.modules = {} - - def DefineModule(self, name): - """Define a module and its parents in module space. - - Modules that are already defined in self.modules are not re-created. - - Args: - name: Fully qualified name of modules to create. - - Returns: - Deepest nested module. For example: - - DefineModule('a.b.c') # Returns c. - """ - name_path = name.split('.') - full_path = [] - for node in name_path: - full_path.append(node) - full_name = '.'.join(full_path) - self.modules.setdefault(full_name, types.ModuleType(full_name)) - return self.modules[name] - - def DefineMessage(self, module, name, children={}, add_to_module=True): - """Define a new Message class in the context of a module. - - Used for easily describing complex Message hierarchy. Message is defined - including all child definitions. - - Args: - module: Fully qualified name of module to place Message class in. - name: Name of Message to define within module. - children: Define any level of nesting of children definitions. To define - a message, map the name to another dictionary. The dictionary can - itself contain additional definitions, and so on. To map to an Enum, - define the Enum class separately and map it by name. - add_to_module: If True, new Message class is added to module. If False, - new Message is not added. - """ - # Make sure module exists. - module_instance = self.DefineModule(module) - - # Recursively define all child messages. - for attribute, value in children.items(): - if isinstance(value, dict): - children[attribute] = self.DefineMessage( - module, attribute, value, False) - - # Override default __module__ variable. - children['__module__'] = module - - # Instantiate and possibly add to module. - message_class = type(name, (messages.Message,), dict(children)) - if add_to_module: - setattr(module_instance, name, message_class) - return message_class - - def Importer(self, module, globals='', locals='', fromlist=None): - """Importer function. - - Acts like __import__. Only loads modules from self.modules. Does not - try to load real modules defined elsewhere. Does not try to handle relative - imports. - - Args: - module: Fully qualified name of module to load from self.modules. - """ - if fromlist is None: - module = module.split('.')[0] - try: - return self.modules[module] - except KeyError: - raise ImportError() - - def testNoSuchModule(self): - """Test searching for definitions that do no exist.""" - self.assertRaises(messages.DefinitionNotFoundError, - messages.find_definition, - 'does.not.exist', - importer=self.Importer) - - def testRefersToModule(self): - """Test that referring to a module does not return that module.""" - self.DefineModule('i.am.a.module') - self.assertRaises(messages.DefinitionNotFoundError, - messages.find_definition, - 'i.am.a.module', - importer=self.Importer) - - def testNoDefinition(self): - """Test not finding a definition in an existing module.""" - self.DefineModule('i.am.a.module') - self.assertRaises(messages.DefinitionNotFoundError, - messages.find_definition, - 'i.am.a.module.MyMessage', - importer=self.Importer) - - def testNotADefinition(self): - """Test trying to fetch something that is not a definition.""" - module = self.DefineModule('i.am.a.module') - setattr(module, 'A', 'a string') - self.assertRaises(messages.DefinitionNotFoundError, - messages.find_definition, - 'i.am.a.module.A', - importer=self.Importer) - - def testGlobalFind(self): - """Test finding definitions from fully qualified module names.""" - A = self.DefineMessage('a.b.c', 'A', {}) - self.assertEquals(A, messages.find_definition('a.b.c.A', - importer=self.Importer)) - B = self.DefineMessage('a.b.c', 'B', {'C':{}}) - self.assertEquals(B.C, messages.find_definition('a.b.c.B.C', - importer=self.Importer)) - - def testRelativeToModule(self): - """Test finding definitions relative to modules.""" - # Define modules. - a = self.DefineModule('a') - b = self.DefineModule('a.b') - c = self.DefineModule('a.b.c') - - # Define messages. - A = self.DefineMessage('a', 'A') - B = self.DefineMessage('a.b', 'B') - C = self.DefineMessage('a.b.c', 'C') - D = self.DefineMessage('a.b.d', 'D') - - # Find A, B, C and D relative to a. - self.assertEquals(A, messages.find_definition( - 'A', a, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'b.B', a, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'b.c.C', a, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'b.d.D', a, importer=self.Importer)) - - # Find A, B, C and D relative to b. - self.assertEquals(A, messages.find_definition( - 'A', b, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'B', b, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'c.C', b, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'd.D', b, importer=self.Importer)) - - # Find A, B, C and D relative to c. Module d is the same case as c. - self.assertEquals(A, messages.find_definition( - 'A', c, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'B', c, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'C', c, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'd.D', c, importer=self.Importer)) - - def testRelativeToMessages(self): - """Test finding definitions relative to Message definitions.""" - A = self.DefineMessage('a.b', 'A', {'B': {'C': {}, 'D': {}}}) - B = A.B - C = A.B.C - D = A.B.D - - # Find relative to A. - self.assertEquals(A, messages.find_definition( - 'A', A, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'B', A, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'B.C', A, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'B.D', A, importer=self.Importer)) - - # Find relative to B. - self.assertEquals(A, messages.find_definition( - 'A', B, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'B', B, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'C', B, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'D', B, importer=self.Importer)) - - # Find relative to C. - self.assertEquals(A, messages.find_definition( - 'A', C, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'B', C, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'C', C, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'D', C, importer=self.Importer)) - - # Find relative to C searching from c. - self.assertEquals(A, messages.find_definition( - 'b.A', C, importer=self.Importer)) - self.assertEquals(B, messages.find_definition( - 'b.A.B', C, importer=self.Importer)) - self.assertEquals(C, messages.find_definition( - 'b.A.B.C', C, importer=self.Importer)) - self.assertEquals(D, messages.find_definition( - 'b.A.B.D', C, importer=self.Importer)) - - def testAbsoluteReference(self): - """Test finding absolute definition names.""" - # Define modules. - a = self.DefineModule('a') - b = self.DefineModule('a.a') - - # Define messages. - aA = self.DefineMessage('a', 'A') - aaA = self.DefineMessage('a.a', 'A') - - # Always find a.A. - self.assertEquals(aA, messages.find_definition('.a.A', None, - importer=self.Importer)) - self.assertEquals(aA, messages.find_definition('.a.A', a, - importer=self.Importer)) - self.assertEquals(aA, messages.find_definition('.a.A', aA, - importer=self.Importer)) - self.assertEquals(aA, messages.find_definition('.a.A', aaA, - importer=self.Importer)) - - def testFindEnum(self): - """Test that Enums are found.""" - class Color(messages.Enum): - pass - A = self.DefineMessage('a', 'A', {'Color': Color}) - - self.assertEquals( - Color, - messages.find_definition('Color', A, importer=self.Importer)) - - def testFalseScope(self): - """Test that Message definitions nested in strange objects are hidden.""" - global X - class X(object): - class A(messages.Message): - pass - - self.assertRaises(TypeError, messages.find_definition, 'A', X) - self.assertRaises(messages.DefinitionNotFoundError, - messages.find_definition, - 'X.A', sys.modules[__name__]) - - def testSearchAttributeFirst(self): - """Make sure not faked out by module, but continues searching.""" - A = self.DefineMessage('a', 'A') - module_A = self.DefineModule('a.A') - - self.assertEquals(A, messages.find_definition( - 'a.A', None, importer=self.Importer)) + """Test finding definitions relative to various definitions and modules.""" + + def setUp(self): + """Set up module-space. Starts off empty.""" + self.modules = {} + + def DefineModule(self, name): + """Define a module and its parents in module space. + + Modules that are already defined in self.modules are not re-created. + + Args: + name: Fully qualified name of modules to create. + + Returns: + Deepest nested module. For example: + + DefineModule('a.b.c') # Returns c. + """ + name_path = name.split('.') + full_path = [] + for node in name_path: + full_path.append(node) + full_name = '.'.join(full_path) + self.modules.setdefault(full_name, types.ModuleType(full_name)) + return self.modules[name] + + def DefineMessage(self, module, name, children={}, add_to_module=True): + """Define a new Message class in the context of a module. + + Used for easily describing complex Message hierarchy. Message is defined + including all child definitions. + + Args: + module: Fully qualified name of module to place Message class in. + name: Name of Message to define within module. + children: Define any level of nesting of children definitions. To define + a message, map the name to another dictionary. The dictionary can + itself contain additional definitions, and so on. To map to an Enum, + define the Enum class separately and map it by name. + add_to_module: If True, new Message class is added to module. If False, + new Message is not added. + """ + # Make sure module exists. + module_instance = self.DefineModule(module) + + # Recursively define all child messages. + for attribute, value in children.items(): + if isinstance(value, dict): + children[attribute] = self.DefineMessage( + module, attribute, value, False) + + # Override default __module__ variable. + children['__module__'] = module + + # Instantiate and possibly add to module. + message_class = type(name, (messages.Message,), dict(children)) + if add_to_module: + setattr(module_instance, name, message_class) + return message_class + + def Importer(self, module, globals='', locals='', fromlist=None): + """Importer function. + + Acts like __import__. Only loads modules from self.modules. Does not + try to load real modules defined elsewhere. Does not try to handle relative + imports. + + Args: + module: Fully qualified name of module to load from self.modules. + """ + if fromlist is None: + module = module.split('.')[0] + try: + return self.modules[module] + except KeyError: + raise ImportError() + + def testNoSuchModule(self): + """Test searching for definitions that do no exist.""" + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'does.not.exist', + importer=self.Importer) + + def testRefersToModule(self): + """Test that referring to a module does not return that module.""" + self.DefineModule('i.am.a.module') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module', + importer=self.Importer) + + def testNoDefinition(self): + """Test not finding a definition in an existing module.""" + self.DefineModule('i.am.a.module') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module.MyMessage', + importer=self.Importer) + + def testNotADefinition(self): + """Test trying to fetch something that is not a definition.""" + module = self.DefineModule('i.am.a.module') + setattr(module, 'A', 'a string') + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'i.am.a.module.A', + importer=self.Importer) + + def testGlobalFind(self): + """Test finding definitions from fully qualified module names.""" + A = self.DefineMessage('a.b.c', 'A', {}) + self.assertEquals(A, messages.find_definition('a.b.c.A', + importer=self.Importer)) + B = self.DefineMessage('a.b.c', 'B', {'C': {}}) + self.assertEquals(B.C, messages.find_definition('a.b.c.B.C', + importer=self.Importer)) + + def testRelativeToModule(self): + """Test finding definitions relative to modules.""" + # Define modules. + a = self.DefineModule('a') + b = self.DefineModule('a.b') + c = self.DefineModule('a.b.c') + + # Define messages. + A = self.DefineMessage('a', 'A') + B = self.DefineMessage('a.b', 'B') + C = self.DefineMessage('a.b.c', 'C') + D = self.DefineMessage('a.b.d', 'D') + + # Find A, B, C and D relative to a. + self.assertEquals(A, messages.find_definition( + 'A', a, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'b.B', a, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'b.c.C', a, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'b.d.D', a, importer=self.Importer)) + + # Find A, B, C and D relative to b. + self.assertEquals(A, messages.find_definition( + 'A', b, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', b, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'c.C', b, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'd.D', b, importer=self.Importer)) + + # Find A, B, C and D relative to c. Module d is the same case as c. + self.assertEquals(A, messages.find_definition( + 'A', c, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', c, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', c, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'd.D', c, importer=self.Importer)) + + def testRelativeToMessages(self): + """Test finding definitions relative to Message definitions.""" + A = self.DefineMessage('a.b', 'A', {'B': {'C': {}, 'D': {}}}) + B = A.B + C = A.B.C + D = A.B.D + + # Find relative to A. + self.assertEquals(A, messages.find_definition( + 'A', A, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', A, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'B.C', A, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'B.D', A, importer=self.Importer)) + + # Find relative to B. + self.assertEquals(A, messages.find_definition( + 'A', B, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', B, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', B, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'D', B, importer=self.Importer)) + + # Find relative to C. + self.assertEquals(A, messages.find_definition( + 'A', C, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'B', C, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'C', C, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'D', C, importer=self.Importer)) + + # Find relative to C searching from c. + self.assertEquals(A, messages.find_definition( + 'b.A', C, importer=self.Importer)) + self.assertEquals(B, messages.find_definition( + 'b.A.B', C, importer=self.Importer)) + self.assertEquals(C, messages.find_definition( + 'b.A.B.C', C, importer=self.Importer)) + self.assertEquals(D, messages.find_definition( + 'b.A.B.D', C, importer=self.Importer)) + + def testAbsoluteReference(self): + """Test finding absolute definition names.""" + # Define modules. + a = self.DefineModule('a') + b = self.DefineModule('a.a') + + # Define messages. + aA = self.DefineMessage('a', 'A') + aaA = self.DefineMessage('a.a', 'A') + + # Always find a.A. + self.assertEquals(aA, messages.find_definition('.a.A', None, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', a, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', aA, + importer=self.Importer)) + self.assertEquals(aA, messages.find_definition('.a.A', aaA, + importer=self.Importer)) + + def testFindEnum(self): + """Test that Enums are found.""" + class Color(messages.Enum): + pass + A = self.DefineMessage('a', 'A', {'Color': Color}) + + self.assertEquals( + Color, + messages.find_definition('Color', A, importer=self.Importer)) + + def testFalseScope(self): + """Test that Message definitions nested in strange objects are hidden.""" + global X + + class X(object): + + class A(messages.Message): + pass + + self.assertRaises(TypeError, messages.find_definition, 'A', X) + self.assertRaises(messages.DefinitionNotFoundError, + messages.find_definition, + 'X.A', sys.modules[__name__]) + + def testSearchAttributeFirst(self): + """Make sure not faked out by module, but continues searching.""" + A = self.DefineMessage('a', 'A') + module_A = self.DefineModule('a.A') + + self.assertEquals(A, messages.find_definition( + 'a.A', None, importer=self.Importer)) def main(): - unittest.main() + unittest.main() if __name__ == '__main__': - main() + main() diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py index 0e9c6bb..acd95c2 100644 --- a/apitools/base/protorpclite/protojson.py +++ b/apitools/base/protorpclite/protojson.py @@ -47,315 +47,319 @@ __all__ = [ def _load_json_module(): - """Try to load a valid json module. - - There are more than one json modules that might be installed. They are - mostly compatible with one another but some versions may be different. - This function attempts to load various json modules in a preferred order. - It does a basic check to guess if a loaded version of json is compatible. - - Returns: - Compatible json module. - - Raises: - ImportError if there are no json modules or the loaded json module is - not compatible with ProtoRPC. - """ - first_import_error = None - for module_name in ['json', - 'simplejson']: - try: - module = __import__(module_name, {}, {}, 'json') - if not hasattr(module, 'JSONEncoder'): - message = ('json library "%s" is not compatible with ProtoRPC' % - module_name) - logging.warning(message) - raise ImportError(message) - else: - return module - except ImportError as err: - if not first_import_error: - first_import_error = err - - logging.error('Must use valid json library (Python 2.6 json or simplejson)') - raise first_import_error -json = _load_json_module() - - -# TODO: Rename this to MessageJsonEncoder. -class MessageJSONEncoder(json.JSONEncoder): - """Message JSON encoder class. - - Extension of JSONEncoder that can build JSON from a message object. - """ - - def __init__(self, protojson_protocol=None, **kwargs): - """Constructor. - - Args: - protojson_protocol: ProtoJson instance. - """ - super(MessageJSONEncoder, self).__init__(**kwargs) - self.__protojson_protocol = protojson_protocol or ProtoJson.get_default() - - def default(self, value): - """Return dictionary instance from a message object. - - Args: - value: Value to get dictionary for. If not encodable, will - call superclasses default method. - """ - if isinstance(value, messages.Enum): - return str(value) - - if six.PY3 and isinstance(value, bytes): - return value.decode('utf8') - - if isinstance(value, messages.Message): - result = {} - for field in value.all_fields(): - item = value.get_assigned_value(field.name) - if item not in (None, [], ()): - result[field.name] = self.__protojson_protocol.encode_field( - field, item) - # Handle unrecognized fields, so they're included when a message is - # decoded then encoded. - for unknown_key in value.all_unrecognized_fields(): - unrecognized_field, _ = value.get_unrecognized_field_info(unknown_key) - result[unknown_key] = unrecognized_field - return result - else: - return super(MessageJSONEncoder, self).default(value) - - -class ProtoJson(object): - """ProtoRPC JSON implementation class. - - Implementation of JSON based protocol used for serializing and deserializing - message objects. Instances of remote.ProtocolConfig constructor or used with - remote.Protocols.add_protocol. See the remote.py module for more details. - """ - - CONTENT_TYPE = 'application/json' - ALTERNATIVE_CONTENT_TYPES = [ - 'application/x-javascript', - 'text/javascript', - 'text/x-javascript', - 'text/x-json', - 'text/json', - ] - - def encode_field(self, field, value): - """Encode a python field value to a JSON value. - - Args: - field: A ProtoRPC field instance. - value: A python value supported by field. + """Try to load a valid json module. - Returns: - A JSON serializable value appropriate for field. - """ - if isinstance(field, messages.BytesField): - if field.repeated: - value = [base64.b64encode(byte) for byte in value] - else: - value = base64.b64encode(value) - elif isinstance(field, message_types.DateTimeField): - # DateTimeField stores its data as a RFC 3339 compliant string. - if field.repeated: - value = [i.isoformat() for i in value] - else: - value = value.isoformat() - return value - - def encode_message(self, message): - """Encode Message instance to JSON string. - - Args: - Message instance to encode in to JSON string. + There are more than one json modules that might be installed. They are + mostly compatible with one another but some versions may be different. + This function attempts to load various json modules in a preferred order. + It does a basic check to guess if a loaded version of json is compatible. Returns: - String encoding of Message instance in protocol JSON format. + Compatible json module. Raises: - messages.ValidationError if message is not initialized. + ImportError if there are no json modules or the loaded json module is + not compatible with ProtoRPC. """ - message.check_initialized() - - return json.dumps(message, cls=MessageJSONEncoder, protojson_protocol=self) - - def decode_message(self, message_type, encoded_message): - """Merge JSON structure to Message instance. + first_import_error = None + for module_name in ['json', + 'simplejson']: + try: + module = __import__(module_name, {}, {}, 'json') + if not hasattr(module, 'JSONEncoder'): + message = ('json library "%s" is not compatible with ProtoRPC' % + module_name) + logging.warning(message) + raise ImportError(message) + else: + return module + except ImportError as err: + if not first_import_error: + first_import_error = err + + logging.error( + 'Must use valid json library (Python 2.6 json or simplejson)') + raise first_import_error +json = _load_json_module() - Args: - message_type: Message to decode data to. - encoded_message: JSON encoded version of message. - Returns: - Decoded instance of message_type. +# TODO: Rename this to MessageJsonEncoder. +class MessageJSONEncoder(json.JSONEncoder): + """Message JSON encoder class. - Raises: - ValueError: If encoded_message is not valid JSON. - messages.ValidationError if merged message is not initialized. + Extension of JSONEncoder that can build JSON from a message object. """ - if not encoded_message.strip(): - return message_type() - dictionary = json.loads(encoded_message) - message = self.__decode_dictionary(message_type, dictionary) - message.check_initialized() - return message + def __init__(self, protojson_protocol=None, **kwargs): + """Constructor. + + Args: + protojson_protocol: ProtoJson instance. + """ + super(MessageJSONEncoder, self).__init__(**kwargs) + self.__protojson_protocol = protojson_protocol or ProtoJson.get_default() + + def default(self, value): + """Return dictionary instance from a message object. + + Args: + value: Value to get dictionary for. If not encodable, will + call superclasses default method. + """ + if isinstance(value, messages.Enum): + return str(value) + + if six.PY3 and isinstance(value, bytes): + return value.decode('utf8') + + if isinstance(value, messages.Message): + result = {} + for field in value.all_fields(): + item = value.get_assigned_value(field.name) + if item not in (None, [], ()): + result[field.name] = self.__protojson_protocol.encode_field( + field, item) + # Handle unrecognized fields, so they're included when a message is + # decoded then encoded. + for unknown_key in value.all_unrecognized_fields(): + unrecognized_field, _ = value.get_unrecognized_field_info( + unknown_key) + result[unknown_key] = unrecognized_field + return result + else: + return super(MessageJSONEncoder, self).default(value) - def __find_variant(self, value): - """Find the messages.Variant type that describes this value. - Args: - value: The value whose variant type is being determined. +class ProtoJson(object): + """ProtoRPC JSON implementation class. - Returns: - The messages.Variant value that best describes value's type, or None if - it's a type we don't know how to handle. - """ - if isinstance(value, bool): - return messages.Variant.BOOL - elif isinstance(value, six.integer_types): - return messages.Variant.INT64 - elif isinstance(value, float): - return messages.Variant.DOUBLE - elif isinstance(value, six.string_types): - return messages.Variant.STRING - elif isinstance(value, (list, tuple)): - # Find the most specific variant that covers all elements. - variant_priority = [None, messages.Variant.INT64, messages.Variant.DOUBLE, - messages.Variant.STRING] - chosen_priority = 0 - for v in value: - variant = self.__find_variant(v) - try: - priority = variant_priority.index(variant) - except IndexError: - priority = -1 - if priority > chosen_priority: - chosen_priority = priority - return variant_priority[chosen_priority] - # Unrecognized type. - return None - - def __decode_dictionary(self, message_type, dictionary): - """Merge dictionary in to message. - - Args: - message: Message to merge dictionary in to. - dictionary: Dictionary to extract information from. Dictionary - is as parsed from JSON. Nested objects will also be dictionaries. + Implementation of JSON based protocol used for serializing and deserializing + message objects. Instances of remote.ProtocolConfig constructor or used with + remote.Protocols.add_protocol. See the remote.py module for more details. """ - message = message_type() - for key, value in six.iteritems(dictionary): - if value is None: + + CONTENT_TYPE = 'application/json' + ALTERNATIVE_CONTENT_TYPES = [ + 'application/x-javascript', + 'text/javascript', + 'text/x-javascript', + 'text/x-json', + 'text/json', + ] + + def encode_field(self, field, value): + """Encode a python field value to a JSON value. + + Args: + field: A ProtoRPC field instance. + value: A python value supported by field. + + Returns: + A JSON serializable value appropriate for field. + """ + if isinstance(field, messages.BytesField): + if field.repeated: + value = [base64.b64encode(byte) for byte in value] + else: + value = base64.b64encode(value) + elif isinstance(field, message_types.DateTimeField): + # DateTimeField stores its data as a RFC 3339 compliant string. + if field.repeated: + value = [i.isoformat() for i in value] + else: + value = value.isoformat() + return value + + def encode_message(self, message): + """Encode Message instance to JSON string. + + Args: + Message instance to encode in to JSON string. + + Returns: + String encoding of Message instance in protocol JSON format. + + Raises: + messages.ValidationError if message is not initialized. + """ + message.check_initialized() + + return json.dumps(message, cls=MessageJSONEncoder, protojson_protocol=self) + + def decode_message(self, message_type, encoded_message): + """Merge JSON structure to Message instance. + + Args: + message_type: Message to decode data to. + encoded_message: JSON encoded version of message. + + Returns: + Decoded instance of message_type. + + Raises: + ValueError: If encoded_message is not valid JSON. + messages.ValidationError if merged message is not initialized. + """ + if not encoded_message.strip(): + return message_type() + + dictionary = json.loads(encoded_message) + message = self.__decode_dictionary(message_type, dictionary) + message.check_initialized() + return message + + def __find_variant(self, value): + """Find the messages.Variant type that describes this value. + + Args: + value: The value whose variant type is being determined. + + Returns: + The messages.Variant value that best describes value's type, or None if + it's a type we don't know how to handle. + """ + if isinstance(value, bool): + return messages.Variant.BOOL + elif isinstance(value, six.integer_types): + return messages.Variant.INT64 + elif isinstance(value, float): + return messages.Variant.DOUBLE + elif isinstance(value, six.string_types): + return messages.Variant.STRING + elif isinstance(value, (list, tuple)): + # Find the most specific variant that covers all elements. + variant_priority = [None, messages.Variant.INT64, messages.Variant.DOUBLE, + messages.Variant.STRING] + chosen_priority = 0 + for v in value: + variant = self.__find_variant(v) + try: + priority = variant_priority.index(variant) + except IndexError: + priority = -1 + if priority > chosen_priority: + chosen_priority = priority + return variant_priority[chosen_priority] + # Unrecognized type. + return None + + def __decode_dictionary(self, message_type, dictionary): + """Merge dictionary in to message. + + Args: + message: Message to merge dictionary in to. + dictionary: Dictionary to extract information from. Dictionary + is as parsed from JSON. Nested objects will also be dictionaries. + """ + message = message_type() + for key, value in six.iteritems(dictionary): + if value is None: + try: + message.reset(key) + except AttributeError: + pass # This is an unrecognized field, skip it. + continue + + try: + field = message.field_by_name(key) + except KeyError: + # Save unknown values. + variant = self.__find_variant(value) + if variant: + if key.isdigit(): + key = int(key) + message.set_unrecognized_field(key, value, variant) + else: + logging.warning( + 'No variant found for unrecognized field: %s', key) + continue + + # Normalize values in to a list. + if isinstance(value, list): + if not value: + continue + else: + value = [value] + + valid_value = [] + for item in value: + valid_value.append(self.decode_field(field, item)) + + if field.repeated: + existing_value = getattr(message, field.name) + setattr(message, field.name, valid_value) + else: + setattr(message, field.name, valid_value[-1]) + return message + + def decode_field(self, field, value): + """Decode a JSON value to a python value. + + Args: + field: A ProtoRPC field instance. + value: A serialized JSON value. + + Return: + A Python value compatible with field. + """ + if isinstance(field, messages.EnumField): + try: + return field.type(value) + except TypeError: + raise messages.DecodeError( + 'Invalid enum value "%s"' % (value or '')) + + elif isinstance(field, messages.BytesField): + try: + return base64.b64decode(value) + except (binascii.Error, TypeError) as err: + raise messages.DecodeError('Base64 decoding error: %s' % err) + + elif isinstance(field, message_types.DateTimeField): + try: + return util.decode_datetime(value) + except ValueError as err: + raise messages.DecodeError(err) + + elif (isinstance(field, messages.MessageField) and + issubclass(field.type, messages.Message)): + return self.__decode_dictionary(field.type, value) + + elif (isinstance(field, messages.FloatField) and + isinstance(value, (six.integer_types, six.string_types))): + try: + return float(value) + except: + pass + + elif (isinstance(field, messages.IntegerField) and + isinstance(value, six.string_types)): + try: + return int(value) + except: + pass + + return value + + @staticmethod + def get_default(): + """Get default instanceof ProtoJson.""" try: - message.reset(key) + return ProtoJson.__default except AttributeError: - pass # This is an unrecognized field, skip it. - continue - - try: - field = message.field_by_name(key) - except KeyError: - # Save unknown values. - variant = self.__find_variant(value) - if variant: - if key.isdigit(): - key = int(key) - message.set_unrecognized_field(key, value, variant) - else: - logging.warning('No variant found for unrecognized field: %s', key) - continue - - # Normalize values in to a list. - if isinstance(value, list): - if not value: - continue - else: - value = [value] - - valid_value = [] - for item in value: - valid_value.append(self.decode_field(field, item)) - - if field.repeated: - existing_value = getattr(message, field.name) - setattr(message, field.name, valid_value) - else: - setattr(message, field.name, valid_value[-1]) - return message - - def decode_field(self, field, value): - """Decode a JSON value to a python value. - - Args: - field: A ProtoRPC field instance. - value: A serialized JSON value. - - Return: - A Python value compatible with field. - """ - if isinstance(field, messages.EnumField): - try: - return field.type(value) - except TypeError: - raise messages.DecodeError('Invalid enum value "%s"' % (value or '')) - - elif isinstance(field, messages.BytesField): - try: - return base64.b64decode(value) - except (binascii.Error, TypeError) as err: - raise messages.DecodeError('Base64 decoding error: %s' % err) - - elif isinstance(field, message_types.DateTimeField): - try: - return util.decode_datetime(value) - except ValueError as err: - raise messages.DecodeError(err) - - elif (isinstance(field, messages.MessageField) and - issubclass(field.type, messages.Message)): - return self.__decode_dictionary(field.type, value) - - elif (isinstance(field, messages.FloatField) and - isinstance(value, (six.integer_types, six.string_types))): - try: - return float(value) - except: - pass - - elif (isinstance(field, messages.IntegerField) and - isinstance(value, six.string_types)): - try: - return int(value) - except: - pass - - return value - - @staticmethod - def get_default(): - """Get default instanceof ProtoJson.""" - try: - return ProtoJson.__default - except AttributeError: - ProtoJson.__default = ProtoJson() - return ProtoJson.__default - - @staticmethod - def set_default(protocol): - """Set the default instance of ProtoJson. - - Args: - protocol: A ProtoJson instance. - """ - if not isinstance(protocol, ProtoJson): - raise TypeError('Expected protocol of type ProtoJson') - ProtoJson.__default = protocol + ProtoJson.__default = ProtoJson() + return ProtoJson.__default + + @staticmethod + def set_default(protocol): + """Set the default instance of ProtoJson. + + Args: + protocol: A ProtoJson instance. + """ + if not isinstance(protocol, ProtoJson): + raise TypeError('Expected protocol of type ProtoJson') + ProtoJson.__default = protocol CONTENT_TYPE = ProtoJson.CONTENT_TYPE diff --git a/apitools/base/protorpclite/protojson_test.py b/apitools/base/protorpclite/protojson_test.py index a0349d4..3c08619 100644 --- a/apitools/base/protorpclite/protojson_test.py +++ b/apitools/base/protorpclite/protojson_test.py @@ -31,66 +31,66 @@ from apitools.base.protorpclite import test_util class CustomField(messages.MessageField): - """Custom MessageField class.""" + """Custom MessageField class.""" - type = int - message_type = message_types.VoidMessage + type = int + message_type = message_types.VoidMessage - def __init__(self, number, **kwargs): - super(CustomField, self).__init__(self.message_type, number, **kwargs) + def __init__(self, number, **kwargs): + super(CustomField, self).__init__(self.message_type, number, **kwargs) - def value_to_message(self, value): - return self.message_type() + def value_to_message(self, value): + return self.message_type() class MyMessage(messages.Message): - """Test message containing various types.""" + """Test message containing various types.""" - class Color(messages.Enum): + class Color(messages.Enum): - RED = 1 - GREEN = 2 - BLUE = 3 + RED = 1 + GREEN = 2 + BLUE = 3 - class Nested(messages.Message): + class Nested(messages.Message): - nested_value = messages.StringField(1) + nested_value = messages.StringField(1) - a_string = messages.StringField(2) - an_integer = messages.IntegerField(3) - a_float = messages.FloatField(4) - a_boolean = messages.BooleanField(5) - an_enum = messages.EnumField(Color, 6) - a_nested = messages.MessageField(Nested, 7) - a_repeated = messages.IntegerField(8, repeated=True) - a_repeated_float = messages.FloatField(9, repeated=True) - a_datetime = message_types.DateTimeField(10) - a_repeated_datetime = message_types.DateTimeField(11, repeated=True) - a_custom = CustomField(12) - a_repeated_custom = CustomField(13, repeated=True) + a_string = messages.StringField(2) + an_integer = messages.IntegerField(3) + a_float = messages.FloatField(4) + a_boolean = messages.BooleanField(5) + an_enum = messages.EnumField(Color, 6) + a_nested = messages.MessageField(Nested, 7) + a_repeated = messages.IntegerField(8, repeated=True) + a_repeated_float = messages.FloatField(9, repeated=True) + a_datetime = message_types.DateTimeField(10) + a_repeated_datetime = message_types.DateTimeField(11, repeated=True) + a_custom = CustomField(12) + a_repeated_custom = CustomField(13, repeated=True) class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): - MODULE = protojson + MODULE = protojson # TODO(rafek): Convert this test to the compliance test in test_util. class ProtojsonTest(test_util.TestCase, test_util.ProtoConformanceTestBase): - """Test JSON encoding and decoding.""" + """Test JSON encoding and decoding.""" - PROTOLIB = protojson + PROTOLIB = protojson - def CompareEncoded(self, expected_encoded, actual_encoded): - """JSON encoding will be laundered to remove string differences.""" - self.assertEquals(json.loads(expected_encoded), - json.loads(actual_encoded)) + def CompareEncoded(self, expected_encoded, actual_encoded): + """JSON encoding will be laundered to remove string differences.""" + self.assertEquals(json.loads(expected_encoded), + json.loads(actual_encoded)) - encoded_empty_message = '{}' + encoded_empty_message = '{}' - encoded_partial = """{ + encoded_partial = """{ "double_value": 1.23, "int64_value": -100000000000, "int32_value": 1020, @@ -99,7 +99,7 @@ class ProtojsonTest(test_util.TestCase, } """ - encoded_full = """{ + encoded_full = """{ "double_value": 1.23, "float_value": -2.5, "int64_value": -100000000000, @@ -112,7 +112,7 @@ class ProtojsonTest(test_util.TestCase, } """ - encoded_repeated = """{ + encoded_repeated = """{ "double_value": [1.23, 2.3], "float_value": [-2.5, 0.5], "int64_value": [-100000000000, 20], @@ -125,317 +125,320 @@ class ProtojsonTest(test_util.TestCase, } """ - encoded_nested = """{ + encoded_nested = """{ "nested": { "a_value": "a string" } } """ - encoded_repeated_nested = """{ + encoded_repeated_nested = """{ "repeated_nested": [{"a_value": "a string"}, {"a_value": "another string"}] } """ - unexpected_tag_message = '{"unknown": "value"}' + unexpected_tag_message = '{"unknown": "value"}' - encoded_default_assigned = '{"a_value": "a default"}' + encoded_default_assigned = '{"a_value": "a default"}' - encoded_nested_empty = '{"nested": {}}' + encoded_nested_empty = '{"nested": {}}' - encoded_repeated_nested_empty = '{"repeated_nested": [{}, {}]}' + encoded_repeated_nested_empty = '{"repeated_nested": [{}, {}]}' - encoded_extend_message = '{"int64_value": [400, 50, 6000]}' + encoded_extend_message = '{"int64_value": [400, 50, 6000]}' - encoded_string_types = '{"string_value": "Latin"}' + encoded_string_types = '{"string_value": "Latin"}' - encoded_invalid_enum = '{"enum_value": "undefined"}' + encoded_invalid_enum = '{"enum_value": "undefined"}' - def testConvertIntegerToFloat(self): - """Test that integers passed in to float fields are converted. + def testConvertIntegerToFloat(self): + """Test that integers passed in to float fields are converted. - This is necessary because JSON outputs integers for numbers with 0 decimals. - """ - message = protojson.decode_message(MyMessage, '{"a_float": 10}') + This is necessary because JSON outputs integers for numbers with 0 decimals. + """ + message = protojson.decode_message(MyMessage, '{"a_float": 10}') - self.assertTrue(isinstance(message.a_float, float)) - self.assertEquals(10.0, message.a_float) + self.assertTrue(isinstance(message.a_float, float)) + self.assertEquals(10.0, message.a_float) - def testConvertStringToNumbers(self): - """Test that strings passed to integer fields are converted.""" - message = protojson.decode_message(MyMessage, - """{"an_integer": "10", + def testConvertStringToNumbers(self): + """Test that strings passed to integer fields are converted.""" + message = protojson.decode_message(MyMessage, + """{"an_integer": "10", "a_float": "3.5", "a_repeated": ["1", "2"], "a_repeated_float": ["1.5", "2", 10] }""") - self.assertEquals(MyMessage(an_integer=10, - a_float=3.5, - a_repeated=[1, 2], - a_repeated_float=[1.5, 2.0, 10.0]), - message) - - def testWrongTypeAssignment(self): - """Test when wrong type is assigned to a field.""" - self.assertRaises(messages.ValidationError, - protojson.decode_message, - MyMessage, '{"a_string": 10}') - self.assertRaises(messages.ValidationError, - protojson.decode_message, - MyMessage, '{"an_integer": 10.2}') - self.assertRaises(messages.ValidationError, - protojson.decode_message, - MyMessage, '{"an_integer": "10.2"}') - - def testNumericEnumeration(self): - """Test that numbers work for enum values.""" - message = protojson.decode_message(MyMessage, '{"an_enum": 2}') - - expected_message = MyMessage() - expected_message.an_enum = MyMessage.Color.GREEN - - self.assertEquals(expected_message, message) - - def testNumericEnumerationNegativeTest(self): - """Test with an invalid number for the enum value.""" - self.assertRaisesRegexp( - messages.DecodeError, - 'Invalid enum value "89"', - protojson.decode_message, - MyMessage, - '{"an_enum": 89}') - - def testAlphaEnumeration(self): - """Test that alpha enum values work.""" - message = protojson.decode_message(MyMessage, '{"an_enum": "RED"}') - - expected_message = MyMessage() - expected_message.an_enum = MyMessage.Color.RED - - self.assertEquals(expected_message, message) - - def testAlphaEnumerationNegativeTest(self): - """The alpha enum value is invalid.""" - self.assertRaisesRegexp( - messages.DecodeError, - 'Invalid enum value "IAMINVALID"', - protojson.decode_message, - MyMessage, - '{"an_enum": "IAMINVALID"}') - - def testEnumerationNegativeTestWithEmptyString(self): - """The enum value is an empty string.""" - self.assertRaisesRegexp( - messages.DecodeError, - 'Invalid enum value ""', - protojson.decode_message, - MyMessage, - '{"an_enum": ""}') - - def testNullValues(self): - """Test that null values overwrite existing values.""" - self.assertEquals(MyMessage(), - protojson.decode_message(MyMessage, - ('{"an_integer": null,' - ' "a_nested": null,' - ' "an_enum": null' - '}'))) - - def testEmptyList(self): - """Test that empty lists are ignored.""" - self.assertEquals(MyMessage(), - protojson.decode_message(MyMessage, - '{"a_repeated": []}')) - - def testNotJSON(self): - """Test error when string is not valid JSON.""" - self.assertRaises(ValueError, - protojson.decode_message, MyMessage, '{this is not json}') - - def testDoNotEncodeStrangeObjects(self): - """Test trying to encode a strange object. - - The main purpose of this test is to complete coverage. It ensures that - the default behavior of the JSON encoder is preserved when someone tries to - serialized an unexpected type. - """ - class BogusObject(object): - - def check_initialized(self): - pass - - self.assertRaises(TypeError, - protojson.encode_message, - BogusObject()) - - def testMergeEmptyString(self): - """Test merging the empty or space only string.""" - message = protojson.decode_message(test_util.OptionalMessage, '') - self.assertEquals(test_util.OptionalMessage(), message) - - message = protojson.decode_message(test_util.OptionalMessage, ' ') - self.assertEquals(test_util.OptionalMessage(), message) - - def testProtojsonUnrecognizedFieldName(self): - """Test that unrecognized fields are saved and can be accessed.""" - decoded = protojson.decode_message(MyMessage, - ('{"an_integer": 1, "unknown_val": 2}')) - self.assertEquals(decoded.an_integer, 1) - self.assertEquals(1, len(decoded.all_unrecognized_fields())) - self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) - self.assertEquals((2, messages.Variant.INT64), - decoded.get_unrecognized_field_info('unknown_val')) - - def testProtojsonUnrecognizedFieldNumber(self): - """Test that unrecognized fields are saved and can be accessed.""" - decoded = protojson.decode_message( - MyMessage, - '{"an_integer": 1, "1001": "unknown", "-123": "negative", ' - '"456_mixed": 2}') - self.assertEquals(decoded.an_integer, 1) - self.assertEquals(3, len(decoded.all_unrecognized_fields())) - self.assertTrue(1001 in decoded.all_unrecognized_fields()) - self.assertEquals(('unknown', messages.Variant.STRING), - decoded.get_unrecognized_field_info(1001)) - self.assertTrue('-123' in decoded.all_unrecognized_fields()) - self.assertEquals(('negative', messages.Variant.STRING), - decoded.get_unrecognized_field_info('-123')) - self.assertTrue('456_mixed' in decoded.all_unrecognized_fields()) - self.assertEquals((2, messages.Variant.INT64), - decoded.get_unrecognized_field_info('456_mixed')) - - def testProtojsonUnrecognizedNull(self): - """Test that unrecognized fields that are None are skipped.""" - decoded = protojson.decode_message( - MyMessage, - '{"an_integer": 1, "unrecognized_null": null}') - self.assertEquals(decoded.an_integer, 1) - self.assertEquals(decoded.all_unrecognized_fields(), []) - - def testUnrecognizedFieldVariants(self): - """Test that unrecognized fields are mapped to the right variants.""" - for encoded, expected_variant in ( - ('{"an_integer": 1, "unknown_val": 2}', messages.Variant.INT64), - ('{"an_integer": 1, "unknown_val": 2.0}', messages.Variant.DOUBLE), - ('{"an_integer": 1, "unknown_val": "string value"}', - messages.Variant.STRING), - ('{"an_integer": 1, "unknown_val": [1, 2, 3]}', messages.Variant.INT64), - ('{"an_integer": 1, "unknown_val": [1, 2.0, 3]}', - messages.Variant.DOUBLE), - ('{"an_integer": 1, "unknown_val": [1, "foo", 3]}', - messages.Variant.STRING), - ('{"an_integer": 1, "unknown_val": true}', messages.Variant.BOOL)): - decoded = protojson.decode_message(MyMessage, encoded) - self.assertEquals(decoded.an_integer, 1) - self.assertEquals(1, len(decoded.all_unrecognized_fields())) - self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) - _, decoded_variant = decoded.get_unrecognized_field_info('unknown_val') - self.assertEquals(expected_variant, decoded_variant) - - def testDecodeDateTime(self): - for datetime_string, datetime_vals in ( - ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), - ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): - message = protojson.decode_message( - MyMessage, '{"a_datetime": "%s"}' % datetime_string) - expected_message = MyMessage( - a_datetime=datetime.datetime(*datetime_vals)) - - self.assertEquals(expected_message, message) - - def testDecodeInvalidDateTime(self): - self.assertRaises(messages.DecodeError, protojson.decode_message, - MyMessage, '{"a_datetime": "invalid"}') - - def testEncodeDateTime(self): - for datetime_string, datetime_vals in ( - ('2012-09-30T15:31:50.262000', (2012, 9, 30, 15, 31, 50, 262000)), - ('2012-09-30T15:31:50.262123', (2012, 9, 30, 15, 31, 50, 262123)), - ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): - decoded_message = protojson.encode_message( - MyMessage(a_datetime=datetime.datetime(*datetime_vals))) - expected_decoding = '{"a_datetime": "%s"}' % datetime_string - self.CompareEncoded(expected_decoding, decoded_message) - - def testDecodeRepeatedDateTime(self): - message = protojson.decode_message( - MyMessage, - '{"a_repeated_datetime": ["2012-09-30T15:31:50.262", ' - '"2010-01-21T09:52:00", "2000-01-01T01:00:59.999999"]}') - expected_message = MyMessage( - a_repeated_datetime=[ - datetime.datetime(2012, 9, 30, 15, 31, 50, 262000), - datetime.datetime(2010, 1, 21, 9, 52), - datetime.datetime(2000, 1, 1, 1, 0, 59, 999999)]) - - self.assertEquals(expected_message, message) - - def testDecodeCustom(self): - message = protojson.decode_message(MyMessage, '{"a_custom": 1}') - self.assertEquals(MyMessage(a_custom=1), message) - - def testDecodeInvalidCustom(self): - self.assertRaises(messages.ValidationError, protojson.decode_message, - MyMessage, '{"a_custom": "invalid"}') - - def testEncodeCustom(self): - decoded_message = protojson.encode_message(MyMessage(a_custom=1)) - self.CompareEncoded('{"a_custom": 1}', decoded_message) - - def testDecodeRepeatedCustom(self): - message = protojson.decode_message( - MyMessage, '{"a_repeated_custom": [1, 2, 3]}') - self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message) - - def testDecodeBadBase64BytesField(self): - """Test decoding improperly encoded base64 bytes value.""" - self.assertRaisesWithRegexpMatch( - messages.DecodeError, - 'Base64 decoding error: Incorrect padding', - protojson.decode_message, - test_util.OptionalMessage, - '{"bytes_value": "abcdefghijklmnopq"}') + self.assertEquals(MyMessage(an_integer=10, + a_float=3.5, + a_repeated=[1, 2], + a_repeated_float=[1.5, 2.0, 10.0]), + message) + + def testWrongTypeAssignment(self): + """Test when wrong type is assigned to a field.""" + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"a_string": 10}') + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"an_integer": 10.2}') + self.assertRaises(messages.ValidationError, + protojson.decode_message, + MyMessage, '{"an_integer": "10.2"}') + + def testNumericEnumeration(self): + """Test that numbers work for enum values.""" + message = protojson.decode_message(MyMessage, '{"an_enum": 2}') + + expected_message = MyMessage() + expected_message.an_enum = MyMessage.Color.GREEN + + self.assertEquals(expected_message, message) + + def testNumericEnumerationNegativeTest(self): + """Test with an invalid number for the enum value.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value "89"', + protojson.decode_message, + MyMessage, + '{"an_enum": 89}') + + def testAlphaEnumeration(self): + """Test that alpha enum values work.""" + message = protojson.decode_message(MyMessage, '{"an_enum": "RED"}') + + expected_message = MyMessage() + expected_message.an_enum = MyMessage.Color.RED + + self.assertEquals(expected_message, message) + + def testAlphaEnumerationNegativeTest(self): + """The alpha enum value is invalid.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value "IAMINVALID"', + protojson.decode_message, + MyMessage, + '{"an_enum": "IAMINVALID"}') + + def testEnumerationNegativeTestWithEmptyString(self): + """The enum value is an empty string.""" + self.assertRaisesRegexp( + messages.DecodeError, + 'Invalid enum value ""', + protojson.decode_message, + MyMessage, + '{"an_enum": ""}') + + def testNullValues(self): + """Test that null values overwrite existing values.""" + self.assertEquals(MyMessage(), + protojson.decode_message(MyMessage, + ('{"an_integer": null,' + ' "a_nested": null,' + ' "an_enum": null' + '}'))) + + def testEmptyList(self): + """Test that empty lists are ignored.""" + self.assertEquals(MyMessage(), + protojson.decode_message(MyMessage, + '{"a_repeated": []}')) + + def testNotJSON(self): + """Test error when string is not valid JSON.""" + self.assertRaises(ValueError, + protojson.decode_message, MyMessage, '{this is not json}') + + def testDoNotEncodeStrangeObjects(self): + """Test trying to encode a strange object. + + The main purpose of this test is to complete coverage. It ensures that + the default behavior of the JSON encoder is preserved when someone tries to + serialized an unexpected type. + """ + class BogusObject(object): + + def check_initialized(self): + pass + + self.assertRaises(TypeError, + protojson.encode_message, + BogusObject()) + + def testMergeEmptyString(self): + """Test merging the empty or space only string.""" + message = protojson.decode_message(test_util.OptionalMessage, '') + self.assertEquals(test_util.OptionalMessage(), message) + + message = protojson.decode_message(test_util.OptionalMessage, ' ') + self.assertEquals(test_util.OptionalMessage(), message) + + def testProtojsonUnrecognizedFieldName(self): + """Test that unrecognized fields are saved and can be accessed.""" + decoded = protojson.decode_message(MyMessage, + ('{"an_integer": 1, "unknown_val": 2}')) + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(1, len(decoded.all_unrecognized_fields())) + self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) + self.assertEquals((2, messages.Variant.INT64), + decoded.get_unrecognized_field_info('unknown_val')) + + def testProtojsonUnrecognizedFieldNumber(self): + """Test that unrecognized fields are saved and can be accessed.""" + decoded = protojson.decode_message( + MyMessage, + '{"an_integer": 1, "1001": "unknown", "-123": "negative", ' + '"456_mixed": 2}') + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(3, len(decoded.all_unrecognized_fields())) + self.assertTrue(1001 in decoded.all_unrecognized_fields()) + self.assertEquals(('unknown', messages.Variant.STRING), + decoded.get_unrecognized_field_info(1001)) + self.assertTrue('-123' in decoded.all_unrecognized_fields()) + self.assertEquals(('negative', messages.Variant.STRING), + decoded.get_unrecognized_field_info('-123')) + self.assertTrue('456_mixed' in decoded.all_unrecognized_fields()) + self.assertEquals((2, messages.Variant.INT64), + decoded.get_unrecognized_field_info('456_mixed')) + + def testProtojsonUnrecognizedNull(self): + """Test that unrecognized fields that are None are skipped.""" + decoded = protojson.decode_message( + MyMessage, + '{"an_integer": 1, "unrecognized_null": null}') + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(decoded.all_unrecognized_fields(), []) + + def testUnrecognizedFieldVariants(self): + """Test that unrecognized fields are mapped to the right variants.""" + for encoded, expected_variant in ( + ('{"an_integer": 1, "unknown_val": 2}', messages.Variant.INT64), + ('{"an_integer": 1, "unknown_val": 2.0}', messages.Variant.DOUBLE), + ('{"an_integer": 1, "unknown_val": "string value"}', + messages.Variant.STRING), + ('{"an_integer": 1, "unknown_val": [1, 2, 3]}', + messages.Variant.INT64), + ('{"an_integer": 1, "unknown_val": [1, 2.0, 3]}', + messages.Variant.DOUBLE), + ('{"an_integer": 1, "unknown_val": [1, "foo", 3]}', + messages.Variant.STRING), + ('{"an_integer": 1, "unknown_val": true}', messages.Variant.BOOL)): + decoded = protojson.decode_message(MyMessage, encoded) + self.assertEquals(decoded.an_integer, 1) + self.assertEquals(1, len(decoded.all_unrecognized_fields())) + self.assertEquals( + 'unknown_val', decoded.all_unrecognized_fields()[0]) + _, decoded_variant = decoded.get_unrecognized_field_info( + 'unknown_val') + self.assertEquals(expected_variant, decoded_variant) + + def testDecodeDateTime(self): + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + message = protojson.decode_message( + MyMessage, '{"a_datetime": "%s"}' % datetime_string) + expected_message = MyMessage( + a_datetime=datetime.datetime(*datetime_vals)) + + self.assertEquals(expected_message, message) + + def testDecodeInvalidDateTime(self): + self.assertRaises(messages.DecodeError, protojson.decode_message, + MyMessage, '{"a_datetime": "invalid"}') + + def testEncodeDateTime(self): + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262000', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50.262123', (2012, 9, 30, 15, 31, 50, 262123)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + decoded_message = protojson.encode_message( + MyMessage(a_datetime=datetime.datetime(*datetime_vals))) + expected_decoding = '{"a_datetime": "%s"}' % datetime_string + self.CompareEncoded(expected_decoding, decoded_message) + + def testDecodeRepeatedDateTime(self): + message = protojson.decode_message( + MyMessage, + '{"a_repeated_datetime": ["2012-09-30T15:31:50.262", ' + '"2010-01-21T09:52:00", "2000-01-01T01:00:59.999999"]}') + expected_message = MyMessage( + a_repeated_datetime=[ + datetime.datetime(2012, 9, 30, 15, 31, 50, 262000), + datetime.datetime(2010, 1, 21, 9, 52), + datetime.datetime(2000, 1, 1, 1, 0, 59, 999999)]) + + self.assertEquals(expected_message, message) + + def testDecodeCustom(self): + message = protojson.decode_message(MyMessage, '{"a_custom": 1}') + self.assertEquals(MyMessage(a_custom=1), message) + + def testDecodeInvalidCustom(self): + self.assertRaises(messages.ValidationError, protojson.decode_message, + MyMessage, '{"a_custom": "invalid"}') + + def testEncodeCustom(self): + decoded_message = protojson.encode_message(MyMessage(a_custom=1)) + self.CompareEncoded('{"a_custom": 1}', decoded_message) + + def testDecodeRepeatedCustom(self): + message = protojson.decode_message( + MyMessage, '{"a_repeated_custom": [1, 2, 3]}') + self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message) + + def testDecodeBadBase64BytesField(self): + """Test decoding improperly encoded base64 bytes value.""" + self.assertRaisesWithRegexpMatch( + messages.DecodeError, + 'Base64 decoding error: Incorrect padding', + protojson.decode_message, + test_util.OptionalMessage, + '{"bytes_value": "abcdefghijklmnopq"}') class CustomProtoJson(protojson.ProtoJson): - def encode_field(self, field, value): - return '{encoded}' + value + def encode_field(self, field, value): + return '{encoded}' + value - def decode_field(self, field, value): - return '{decoded}' + value + def decode_field(self, field, value): + return '{decoded}' + value class CustomProtoJsonTest(test_util.TestCase): - """Tests for serialization overriding functionality.""" + """Tests for serialization overriding functionality.""" - def setUp(self): - self.protojson = CustomProtoJson() + def setUp(self): + self.protojson = CustomProtoJson() - def testEncode(self): - self.assertEqual('{"a_string": "{encoded}xyz"}', - self.protojson.encode_message(MyMessage(a_string='xyz'))) + def testEncode(self): + self.assertEqual('{"a_string": "{encoded}xyz"}', + self.protojson.encode_message(MyMessage(a_string='xyz'))) - def testDecode(self): - self.assertEqual( - MyMessage(a_string='{decoded}xyz'), - self.protojson.decode_message(MyMessage, '{"a_string": "xyz"}')) + def testDecode(self): + self.assertEqual( + MyMessage(a_string='{decoded}xyz'), + self.protojson.decode_message(MyMessage, '{"a_string": "xyz"}')) - def testDecodeEmptyMessage(self): - self.assertEqual( - MyMessage(a_string='{decoded}'), - self.protojson.decode_message(MyMessage, '{"a_string": ""}')) + def testDecodeEmptyMessage(self): + self.assertEqual( + MyMessage(a_string='{decoded}'), + self.protojson.decode_message(MyMessage, '{"a_string": ""}')) - def testDefault(self): - self.assertTrue(protojson.ProtoJson.get_default(), - protojson.ProtoJson.get_default()) + def testDefault(self): + self.assertTrue(protojson.ProtoJson.get_default(), + protojson.ProtoJson.get_default()) - instance = CustomProtoJson() - protojson.ProtoJson.set_default(instance) - self.assertTrue(instance is protojson.ProtoJson.get_default()) + instance = CustomProtoJson() + protojson.ProtoJson.set_default(instance) + self.assertTrue(instance is protojson.ProtoJson.get_default()) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/apitools/base/protorpclite/test_util.py b/apitools/base/protorpclite/test_util.py index b6b3537..b50aa88 100644 --- a/apitools/base/protorpclite/test_util.py +++ b/apitools/base/protorpclite/test_util.py @@ -53,613 +53,616 @@ BINARY = b''.join(six.int2byte(value) + b'\0' for value in range(256)) class TestCase(unittest.TestCase): - def assertRaisesWithRegexpMatch(self, - exception, - regexp, - function, - *params, - **kwargs): - """Check that exception is raised and text matches regular expression. - - Args: - exception: Exception type that is expected. - regexp: String regular expression that is expected in error message. - function: Callable to test. - params: Parameters to forward to function. - kwargs: Keyword arguments to forward to function. - """ - try: - function(*params, **kwargs) - self.fail('Expected exception %s was not raised' % exception.__name__) - except exception as err: - match = bool(re.match(regexp, str(err))) - self.assertTrue(match, 'Expected match "%s", found "%s"' % (regexp, - err)) - - def assertHeaderSame(self, header1, header2): - """Check that two HTTP headers are the same. - - Args: - header1: Header value string 1. - header2: header value string 2. - """ - value1, params1 = cgi.parse_header(header1) - value2, params2 = cgi.parse_header(header2) - self.assertEqual(value1, value2) - self.assertEqual(params1, params2) - - def assertIterEqual(self, iter1, iter2): - """Check that two iterators or iterables are equal independent of order. - - Similar to Python 2.7 assertItemsEqual. Named differently in order to - avoid potential conflict. - - Args: - iter1: An iterator or iterable. - iter2: An iterator or iterable. - """ - list1 = list(iter1) - list2 = list(iter2) - - unmatched1 = list() - - while list1: - item1 = list1[0] - del list1[0] - for index in range(len(list2)): - if item1 == list2[index]: - del list2[index] - break - else: - unmatched1.append(item1) - - error_message = [] - for item in unmatched1: - error_message.append( - ' Item from iter1 not found in iter2: %r' % item) - for item in list2: - error_message.append( - ' Item from iter2 not found in iter1: %r' % item) - if error_message: - self.fail('Collections not equivalent:\n' + '\n'.join(error_message)) + def assertRaisesWithRegexpMatch(self, + exception, + regexp, + function, + *params, + **kwargs): + """Check that exception is raised and text matches regular expression. + + Args: + exception: Exception type that is expected. + regexp: String regular expression that is expected in error message. + function: Callable to test. + params: Parameters to forward to function. + kwargs: Keyword arguments to forward to function. + """ + try: + function(*params, **kwargs) + self.fail('Expected exception %s was not raised' % + exception.__name__) + except exception as err: + match = bool(re.match(regexp, str(err))) + self.assertTrue(match, 'Expected match "%s", found "%s"' % (regexp, + err)) + + def assertHeaderSame(self, header1, header2): + """Check that two HTTP headers are the same. + + Args: + header1: Header value string 1. + header2: header value string 2. + """ + value1, params1 = cgi.parse_header(header1) + value2, params2 = cgi.parse_header(header2) + self.assertEqual(value1, value2) + self.assertEqual(params1, params2) + + def assertIterEqual(self, iter1, iter2): + """Check that two iterators or iterables are equal independent of order. + + Similar to Python 2.7 assertItemsEqual. Named differently in order to + avoid potential conflict. + + Args: + iter1: An iterator or iterable. + iter2: An iterator or iterable. + """ + list1 = list(iter1) + list2 = list(iter2) + + unmatched1 = list() + + while list1: + item1 = list1[0] + del list1[0] + for index in range(len(list2)): + if item1 == list2[index]: + del list2[index] + break + else: + unmatched1.append(item1) + + error_message = [] + for item in unmatched1: + error_message.append( + ' Item from iter1 not found in iter2: %r' % item) + for item in list2: + error_message.append( + ' Item from iter2 not found in iter1: %r' % item) + if error_message: + self.fail('Collections not equivalent:\n' + + '\n'.join(error_message)) class ModuleInterfaceTest(object): - """Test to ensure module interface is carefully constructed. - - A module interface is the set of public objects listed in the module __all__ - attribute. Modules that that are considered public should have this interface - carefully declared. At all times, the __all__ attribute should have objects - intended to be publically used and all other objects in the module should be - considered unused. + """Test to ensure module interface is carefully constructed. - Protected attributes (those beginning with '_') and other imported modules - should not be part of this set of variables. An exception is for variables - that begin and end with '__' which are implicitly part of the interface - (eg. __name__, __file__, __all__ itself, etc.). + A module interface is the set of public objects listed in the module __all__ + attribute. Modules that that are considered public should have this interface + carefully declared. At all times, the __all__ attribute should have objects + intended to be publically used and all other objects in the module should be + considered unused. - Modules that are imported in to the tested modules are an exception and may - be left out of the __all__ definition. The test is done by checking the value - of what would otherwise be a public name and not allowing it to be exported - if it is an instance of a module. Modules that are explicitly exported are - for the time being not permitted. + Protected attributes (those beginning with '_') and other imported modules + should not be part of this set of variables. An exception is for variables + that begin and end with '__' which are implicitly part of the interface + (eg. __name__, __file__, __all__ itself, etc.). - To use this test class a module should define a new class that inherits first - from ModuleInterfaceTest and then from test_util.TestCase. No other tests - should be added to this test case, making the order of inheritance less - important, but if setUp for some reason is overidden, it is important that - ModuleInterfaceTest is first in the list so that its setUp method is - invoked. + Modules that are imported in to the tested modules are an exception and may + be left out of the __all__ definition. The test is done by checking the value + of what would otherwise be a public name and not allowing it to be exported + if it is an instance of a module. Modules that are explicitly exported are + for the time being not permitted. - Multiple inheretance is required so that ModuleInterfaceTest is not itself - a test, and is not itself executed as one. + To use this test class a module should define a new class that inherits first + from ModuleInterfaceTest and then from test_util.TestCase. No other tests + should be added to this test case, making the order of inheritance less + important, but if setUp for some reason is overidden, it is important that + ModuleInterfaceTest is first in the list so that its setUp method is + invoked. - The test class is expected to have the following class attributes defined: + Multiple inheretance is required so that ModuleInterfaceTest is not itself + a test, and is not itself executed as one. - MODULE: A reference to the module that is being validated for interface - correctness. + The test class is expected to have the following class attributes defined: - Example: - Module definition (hello.py): + MODULE: A reference to the module that is being validated for interface + correctness. - import sys + Example: + Module definition (hello.py): - __all__ = ['hello'] + import sys - def _get_outputter(): - return sys.stdout + __all__ = ['hello'] - def hello(): - _get_outputter().write('Hello\n') + def _get_outputter(): + return sys.stdout - Test definition: + def hello(): + _get_outputter().write('Hello\n') - import unittest - from protorpc import test_util + Test definition: - import hello + import unittest + from protorpc import test_util - class ModuleInterfaceTest(test_util.ModuleInterfaceTest, - test_util.TestCase): + import hello - MODULE = hello + class ModuleInterfaceTest(test_util.ModuleInterfaceTest, + test_util.TestCase): + MODULE = hello - class HelloTest(test_util.TestCase): - ... Test 'hello' module ... + class HelloTest(test_util.TestCase): + ... Test 'hello' module ... - if __name__ == '__main__': - unittest.main() - """ - def setUp(self): - """Set up makes sure that MODULE and IMPORTED_MODULES is defined. - - This is a basic configuration test for the test itself so does not - get it's own test case. + if __name__ == '__main__': + unittest.main() """ - if not hasattr(self, 'MODULE'): - self.fail( - "You must define 'MODULE' on ModuleInterfaceTest sub-class %s." % - type(self).__name__) - - def testAllExist(self): - """Test that all attributes defined in __all__ exist.""" - missing_attributes = [] - for attribute in self.MODULE.__all__: - if not hasattr(self.MODULE, attribute): - missing_attributes.append(attribute) - if missing_attributes: - self.fail('%s of __all__ are not defined in module.' % - missing_attributes) - - def testAllExported(self): - """Test that all public attributes not imported are in __all__.""" - missing_attributes = [] - for attribute in dir(self.MODULE): - if not attribute.startswith('_'): - if (attribute not in self.MODULE.__all__ and - not isinstance(getattr(self.MODULE, attribute), - types.ModuleType) and - attribute != 'with_statement'): - missing_attributes.append(attribute) - if missing_attributes: - self.fail('%s are not modules and not defined in __all__.' % - missing_attributes) - - def testNoExportedProtectedVariables(self): - """Test that there are no protected variables listed in __all__.""" - protected_variables = [] - for attribute in self.MODULE.__all__: - if attribute.startswith('_'): - protected_variables.append(attribute) - if protected_variables: - self.fail('%s are protected variables and may not be exported.' % - protected_variables) - - def testNoExportedModules(self): - """Test that no modules exist in __all__.""" - exported_modules = [] - for attribute in self.MODULE.__all__: - try: - value = getattr(self.MODULE, attribute) - except AttributeError: - # This is a different error case tested for in testAllExist. - pass - else: - if isinstance(value, types.ModuleType): - exported_modules.append(attribute) - if exported_modules: - self.fail('%s are modules and may not be exported.' % exported_modules) + + def setUp(self): + """Set up makes sure that MODULE and IMPORTED_MODULES is defined. + + This is a basic configuration test for the test itself so does not + get it's own test case. + """ + if not hasattr(self, 'MODULE'): + self.fail( + "You must define 'MODULE' on ModuleInterfaceTest sub-class %s." % + type(self).__name__) + + def testAllExist(self): + """Test that all attributes defined in __all__ exist.""" + missing_attributes = [] + for attribute in self.MODULE.__all__: + if not hasattr(self.MODULE, attribute): + missing_attributes.append(attribute) + if missing_attributes: + self.fail('%s of __all__ are not defined in module.' % + missing_attributes) + + def testAllExported(self): + """Test that all public attributes not imported are in __all__.""" + missing_attributes = [] + for attribute in dir(self.MODULE): + if not attribute.startswith('_'): + if (attribute not in self.MODULE.__all__ and + not isinstance(getattr(self.MODULE, attribute), + types.ModuleType) and + attribute != 'with_statement'): + missing_attributes.append(attribute) + if missing_attributes: + self.fail('%s are not modules and not defined in __all__.' % + missing_attributes) + + def testNoExportedProtectedVariables(self): + """Test that there are no protected variables listed in __all__.""" + protected_variables = [] + for attribute in self.MODULE.__all__: + if attribute.startswith('_'): + protected_variables.append(attribute) + if protected_variables: + self.fail('%s are protected variables and may not be exported.' % + protected_variables) + + def testNoExportedModules(self): + """Test that no modules exist in __all__.""" + exported_modules = [] + for attribute in self.MODULE.__all__: + try: + value = getattr(self.MODULE, attribute) + except AttributeError: + # This is a different error case tested for in testAllExist. + pass + else: + if isinstance(value, types.ModuleType): + exported_modules.append(attribute) + if exported_modules: + self.fail('%s are modules and may not be exported.' % + exported_modules) class NestedMessage(messages.Message): - """Simple message that gets nested in another message.""" + """Simple message that gets nested in another message.""" - a_value = messages.StringField(1, required=True) + a_value = messages.StringField(1, required=True) class HasNestedMessage(messages.Message): - """Message that has another message nested in it.""" + """Message that has another message nested in it.""" - nested = messages.MessageField(NestedMessage, 1) - repeated_nested = messages.MessageField(NestedMessage, 2, repeated=True) + nested = messages.MessageField(NestedMessage, 1) + repeated_nested = messages.MessageField(NestedMessage, 2, repeated=True) class HasDefault(messages.Message): - """Has a default value.""" + """Has a default value.""" - a_value = messages.StringField(1, default=u'a default') + a_value = messages.StringField(1, default=u'a default') class OptionalMessage(messages.Message): - """Contains all message types.""" + """Contains all message types.""" - class SimpleEnum(messages.Enum): - """Simple enumeration type.""" - VAL1 = 1 - VAL2 = 2 + class SimpleEnum(messages.Enum): + """Simple enumeration type.""" + VAL1 = 1 + VAL2 = 2 - double_value = messages.FloatField(1, variant=messages.Variant.DOUBLE) - float_value = messages.FloatField(2, variant=messages.Variant.FLOAT) - int64_value = messages.IntegerField(3, variant=messages.Variant.INT64) - uint64_value = messages.IntegerField(4, variant=messages.Variant.UINT64) - int32_value = messages.IntegerField(5, variant=messages.Variant.INT32) - bool_value = messages.BooleanField(6, variant=messages.Variant.BOOL) - string_value = messages.StringField(7, variant=messages.Variant.STRING) - bytes_value = messages.BytesField(8, variant=messages.Variant.BYTES) - enum_value = messages.EnumField(SimpleEnum, 10) + double_value = messages.FloatField(1, variant=messages.Variant.DOUBLE) + float_value = messages.FloatField(2, variant=messages.Variant.FLOAT) + int64_value = messages.IntegerField(3, variant=messages.Variant.INT64) + uint64_value = messages.IntegerField(4, variant=messages.Variant.UINT64) + int32_value = messages.IntegerField(5, variant=messages.Variant.INT32) + bool_value = messages.BooleanField(6, variant=messages.Variant.BOOL) + string_value = messages.StringField(7, variant=messages.Variant.STRING) + bytes_value = messages.BytesField(8, variant=messages.Variant.BYTES) + enum_value = messages.EnumField(SimpleEnum, 10) - # TODO(rafek): Add support for these variants. - # uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) - # sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) - # sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) + # TODO(rafek): Add support for these variants. + # uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) + # sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) + # sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) class RepeatedMessage(messages.Message): - """Contains all message types as repeated fields.""" - - class SimpleEnum(messages.Enum): - """Simple enumeration type.""" - VAL1 = 1 - VAL2 = 2 - - double_value = messages.FloatField(1, - variant=messages.Variant.DOUBLE, - repeated=True) - float_value = messages.FloatField(2, - variant=messages.Variant.FLOAT, - repeated=True) - int64_value = messages.IntegerField(3, - variant=messages.Variant.INT64, - repeated=True) - uint64_value = messages.IntegerField(4, - variant=messages.Variant.UINT64, + """Contains all message types as repeated fields.""" + + class SimpleEnum(messages.Enum): + """Simple enumeration type.""" + VAL1 = 1 + VAL2 = 2 + + double_value = messages.FloatField(1, + variant=messages.Variant.DOUBLE, repeated=True) - int32_value = messages.IntegerField(5, - variant=messages.Variant.INT32, + float_value = messages.FloatField(2, + variant=messages.Variant.FLOAT, repeated=True) - bool_value = messages.BooleanField(6, - variant=messages.Variant.BOOL, - repeated=True) - string_value = messages.StringField(7, - variant=messages.Variant.STRING, + int64_value = messages.IntegerField(3, + variant=messages.Variant.INT64, + repeated=True) + uint64_value = messages.IntegerField(4, + variant=messages.Variant.UINT64, + repeated=True) + int32_value = messages.IntegerField(5, + variant=messages.Variant.INT32, + repeated=True) + bool_value = messages.BooleanField(6, + variant=messages.Variant.BOOL, + repeated=True) + string_value = messages.StringField(7, + variant=messages.Variant.STRING, + repeated=True) + bytes_value = messages.BytesField(8, + variant=messages.Variant.BYTES, repeated=True) - bytes_value = messages.BytesField(8, - variant=messages.Variant.BYTES, + #uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) + enum_value = messages.EnumField(SimpleEnum, + 10, repeated=True) - #uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) - enum_value = messages.EnumField(SimpleEnum, - 10, - repeated=True) - #sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) - #sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) + #sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) + #sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) class HasOptionalNestedMessage(messages.Message): - nested = messages.MessageField(OptionalMessage, 1) - repeated_nested = messages.MessageField(OptionalMessage, 2, repeated=True) + nested = messages.MessageField(OptionalMessage, 1) + repeated_nested = messages.MessageField(OptionalMessage, 2, repeated=True) class ProtoConformanceTestBase(object): - """Protocol conformance test base class. - - Each supported protocol should implement two methods that support encoding - and decoding of Message objects in that format: - - encode_message(message) - Serialize to encoding. - encode_message(message, encoded_message) - Deserialize from encoding. - - Tests for the modules where these functions are implemented should extend - this class in order to support basic behavioral expectations. This ensures - that protocols correctly encode and decode message transparently to the - caller. - - In order to support these test, the base class should also extend the TestCase - class and implement the following class attributes which define the encoded - version of certain protocol buffers: - - encoded_partial: - - - encoded_full: - - - encoded_repeated: - - - encoded_nested: - - > + """Protocol conformance test base class. - encoded_repeated_nested: - , - - ] - > + Each supported protocol should implement two methods that support encoding + and decoding of Message objects in that format: - unexpected_tag_message: - An encoded message that has an undefined tag or number in the stream. + encode_message(message) - Serialize to encoding. + encode_message(message, encoded_message) - Deserialize from encoding. - encoded_default_assigned: - + Tests for the modules where these functions are implemented should extend + this class in order to support basic behavioral expectations. This ensures + that protocols correctly encode and decode message transparently to the + caller. - encoded_nested_empty: - - > + In order to support these test, the base class should also extend the TestCase + class and implement the following class attributes which define the encoded + version of certain protocol buffers: - encoded_invalid_enum: - - """ + encoded_partial: + - encoded_empty_message = '' + encoded_full: + - def testEncodeInvalidMessage(self): - message = NestedMessage() - self.assertRaises(messages.ValidationError, - self.PROTOLIB.encode_message, message) + encoded_repeated: + - def CompareEncoded(self, expected_encoded, actual_encoded): - """Compare two encoded protocol values. + encoded_nested: + + > - Can be overridden by sub-classes to special case comparison. - For example, to eliminate white space from output that is not - relevant to encoding. + encoded_repeated_nested: + , + + ] + > - Args: - expected_encoded: Expected string encoded value. - actual_encoded: Actual string encoded value. + unexpected_tag_message: + An encoded message that has an undefined tag or number in the stream. + + encoded_default_assigned: + + + encoded_nested_empty: + + > + + encoded_invalid_enum: + """ - self.assertEquals(expected_encoded, actual_encoded) - - def EncodeDecode(self, encoded, expected_message): - message = self.PROTOLIB.decode_message(type(expected_message), encoded) - self.assertEquals(expected_message, message) - self.CompareEncoded(encoded, self.PROTOLIB.encode_message(message)) - - def testEmptyMessage(self): - self.EncodeDecode(self.encoded_empty_message, OptionalMessage()) - - def testPartial(self): - """Test message with a few values set.""" - message = OptionalMessage() - message.double_value = 1.23 - message.int64_value = -100000000000 - message.int32_value = 1020 - message.string_value = u'a string' - message.enum_value = OptionalMessage.SimpleEnum.VAL2 - - self.EncodeDecode(self.encoded_partial, message) - - def testFull(self): - """Test all types.""" - message = OptionalMessage() - message.double_value = 1.23 - message.float_value = -2.5 - message.int64_value = -100000000000 - message.uint64_value = 102020202020 - message.int32_value = 1020 - message.bool_value = True - message.string_value = u'a string\u044f' - message.bytes_value = b'a bytes\xff\xfe' - message.enum_value = OptionalMessage.SimpleEnum.VAL2 - - self.EncodeDecode(self.encoded_full, message) - - def testRepeated(self): - """Test repeated fields.""" - message = RepeatedMessage() - message.double_value = [1.23, 2.3] - message.float_value = [-2.5, 0.5] - message.int64_value = [-100000000000, 20] - message.uint64_value = [102020202020, 10] - message.int32_value = [1020, 718] - message.bool_value = [True, False] - message.string_value = [u'a string\u044f', u'another string'] - message.bytes_value = [b'a bytes\xff\xfe', b'another bytes'] - message.enum_value = [RepeatedMessage.SimpleEnum.VAL2, - RepeatedMessage.SimpleEnum.VAL1] - - self.EncodeDecode(self.encoded_repeated, message) - - def testNested(self): - """Test nested messages.""" - nested_message = NestedMessage() - nested_message.a_value = u'a string' - - message = HasNestedMessage() - message.nested = nested_message - - self.EncodeDecode(self.encoded_nested, message) - - def testRepeatedNested(self): - """Test repeated nested messages.""" - nested_message1 = NestedMessage() - nested_message1.a_value = u'a string' - nested_message2 = NestedMessage() - nested_message2.a_value = u'another string' - - message = HasNestedMessage() - message.repeated_nested = [nested_message1, nested_message2] - - self.EncodeDecode(self.encoded_repeated_nested, message) - - def testStringTypes(self): - """Test that encoding str on StringField works.""" - message = OptionalMessage() - message.string_value = 'Latin' - self.EncodeDecode(self.encoded_string_types, message) - - def testEncodeUninitialized(self): - """Test that cannot encode uninitialized message.""" - required = NestedMessage() - self.assertRaisesWithRegexpMatch(messages.ValidationError, - "Message NestedMessage is missing " - "required field a_value", - self.PROTOLIB.encode_message, - required) - - def testUnexpectedField(self): - """Test decoding and encoding unexpected fields.""" - loaded_message = self.PROTOLIB.decode_message(OptionalMessage, - self.unexpected_tag_message) - # Message should be equal to an empty message, since unknown values aren't - # included in equality. - self.assertEquals(OptionalMessage(), loaded_message) - # Verify that the encoded message matches the source, including the - # unknown value. - self.assertEquals(self.unexpected_tag_message, - self.PROTOLIB.encode_message(loaded_message)) - - def testDoNotSendDefault(self): - """Test that default is not sent when nothing is assigned.""" - self.EncodeDecode(self.encoded_empty_message, HasDefault()) - - def testSendDefaultExplicitlyAssigned(self): - """Test that default is sent when explcitly assigned.""" - message = HasDefault() - - message.a_value = HasDefault.a_value.default - - self.EncodeDecode(self.encoded_default_assigned, message) - - def testEncodingNestedEmptyMessage(self): - """Test encoding a nested empty message.""" - message = HasOptionalNestedMessage() - message.nested = OptionalMessage() - - self.EncodeDecode(self.encoded_nested_empty, message) - - def testEncodingRepeatedNestedEmptyMessage(self): - """Test encoding a nested empty message.""" - message = HasOptionalNestedMessage() - message.repeated_nested = [OptionalMessage(), OptionalMessage()] - - self.EncodeDecode(self.encoded_repeated_nested_empty, message) - - def testContentType(self): - self.assertTrue(isinstance(self.PROTOLIB.CONTENT_TYPE, str)) - - def testDecodeInvalidEnumType(self): - self.assertRaisesWithRegexpMatch(messages.DecodeError, - 'Invalid enum value ', - self.PROTOLIB.decode_message, - OptionalMessage, - self.encoded_invalid_enum) - - def testDateTimeNoTimeZone(self): - """Test that DateTimeFields are encoded/decoded correctly.""" - - class MyMessage(messages.Message): - value = message_types.DateTimeField(1) - - value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000) - message = MyMessage(value=value) - decoded = self.PROTOLIB.decode_message( - MyMessage, self.PROTOLIB.encode_message(message)) - self.assertEquals(decoded.value, value) - def testDateTimeWithTimeZone(self): - """Test DateTimeFields with time zones.""" - - class MyMessage(messages.Message): - value = message_types.DateTimeField(1) - - value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000, - util.TimeZoneOffset(8 * 60)) - message = MyMessage(value=value) - decoded = self.PROTOLIB.decode_message( - MyMessage, self.PROTOLIB.encode_message(message)) - self.assertEquals(decoded.value, value) + encoded_empty_message = '' + + def testEncodeInvalidMessage(self): + message = NestedMessage() + self.assertRaises(messages.ValidationError, + self.PROTOLIB.encode_message, message) + + def CompareEncoded(self, expected_encoded, actual_encoded): + """Compare two encoded protocol values. + + Can be overridden by sub-classes to special case comparison. + For example, to eliminate white space from output that is not + relevant to encoding. + + Args: + expected_encoded: Expected string encoded value. + actual_encoded: Actual string encoded value. + """ + self.assertEquals(expected_encoded, actual_encoded) + + def EncodeDecode(self, encoded, expected_message): + message = self.PROTOLIB.decode_message(type(expected_message), encoded) + self.assertEquals(expected_message, message) + self.CompareEncoded(encoded, self.PROTOLIB.encode_message(message)) + + def testEmptyMessage(self): + self.EncodeDecode(self.encoded_empty_message, OptionalMessage()) + + def testPartial(self): + """Test message with a few values set.""" + message = OptionalMessage() + message.double_value = 1.23 + message.int64_value = -100000000000 + message.int32_value = 1020 + message.string_value = u'a string' + message.enum_value = OptionalMessage.SimpleEnum.VAL2 + + self.EncodeDecode(self.encoded_partial, message) + + def testFull(self): + """Test all types.""" + message = OptionalMessage() + message.double_value = 1.23 + message.float_value = -2.5 + message.int64_value = -100000000000 + message.uint64_value = 102020202020 + message.int32_value = 1020 + message.bool_value = True + message.string_value = u'a string\u044f' + message.bytes_value = b'a bytes\xff\xfe' + message.enum_value = OptionalMessage.SimpleEnum.VAL2 + + self.EncodeDecode(self.encoded_full, message) + + def testRepeated(self): + """Test repeated fields.""" + message = RepeatedMessage() + message.double_value = [1.23, 2.3] + message.float_value = [-2.5, 0.5] + message.int64_value = [-100000000000, 20] + message.uint64_value = [102020202020, 10] + message.int32_value = [1020, 718] + message.bool_value = [True, False] + message.string_value = [u'a string\u044f', u'another string'] + message.bytes_value = [b'a bytes\xff\xfe', b'another bytes'] + message.enum_value = [RepeatedMessage.SimpleEnum.VAL2, + RepeatedMessage.SimpleEnum.VAL1] + + self.EncodeDecode(self.encoded_repeated, message) + + def testNested(self): + """Test nested messages.""" + nested_message = NestedMessage() + nested_message.a_value = u'a string' + + message = HasNestedMessage() + message.nested = nested_message + + self.EncodeDecode(self.encoded_nested, message) + + def testRepeatedNested(self): + """Test repeated nested messages.""" + nested_message1 = NestedMessage() + nested_message1.a_value = u'a string' + nested_message2 = NestedMessage() + nested_message2.a_value = u'another string' + + message = HasNestedMessage() + message.repeated_nested = [nested_message1, nested_message2] + + self.EncodeDecode(self.encoded_repeated_nested, message) + + def testStringTypes(self): + """Test that encoding str on StringField works.""" + message = OptionalMessage() + message.string_value = 'Latin' + self.EncodeDecode(self.encoded_string_types, message) + + def testEncodeUninitialized(self): + """Test that cannot encode uninitialized message.""" + required = NestedMessage() + self.assertRaisesWithRegexpMatch(messages.ValidationError, + "Message NestedMessage is missing " + "required field a_value", + self.PROTOLIB.encode_message, + required) + + def testUnexpectedField(self): + """Test decoding and encoding unexpected fields.""" + loaded_message = self.PROTOLIB.decode_message(OptionalMessage, + self.unexpected_tag_message) + # Message should be equal to an empty message, since unknown values aren't + # included in equality. + self.assertEquals(OptionalMessage(), loaded_message) + # Verify that the encoded message matches the source, including the + # unknown value. + self.assertEquals(self.unexpected_tag_message, + self.PROTOLIB.encode_message(loaded_message)) + + def testDoNotSendDefault(self): + """Test that default is not sent when nothing is assigned.""" + self.EncodeDecode(self.encoded_empty_message, HasDefault()) + + def testSendDefaultExplicitlyAssigned(self): + """Test that default is sent when explcitly assigned.""" + message = HasDefault() + + message.a_value = HasDefault.a_value.default + + self.EncodeDecode(self.encoded_default_assigned, message) + + def testEncodingNestedEmptyMessage(self): + """Test encoding a nested empty message.""" + message = HasOptionalNestedMessage() + message.nested = OptionalMessage() + + self.EncodeDecode(self.encoded_nested_empty, message) + + def testEncodingRepeatedNestedEmptyMessage(self): + """Test encoding a nested empty message.""" + message = HasOptionalNestedMessage() + message.repeated_nested = [OptionalMessage(), OptionalMessage()] + + self.EncodeDecode(self.encoded_repeated_nested_empty, message) + + def testContentType(self): + self.assertTrue(isinstance(self.PROTOLIB.CONTENT_TYPE, str)) + + def testDecodeInvalidEnumType(self): + self.assertRaisesWithRegexpMatch(messages.DecodeError, + 'Invalid enum value ', + self.PROTOLIB.decode_message, + OptionalMessage, + self.encoded_invalid_enum) + + def testDateTimeNoTimeZone(self): + """Test that DateTimeFields are encoded/decoded correctly.""" + + class MyMessage(messages.Message): + value = message_types.DateTimeField(1) + + value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000) + message = MyMessage(value=value) + decoded = self.PROTOLIB.decode_message( + MyMessage, self.PROTOLIB.encode_message(message)) + self.assertEquals(decoded.value, value) + + def testDateTimeWithTimeZone(self): + """Test DateTimeFields with time zones.""" + + class MyMessage(messages.Message): + value = message_types.DateTimeField(1) + + value = datetime.datetime(2013, 1, 3, 11, 36, 30, 123000, + util.TimeZoneOffset(8 * 60)) + message = MyMessage(value=value) + decoded = self.PROTOLIB.decode_message( + MyMessage, self.PROTOLIB.encode_message(message)) + self.assertEquals(decoded.value, value) def do_with(context, function, *args, **kwargs): - """Simulate a with statement. + """Simulate a with statement. - Avoids need to import with from future. + Avoids need to import with from future. - Does not support simulation of 'as'. + Does not support simulation of 'as'. - Args: - context: Context object normally used with 'with'. - function: Callable to evoke. Replaces with-block. - """ - context.__enter__() - try: - function(*args, **kwargs) - except: - context.__exit__(*sys.exc_info()) - finally: - context.__exit__(None, None, None) + Args: + context: Context object normally used with 'with'. + function: Callable to evoke. Replaces with-block. + """ + context.__enter__() + try: + function(*args, **kwargs) + except: + context.__exit__(*sys.exc_info()) + finally: + context.__exit__(None, None, None) def pick_unused_port(): - """Find an unused port to use in tests. + """Find an unused port to use in tests. - Derived from Damon Kohlers example: + Derived from Damon Kohlers example: - http://code.activestate.com/recipes/531822-pick-unused-port - """ - temp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - temp.bind(('localhost', 0)) - port = temp.getsockname()[1] - finally: - temp.close() - return port + http://code.activestate.com/recipes/531822-pick-unused-port + """ + temp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + temp.bind(('localhost', 0)) + port = temp.getsockname()[1] + finally: + temp.close() + return port def get_module_name(module_attribute): - """Get the module name. - - Args: - module_attribute: An attribute of the module. - - Returns: - The fully qualified module name or simple module name where - 'module_attribute' is defined if the module name is "__main__". - """ - if module_attribute.__module__ == '__main__': - module_file = inspect.getfile(module_attribute) - default = os.path.basename(module_file).split('.')[0] - return default - else: - return module_attribute.__module__ + """Get the module name. + + Args: + module_attribute: An attribute of the module. + + Returns: + The fully qualified module name or simple module name where + 'module_attribute' is defined if the module name is "__main__". + """ + if module_attribute.__module__ == '__main__': + module_file = inspect.getfile(module_attribute) + default = os.path.basename(module_file).split('.')[0] + return default + else: + return module_attribute.__module__ diff --git a/apitools/base/protorpclite/util.py b/apitools/base/protorpclite/util.py index 4881889..0d3db2f 100644 --- a/apitools/base/protorpclite/util.py +++ b/apitools/base/protorpclite/util.py @@ -22,7 +22,7 @@ import six __author__ = ['rafek@google.com (Rafe Kaplan)', 'guido@google.com (Guido van Rossum)', -] + ] import cgi import datetime @@ -38,11 +38,11 @@ __all__ = ['Error', 'positional', 'TimeZoneOffset', 'total_seconds', -] + ] class Error(Exception): - """Base class for protorpc exceptions.""" + """Base class for protorpc exceptions.""" _TIME_ZONE_RE_STRING = r""" @@ -58,238 +58,238 @@ _TIME_ZONE_RE = re.compile(_TIME_ZONE_RE_STRING, re.IGNORECASE | re.VERBOSE) def positional(max_positional_args): - """A decorator to declare that only the first N arguments may be positional. + """A decorator to declare that only the first N arguments may be positional. - This decorator makes it easy to support Python 3 style keyword-only - parameters. For example, in Python 3 it is possible to write: + This decorator makes it easy to support Python 3 style keyword-only + parameters. For example, in Python 3 it is possible to write: - def fn(pos1, *, kwonly1=None, kwonly1=None): - ... + def fn(pos1, *, kwonly1=None, kwonly1=None): + ... - All named parameters after * must be a keyword: + All named parameters after * must be a keyword: - fn(10, 'kw1', 'kw2') # Raises exception. - fn(10, kwonly1='kw1') # Ok. + fn(10, 'kw1', 'kw2') # Raises exception. + fn(10, kwonly1='kw1') # Ok. - Example: - To define a function like above, do: + Example: + To define a function like above, do: - @positional(1) - def fn(pos1, kwonly1=None, kwonly2=None): - ... + @positional(1) + def fn(pos1, kwonly1=None, kwonly2=None): + ... - If no default value is provided to a keyword argument, it becomes a required - keyword argument: + If no default value is provided to a keyword argument, it becomes a required + keyword argument: - @positional(0) - def fn(required_kw): - ... + @positional(0) + def fn(required_kw): + ... - This must be called with the keyword parameter: + This must be called with the keyword parameter: - fn() # Raises exception. - fn(10) # Raises exception. - fn(required_kw=10) # Ok. + fn() # Raises exception. + fn(10) # Raises exception. + fn(required_kw=10) # Ok. - When defining instance or class methods always remember to account for - 'self' and 'cls': + When defining instance or class methods always remember to account for + 'self' and 'cls': - class MyClass(object): + class MyClass(object): - @positional(2) - def my_method(self, pos1, kwonly1=None): - ... + @positional(2) + def my_method(self, pos1, kwonly1=None): + ... + + @classmethod + @positional(2) + def my_method(cls, pos1, kwonly1=None): + ... + + One can omit the argument to 'positional' altogether, and then no + arguments with default values may be passed positionally. This + would be equivalent to placing a '*' before the first argument + with a default value in Python 3. If there are no arguments with + default values, and no argument is given to 'positional', an error + is raised. - @classmethod - @positional(2) - def my_method(cls, pos1, kwonly1=None): + @positional + def fn(arg1, arg2, required_kw1=None, required_kw2=0): ... - One can omit the argument to 'positional' altogether, and then no - arguments with default values may be passed positionally. This - would be equivalent to placing a '*' before the first argument - with a default value in Python 3. If there are no arguments with - default values, and no argument is given to 'positional', an error - is raised. + fn(1, 3, 5) # Raises exception. + fn(1, 3) # Ok. + fn(1, 3, required_kw1=5) # Ok. - @positional - def fn(arg1, arg2, required_kw1=None, required_kw2=0): - ... + Args: + max_positional_arguments: Maximum number of positional arguments. All + parameters after the this index must be keyword only. - fn(1, 3, 5) # Raises exception. - fn(1, 3) # Ok. - fn(1, 3, required_kw1=5) # Ok. - - Args: - max_positional_arguments: Maximum number of positional arguments. All - parameters after the this index must be keyword only. - - Returns: - A decorator that prevents using arguments after max_positional_args from - being used as positional parameters. - - Raises: - TypeError if a keyword-only argument is provided as a positional parameter. - ValueError if no maximum number of arguments is provided and the function - has no arguments with default values. - """ - def positional_decorator(wrapped): - @functools.wraps(wrapped) - def positional_wrapper(*args, **kwargs): - if len(args) > max_positional_args: - plural_s = '' - if max_positional_args != 1: - plural_s = 's' - raise TypeError('%s() takes at most %d positional argument%s ' - '(%d given)' % (wrapped.__name__, - max_positional_args, - plural_s, len(args))) - return wrapped(*args, **kwargs) - return positional_wrapper - - if isinstance(max_positional_args, six.integer_types): - return positional_decorator - else: - args, _, _, defaults = inspect.getargspec(max_positional_args) - if defaults is None: - raise ValueError( - 'Functions with no keyword arguments must specify ' - 'max_positional_args') - return positional(len(args) - len(defaults))(max_positional_args) + Returns: + A decorator that prevents using arguments after max_positional_args from + being used as positional parameters. + + Raises: + TypeError if a keyword-only argument is provided as a positional parameter. + ValueError if no maximum number of arguments is provided and the function + has no arguments with default values. + """ + def positional_decorator(wrapped): + @functools.wraps(wrapped) + def positional_wrapper(*args, **kwargs): + if len(args) > max_positional_args: + plural_s = '' + if max_positional_args != 1: + plural_s = 's' + raise TypeError('%s() takes at most %d positional argument%s ' + '(%d given)' % (wrapped.__name__, + max_positional_args, + plural_s, len(args))) + return wrapped(*args, **kwargs) + return positional_wrapper + + if isinstance(max_positional_args, six.integer_types): + return positional_decorator + else: + args, _, _, defaults = inspect.getargspec(max_positional_args) + if defaults is None: + raise ValueError( + 'Functions with no keyword arguments must specify ' + 'max_positional_args') + return positional(len(args) - len(defaults))(max_positional_args) @positional(1) def get_package_for_module(module): - """Get package name for a module. + """Get package name for a module. - Helper calculates the package name of a module. + Helper calculates the package name of a module. - Args: - module: Module to get name for. If module is a string, try to find - module in sys.modules. + Args: + module: Module to get name for. If module is a string, try to find + module in sys.modules. + + Returns: + If module contains 'package' attribute, uses that as package name. + Else, if module is not the '__main__' module, the module __name__. + Else, the base name of the module file name. Else None. + """ + if isinstance(module, six.string_types): + try: + module = sys.modules[module] + except KeyError: + return None - Returns: - If module contains 'package' attribute, uses that as package name. - Else, if module is not the '__main__' module, the module __name__. - Else, the base name of the module file name. Else None. - """ - if isinstance(module, six.string_types): try: - module = sys.modules[module] - except KeyError: - return None - - try: - return six.text_type(module.package) - except AttributeError: - if module.__name__ == '__main__': - try: - file_name = module.__file__ - except AttributeError: - pass - else: - base_name = os.path.basename(file_name) - split_name = os.path.splitext(base_name) - if len(split_name) == 1: - return six.text_type(base_name) - else: - return u'.'.join(split_name[:-1]) - - return six.text_type(module.__name__) + return six.text_type(module.package) + except AttributeError: + if module.__name__ == '__main__': + try: + file_name = module.__file__ + except AttributeError: + pass + else: + base_name = os.path.basename(file_name) + split_name = os.path.splitext(base_name) + if len(split_name) == 1: + return six.text_type(base_name) + else: + return u'.'.join(split_name[:-1]) + + return six.text_type(module.__name__) def total_seconds(offset): - """Backport of offset.total_seconds() from python 2.7+.""" - seconds = offset.days * 24 * 60 * 60 + offset.seconds - microseconds = seconds * 10**6 + offset.microseconds - return microseconds / (10**6 * 1.0) + """Backport of offset.total_seconds() from python 2.7+.""" + seconds = offset.days * 24 * 60 * 60 + offset.seconds + microseconds = seconds * 10**6 + offset.microseconds + return microseconds / (10**6 * 1.0) class TimeZoneOffset(datetime.tzinfo): - """Time zone information as encoded/decoded for DateTimeFields.""" + """Time zone information as encoded/decoded for DateTimeFields.""" - def __init__(self, offset): - """Initialize a time zone offset. + def __init__(self, offset): + """Initialize a time zone offset. - Args: - offset: Integer or timedelta time zone offset, in minutes from UTC. This - can be negative. - """ - super(TimeZoneOffset, self).__init__() - if isinstance(offset, datetime.timedelta): - offset = total_seconds(offset) / 60 - self.__offset = offset + Args: + offset: Integer or timedelta time zone offset, in minutes from UTC. This + can be negative. + """ + super(TimeZoneOffset, self).__init__() + if isinstance(offset, datetime.timedelta): + offset = total_seconds(offset) / 60 + self.__offset = offset - def utcoffset(self, dt): - """Get the a timedelta with the time zone's offset from UTC. + def utcoffset(self, dt): + """Get the a timedelta with the time zone's offset from UTC. - Returns: - The time zone offset from UTC, as a timedelta. - """ - return datetime.timedelta(minutes=self.__offset) + Returns: + The time zone offset from UTC, as a timedelta. + """ + return datetime.timedelta(minutes=self.__offset) - def dst(self, dt): - """Get the daylight savings time offset. + def dst(self, dt): + """Get the daylight savings time offset. - The formats that ProtoRPC uses to encode/decode time zone information don't - contain any information about daylight savings time. So this always - returns a timedelta of 0. + The formats that ProtoRPC uses to encode/decode time zone information don't + contain any information about daylight savings time. So this always + returns a timedelta of 0. - Returns: - A timedelta of 0. - """ - return datetime.timedelta(0) + Returns: + A timedelta of 0. + """ + return datetime.timedelta(0) def decode_datetime(encoded_datetime): - """Decode a DateTimeField parameter from a string to a python datetime. - - Args: - encoded_datetime: A string in RFC 3339 format. - - Returns: - A datetime object with the date and time specified in encoded_datetime. - - Raises: - ValueError: If the string is not in a recognized format. - """ - # Check if the string includes a time zone offset. Break out the - # part that doesn't include time zone info. Convert to uppercase - # because all our comparisons should be case-insensitive. - time_zone_match = _TIME_ZONE_RE.search(encoded_datetime) - if time_zone_match: - time_string = encoded_datetime[:time_zone_match.start(1)].upper() - else: - time_string = encoded_datetime.upper() - - if '.' in time_string: - format_string = '%Y-%m-%dT%H:%M:%S.%f' - else: - format_string = '%Y-%m-%dT%H:%M:%S' - - decoded_datetime = datetime.datetime.strptime(time_string, format_string) - - if not time_zone_match: - return decoded_datetime - - # Time zone info was included in the parameter. Add a tzinfo - # object to the datetime. Datetimes can't be changed after they're - # created, so we'll need to create a new one. - if time_zone_match.group('z'): - offset_minutes = 0 - else: - sign = time_zone_match.group('sign') - hours, minutes = [int(value) for value in - time_zone_match.group('hours', 'minutes')] - offset_minutes = hours * 60 + minutes - if sign == '-': - offset_minutes *= -1 - - return datetime.datetime(decoded_datetime.year, - decoded_datetime.month, - decoded_datetime.day, - decoded_datetime.hour, - decoded_datetime.minute, - decoded_datetime.second, - decoded_datetime.microsecond, - TimeZoneOffset(offset_minutes)) + """Decode a DateTimeField parameter from a string to a python datetime. + + Args: + encoded_datetime: A string in RFC 3339 format. + + Returns: + A datetime object with the date and time specified in encoded_datetime. + + Raises: + ValueError: If the string is not in a recognized format. + """ + # Check if the string includes a time zone offset. Break out the + # part that doesn't include time zone info. Convert to uppercase + # because all our comparisons should be case-insensitive. + time_zone_match = _TIME_ZONE_RE.search(encoded_datetime) + if time_zone_match: + time_string = encoded_datetime[:time_zone_match.start(1)].upper() + else: + time_string = encoded_datetime.upper() + + if '.' in time_string: + format_string = '%Y-%m-%dT%H:%M:%S.%f' + else: + format_string = '%Y-%m-%dT%H:%M:%S' + + decoded_datetime = datetime.datetime.strptime(time_string, format_string) + + if not time_zone_match: + return decoded_datetime + + # Time zone info was included in the parameter. Add a tzinfo + # object to the datetime. Datetimes can't be changed after they're + # created, so we'll need to create a new one. + if time_zone_match.group('z'): + offset_minutes = 0 + else: + sign = time_zone_match.group('sign') + hours, minutes = [int(value) for value in + time_zone_match.group('hours', 'minutes')] + offset_minutes = hours * 60 + minutes + if sign == '-': + offset_minutes *= -1 + + return datetime.datetime(decoded_datetime.year, + decoded_datetime.month, + decoded_datetime.day, + decoded_datetime.hour, + decoded_datetime.minute, + decoded_datetime.second, + decoded_datetime.microsecond, + TimeZoneOffset(offset_minutes)) diff --git a/apitools/base/protorpclite/util_test.py b/apitools/base/protorpclite/util_test.py index a61a94e..b7eb0e4 100644 --- a/apitools/base/protorpclite/util_test.py +++ b/apitools/base/protorpclite/util_test.py @@ -34,199 +34,206 @@ from apitools.base.protorpclite import util class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): - MODULE = util + MODULE = util class UtilTest(test_util.TestCase): - def testDecoratedFunction_LengthZero(self): - @util.positional(0) - def fn(kwonly=1): - return [kwonly] - self.assertEquals([1], fn()) - self.assertEquals([2], fn(kwonly=2)) - self.assertRaisesWithRegexpMatch(TypeError, - r'fn\(\) takes at most 0 positional ' - r'arguments \(1 given\)', - fn, 1) - - def testDecoratedFunction_LengthOne(self): - @util.positional(1) - def fn(pos, kwonly=1): - return [pos, kwonly] - self.assertEquals([1, 1], fn(1)) - self.assertEquals([2, 2], fn(2, kwonly=2)) - self.assertRaisesWithRegexpMatch(TypeError, - r'fn\(\) takes at most 1 positional ' - r'argument \(2 given\)', - fn, 2, 3) - - def testDecoratedFunction_LengthTwoWithDefault(self): - @util.positional(2) - def fn(pos1, pos2=1, kwonly=1): - return [pos1, pos2, kwonly] - self.assertEquals([1, 1, 1], fn(1)) - self.assertEquals([2, 2, 1], fn(2, 2)) - self.assertEquals([2, 3, 4], fn(2, 3, kwonly=4)) - self.assertRaisesWithRegexpMatch(TypeError, - r'fn\(\) takes at most 2 positional ' - r'arguments \(3 given\)', - fn, 2, 3, 4) - - def testDecoratedMethod(self): - class MyClass(object): - @util.positional(2) - def meth(self, pos1, kwonly=1): - return [pos1, kwonly] - self.assertEquals([1, 1], MyClass().meth(1)) - self.assertEquals([2, 2], MyClass().meth(2, kwonly=2)) - self.assertRaisesWithRegexpMatch(TypeError, - r'meth\(\) takes at most 2 positional ' - r'arguments \(3 given\)', - MyClass().meth, 2, 3) - - def testDefaultDecoration(self): - @util.positional - def fn(a, b, c=None): - return a, b, c - self.assertEquals((1, 2, 3), fn(1, 2, c=3)) - self.assertEquals((3, 4, None), fn(3, b=4)) - self.assertRaisesWithRegexpMatch(TypeError, - r'fn\(\) takes at most 2 positional ' - r'arguments \(3 given\)', - fn, 2, 3, 4) - - def testDefaultDecorationNoKwdsFails(self): - def fn(a): - return a - self.assertRaisesRegexp( - ValueError, - 'Functions with no keyword arguments must specify max_positional_args', - util.positional, fn) - - def testDecoratedFunctionDocstring(self): - @util.positional(0) - def fn(kwonly=1): - """fn docstring.""" - return [kwonly] - self.assertEquals('fn docstring.', fn.__doc__) + def testDecoratedFunction_LengthZero(self): + @util.positional(0) + def fn(kwonly=1): + return [kwonly] + self.assertEquals([1], fn()) + self.assertEquals([2], fn(kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 0 positional ' + r'arguments \(1 given\)', + fn, 1) + + def testDecoratedFunction_LengthOne(self): + @util.positional(1) + def fn(pos, kwonly=1): + return [pos, kwonly] + self.assertEquals([1, 1], fn(1)) + self.assertEquals([2, 2], fn(2, kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 1 positional ' + r'argument \(2 given\)', + fn, 2, 3) + + def testDecoratedFunction_LengthTwoWithDefault(self): + @util.positional(2) + def fn(pos1, pos2=1, kwonly=1): + return [pos1, pos2, kwonly] + self.assertEquals([1, 1, 1], fn(1)) + self.assertEquals([2, 2, 1], fn(2, 2)) + self.assertEquals([2, 3, 4], fn(2, 3, kwonly=4)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + fn, 2, 3, 4) + + def testDecoratedMethod(self): + class MyClass(object): + + @util.positional(2) + def meth(self, pos1, kwonly=1): + return [pos1, kwonly] + self.assertEquals([1, 1], MyClass().meth(1)) + self.assertEquals([2, 2], MyClass().meth(2, kwonly=2)) + self.assertRaisesWithRegexpMatch(TypeError, + r'meth\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + MyClass().meth, 2, 3) + + def testDefaultDecoration(self): + @util.positional + def fn(a, b, c=None): + return a, b, c + self.assertEquals((1, 2, 3), fn(1, 2, c=3)) + self.assertEquals((3, 4, None), fn(3, b=4)) + self.assertRaisesWithRegexpMatch(TypeError, + r'fn\(\) takes at most 2 positional ' + r'arguments \(3 given\)', + fn, 2, 3, 4) + + def testDefaultDecorationNoKwdsFails(self): + def fn(a): + return a + self.assertRaisesRegexp( + ValueError, + 'Functions with no keyword arguments must specify max_positional_args', + util.positional, fn) + + def testDecoratedFunctionDocstring(self): + @util.positional(0) + def fn(kwonly=1): + """fn docstring.""" + return [kwonly] + self.assertEquals('fn docstring.', fn.__doc__) class GetPackageForModuleTest(test_util.TestCase): - def setUp(self): - self.original_modules = dict(sys.modules) - - def tearDown(self): - sys.modules.clear() - sys.modules.update(self.original_modules) - - def CreateModule(self, name, file_name=None): - if file_name is None: - file_name = '%s.py' % name - module = types.ModuleType(name) - sys.modules[name] = module - return module - - def assertPackageEquals(self, expected, actual): - self.assertEquals(expected, actual) - if actual is not None: - self.assertTrue(isinstance(actual, six.text_type)) - - def testByString(self): - module = self.CreateModule('service_module') - module.package = 'my_package' - self.assertPackageEquals('my_package', - util.get_package_for_module('service_module')) - - def testModuleNameNotInSys(self): - self.assertPackageEquals(None, - util.get_package_for_module('service_module')) - - def testHasPackage(self): - module = self.CreateModule('service_module') - module.package = 'my_package' - self.assertPackageEquals('my_package', util.get_package_for_module(module)) - - def testHasModuleName(self): - module = self.CreateModule('service_module') - self.assertPackageEquals('service_module', - util.get_package_for_module(module)) - - def testIsMain(self): - module = self.CreateModule('__main__') - module.__file__ = '/bing/blam/bloom/blarm/my_file.py' - self.assertPackageEquals('my_file', util.get_package_for_module(module)) - - def testIsMainCompiled(self): - module = self.CreateModule('__main__') - module.__file__ = '/bing/blam/bloom/blarm/my_file.pyc' - self.assertPackageEquals('my_file', util.get_package_for_module(module)) - - def testNoExtension(self): - module = self.CreateModule('__main__') - module.__file__ = '/bing/blam/bloom/blarm/my_file' - self.assertPackageEquals('my_file', util.get_package_for_module(module)) - - def testNoPackageAtAll(self): - module = self.CreateModule('__main__') - self.assertPackageEquals('__main__', util.get_package_for_module(module)) + def setUp(self): + self.original_modules = dict(sys.modules) + + def tearDown(self): + sys.modules.clear() + sys.modules.update(self.original_modules) + + def CreateModule(self, name, file_name=None): + if file_name is None: + file_name = '%s.py' % name + module = types.ModuleType(name) + sys.modules[name] = module + return module + + def assertPackageEquals(self, expected, actual): + self.assertEquals(expected, actual) + if actual is not None: + self.assertTrue(isinstance(actual, six.text_type)) + + def testByString(self): + module = self.CreateModule('service_module') + module.package = 'my_package' + self.assertPackageEquals('my_package', + util.get_package_for_module('service_module')) + + def testModuleNameNotInSys(self): + self.assertPackageEquals(None, + util.get_package_for_module('service_module')) + + def testHasPackage(self): + module = self.CreateModule('service_module') + module.package = 'my_package' + self.assertPackageEquals( + 'my_package', util.get_package_for_module(module)) + + def testHasModuleName(self): + module = self.CreateModule('service_module') + self.assertPackageEquals('service_module', + util.get_package_for_module(module)) + + def testIsMain(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file.py' + self.assertPackageEquals( + 'my_file', util.get_package_for_module(module)) + + def testIsMainCompiled(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file.pyc' + self.assertPackageEquals( + 'my_file', util.get_package_for_module(module)) + + def testNoExtension(self): + module = self.CreateModule('__main__') + module.__file__ = '/bing/blam/bloom/blarm/my_file' + self.assertPackageEquals( + 'my_file', util.get_package_for_module(module)) + + def testNoPackageAtAll(self): + module = self.CreateModule('__main__') + self.assertPackageEquals( + '__main__', util.get_package_for_module(module)) class DateTimeTests(test_util.TestCase): - def testDecodeDateTime(self): - """Test that a RFC 3339 datetime string is decoded properly.""" - for datetime_string, datetime_vals in ( - ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), - ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): - decoded = util.decode_datetime(datetime_string) - expected = datetime.datetime(*datetime_vals) - self.assertEquals(expected, decoded) - - def testDateTimeTimeZones(self): - """Test that a datetime string with a timezone is decoded correctly.""" - for datetime_string, datetime_vals in ( - ('2012-09-30T15:31:50.262-06:00', - (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(-360))), - ('2012-09-30T15:31:50.262+01:30', - (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(90))), - ('2012-09-30T15:31:50+00:05', - (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(5))), - ('2012-09-30T15:31:50+00:00', - (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), - ('2012-09-30t15:31:50-00:00', - (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), - ('2012-09-30t15:31:50z', - (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), - ('2012-09-30T15:31:50-23:00', - (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(-1380)))): - decoded = util.decode_datetime(datetime_string) - expected = datetime.datetime(*datetime_vals) - self.assertEquals(expected, decoded) - - def testDecodeDateTimeInvalid(self): - """Test that decoding malformed datetime strings raises execptions.""" - for datetime_string in ('invalid', - '2012-09-30T15:31:50.', - '-08:00 2012-09-30T15:31:50.262', - '2012-09-30T15:31', - '2012-09-30T15:31Z', - '2012-09-30T15:31:50ZZ', - '2012-09-30T15:31:50.262 blah blah -08:00', - '1000-99-99T25:99:99.999-99:99'): - self.assertRaises(ValueError, util.decode_datetime, datetime_string) - - def testTimeZoneOffsetDelta(self): - """Test that delta works with TimeZoneOffset.""" - time_zone = util.TimeZoneOffset(datetime.timedelta(minutes=3)) - epoch = time_zone.utcoffset(datetime.datetime.utcfromtimestamp(0)) - self.assertEqual(180, util.total_seconds(epoch)) + def testDecodeDateTime(self): + """Test that a RFC 3339 datetime string is decoded properly.""" + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262', (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + decoded = util.decode_datetime(datetime_string) + expected = datetime.datetime(*datetime_vals) + self.assertEquals(expected, decoded) + + def testDateTimeTimeZones(self): + """Test that a datetime string with a timezone is decoded correctly.""" + for datetime_string, datetime_vals in ( + ('2012-09-30T15:31:50.262-06:00', + (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(-360))), + ('2012-09-30T15:31:50.262+01:30', + (2012, 9, 30, 15, 31, 50, 262000, util.TimeZoneOffset(90))), + ('2012-09-30T15:31:50+00:05', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(5))), + ('2012-09-30T15:31:50+00:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30t15:31:50-00:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30t15:31:50z', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(0))), + ('2012-09-30T15:31:50-23:00', + (2012, 9, 30, 15, 31, 50, 0, util.TimeZoneOffset(-1380)))): + decoded = util.decode_datetime(datetime_string) + expected = datetime.datetime(*datetime_vals) + self.assertEquals(expected, decoded) + + def testDecodeDateTimeInvalid(self): + """Test that decoding malformed datetime strings raises execptions.""" + for datetime_string in ('invalid', + '2012-09-30T15:31:50.', + '-08:00 2012-09-30T15:31:50.262', + '2012-09-30T15:31', + '2012-09-30T15:31Z', + '2012-09-30T15:31:50ZZ', + '2012-09-30T15:31:50.262 blah blah -08:00', + '1000-99-99T25:99:99.999-99:99'): + self.assertRaises( + ValueError, util.decode_datetime, datetime_string) + + def testTimeZoneOffsetDelta(self): + """Test that delta works with TimeZoneOffset.""" + time_zone = util.TimeZoneOffset(datetime.timedelta(minutes=3)) + epoch = time_zone.utcoffset(datetime.datetime.utcfromtimestamp(0)) + self.assertEqual(180, util.total_seconds(epoch)) def main(): - unittest.main() + unittest.main() if __name__ == '__main__': - main() + main() -- GitLab From 976dcb134a4e21b922ee266dbaf13bc58f02b6e8 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 13:49:53 -0700 Subject: [PATCH 172/295] Delint the newly-imported protorpclite code. --- apitools/base/protorpclite/descriptor.py | 78 +-- apitools/base/protorpclite/descriptor_test.py | 28 +- apitools/base/protorpclite/message_types.py | 6 +- .../base/protorpclite/message_types_test.py | 21 +- apitools/base/protorpclite/messages.py | 447 ++++++++++-------- apitools/base/protorpclite/messages_test.py | 215 +++++---- apitools/base/protorpclite/protojson.py | 49 +- apitools/base/protorpclite/protojson_test.py | 73 +-- apitools/base/protorpclite/test_util.py | 113 ++--- apitools/base/protorpclite/util.py | 47 +- apitools/base/protorpclite/util_test.py | 24 +- apitools/gen/message_registry.py | 3 +- default.pylintrc | 2 + 13 files changed, 583 insertions(+), 523 deletions(-) diff --git a/apitools/base/protorpclite/descriptor.py b/apitools/base/protorpclite/descriptor.py index de7a17f..add0e4c 100644 --- a/apitools/base/protorpclite/descriptor.py +++ b/apitools/base/protorpclite/descriptor.py @@ -27,15 +27,15 @@ to know the description of an enum value, enum, field or message without needing to download the source code. This format is also compatible with other, non-Python languages. -The descriptors are modeled to be binary compatible with: - - http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto +The descriptors are modeled to be binary compatible with + https://github.com/google/protobuf NOTE: The names of types and fields are not always the same between these descriptors and the ones defined in descriptor.proto. This was done in order to make source code files that use these descriptors easier to read. For example, it is not necessary to prefix TYPE to all the values in -FieldDescriptor.Variant as is done in descriptor.proto FieldDescriptorProto.Type. +FieldDescriptor.Variant as is done in descriptor.proto +FieldDescriptorProto.Type. Example: @@ -93,34 +93,33 @@ Public Functions: describe_file_set: Describe a file set from a list of modules or objects. describe_message: Describe a Message definition. """ -import six - -__author__ = 'rafek@google.com (Rafe Kaplan)' - import codecs import types +import six + from apitools.base.protorpclite import messages from apitools.base.protorpclite import util -__all__ = ['EnumDescriptor', - 'EnumValueDescriptor', - 'FieldDescriptor', - 'MessageDescriptor', - 'FileDescriptor', - 'FileSet', - 'DescriptorLibrary', +__all__ = [ + 'EnumDescriptor', + 'EnumValueDescriptor', + 'FieldDescriptor', + 'MessageDescriptor', + 'FileDescriptor', + 'FileSet', + 'DescriptorLibrary', - 'describe_enum', - 'describe_enum_value', - 'describe_field', - 'describe_message', - 'describe_file', - 'describe_file_set', - 'describe', - 'import_descriptor_loader', - ] + 'describe_enum', + 'describe_enum_value', + 'describe_field', + 'describe_message', + 'describe_file', + 'describe_file_set', + 'describe', + 'import_descriptor_loader', +] # NOTE: MessageField is missing because message fields cannot have @@ -192,7 +191,7 @@ class FieldDescriptor(messages.Message): default_value: String representation of default value. """ - Variant = messages.Variant + Variant = messages.Variant # pylint:disable=invalid-name class Label(messages.Enum): """Field label.""" @@ -209,9 +208,11 @@ class FieldDescriptor(messages.Message): variant = messages.EnumField(Variant, 5) type_name = messages.StringField(6) - # For numeric types, contains the original text representation of the value. + # For numeric types, contains the original text representation of + # the value. # For booleans, "true" or "false". - # For strings, contains the default text contents (not escaped in any way). + # For strings, contains the default text contents (not escaped in any + # way). # For bytes, contains the C escaped value. All bytes < 128 are that are # traditionally considered unprintable are also escaped. default_value = messages.StringField(7) @@ -231,7 +232,8 @@ class MessageDescriptor(messages.Message): fields = messages.MessageField(FieldDescriptor, 2, repeated=True) message_types = messages.MessageField( - 'apitools.base.protorpclite.descriptor.MessageDescriptor', 3, repeated=True) + 'apitools.base.protorpclite.descriptor.MessageDescriptor', 3, + repeated=True) enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True) @@ -318,7 +320,8 @@ def describe_field(field_definition): field_descriptor.type_name = field_definition.type.definition_name() if isinstance(field_definition, messages.MessageField): - field_descriptor.type_name = field_definition.message_type.definition_name() + field_descriptor.type_name = ( + field_definition.message_type.definition_name()) if field_definition.default is not None: field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[ @@ -389,11 +392,6 @@ def describe_file(module): Returns: Initialized FileDescriptor instance describing the module. """ - # May not import remote at top of file because remote depends on this - # file - # TODO(rafek): Straighten out this dependency. Possibly move these functions - # from descriptor to their own module. - descriptor = FileDescriptor() descriptor.package = util.get_package_for_module(module) @@ -510,8 +508,9 @@ def import_descriptor_loader(definition_name, importer=__import__): return describe(messages.find_definition(definition_name, importer=__import__)) except messages.DefinitionNotFoundError as err: - # There are things that find_definition will not find, but if the parent - # is loaded, its children can be searched for a match. + # There are things that find_definition will not find, but if + # the parent is loaded, its children can be searched for a + # match. split_name = definition_name.rsplit('.', 1) if len(split_name) > 1: parent, child = split_name @@ -597,12 +596,13 @@ class DescriptorLibrary(object): def lookup_package(self, definition_name): """Determines the package name for any definition. - Determine the package that any definition name belongs to. May check - parent for package name and will resolve missing descriptors if provided - descriptor loader. + Determine the package that any definition name belongs to. May + check parent for package name and will resolve missing + descriptors if provided descriptor loader. Args: definition_name: Definition name to find package for. + """ while True: descriptor = self.lookup_descriptor(definition_name) diff --git a/apitools/base/protorpclite/descriptor_test.py b/apitools/base/protorpclite/descriptor_test.py index c9943b8..e6b3055 100644 --- a/apitools/base/protorpclite/descriptor_test.py +++ b/apitools/base/protorpclite/descriptor_test.py @@ -16,10 +16,6 @@ # """Tests for apitools.base.protorpclite.descriptor.""" - -__author__ = 'rafek@google.com (Rafe Kaplan)' - - import types import unittest @@ -131,7 +127,7 @@ class DescribeFieldTest(test_util.TestCase): self.assertEquals(expected, described) def testDefault(self): - for field_class, default, expected_default in ( + test_cases = ( (messages.IntegerField, 200, '200'), (messages.FloatField, 1.5, '1.5'), (messages.FloatField, 1e6, '1000000.0'), @@ -141,7 +137,8 @@ class DescribeFieldTest(test_util.TestCase): b''.join([six.int2byte(x) for x in (31, 32, 33)]), b'\\x1f !'), (messages.StringField, RUSSIA, RUSSIA), - ): + ) + for field_class, default, expected_default in test_cases: field = field_class(10, default=default) field.name = u'a_field' @@ -288,9 +285,10 @@ class DescribeFileTest(test_util.TestCase): """Test describing modules.""" def LoadModule(self, module_name, source): - result = {'__name__': module_name, - 'messages': messages, - } + result = { + '__name__': module_name, + 'messages': messages, + } exec(source, result) module = types.ModuleType(module_name) @@ -472,7 +470,8 @@ class ModuleFinderTest(test_util.TestCase): descriptor.describe_enum_value( test_util.OptionalMessage.SimpleEnum.VAL1), descriptor.import_descriptor_loader( - 'apitools.base.protorpclite.test_util.OptionalMessage.SimpleEnum.VAL1')) + 'apitools.base.protorpclite.test_util.' + 'OptionalMessage.SimpleEnum.VAL1')) class DescriptorLibraryTest(test_util.TestCase): @@ -488,12 +487,13 @@ class DescriptorLibraryTest(test_util.TestCase): def testLookupPackage(self): self.assertEquals('csv', self.library.lookup_package('csv')) - self.assertEquals('apitools.base.protorpclite', - self.library.lookup_package('apitools.base.protorpclite')) + self.assertEquals( + 'apitools.base.protorpclite', + self.library.lookup_package('apitools.base.protorpclite')) def testLookupNonPackages(self): - for name in ('', 'a', - 'apitools.base.protorpclite.descriptor.DescriptorLibrary'): + lib = 'apitools.base.protorpclite.descriptor.DescriptorLibrary' + for name in ('', 'a', lib): self.assertRaisesWithRegexpMatch( messages.DefinitionNotFoundError, 'Could not find definition for %s' % name, diff --git a/apitools/base/protorpclite/message_types.py b/apitools/base/protorpclite/message_types.py index fb55ddc..1bbac38 100644 --- a/apitools/base/protorpclite/message_types.py +++ b/apitools/base/protorpclite/message_types.py @@ -20,9 +20,6 @@ Includes new message and field types that are outside what is defined by the protocol buffers standard. """ - -__author__ = 'rafek@google.com (Rafe Kaplan)' - import datetime from apitools.base.protorpclite import messages @@ -81,7 +78,8 @@ class DateTimeField(messages.MessageField): """ message = super(DateTimeField, self).value_from_message(message) if message.time_zone_offset is None: - return datetime.datetime.utcfromtimestamp(message.milliseconds / 1000.0) + return datetime.datetime.utcfromtimestamp( + message.milliseconds / 1000.0) # Need to subtract the time zone offset, because when we call # datetime.fromtimestamp, it will add the time zone offset to the diff --git a/apitools/base/protorpclite/message_types_test.py b/apitools/base/protorpclite/message_types_test.py index 39f1f4a..8a5afdb 100644 --- a/apitools/base/protorpclite/message_types_test.py +++ b/apitools/base/protorpclite/message_types_test.py @@ -16,12 +16,7 @@ # """Tests for apitools.base.protorpclite.message_types.""" - -__author__ = 'rafek@google.com (Rafe Kaplan)' - - import datetime - import unittest from apitools.base.protorpclite import message_types @@ -42,8 +37,8 @@ class DateTimeFieldTest(test_util.TestCase): field = message_types.DateTimeField(1) message = field.value_to_message( datetime.datetime(2033, 2, 4, 11, 22, 10)) - self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000), - message) + self.assertEqual( + message_types.DateTimeMessage(milliseconds=1991128930000), message) def testValueToMessageBadValue(self): field = message_types.DateTimeField(1) @@ -57,9 +52,10 @@ class DateTimeFieldTest(test_util.TestCase): field = message_types.DateTimeField(1) message = field.value_to_message( datetime.datetime(2033, 2, 4, 11, 22, 10, tzinfo=time_zone)) - self.assertEqual(message_types.DateTimeMessage(milliseconds=1991128930000, - time_zone_offset=600), - message) + self.assertEqual( + message_types.DateTimeMessage(milliseconds=1991128930000, + time_zone_offset=600), + message) def testValueFromMessage(self): message = message_types.DateTimeMessage(milliseconds=1991128000000) @@ -81,8 +77,9 @@ class DateTimeFieldTest(test_util.TestCase): field = message_types.DateTimeField(1) timestamp = field.value_from_message(message) time_zone = util.TimeZoneOffset(60 * 5) - self.assertEqual(datetime.datetime(2033, 2, 4, 11, 6, 40, tzinfo=time_zone), - timestamp) + self.assertEqual( + datetime.datetime(2033, 2, 4, 11, 6, 40, tzinfo=time_zone), + timestamp) if __name__ == '__main__': diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index 196a640..9880c0c 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -40,48 +40,49 @@ Public Exceptions (indentation indications class hierarchy): ValidationError: Raised when a message or field is not valid. DefinitionNotFoundError: Raised when definition not found. """ -import six - -__author__ = 'rafek@google.com (Rafe Kaplan)' - - import types import weakref +import six + from apitools.base.protorpclite import util -__all__ = ['MAX_ENUM_VALUE', - 'MAX_FIELD_NUMBER', - 'FIRST_RESERVED_FIELD_NUMBER', - 'LAST_RESERVED_FIELD_NUMBER', - - 'Enum', - 'Field', - 'FieldList', - 'Variant', - 'Message', - 'IntegerField', - 'FloatField', - 'BooleanField', - 'BytesField', - 'StringField', - 'MessageField', - 'EnumField', - 'find_definition', - - 'Error', - 'DecodeError', - 'EncodeError', - 'EnumDefinitionError', - 'FieldDefinitionError', - 'InvalidVariantError', - 'InvalidDefaultError', - 'InvalidNumberError', - 'MessageDefinitionError', - 'DuplicateNumberError', - 'ValidationError', - 'DefinitionNotFoundError', - ] +__all__ = [ + 'MAX_ENUM_VALUE', + 'MAX_FIELD_NUMBER', + 'FIRST_RESERVED_FIELD_NUMBER', + 'LAST_RESERVED_FIELD_NUMBER', + + 'Enum', + 'Field', + 'FieldList', + 'Variant', + 'Message', + 'IntegerField', + 'FloatField', + 'BooleanField', + 'BytesField', + 'StringField', + 'MessageField', + 'EnumField', + 'find_definition', + + 'Error', + 'DecodeError', + 'EncodeError', + 'EnumDefinitionError', + 'FieldDefinitionError', + 'InvalidVariantError', + 'InvalidDefaultError', + 'InvalidNumberError', + 'MessageDefinitionError', + 'DuplicateNumberError', + 'ValidationError', + 'DefinitionNotFoundError', +] + +# pylint:disable=attribute-defined-outside-init +# pylint:disable=protected-access # TODO(rafek): Add extended module test to ensure all exceptions @@ -134,13 +135,7 @@ class ValidationError(Error): def __str__(self): """Prints string with field name if present on exception.""" - message = Error.__str__(self) - try: - field_name = self.field_name - except AttributeError: - return message - else: - return message + return Error.__str__(self) # Attributes that are reserved by a class definition that @@ -182,7 +177,7 @@ class _DefinitionClass(type): may change only once. """ - __initialized = False + __initialized = False # pylint:disable=invalid-name def __init__(cls, name, bases, dct): """Constructor.""" @@ -204,16 +199,18 @@ class _DefinitionClass(type): return None def __setattr__(cls, name, value): - """Overridden so that cannot set variables on definition classes after init. + """Overridden to avoid setting variables after init. - Setting attributes on a class must work during the period of initialization - to set the enumation value class variables and build the name/number maps. - Once __init__ has set the __initialized flag to True prohibits setting any - more values on the class. The class is in effect frozen. + Setting attributes on a class must work during the period of + initialization to set the enumation value class variables and + build the name/number maps. Once __init__ has set the + __initialized flag to True prohibits setting any more values + on the class. The class is in effect frozen. Args: name: Name of value to set. value: Value to set. + """ if cls.__initialized and name not in _POST_INIT_ATTRIBUTE_NAMES: raise AttributeError('May not change values: %s' % name) @@ -227,12 +224,14 @@ class _DefinitionClass(type): def definition_name(cls): """Helper method for creating definition name. - Names will be generated to include the classes package name, scope (if the - class is nested in another definition) and class name. + Names will be generated to include the classes package name, + scope (if the class is nested in another definition) and class + name. - By default, the package name for a definition is derived from its module - name. However, this value can be overriden by placing a 'package' attribute - in the module that contains the definition class. For example: + By default, the package name for a definition is derived from + its module name. However, this value can be overriden by + placing a 'package' attribute in the module that contains the + definition class. For example: package = 'some.alternate.package' @@ -244,6 +243,7 @@ class _DefinitionClass(type): Returns: Dot-separated fully qualified name of definition. + """ outer_definition_name = cls.outer_definition_name() if outer_definition_name is None: @@ -255,8 +255,9 @@ class _DefinitionClass(type): """Helper method for creating outer definition name. Returns: - If definition is nested, will return the outer definitions name, else the - package name. + If definition is nested, will return the outer definitions + name, else the package name. + """ outer_definition = cls.message_definition() if not outer_definition: @@ -298,8 +299,8 @@ class _EnumClass(_DefinitionClass): def __init__(cls, name, bases, dct): # Can only define one level of sub-classes below Enum. if not (bases == (object,) or bases == (Enum,)): - raise EnumDefinitionError('Enum type %s may only inherit from Enum' % - (name,)) + raise EnumDefinitionError( + 'Enum type %s may only inherit from Enum' % name) cls.__by_number = {} cls.__by_name = {} @@ -316,7 +317,8 @@ class _EnumClass(_DefinitionClass): # Reject anything that is not an int. if not isinstance(value, six.integer_types): raise EnumDefinitionError( - 'May only use integers in Enum definitions. Found: %s = %s' % + 'May only use integers in Enum definitions. ' + 'Found: %s = %s' % (attribute, value)) # Protocol buffer standard recommends non-negative values. @@ -328,7 +330,8 @@ class _EnumClass(_DefinitionClass): if value > MAX_ENUM_VALUE: raise EnumDefinitionError( - 'Must use enum values less than or equal %d. Found: %s = %d' % + 'Must use enum values less than or equal %d. ' + 'Found: %s = %d' % (MAX_ENUM_VALUE, attribute, value)) if value in cls.__by_number: @@ -338,6 +341,7 @@ class _EnumClass(_DefinitionClass): # Create enum instance and list in new Enum type. instance = object.__new__(cls) + # pylint:disable=non-parent-init-called cls.__init__(instance, attribute, value) cls.__by_name[instance.name] = instance cls.__by_number[instance.number] = instance @@ -466,8 +470,9 @@ class Enum(six.with_metaclass(_EnumClass, object)): """Enable pickling. Returns: - A 2-tuple containing the class and __new__ args to be used for restoring - a pickled instance. + A 2-tuple containing the class and __new__ args to be used + for restoring a pickled instance. + """ return self.__class__, (self.number,) @@ -540,9 +545,7 @@ class Enum(six.with_metaclass(_EnumClass, object)): # TODO(rafek): Determine to what degree this enumeration should be compatible -# with FieldDescriptor.Type in: -# -# http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto +# with FieldDescriptor.Type in https://github.com/google/protobuf. class Variant(Enum): """Wire format variant. @@ -588,21 +591,23 @@ class _MessageClass(_DefinitionClass): Information contained there may help understanding this class. Meta-class enables very specific behavior for any defined Message - class. All attributes defined on an Message sub-class must be field - instances, Enum class definitions or other Message class definitions. Each - field attribute defined on an Message sub-class is added to the set of - field definitions and the attribute is translated in to a slot. It also - ensures that only one level of Message class hierarchy is possible. In other - words it is not possible to declare sub-classes of sub-classes of - Message. + class. All attributes defined on an Message sub-class must be + field instances, Enum class definitions or other Message class + definitions. Each field attribute defined on an Message sub-class + is added to the set of field definitions and the attribute is + translated in to a slot. It also ensures that only one level of + Message class hierarchy is possible. In other words it is not + possible to declare sub-classes of sub-classes of Message. This class also defines some functions in order to restrict the - behavior of the Message class and its sub-classes. It is not possible - to change the behavior of the Message class in later classes since - any new classes may be defined with only field, Enums and Messages, and - no methods. + behavior of the Message class and its sub-classes. It is not + possible to change the behavior of the Message class in later + classes since any new classes may be defined with only field, + Enums and Messages, and no methods. + """ + # pylint:disable=bad-mcs-classmethod-argument def __new__(cls, name, bases, dct): """Create new Message class instance. @@ -612,7 +617,7 @@ class _MessageClass(_DefinitionClass): by_number = {} by_name = {} - variant_map = {} + variant_map = {} # pylint:disable=unused-variable if bases != (object,): # Can only define one level of sub-classes below Message. @@ -640,9 +645,11 @@ class _MessageClass(_DefinitionClass): continue # Reject anything that is not a field. + # pylint:disable=unidiomatic-typecheck if type(field) is Field or not isinstance(field, Field): raise MessageDefinitionError( - 'May only use fields in message definitions. Found: %s = %s' % + 'May only use fields in message definitions. ' + 'Found: %s = %s' % (key, field)) if field.number in by_number: @@ -672,9 +679,9 @@ class _MessageClass(_DefinitionClass): def __init__(cls, name, bases, dct): """Initializer required to assign references to new class.""" if bases != (object,): - for value in dct.values(): - if isinstance(value, _DefinitionClass) and not value is Message: - value._message_definition = weakref.ref(cls) + for v in dct.values(): + if isinstance(v, _DefinitionClass) and v is not Message: + v._message_definition = weakref.ref(cls) for field in cls.all_fields(): field._message_definition = weakref.ref(cls) @@ -689,13 +696,14 @@ class Message(six.with_metaclass(_MessageClass, object)): process space. Messages are defined using the field classes (IntegerField, FloatField, EnumField, etc.). - Messages are more restricted than normal classes in that they may only - contain field attributes and other Message and Enum definitions. These - restrictions are in place because the structure of the Message class is - intentended to itself be transmitted across network or process space and - used directly by clients or even other servers. As such methods and - non-field attributes could not be transmitted with the structural information - causing discrepancies between different languages and implementations. + Messages are more restricted than normal classes in that they may + only contain field attributes and other Message and Enum + definitions. These restrictions are in place because the structure + of the Message class is intentended to itself be transmitted + across network or process space and used directly by clients or + even other servers. As such methods and non-field attributes could + not be transmitted with the structural information causing + discrepancies between different languages and implementations. Initialization and validation: @@ -747,14 +755,15 @@ class Message(six.with_metaclass(_MessageClass, object)): # Now object is initialized! order.check_initialized() + """ def __init__(self, **kwargs): """Initialize internal messages state. Args: - A message can be initialized via the constructor by passing in keyword - arguments corresponding to fields. For example: + A message can be initialized via the constructor by passing + in keyword arguments corresponding to fields. For example: class Date(Message): day = IntegerField(1) @@ -771,6 +780,7 @@ class Message(six.with_metaclass(_MessageClass, object)): date.day = 6 date.month = 6 date.year = 1911 + """ # Tag being an essential implementation detail must be private. self.__tags = {} @@ -798,8 +808,9 @@ class Message(six.with_metaclass(_MessageClass, object)): value = getattr(self, name) if value is None: if field.required: - raise ValidationError("Message %s is missing required field %s" % - (type(self).__name__, name)) + raise ValidationError( + "Message %s is missing required field %s" % + (type(self).__name__, name)) else: try: if (isinstance(field, MessageField) and @@ -868,14 +879,15 @@ class Message(six.with_metaclass(_MessageClass, object)): def get_assigned_value(self, name): """Get the assigned value of an attribute. - Get the underlying value of an attribute. If value has not been set, will - not return the default for the field. + Get the underlying value of an attribute. If value has not + been set, will not return the default for the field. Args: name: Name of attribute to get. Returns: Value of attribute, None if it has not been set. + """ message_type = type(self) try: @@ -933,7 +945,8 @@ class Message(six.with_metaclass(_MessageClass, object)): Args: key: The name or number used to refer to this unknown value. value: The value of the field. - variant: Type information needed to interpret the value or re-encode it. + variant: Type information needed to interpret the value or re-encode + it. Raises: TypeError: If the variant is not an instance of messages.Variant. @@ -1049,9 +1062,11 @@ class Message(six.with_metaclass(_MessageClass, object)): class FieldList(list): """List implementation that validates field values. - This list implementation overrides all methods that add values in to a list - in order to validate those new elements. Attempting to add or set list - values that are not of the correct type will raise ValidationError. + This list implementation overrides all methods that add values in + to a list in order to validate those new elements. Attempting to + add or set list values that are not of the correct type will raise + ValidationError. + """ def __init__(self, field_instance, sequence): @@ -1071,15 +1086,17 @@ class FieldList(list): def __getstate__(self): """Enable pickling. - The assigned field instance can't be pickled if it belongs to a Message - definition (message_definition uses a weakref), so the Message class and - field number are returned in that case. + The assigned field instance can't be pickled if it belongs to + a Message definition (message_definition uses a weakref), so + the Message class and field number are returned in that case. Returns: A 3-tuple containing: - The field instance, or None if it belongs to a Message class. - The Message class that the field instance belongs to, or None. - - The field instance number of the Message class it belongs to, or None. + - The field instance number of the Message class it belongs to, or + None. + """ message_class = self.__field.message_definition() if message_class is None: @@ -1094,7 +1111,8 @@ class FieldList(list): state: A 3-tuple containing: - The field instance, or None if it belongs to a Message class. - The Message class that the field instance belongs to, or None. - - The field instance number of the Message class it belongs to, or None. + - The field instance number of the Message class it belongs to, or + None. """ field_instance, message_class, number = state if field_instance is None: @@ -1147,8 +1165,8 @@ class _FieldMeta(type): # TODO(rafek): Prevent additional field subclasses. class Field(six.with_metaclass(_FieldMeta, object)): - __initialized = False - __variant_to_type = {} + __initialized = False # pylint:disable=invalid-name + __variant_to_type = {} # pylint:disable=invalid-name # TODO(craigcitro): Remove this alias. # @@ -1165,13 +1183,14 @@ class Field(six.with_metaclass(_FieldMeta, object)): default=None): """Constructor. - The required and repeated parameters are mutually exclusive. Setting both - to True will raise a FieldDefinitionError. + The required and repeated parameters are mutually exclusive. + Setting both to True will raise a FieldDefinitionError. Sub-class Attributes: Each sub-class of Field must define the following: VARIANTS: Set of variant types accepted by that field. - DEFAULT_VARIANT: Default variant type if not specified in constructor. + DEFAULT_VARIANT: Default variant type if not specified in + constructor. Args: number: Number of field. Must be unique per message class. @@ -1185,14 +1204,16 @@ class Field(six.with_metaclass(_FieldMeta, object)): Raises: InvalidVariantError when invalid variant for field is provided. InvalidDefaultError when invalid default for field is provided. - FieldDefinitionError when invalid number provided or mutually exclusive - fields are used. + FieldDefinitionError when invalid number provided or mutually + exclusive fields are used. InvalidNumberError when the field number is out of range or reserved. + """ if not isinstance(number, int) or not 1 <= number <= MAX_FIELD_NUMBER: - raise InvalidNumberError('Invalid number for field: %s\n' - 'Number must be 1 or greater and %d or less' % - (number, MAX_FIELD_NUMBER)) + raise InvalidNumberError( + 'Invalid number for field: %s\n' + 'Number must be 1 or greater and %d or less' % + (number, MAX_FIELD_NUMBER)) if FIRST_RESERVED_FIELD_NUMBER <= number <= LAST_RESERVED_FIELD_NUMBER: raise InvalidNumberError('Tag number %d is a reserved number.\n' @@ -1227,11 +1248,13 @@ class Field(six.with_metaclass(_FieldMeta, object)): name = self.name except AttributeError: # For when raising error before name initialization. - raise InvalidDefaultError('Invalid default value for %s: %r: %s' % - (self.__class__.__name__, default, err)) + raise InvalidDefaultError( + 'Invalid default value for %s: %r: %s' % + (self.__class__.__name__, default, err)) else: - raise InvalidDefaultError('Invalid default value for field %s: ' - '%r: %s' % (name, default, err)) + raise InvalidDefaultError( + 'Invalid default value for field %s: ' + '%r: %s' % (name, default, err)) self.__default = default self.__initialized = True @@ -1311,14 +1334,15 @@ class Field(six.with_metaclass(_FieldMeta, object)): (self.type, self.__class__.__name__, value, type(value))) else: - raise ValidationError('Expected type %s for field %s, ' - 'found %s (type %s)' % - (self.type, name, value, type(value))) + raise ValidationError( + 'Expected type %s for field %s, found %s (type %s)' % + (self.type, name, value, type(value))) def __validate(self, value, validate_element): """Internal validation function. - Validate an internal value using a function to validate individual elements. + Validate an internal value using a function to validate + individual elements. Args: value: Value to validate. @@ -1326,6 +1350,7 @@ class Field(six.with_metaclass(_FieldMeta, object)): Raises: ValidationError if value is not expected type. + """ if not self.repeated: validate_element(value) @@ -1337,11 +1362,13 @@ class Field(six.with_metaclass(_FieldMeta, object)): try: name = self.name except AttributeError: - raise ValidationError('Repeated values for %s ' - 'may not be None' % self.__class__.__name__) + raise ValidationError( + 'Repeated values for %s ' + 'may not be None' % self.__class__.__name__) else: - raise ValidationError('Repeated values for field %s ' - 'may not be None' % name) + raise ValidationError( + 'Repeated values for field %s ' + 'may not be None' % name) validate_element(element) elif value is not None: try: @@ -1350,8 +1377,8 @@ class Field(six.with_metaclass(_FieldMeta, object)): raise ValidationError('%s is repeated. Found: %s' % ( self.__class__.__name__, value)) else: - raise ValidationError('Field %s is repeated. Found: %s' % (name, - value)) + raise ValidationError( + 'Field %s is repeated. Found: %s' % (name, value)) def validate(self, value): """Validate value assigned to field. @@ -1367,16 +1394,18 @@ class Field(six.with_metaclass(_FieldMeta, object)): def validate_default_element(self, value): """Validate value as assigned to field default field. - Some fields may allow for delayed resolution of default types necessary - in the case of circular definition references. In this case, the default - value might be a place holder that is resolved when needed after all the - message classes are defined. + Some fields may allow for delayed resolution of default types + necessary in the case of circular definition references. In + this case, the default value might be a place holder that is + resolved when needed after all the message classes are + defined. Args: value: Default value to validate. Raises: ValidationError if value is not expected type. + """ self.validate_element(value) @@ -1395,8 +1424,9 @@ class Field(six.with_metaclass(_FieldMeta, object)): """Get Message definition that contains this Field definition. Returns: - Containing Message definition for Field. Will return None if for - some reason Field is defined outside of a Message class. + Containing Message definition for Field. Will return None if + for some reason Field is defined outside of a Message class. + """ try: return self._message_definition() @@ -1416,13 +1446,14 @@ class Field(six.with_metaclass(_FieldMeta, object)): class IntegerField(Field): """Field definition for integer values.""" - VARIANTS = frozenset([Variant.INT32, - Variant.INT64, - Variant.UINT32, - Variant.UINT64, - Variant.SINT32, - Variant.SINT64, - ]) + VARIANTS = frozenset([ + Variant.INT32, + Variant.INT64, + Variant.UINT32, + Variant.UINT64, + Variant.SINT32, + Variant.SINT64, + ]) DEFAULT_VARIANT = Variant.INT64 @@ -1432,9 +1463,10 @@ class IntegerField(Field): class FloatField(Field): """Field definition for float values.""" - VARIANTS = frozenset([Variant.FLOAT, - Variant.DOUBLE, - ]) + VARIANTS = frozenset([ + Variant.FLOAT, + Variant.DOUBLE, + ]) DEFAULT_VARIANT = Variant.DOUBLE @@ -1482,16 +1514,15 @@ class StringField(Field): six.text_type(value, 'ascii') except UnicodeDecodeError as err: try: - name = self.name + _ = self.name except AttributeError: validation_error = ValidationError( 'Field encountered non-ASCII string %r: %s' % (value, err)) else: validation_error = ValidationError( - 'Field %s encountered non-ASCII string %r: %s' % (self.name, - value, - err)) + 'Field %s encountered non-ASCII string %r: %s' % ( + self.name, value, err)) validation_error.field_name = self.name raise validation_error else: @@ -1510,14 +1541,15 @@ class MessageField(Field): Normally message field are defined by passing the referenced message class in to the constructor. - It is possible to define a message field for a type that does not yet - exist by passing the name of the message in to the constructor instead - of a message class. Resolution of the actual type of the message is - deferred until it is needed, for example, during message verification. - Names provided to the constructor must refer to a class within the same - python module as the class that is using it. Names refer to messages - relative to the containing messages scope. For example, the two fields - of OuterMessage refer to the same message type: + It is possible to define a message field for a type that does not + yet exist by passing the name of the message in to the constructor + instead of a message class. Resolution of the actual type of the + message is deferred until it is needed, for example, during + message verification. Names provided to the constructor must refer + to a class within the same python module as the class that is + using it. Names refer to messages relative to the containing + messages scope. For example, the two fields of OuterMessage refer + to the same message type: class Outer(Message): @@ -1527,9 +1559,9 @@ class MessageField(Field): class Inner(Message): ... - When resolving an actual type, MessageField will traverse the entire - scope of nested messages to match a message name. This makes it easy - for siblings to reference siblings: + When resolving an actual type, MessageField will traverse the + entire scope of nested messages to match a message name. This + makes it easy for siblings to reference siblings: class Outer(Message): @@ -1539,6 +1571,7 @@ class MessageField(Field): class Sibling(Message): ... + """ VARIANTS = frozenset([Variant.MESSAGE]) @@ -1593,14 +1626,14 @@ class MessageField(Field): message_instance: Message instance to set value on. value: Value to set on message. """ - message_type = self.type - if isinstance(message_type, type) and issubclass(message_type, Message): + t = self.type + if isinstance(t, type) and issubclass(t, Message): if self.repeated: if value and isinstance(value, (list, tuple)): - value = [(message_type(**v) if isinstance(v, dict) else v) + value = [(t(**v) if isinstance(v, dict) else v) for v in value] elif isinstance(value, dict): - value = message_type(**value) + value = t(**value) super(MessageField, self).__set__(message_instance, value) @property @@ -1669,8 +1702,9 @@ class MessageField(Field): class EnumField(Field): """Field definition for enum values. - Enum fields may have default values that are delayed until the associated enum - type is resolved. This is necessary to support certain circular references. + Enum fields may have default values that are delayed until the + associated enum type is resolved. This is necessary to support + certain circular references. For example: @@ -1693,8 +1727,8 @@ class EnumField(Field): CAT = 2 HORSE = 3 - # This fields default value will be validated right away since Color is - # already fully resolved. + # This fields default value will be validated right away since Color + # is already fully resolved. color = EnumField(Message1.Color, 1, default='RED') """ @@ -1737,16 +1771,18 @@ class EnumField(Field): def validate_default_element(self, value): """Validate default element of Enum field. - Enum fields allow for delayed resolution of default values when the type - of the field has not been resolved. The default value of a field may be - a string or an integer. If the Enum type of the field has been resolved, - the default value is validated against that type. + Enum fields allow for delayed resolution of default values + when the type of the field has not been resolved. The default + value of a field may be a string or an integer. If the Enum + type of the field has been resolved, the default value is + validated against that type. Args: value: Value to validate. Raises: ValidationError if value is not expected message type. + """ if isinstance(value, (six.string_types, six.integer_types)): # Validation of the value does not happen for delayed resolution @@ -1782,7 +1818,9 @@ class EnumField(Field): return self.__resolved_default except AttributeError: resolved_default = super(EnumField, self).default - if isinstance(resolved_default, (six.string_types, six.integer_types)): + if isinstance(resolved_default, (six.string_types, + six.integer_types)): + # pylint:disable=not-callable resolved_default = self.type(resolved_default) self.__resolved_default = resolved_default return self.__resolved_default @@ -1792,19 +1830,20 @@ class EnumField(Field): def find_definition(name, relative_to=None, importer=__import__): """Find definition by name in module-space. - The find algorthm will look for definitions by name relative to a message - definition or by fully qualfied name. If no definition is found relative - to the relative_to parameter it will do the same search against the container - of relative_to. If relative_to is a nested Message, it will search its - message_definition(). If that message has no message_definition() it will - search its module. If relative_to is a module, it will attempt to look for - the containing module and search relative to it. If the module is a top-level - module, it will look for the a message using a fully qualified name. If - no message is found then, the search fails and DefinitionNotFoundError is - raised. - - For example, when looking for any definition 'foo.bar.ADefinition' relative to - an actual message definition abc.xyz.SomeMessage: + The find algorthm will look for definitions by name relative to a + message definition or by fully qualfied name. If no definition is + found relative to the relative_to parameter it will do the same + search against the container of relative_to. If relative_to is a + nested Message, it will search its message_definition(). If that + message has no message_definition() it will search its module. If + relative_to is a module, it will attempt to look for the + containing module and search relative to it. If the module is a + top-level module, it will look for the a message using a fully + qualified name. If no message is found then, the search fails and + DefinitionNotFoundError is raised. + + For example, when looking for any definition 'foo.bar.ADefinition' + relative to an actual message definition abc.xyz.SomeMessage: find_definition('foo.bar.ADefinition', SomeMessage) @@ -1819,9 +1858,9 @@ def find_definition(name, relative_to=None, importer=__import__): algorithm searches any Messages or sub-modules found in its path. Non-Message values are not searched. - A name that begins with '.' is considered to be a fully qualified name. The - name is always searched for from the topmost package. For example, assume - two message types: + A name that begins with '.' is considered to be a fully qualified + name. The name is always searched for from the topmost package. + For example, assume two message types: abc.xyz.SomeMessage xyz.SomeMessage @@ -1835,9 +1874,10 @@ def find_definition(name, relative_to=None, importer=__import__): http://code.google.com/apis/protocolbuffers/docs/proto.html#packages Args: - name: Name of definition to find. May be fully qualified or relative name. - relative_to: Search for definition relative to message definition or module. - None will cause a fully qualified name search. + name: Name of definition to find. May be fully qualified or relative + name. + relative_to: Search for definition relative to message definition or + module. None will cause a fully qualified name search. importer: Import function to use for resolving modules. Returns: @@ -1845,13 +1885,16 @@ def find_definition(name, relative_to=None, importer=__import__): Raises: DefinitionNotFoundError if no definition is found in any search path. + """ # Check parameters. if not (relative_to is None or isinstance(relative_to, types.ModuleType) or - isinstance(relative_to, type) and issubclass(relative_to, Message)): - raise TypeError('relative_to must be None, Message definition or module. ' - 'Found: %s' % relative_to) + isinstance(relative_to, type) and + issubclass(relative_to, Message)): + raise TypeError( + 'relative_to must be None, Message definition or module.' + ' Found: %s' % relative_to) name_path = name.split('.') @@ -1897,10 +1940,10 @@ def find_definition(name, relative_to=None, importer=__import__): else: return None - if (not isinstance(next, types.ModuleType) and - not (isinstance(next, type) and - issubclass(next, (Message, Enum)))): - return None + if not isinstance(next, types.ModuleType): + if not (isinstance(next, type) and + issubclass(next, (Message, Enum))): + return None return next @@ -1918,8 +1961,8 @@ def find_definition(name, relative_to=None, importer=__import__): # does this part of search if relative_to is None: # Fully qualified search was done. Nothing found. Fail. - raise DefinitionNotFoundError('Could not find definition for %s' - % (name,)) + raise DefinitionNotFoundError( + 'Could not find definition for %s' % name) else: if isinstance(relative_to, types.ModuleType): # Find parent module. @@ -1927,8 +1970,8 @@ def find_definition(name, relative_to=None, importer=__import__): if not module_path: relative_to = None else: - # Should not raise ImportError. If it does... weird and - # unexepected. Propagate. + # Should not raise ImportError. If it does... + # weird and unexpected. Propagate. relative_to = importer( '.'.join(module_path), '', '', [module_path[-1]]) elif (isinstance(relative_to, type) and diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index 31001ab..b802e1e 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -16,22 +16,28 @@ # """Tests for apitools.base.protorpclite.messages.""" -import six - -__author__ = 'rafek@google.com (Rafe Kaplan)' - - import pickle import re import sys import types import unittest +import six + from apitools.base.protorpclite import descriptor from apitools.base.protorpclite import message_types from apitools.base.protorpclite import messages from apitools.base.protorpclite import test_util +# This package plays lots of games with modifying global variables inside +# test cases. Hence: +# pylint:disable=function-redefined +# pylint:disable=global-variable-not-assigned +# pylint:disable=global-variable-undefined +# pylint:disable=redefined-outer-name +# pylint:disable=undefined-variable +# pylint:disable=unused-variable + class ModuleInterfaceTest(test_util.ModuleInterfaceTest, test_util.TestCase): @@ -57,10 +63,11 @@ class EnumTest(test_util.TestCase): def setUp(self): """Set up tests.""" - # Redefine Color class in case so that changes to it (an error) in one test - # does not affect other tests. - global Color + # Redefine Color class in case so that changes to it (an + # error) in one test does not affect other tests. + global Color # pylint:disable=global-variable-not-assigned + # pylint:disable=unused-variable class Color(messages.Enum): RED = 20 ORANGE = 2 @@ -73,7 +80,8 @@ class EnumTest(test_util.TestCase): def testNames(self): """Test that names iterates over enum names.""" self.assertEquals( - set(['BLUE', 'GREEN', 'INDIGO', 'ORANGE', 'RED', 'VIOLET', 'YELLOW']), + set(['BLUE', 'GREEN', 'INDIGO', 'ORANGE', 'RED', + 'VIOLET', 'YELLOW']), set(Color.names())) def testNumbers(self): @@ -289,13 +297,15 @@ class EnumTest(test_util.TestCase): self.assertEquals(module_name, MyMessage.NestedEnum.definition_package()) - self.assertEquals('%s.MyMessage.NestedMessage.NestedEnum' % module_name, - MyMessage.NestedMessage.NestedEnum.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage.NestedEnum' % module_name, + MyMessage.NestedMessage.NestedEnum.definition_name()) self.assertEquals( '%s.MyMessage.NestedMessage' % module_name, MyMessage.NestedMessage.NestedEnum.outer_definition_name()) - self.assertEquals(module_name, - MyMessage.NestedMessage.NestedEnum.definition_package()) + self.assertEquals( + module_name, + MyMessage.NestedMessage.NestedEnum.definition_package()) def testMessageDefinition(self): """Test that enumeration knows its enclosing message definition.""" @@ -397,7 +407,8 @@ class FieldListTest(test_util.TestCase): self.assertRaisesWithRegexpMatch( messages.ValidationError, - "IntegerField is repeated. Found: <(list[_]?|sequence)iterator object", + ("IntegerField is repeated. Found: " + "<(list[_]?|sequence)iterator object"), messages.FieldList, self.integer_field, iter([1, 2, 3])) def testSetSlice(self): @@ -489,7 +500,7 @@ class FieldListTest(test_util.TestCase): insert) def testPickle(self): - """Testing pickling and unpickling of disconnected FieldList instances.""" + """Testing pickling and unpickling of FieldList instances.""" field_list = messages.FieldList(self.integer_field, [1, 2, 3, 4, 5]) unpickled = pickle.loads(pickle.dumps(field_list)) self.assertEquals(field_list, unpickled) @@ -508,12 +519,12 @@ class FieldTest(test_util.TestCase): Args: action: Callable that takes the field class as a parameter. """ - for field_class in (messages.IntegerField, - messages.FloatField, - messages.BooleanField, - messages.BytesField, - messages.StringField, - ): + classes = (messages.IntegerField, + messages.FloatField, + messages.BooleanField, + messages.BytesField, + messages.StringField) + for field_class in classes: action(field_class) def testNumberAttribute(self): @@ -582,12 +593,13 @@ class FieldTest(test_util.TestCase): def testDefaultFields_Single(self): """Test default field is correct type (single).""" - defaults = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: b'abc', - messages.StringField: u'abc', - } + defaults = { + messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } def action(field_class): field_class(1, default=defaults[field_class]) @@ -645,24 +657,26 @@ class FieldTest(test_util.TestCase): def testDefaultFields_EnumStringDelayedResolution(self): """Test that enum fields resolve default strings.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default='OPTIONAL') + field = messages.EnumField( + 'apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default='OPTIONAL') self.assertEquals( descriptor.FieldDescriptor.Label.OPTIONAL, field.default) def testDefaultFields_EnumIntDelayedResolution(self): """Test that enum fields resolve default integers.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default=2) + field = messages.EnumField( + 'apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=2) self.assertEquals( descriptor.FieldDescriptor.Label.REQUIRED, field.default) def testDefaultFields_EnumOkIfTypeKnown(self): - """Test that enum fields accept valid default values when type is known.""" + """Test enum fields accept valid default values when type is known.""" field = messages.EnumField(descriptor.FieldDescriptor.Label, 1, default='REPEATED') @@ -682,9 +696,10 @@ class FieldTest(test_util.TestCase): def testDefaultFields_EnumInvalidDelayedResolution(self): """Test that enum fields raise errors upon delayed resolution error.""" - field = messages.EnumField('apitools.base.protorpclite.descriptor.FieldDescriptor.Label', - 1, - default=200) + field = messages.EnumField( + 'apitools.base.protorpclite.descriptor.FieldDescriptor.Label', + 1, + default=200) self.assertRaisesWithRegexpMatch(TypeError, 'No such value for 200 in Enum Label', @@ -694,12 +709,13 @@ class FieldTest(test_util.TestCase): def testValidate_Valid(self): """Test validation of valid values.""" - values = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: b'abc', - messages.StringField: u'abc', - } + values = { + messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: b'abc', + messages.StringField: u'abc', + } def action(field_class): # Optional. @@ -729,12 +745,13 @@ class FieldTest(test_util.TestCase): def testValidate_Invalid(self): """Test validation of valid values.""" - values = {messages.IntegerField: "10", - messages.FloatField: 1, - messages.BooleanField: 0, - messages.BytesField: 10.20, - messages.StringField: 42, - } + values = { + messages.IntegerField: "10", + messages.FloatField: 1, + messages.BooleanField: 0, + messages.BytesField: 10.20, + messages.StringField: 42, + } def action(field_class): # Optional. @@ -776,11 +793,12 @@ class FieldTest(test_util.TestCase): # Repeated. field = field_class(1, repeated=True) field.validate(None) - self.assertRaisesWithRegexpMatch(messages.ValidationError, - 'Repeated values for %s may ' - 'not be None' % field_class.__name__, - field.validate, - [None]) + self.assertRaisesWithRegexpMatch( + messages.ValidationError, + 'Repeated values for %s may ' + 'not be None' % field_class.__name__, + field.validate, + [None]) self.assertRaises(messages.ValidationError, field.validate, (None,)) @@ -788,12 +806,13 @@ class FieldTest(test_util.TestCase): def testValidateElement(self): """Test validation of valid values.""" - values = {messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: 'abc', - messages.StringField: u'abc', - } + values = { + messages.IntegerField: 10, + messages.FloatField: 1.5, + messages.BooleanField: False, + messages.BytesField: 'abc', + messages.StringField: u'abc', + } def action(field_class): # Optional. @@ -902,7 +921,7 @@ class FieldTest(test_util.TestCase): try: del MyMessage del ForwardMessage - except: + except: # pylint:disable=bare-except pass def testMessageField_WrongType(self): @@ -1084,7 +1103,7 @@ class FieldTest(test_util.TestCase): del MyMessage del ForwardEnum del ForwardMessage - except: + except: # pylint:disable=bare-except pass def testEnumField_WrongType(self): @@ -1111,8 +1130,9 @@ class FieldTest(test_util.TestCase): my_field = messages.StringField(1) - self.assertEquals(MyMessage, - MyMessage.field_by_name('my_field').message_definition()) + self.assertEquals( + MyMessage, + MyMessage.field_by_name('my_field').message_definition()) def testNoneAssignment(self): """Test that assigning None does not change comparison.""" @@ -1223,8 +1243,9 @@ class MessageTest(test_util.TestCase): # Check invalid values. for invalid_value in 10, ['10', '20'], [None], (None,): - self.assertRaises(messages.ValidationError, - setattr, simple_message, 'repeated', invalid_value) + self.assertRaises( + messages.ValidationError, + setattr, simple_message, 'repeated', invalid_value) def testIsInitialized(self): """Tests is_initialized.""" @@ -1305,7 +1326,7 @@ class MessageTest(test_util.TestCase): action) def testNestedAttributesNotAllowed(self): - """Test that attribute assignment on Message classes are not allowed.""" + """Test attribute assignment on Message classes is not allowed.""" def int_attribute(): class WithMethods(messages.Message): not_allowed = 1 @@ -1555,10 +1576,12 @@ class MessageTest(test_util.TestCase): field1 = messages.IntegerField(1) message1 = MyMessage() - self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', - {'unhandled': 'type'}, None) - self.assertRaises(TypeError, message1.set_unrecognized_field, 'unknown4', - {'unhandled': 'type'}, 123) + self.assertRaises( + TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, None) + self.assertRaises( + TypeError, message1.set_unrecognized_field, 'unknown4', + {'unhandled': 'type'}, 123) def testRepr(self): """Test represtation of Message object.""" @@ -1696,8 +1719,9 @@ class MessageTest(test_util.TestCase): self.assertEquals(module_name, MyMessage.NestedMessage.definition_package()) - self.assertEquals('%s.MyMessage.NestedMessage.NestedMessage' % module_name, - MyMessage.NestedMessage.NestedMessage.definition_name()) + self.assertEquals( + '%s.MyMessage.NestedMessage.NestedMessage' % module_name, + MyMessage.NestedMessage.NestedMessage.definition_name()) self.assertEquals( '%s.MyMessage.NestedMessage' % module_name, MyMessage.NestedMessage.NestedMessage.outer_definition_name()) @@ -1734,7 +1758,8 @@ class MessageTest(test_util.TestCase): self.assertRaisesWithRegexpMatch( AttributeError, - 'May not assign arbitrary value does_not_exist to message SomeMessage', + ('May not assign arbitrary value does_not_exist to message ' + 'SomeMessage'), SomeMessage, does_not_exist=10) @@ -1781,8 +1806,9 @@ class MessageTest(test_util.TestCase): self.assertEquals((9.5, messages.Variant.DOUBLE), message.get_unrecognized_field_info('exists', 'type', 1234)) - self.assertEquals((1234, None), - message.get_unrecognized_field_info('doesntexist', 1234)) + self.assertEquals( + (1234, None), + message.get_unrecognized_field_info('doesntexist', 1234)) message.set_unrecognized_field( 'another', 'value', messages.Variant.STRING) @@ -1865,22 +1891,25 @@ class FindDefinitionTest(test_util.TestCase): self.modules.setdefault(full_name, types.ModuleType(full_name)) return self.modules[name] - def DefineMessage(self, module, name, children={}, add_to_module=True): + def DefineMessage(self, module, name, children=None, add_to_module=True): """Define a new Message class in the context of a module. - Used for easily describing complex Message hierarchy. Message is defined - including all child definitions. + Used for easily describing complex Message hierarchy. Message + is defined including all child definitions. Args: module: Fully qualified name of module to place Message class in. name: Name of Message to define within module. - children: Define any level of nesting of children definitions. To define - a message, map the name to another dictionary. The dictionary can - itself contain additional definitions, and so on. To map to an Enum, - define the Enum class separately and map it by name. - add_to_module: If True, new Message class is added to module. If False, - new Message is not added. + children: Define any level of nesting of children + definitions. To define a message, map the name to another + dictionary. The dictionary can itself contain additional + definitions, and so on. To map to an Enum, define the Enum + class separately and map it by name. + add_to_module: If True, new Message class is added to + module. If False, new Message is not added. + """ + children = children or {} # Make sure module exists. module_instance = self.DefineModule(module) @@ -1899,15 +1928,17 @@ class FindDefinitionTest(test_util.TestCase): setattr(module_instance, name, message_class) return message_class + # pylint:disable=unused-argument def Importer(self, module, globals='', locals='', fromlist=None): """Importer function. - Acts like __import__. Only loads modules from self.modules. Does not - try to load real modules defined elsewhere. Does not try to handle relative - imports. + Acts like __import__. Only loads modules from self.modules. + Does not try to load real modules defined elsewhere. Does not + try to handle relative imports. Args: module: Fully qualified name of module to load from self.modules. + """ if fromlist is None: module = module.split('.')[0] @@ -1915,6 +1946,7 @@ class FindDefinitionTest(test_util.TestCase): return self.modules[module] except KeyError: raise ImportError() + # pylint:disable=unused-argument def testNoSuchModule(self): """Test searching for definitions that do no exist.""" @@ -1954,8 +1986,9 @@ class FindDefinitionTest(test_util.TestCase): self.assertEquals(A, messages.find_definition('a.b.c.A', importer=self.Importer)) B = self.DefineMessage('a.b.c', 'B', {'C': {}}) - self.assertEquals(B.C, messages.find_definition('a.b.c.B.C', - importer=self.Importer)) + self.assertEquals( + B.C, + messages.find_definition('a.b.c.B.C', importer=self.Importer)) def testRelativeToModule(self): """Test finding definitions relative to modules.""" @@ -2078,7 +2111,7 @@ class FindDefinitionTest(test_util.TestCase): messages.find_definition('Color', A, importer=self.Importer)) def testFalseScope(self): - """Test that Message definitions nested in strange objects are hidden.""" + """Test Message definitions nested in strange objects are hidden.""" global X class X(object): diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py index acd95c2..6cb52c8 100644 --- a/apitools/base/protorpclite/protojson.py +++ b/apitools/base/protorpclite/protojson.py @@ -24,14 +24,12 @@ Public functions: encode_message: Encodes a message in to a JSON string. decode_message: Merge from a JSON string in to a message. """ -import six - -__author__ = 'rafek@google.com (Rafe Kaplan)' - import base64 import binascii import logging +import six + from apitools.base.protorpclite import message_types from apitools.base.protorpclite import messages from apitools.base.protorpclite import util @@ -67,8 +65,9 @@ def _load_json_module(): try: module = __import__(module_name, {}, {}, 'json') if not hasattr(module, 'JSONEncoder'): - message = ('json library "%s" is not compatible with ProtoRPC' % - module_name) + message = ( + 'json library "%s" is not compatible with ProtoRPC' % + module_name) logging.warning(message) raise ImportError(message) else: @@ -77,9 +76,8 @@ def _load_json_module(): if not first_import_error: first_import_error = err - logging.error( - 'Must use valid json library (Python 2.6 json or simplejson)') - raise first_import_error + logging.error('Must use valid json library (json or simplejson)') + raise first_import_error # pylint:disable=raising-bad-type json = _load_json_module() @@ -97,7 +95,8 @@ class MessageJSONEncoder(json.JSONEncoder): protojson_protocol: ProtoJson instance. """ super(MessageJSONEncoder, self).__init__(**kwargs) - self.__protojson_protocol = protojson_protocol or ProtoJson.get_default() + self.__protojson_protocol = ( + protojson_protocol or ProtoJson.get_default()) def default(self, value): """Return dictionary instance from a message object. @@ -117,8 +116,8 @@ class MessageJSONEncoder(json.JSONEncoder): for field in value.all_fields(): item = value.get_assigned_value(field.name) if item not in (None, [], ()): - result[field.name] = self.__protojson_protocol.encode_field( - field, item) + result[field.name] = ( + self.__protojson_protocol.encode_field(field, item)) # Handle unrecognized fields, so they're included when a message is # decoded then encoded. for unknown_key in value.all_unrecognized_fields(): @@ -133,9 +132,11 @@ class MessageJSONEncoder(json.JSONEncoder): class ProtoJson(object): """ProtoRPC JSON implementation class. - Implementation of JSON based protocol used for serializing and deserializing - message objects. Instances of remote.ProtocolConfig constructor or used with - remote.Protocols.add_protocol. See the remote.py module for more details. + Implementation of JSON based protocol used for serializing and + deserializing message objects. Instances of remote.ProtocolConfig + constructor or used with remote.Protocols.add_protocol. See the + remote.py module for more details. + """ CONTENT_TYPE = 'application/json' @@ -184,7 +185,8 @@ class ProtoJson(object): """ message.check_initialized() - return json.dumps(message, cls=MessageJSONEncoder, protojson_protocol=self) + return json.dumps(message, cls=MessageJSONEncoder, + protojson_protocol=self) def decode_message(self, message_type, encoded_message): """Merge JSON structure to Message instance. @@ -215,8 +217,9 @@ class ProtoJson(object): value: The value whose variant type is being determined. Returns: - The messages.Variant value that best describes value's type, or None if - it's a type we don't know how to handle. + The messages.Variant value that best describes value's type, + or None if it's a type we don't know how to handle. + """ if isinstance(value, bool): return messages.Variant.BOOL @@ -228,7 +231,9 @@ class ProtoJson(object): return messages.Variant.STRING elif isinstance(value, (list, tuple)): # Find the most specific variant that covers all elements. - variant_priority = [None, messages.Variant.INT64, messages.Variant.DOUBLE, + variant_priority = [None, + messages.Variant.INT64, + messages.Variant.DOUBLE, messages.Variant.STRING] chosen_priority = 0 for v in value: @@ -286,7 +291,7 @@ class ProtoJson(object): valid_value.append(self.decode_field(field, item)) if field.repeated: - existing_value = getattr(message, field.name) + _ = getattr(message, field.name) setattr(message, field.name, valid_value) else: setattr(message, field.name, valid_value[-1]) @@ -329,14 +334,14 @@ class ProtoJson(object): isinstance(value, (six.integer_types, six.string_types))): try: return float(value) - except: + except: # pylint:disable=bare-except pass elif (isinstance(field, messages.IntegerField) and isinstance(value, six.string_types)): try: return int(value) - except: + except: # pylint:disable=bare-except pass return value diff --git a/apitools/base/protorpclite/protojson_test.py b/apitools/base/protorpclite/protojson_test.py index 3c08619..c961368 100644 --- a/apitools/base/protorpclite/protojson_test.py +++ b/apitools/base/protorpclite/protojson_test.py @@ -14,12 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # - """Tests for apitools.base.protorpclite.protojson.""" - -__author__ = 'rafek@google.com (Rafe Kaplan)' - - import datetime import json import unittest @@ -40,7 +35,7 @@ class CustomField(messages.MessageField): super(CustomField, self).__init__(self.message_type, number, **kwargs) def value_to_message(self, value): - return self.message_type() + return self.message_type() # pylint:disable=not-callable class MyMessage(messages.Message): @@ -96,9 +91,10 @@ class ProtojsonTest(test_util.TestCase, "int32_value": 1020, "string_value": "a string", "enum_value": "VAL2" - } - """ + } + """ + # pylint:disable=anomalous-unicode-escape-in-string encoded_full = """{ "double_value": 1.23, "float_value": -2.5, @@ -109,8 +105,8 @@ class ProtojsonTest(test_util.TestCase, "string_value": "a string\u044f", "bytes_value": "YSBieXRlc//+", "enum_value": "VAL2" - } - """ + } + """ encoded_repeated = """{ "double_value": [1.23, 2.3], @@ -122,21 +118,21 @@ class ProtojsonTest(test_util.TestCase, "string_value": ["a string\u044f", "another string"], "bytes_value": ["YSBieXRlc//+", "YW5vdGhlciBieXRlcw=="], "enum_value": ["VAL2", "VAL1"] - } - """ + } + """ encoded_nested = """{ "nested": { "a_value": "a string" } - } - """ + } + """ encoded_repeated_nested = """{ "repeated_nested": [{"a_value": "a string"}, {"a_value": "another string"}] - } - """ + } + """ unexpected_tag_message = '{"unknown": "value"}' @@ -155,7 +151,9 @@ class ProtojsonTest(test_util.TestCase, def testConvertIntegerToFloat(self): """Test that integers passed in to float fields are converted. - This is necessary because JSON outputs integers for numbers with 0 decimals. + This is necessary because JSON outputs integers for numbers + with 0 decimals. + """ message = protojson.decode_message(MyMessage, '{"a_float": 10}') @@ -251,15 +249,18 @@ class ProtojsonTest(test_util.TestCase, def testNotJSON(self): """Test error when string is not valid JSON.""" - self.assertRaises(ValueError, - protojson.decode_message, MyMessage, '{this is not json}') + self.assertRaises( + ValueError, + protojson.decode_message, MyMessage, + '{this is not json}') def testDoNotEncodeStrangeObjects(self): """Test trying to encode a strange object. - The main purpose of this test is to complete coverage. It ensures that - the default behavior of the JSON encoder is preserved when someone tries to - serialized an unexpected type. + The main purpose of this test is to complete coverage. It + ensures that the default behavior of the JSON encoder is + preserved when someone tries to serialized an unexpected type. + """ class BogusObject(object): @@ -280,8 +281,9 @@ class ProtojsonTest(test_util.TestCase, def testProtojsonUnrecognizedFieldName(self): """Test that unrecognized fields are saved and can be accessed.""" - decoded = protojson.decode_message(MyMessage, - ('{"an_integer": 1, "unknown_val": 2}')) + decoded = protojson.decode_message( + MyMessage, + ('{"an_integer": 1, "unknown_val": 2}')) self.assertEquals(decoded.an_integer, 1) self.assertEquals(1, len(decoded.all_unrecognized_fields())) self.assertEquals('unknown_val', decoded.all_unrecognized_fields()[0]) @@ -317,8 +319,10 @@ class ProtojsonTest(test_util.TestCase, def testUnrecognizedFieldVariants(self): """Test that unrecognized fields are mapped to the right variants.""" for encoded, expected_variant in ( - ('{"an_integer": 1, "unknown_val": 2}', messages.Variant.INT64), - ('{"an_integer": 1, "unknown_val": 2.0}', messages.Variant.DOUBLE), + ('{"an_integer": 1, "unknown_val": 2}', + messages.Variant.INT64), + ('{"an_integer": 1, "unknown_val": 2.0}', + messages.Variant.DOUBLE), ('{"an_integer": 1, "unknown_val": "string value"}', messages.Variant.STRING), ('{"an_integer": 1, "unknown_val": [1, 2, 3]}', @@ -327,7 +331,8 @@ class ProtojsonTest(test_util.TestCase, messages.Variant.DOUBLE), ('{"an_integer": 1, "unknown_val": [1, "foo", 3]}', messages.Variant.STRING), - ('{"an_integer": 1, "unknown_val": true}', messages.Variant.BOOL)): + ('{"an_integer": 1, "unknown_val": true}', + messages.Variant.BOOL)): decoded = protojson.decode_message(MyMessage, encoded) self.assertEquals(decoded.an_integer, 1) self.assertEquals(1, len(decoded.all_unrecognized_fields())) @@ -354,9 +359,12 @@ class ProtojsonTest(test_util.TestCase, def testEncodeDateTime(self): for datetime_string, datetime_vals in ( - ('2012-09-30T15:31:50.262000', (2012, 9, 30, 15, 31, 50, 262000)), - ('2012-09-30T15:31:50.262123', (2012, 9, 30, 15, 31, 50, 262123)), - ('2012-09-30T15:31:50', (2012, 9, 30, 15, 31, 50, 0))): + ('2012-09-30T15:31:50.262000', + (2012, 9, 30, 15, 31, 50, 262000)), + ('2012-09-30T15:31:50.262123', + (2012, 9, 30, 15, 31, 50, 262123)), + ('2012-09-30T15:31:50', + (2012, 9, 30, 15, 31, 50, 0))): decoded_message = protojson.encode_message( MyMessage(a_datetime=datetime.datetime(*datetime_vals))) expected_decoding = '{"a_datetime": "%s"}' % datetime_string @@ -418,8 +426,9 @@ class CustomProtoJsonTest(test_util.TestCase): self.protojson = CustomProtoJson() def testEncode(self): - self.assertEqual('{"a_string": "{encoded}xyz"}', - self.protojson.encode_message(MyMessage(a_string='xyz'))) + self.assertEqual( + '{"a_string": "{encoded}xyz"}', + self.protojson.encode_message(MyMessage(a_string='xyz'))) def testDecode(self): self.assertEqual( diff --git a/apitools/base/protorpclite/test_util.py b/apitools/base/protorpclite/test_util.py index b50aa88..fad3a53 100644 --- a/apitools/base/protorpclite/test_util.py +++ b/apitools/base/protorpclite/test_util.py @@ -26,8 +26,6 @@ services_test.proto. Includes additional test utilities to make sure encoding/decoding libraries conform. """ -__author__ = 'rafek@google.com (Rafe Kaplan)' - import cgi import datetime import inspect @@ -35,10 +33,10 @@ import os import re import socket import types -import unittest2 as unittest import six from six.moves import range +import unittest2 as unittest from apitools.base.protorpclite import message_types from apitools.base.protorpclite import messages @@ -129,34 +127,39 @@ class TestCase(unittest.TestCase): class ModuleInterfaceTest(object): """Test to ensure module interface is carefully constructed. - A module interface is the set of public objects listed in the module __all__ - attribute. Modules that that are considered public should have this interface - carefully declared. At all times, the __all__ attribute should have objects - intended to be publically used and all other objects in the module should be - considered unused. - - Protected attributes (those beginning with '_') and other imported modules - should not be part of this set of variables. An exception is for variables - that begin and end with '__' which are implicitly part of the interface - (eg. __name__, __file__, __all__ itself, etc.). - - Modules that are imported in to the tested modules are an exception and may - be left out of the __all__ definition. The test is done by checking the value - of what would otherwise be a public name and not allowing it to be exported - if it is an instance of a module. Modules that are explicitly exported are - for the time being not permitted. - - To use this test class a module should define a new class that inherits first - from ModuleInterfaceTest and then from test_util.TestCase. No other tests - should be added to this test case, making the order of inheritance less - important, but if setUp for some reason is overidden, it is important that - ModuleInterfaceTest is first in the list so that its setUp method is - invoked. - - Multiple inheretance is required so that ModuleInterfaceTest is not itself - a test, and is not itself executed as one. - - The test class is expected to have the following class attributes defined: + A module interface is the set of public objects listed in the + module __all__ attribute. Modules that that are considered public + should have this interface carefully declared. At all times, the + __all__ attribute should have objects intended to be publically + used and all other objects in the module should be considered + unused. + + Protected attributes (those beginning with '_') and other imported + modules should not be part of this set of variables. An exception + is for variables that begin and end with '__' which are implicitly + part of the interface (eg. __name__, __file__, __all__ itself, + etc.). + + Modules that are imported in to the tested modules are an + exception and may be left out of the __all__ definition. The test + is done by checking the value of what would otherwise be a public + name and not allowing it to be exported if it is an instance of a + module. Modules that are explicitly exported are for the time + being not permitted. + + To use this test class a module should define a new class that + inherits first from ModuleInterfaceTest and then from + test_util.TestCase. No other tests should be added to this test + case, making the order of inheritance less important, but if setUp + for some reason is overidden, it is important that + ModuleInterfaceTest is first in the list so that its setUp method + is invoked. + + Multiple inheritance is required so that ModuleInterfaceTest is + not itself a test, and is not itself executed as one. + + The test class is expected to have the following class attributes + defined: MODULE: A reference to the module that is being validated for interface correctness. @@ -193,6 +196,7 @@ class ModuleInterfaceTest(object): if __name__ == '__main__': unittest.main() + """ def setUp(self): @@ -203,8 +207,8 @@ class ModuleInterfaceTest(object): """ if not hasattr(self, 'MODULE'): self.fail( - "You must define 'MODULE' on ModuleInterfaceTest sub-class %s." % - type(self).__name__) + "You must define 'MODULE' on ModuleInterfaceTest sub-class " + "%s." % type(self).__name__) def testAllExist(self): """Test that all attributes defined in __all__ exist.""" @@ -294,11 +298,6 @@ class OptionalMessage(messages.Message): bytes_value = messages.BytesField(8, variant=messages.Variant.BYTES) enum_value = messages.EnumField(SimpleEnum, 10) - # TODO(rafek): Add support for these variants. - # uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) - # sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) - # sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) - class RepeatedMessage(messages.Message): """Contains all message types as repeated fields.""" @@ -332,12 +331,9 @@ class RepeatedMessage(messages.Message): bytes_value = messages.BytesField(8, variant=messages.Variant.BYTES, repeated=True) - #uint32_value = messages.IntegerField(9, variant=messages.Variant.UINT32) enum_value = messages.EnumField(SimpleEnum, 10, repeated=True) - #sint32_value = messages.IntegerField(11, variant=messages.Variant.SINT32) - #sint64_value = messages.IntegerField(12, variant=messages.Variant.SINT64) class HasOptionalNestedMessage(messages.Message): @@ -346,6 +342,7 @@ class HasOptionalNestedMessage(messages.Message): repeated_nested = messages.MessageField(OptionalMessage, 2, repeated=True) +# pylint:disable=anomalous-unicode-escape-in-string class ProtoConformanceTestBase(object): """Protocol conformance test base class. @@ -360,9 +357,9 @@ class ProtoConformanceTestBase(object): that protocols correctly encode and decode message transparently to the caller. - In order to support these test, the base class should also extend the TestCase - class and implement the following class attributes which define the encoded - version of certain protocol buffers: + In order to support these test, the base class should also extend + the TestCase class and implement the following class attributes + which define the encoded version of certain protocol buffers: encoded_partial: Date: Fri, 18 Sep 2015 14:12:36 -0700 Subject: [PATCH 173/295] A few more protorpc-move-related tweaks. --- apitools/base/py/extra_types.py | 7 +------ apitools/gen/gen_client_lib.py | 1 - apitools/gen/util.py | 4 +++- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index 0649e57..de0183a 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -1,9 +1,5 @@ #!/usr/bin/env python -"""Extra types understood by apitools. - -This file will be replaced by a .proto file when we switch to proto2 -from protorpc. -""" +"""Extra types understood by apitools.""" import collections import datetime @@ -29,7 +25,6 @@ __all__ = [ 'JsonProtoDecoder', ] -# We import from protorpc. # pylint:disable=invalid-name DateTimeMessage = message_types.DateTimeMessage # pylint:enable=invalid-name diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 2c11fc1..1a066db 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -209,7 +209,6 @@ class DescriptorGenerator(object): printer('"google-apitools>=0.4.8",') printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') - printer('"protorpc>=0.10.0",') printer(']') printer('_PACKAGE = "apitools.clients.%s"' % self.__package) printer() diff --git a/apitools/gen/util.py b/apitools/gen/util.py index bdc570e..a886da6 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -107,7 +107,9 @@ class Names(object): return name # TODO(craigcitro): This is a hack to handle the case of specific # protorpc class names; clean this up. - if name.startswith('protorpc.') or name.startswith('message_types.'): + if name.startswith(('protorpc.', 'message_types.', + 'apitools.base.protorpclite.', + 'apitools.base.protorpclite.message_types.')): return name name = self.__StripName(name) name = self.__ToCamel(name, separator=separator) -- GitLab From 0dbb0c595673612427415062581e7632142a64f9 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 14:13:05 -0700 Subject: [PATCH 174/295] Update storage sample. --- samples/storage_sample/storage/__init__.py | 1 + samples/storage_sample/storage/storage_v1.py | 57 +++++++++++------- .../storage/storage_v1_client.py | 4 +- .../storage/storage_v1_messages.py | 58 +++++++++++++------ 4 files changed, 79 insertions(+), 41 deletions(-) diff --git a/samples/storage_sample/storage/__init__.py b/samples/storage_sample/storage/__init__.py index 2c8e598..6426fac 100644 --- a/samples/storage_sample/storage/__init__.py +++ b/samples/storage_sample/storage/__init__.py @@ -4,6 +4,7 @@ import pkgutil from apitools.base.py import * +from storage_v1 import * from storage_v1_client import * from storage_v1_messages import * diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage/storage_v1.py index 5b9e5a2..abb2d31 100755 --- a/samples/storage_sample/storage/storage_v1.py +++ b/samples/storage_sample/storage/storage_v1.py @@ -7,9 +7,8 @@ import os import platform import sys -import protorpc -from protorpc import message_types -from protorpc import messages +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages from google.apputils import appcommands import gflags as flags @@ -1618,7 +1617,8 @@ class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. @@ -1660,7 +1660,8 @@ class ObjectAccessControlsGet(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. @@ -1707,7 +1708,8 @@ class ObjectAccessControlsInsert(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, selects a specific revision of this object (as @@ -1749,7 +1751,8 @@ class ObjectAccessControlsList(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, selects a specific revision of this object (as @@ -1793,7 +1796,8 @@ class ObjectAccessControlsPatch(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. @@ -1844,7 +1848,8 @@ class ObjectAccessControlsUpdate(apitools_base_cli.NewCmd): Args: bucket: Name of a bucket. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. @@ -1919,7 +1924,8 @@ class ObjectsCompose(apitools_base_cli.NewCmd): Args: destinationBucket: Name of the bucket in which to store the new object. - destinationObject: Name of the new object. + destinationObject: Name of the new object. For information about how to + URL encode object names to be path safe, see Encoding URI Path Parts. Flags: composeRequest: A ComposeRequest resource to be passed as the request @@ -2054,9 +2060,12 @@ class ObjectsCopy(apitools_base_cli.NewCmd): Args: sourceBucket: Name of the bucket in which to find the source object. - sourceObject: Name of the source object. + sourceObject: Name of the source object. For information about how to + URL encode object names to be path safe, see Encoding URI Path Parts. destinationBucket: Name of the bucket in which to store the new object. - Overrides the provided object metadata's bucket value, if any. + Overrides the provided object metadata's bucket value, if any.For + information about how to URL encode object names to be path safe, see + Encoding URI Path Parts. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. @@ -2178,7 +2187,8 @@ class ObjectsDelete(apitools_base_cli.NewCmd): Args: bucket: Name of the bucket in which the object resides. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, permanently deletes a specific revision of this @@ -2272,7 +2282,8 @@ class ObjectsGet(apitools_base_cli.NewCmd): Args: bucket: Name of the bucket in which the object resides. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, selects a specific revision of this object (as @@ -2362,7 +2373,8 @@ class ObjectsInsert(apitools_base_cli.NewCmd): None, u'Name of the object. Required when the object metadata is not ' u"otherwise provided. Overrides the object metadata's name value, if " - u'any.', + u'any. For information about how to URL encode object names to be ' + u'path safe, see Encoding URI Path Parts.', flag_values=fv) flags.DEFINE_string( 'object', @@ -2428,7 +2440,8 @@ class ObjectsInsert(apitools_base_cli.NewCmd): object's current metageneration does not match the given value. name: Name of the object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if - any. + any. For information about how to URL encode object names to be path + safe, see Encoding URI Path Parts. object: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to noAcl, unless the @@ -2630,7 +2643,8 @@ class ObjectsPatch(apitools_base_cli.NewCmd): Args: bucket: Name of the bucket in which the object resides. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, selects a specific revision of this object (as @@ -2782,12 +2796,14 @@ class ObjectsRewrite(apitools_base_cli.NewCmd): Args: sourceBucket: Name of the bucket in which to find the source object. - sourceObject: Name of the source object. + sourceObject: Name of the source object. For information about how to + URL encode object names to be path safe, see Encoding URI Path Parts. destinationBucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's - name value, if any. + name value, if any. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. Flags: destinationPredefinedAcl: Apply a predefined set of access controls to @@ -2941,7 +2957,8 @@ class ObjectsUpdate(apitools_base_cli.NewCmd): Args: bucket: Name of the bucket in which the object resides. - object: Name of the object. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. Flags: generation: If present, selects a specific revision of this object (as diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage/storage_v1_client.py index 18bd33d..01c2317 100644 --- a/samples/storage_sample/storage/storage_v1_client.py +++ b/samples/storage_sample/storage/storage_v1_client.py @@ -10,11 +10,11 @@ class StorageV1(base_api.BaseApiClient): MESSAGES_MODULE = messages _PACKAGE = u'storage' - _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] + _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] _VERSION = u'v1' _CLIENT_ID = '1042881264118.apps.googleusercontent.com' _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' - _USER_AGENT = '' + _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b' _CLIENT_CLASS_NAME = u'StorageV1' _URL_VERSION = u'v1' _API_KEY = None diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py index 7148d0a..dfc2214 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -4,10 +4,10 @@ Lets you store and retrieve potentially-large, immutable data objects. """ # NOTE: This file is autogenerated and should not be edited by hand. +from apitools.base.protorpclite import message_types as _message_types +from apitools.base.protorpclite import messages as _messages from apitools.base.py import encoding from apitools.base.py import extra_types -from protorpc import message_types as _message_types -from protorpc import messages as _messages package = 'storage' @@ -417,14 +417,16 @@ class Object(_messages.Message): contentLanguage: Content-Language of the object data. contentType: Content-Type of the object data. crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded - using base64 in big-endian byte order. + using base64 in big-endian byte order. For more information about using + the CRC32c checksum, see Hashes and ETags: Best Practices. etag: HTTP 1.1 Entity tag for the object. generation: The content generation of this object. Used for object versioning. id: The ID of the object. kind: The kind of item this is. For objects, this is always storage#object. - md5Hash: MD5 hash of the data; encoded using base64. + md5Hash: MD5 hash of the data; encoded using base64. For more information + about using the MD5 hash, see Hashes and ETags: Best Practices. mediaLink: Media download link. metadata: User-provided metadata, in key/value pairs. metageneration: The version of the metadata for this object at this @@ -1110,7 +1112,8 @@ class StorageObjectAccessControlsDeleteRequest(_messages.Message): allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. """ bucket = _messages.StringField(1, required=True) @@ -1133,7 +1136,8 @@ class StorageObjectAccessControlsGetRequest(_messages.Message): allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. """ bucket = _messages.StringField(1, required=True) @@ -1149,7 +1153,8 @@ class StorageObjectAccessControlsInsertRequest(_messages.Message): bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. """ @@ -1167,7 +1172,8 @@ class StorageObjectAccessControlsListRequest(_messages.Message): bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. """ bucket = _messages.StringField(1, required=True) @@ -1185,7 +1191,8 @@ class StorageObjectAccessControlsPatchRequest(_messages.Message): allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. """ @@ -1207,7 +1214,8 @@ class StorageObjectAccessControlsUpdateRequest(_messages.Message): allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. """ @@ -1230,7 +1238,8 @@ class StorageObjectsComposeRequest(_messages.Message): composeRequest: A ComposeRequest resource to be passed as the request body. destinationBucket: Name of the bucket in which to store the new object. - destinationObject: Name of the new object. + destinationObject: Name of the new object. For information about how to + URL encode object names to be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the object's @@ -1282,7 +1291,9 @@ class StorageObjectsCopyRequest(_messages.Message): Fields: destinationBucket: Name of the bucket in which to store the new object. - Overrides the provided object metadata's bucket value, if any. + Overrides the provided object metadata's bucket value, if any.For + information about how to URL encode object names to be path safe, see + Encoding URI Path Parts. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. @@ -1312,7 +1323,8 @@ class StorageObjectsCopyRequest(_messages.Message): sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). - sourceObject: Name of the source object. + sourceObject: Name of the source object. For information about how to URL + encode object names to be path safe, see Encoding URI Path Parts. """ class DestinationPredefinedAclValueValuesEnum(_messages.Enum): @@ -1382,7 +1394,8 @@ class StorageObjectsDeleteRequest(_messages.Message): object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. """ bucket = _messages.StringField(1, required=True) @@ -1416,7 +1429,8 @@ class StorageObjectsGetRequest(_messages.Message): object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. projection: Set of properties to return. Defaults to noAcl. """ @@ -1468,6 +1482,8 @@ class StorageObjectsInsertRequest(_messages.Message): object's current metageneration does not match the given value. name: Name of the object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. + For information about how to URL encode object names to be path safe, + see Encoding URI Path Parts. object: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to noAcl, unless the @@ -1583,7 +1599,8 @@ class StorageObjectsPatchRequest(_messages.Message): object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. @@ -1649,7 +1666,8 @@ class StorageObjectsRewriteRequest(_messages.Message): Overrides the provided object metadata's bucket value, if any. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name - value, if any. + value, if any. For information about how to URL encode object names to + be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the @@ -1689,7 +1707,8 @@ class StorageObjectsRewriteRequest(_messages.Message): sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). - sourceObject: Name of the source object. + sourceObject: Name of the source object. For information about how to URL + encode object names to be path safe, see Encoding URI Path Parts. """ class DestinationPredefinedAclValueValuesEnum(_messages.Enum): @@ -1766,7 +1785,8 @@ class StorageObjectsUpdateRequest(_messages.Message): object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. - object: Name of the object. + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. -- GitLab From 20b17943629a5dad3858149b4323a0e4409d43eb Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 14:47:18 -0700 Subject: [PATCH 175/295] Add a skip for a pypy test. --- apitools/base/protorpclite/descriptor_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apitools/base/protorpclite/descriptor_test.py b/apitools/base/protorpclite/descriptor_test.py index e6b3055..bc2b8ed 100644 --- a/apitools/base/protorpclite/descriptor_test.py +++ b/apitools/base/protorpclite/descriptor_test.py @@ -16,6 +16,7 @@ # """Tests for apitools.base.protorpclite.descriptor.""" +import platform import types import unittest @@ -77,6 +78,8 @@ class DescribeEnumTest(test_util.TestCase): described.check_initialized() self.assertEquals(expected, described) + @unittest.skipIf('PyPy' in platform.python_implementation(), + 'todo: reenable this') def testEnumWithItems(self): class EnumWithItems(messages.Enum): A = 3 -- GitLab From 4d1e56159c45c23ae6d4fd99073a11794c28d77e Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 18 Sep 2015 14:58:35 -0700 Subject: [PATCH 176/295] Update a unittest skip for py26 compatibility. --- apitools/base/protorpclite/descriptor_test.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/apitools/base/protorpclite/descriptor_test.py b/apitools/base/protorpclite/descriptor_test.py index bc2b8ed..fc27ec4 100644 --- a/apitools/base/protorpclite/descriptor_test.py +++ b/apitools/base/protorpclite/descriptor_test.py @@ -18,9 +18,9 @@ """Tests for apitools.base.protorpclite.descriptor.""" import platform import types -import unittest import six +import unittest2 from apitools.base.protorpclite import descriptor from apitools.base.protorpclite import message_types @@ -78,8 +78,8 @@ class DescribeEnumTest(test_util.TestCase): described.check_initialized() self.assertEquals(expected, described) - @unittest.skipIf('PyPy' in platform.python_implementation(), - 'todo: reenable this') + @unittest2.skipIf('PyPy' in platform.python_implementation(), + 'todo: reenable this') def testEnumWithItems(self): class EnumWithItems(messages.Enum): A = 3 @@ -511,9 +511,5 @@ class DescriptorLibraryTest(test_util.TestCase): self.assertEquals(None, self.library.lookup_package('Packageless')) -def main(): - unittest.main() - - if __name__ == '__main__': - main() + unittest2.main() -- GitLab From 90174a4f8152e7406549e729113267f3baf18945 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 20 Sep 2015 21:42:59 -0700 Subject: [PATCH 177/295] Fix an issue with client generation, and a test. Previously, the test of client generation was failing, as the exit code for `./ help` was indistinguishable from a real error. This only showed up as output running tests. First, we fix the test by checking the output for a traceback. Not foolproof, but a good start. Next, we fix the code itself, by no longer letting `--outdir` be used as the value for `--root_package_dir`. We have both flags; it's a bit silly to have wacky logic that tries to use the value of one to outsmart the caller. Instead, we're explicit, and if we have to repeat ourselves, no big deal. This also allows us to delete `apitools.gen.util.GetPackage`. --- apitools/gen/client_generation_test.py | 20 ++++++++++++++------ apitools/gen/gen_client.py | 3 +-- apitools/gen/util.py | 5 ----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index cb58bb4..5de7005 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -45,11 +45,19 @@ class ClientGenerationTest(unittest2.TestCase): self.assertEqual(0, retcode) with tempfile.NamedTemporaryFile() as out: - cmdline_args = [ - os.path.join( - 'generated', api.replace('.', '_') + '.py'), - 'help', - ] - retcode = subprocess.call(cmdline_args, stdout=out) + with tempfile.NamedTemporaryFile() as err: + cmdline_args = [ + os.path.join( + 'generated', api.replace('.', '_') + '.py'), + 'help', + ] + retcode = subprocess.call( + cmdline_args, stdout=out, stderr=err) + with open(err.name, 'rb') as f: + err_output = f.read() # appcommands returns 1 on help self.assertEqual(1, retcode) + if 'Traceback (most recent call last):' in err_output: + err = '\n======\n%s======\n' % err_output + self.fail( + 'Error raised in generated client:' + err) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index d1de17c..5f2ca58 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -85,9 +85,8 @@ def _GetCodegenFromFlags(args): if not os.path.exists(outdir): os.makedirs(outdir) - root_package = args.root_package or util.GetPackage(outdir) return gen_client_lib.DescriptorGenerator( - discovery_doc, client_info, names, root_package, outdir, + discovery_doc, client_info, names, args.root_package, outdir, base_package=args.base_package, generate_cli=args.generate_cli, use_proto2=args.experimental_proto2_output, diff --git a/apitools/gen/util.py b/apitools/gen/util.py index bdc570e..c9973f8 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -223,11 +223,6 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( return '%s.proto' % self.services_rule_name -def GetPackage(path): - path_components = path.split(os.path.sep) - return '.'.join(path_components) - - def CleanDescription(description): """Return a version of description safe for printing in a docstring.""" if not isinstance(description, six.string_types): -- GitLab From 2ac1d7d847434c2672f2e5a60876ce50888c261f Mon Sep 17 00:00:00 2001 From: Brad Daniels Date: Tue, 29 Sep 2015 15:36:07 -0700 Subject: [PATCH 178/295] Convert ValueError to InvalidDataFromServerError in BaseApiClient.DeserializeMessage Currently, the ValueErrors raised by json.loads are not handled by DeserializeMessage, meaning that we're losing the text of the source message in error logs when the JSON is malformed. --- apitools/base/py/base_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 5623c61..f171dd1 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -419,7 +419,7 @@ class BaseApiClient(object): try: message = encoding.JsonToMessage(response_type, data) except (exceptions.InvalidDataFromServerError, - messages.ValidationError) as e: + messages.ValidationError, ValueError) as e: raise exceptions.InvalidDataFromServerError( 'Error decoding response "%s" as type %s: %s' % ( data, response_type.__name__, e)) -- GitLab From 3aeb8114ff27296b2984eab705e254d403059336 Mon Sep 17 00:00:00 2001 From: CH Albach Date: Fri, 16 Oct 2015 15:31:05 -0700 Subject: [PATCH 179/295] Add global_params pass-through support to list_pager. --- apitools/base/py/list_pager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 85c2594..8e15f58 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -9,7 +9,7 @@ __all__ = [ def YieldFromList( - service, request, limit=None, batch_size=100, + service, request, global_params=None, limit=None, batch_size=100, method='List', field='items', predicate=None, current_token_attribute='pageToken', next_token_attribute='nextPageToken', @@ -22,6 +22,8 @@ def YieldFromList( corresponding to the service's .List() method, with all the attributes populated except the .maxResults and .pageToken attributes. + global_params: protorpc.messages.Message, The global query parameters to + provide when calling the given method. limit: int, The maximum number of records to yield. None if all available records should be yielded. batch_size: int, The number of items to retrieve per request. @@ -45,7 +47,8 @@ def YieldFromList( setattr(request, batch_size_attribute, batch_size) setattr(request, current_token_attribute, None) while limit is None or limit: - response = getattr(service, method)(request) + response = getattr(service, method)(request, + global_params=global_params) items = getattr(response, field) if predicate: items = list(filter(predicate, items)) -- GitLab From 9e480e85d522744389ba83966bf4bcc91ba5e37c Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 23 Oct 2015 16:32:52 -0400 Subject: [PATCH 180/295] Tolerate remappings if the mapped value is the same. --- apitools/base/py/encoding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index abac330..62bc96b 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -651,12 +651,12 @@ def _CheckForExistingMappings(mapping_type, message_type, elif mapping_type == 'enum': getter = GetCustomJsonEnumMapping remapping = getter(message_type, python_name=python_name) - if remapping is not None: + if remapping is not None and remapping != json_name: raise exceptions.InvalidDataError( 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( mapping_type, python_name, remapping)) remapping = getter(message_type, json_name=json_name) - if remapping is not None: + if remapping is not None and remapping != python_name: raise exceptions.InvalidDataError( 'Cannot add mapping for %s "%s", already mapped to "%s"' % ( mapping_type, json_name, remapping)) -- GitLab From 254e9a93187a5bd2afe95e8620b3ccc475894f20 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 23 Oct 2015 17:24:30 -0400 Subject: [PATCH 181/295] Update encoding_test.py --- apitools/base/py/encoding_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 7f0005b..69711a5 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -349,6 +349,16 @@ class EncodingTest(unittest2.TestCase): encoding.AddCustomJsonEnumMapping, MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') + def testAllowIdenticalRepeatedRemapping(self): + encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, + 'enum_value', 'wire_name') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'double_encoding', 'doubleEncoding') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'another_field', 'anotherField') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'repeated_field', 'repeatedField') + def testMessageToRepr(self): # Using the same string returned by MessageToRepr, with the # module names fixed. -- GitLab From efc5e336fc3dd5d6dab6fe8d544eb176be54d987 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 23 Oct 2015 17:38:23 -0400 Subject: [PATCH 182/295] Updated the remapping test to test both allowed and disallowed remappings. --- apitools/base/py/encoding_test.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 69711a5..9d8281b 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -331,7 +331,18 @@ class EncodingTest(unittest2.TestCase): self.assertEqual( msg, encoding.JsonToMessage(MessageWithRemappings, json_message)) - def testNoRepeatedRemapping(self): + def testRepeatedRemapping(self): + # Should allow remapping if the mapping remains the same. + encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, + 'enum_value', 'wire_name') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'double_encoding', 'doubleEncoding') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'another_field', 'anotherField') + encoding.AddCustomJsonFieldMapping(MessageWithRemappings, + 'repeated_field', 'repeatedField') + + # Should raise errors if the remapping changes the mapping. self.assertRaises( exceptions.InvalidDataError, encoding.AddCustomJsonFieldMapping, @@ -349,16 +360,6 @@ class EncodingTest(unittest2.TestCase): encoding.AddCustomJsonEnumMapping, MessageWithRemappings.SomeEnum, 'second_value', 'wire_name') - def testAllowIdenticalRepeatedRemapping(self): - encoding.AddCustomJsonEnumMapping(MessageWithRemappings.SomeEnum, - 'enum_value', 'wire_name') - encoding.AddCustomJsonFieldMapping(MessageWithRemappings, - 'double_encoding', 'doubleEncoding') - encoding.AddCustomJsonFieldMapping(MessageWithRemappings, - 'another_field', 'anotherField') - encoding.AddCustomJsonFieldMapping(MessageWithRemappings, - 'repeated_field', 'repeatedField') - def testMessageToRepr(self): # Using the same string returned by MessageToRepr, with the # module names fixed. -- GitLab From 471ac70f750f21feddf81d5934ebc1ae238a445e Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Sat, 24 Oct 2015 07:54:38 -0700 Subject: [PATCH 183/295] Add per-file licenses. Fixes #63. --- apitools/__init__.py | 15 +++++++++++++++ apitools/base/__init__.py | 15 +++++++++++++++ apitools/base/py/__init__.py | 15 +++++++++++++++ apitools/base/py/app2.py | 15 +++++++++++++++ apitools/base/py/base_api.py | 15 +++++++++++++++ apitools/base/py/base_api_test.py | 15 +++++++++++++++ apitools/base/py/base_cli.py | 15 +++++++++++++++ apitools/base/py/batch.py | 15 +++++++++++++++ apitools/base/py/batch_test.py | 17 ++++++++++++++++- apitools/base/py/buffered_stream.py | 15 +++++++++++++++ apitools/base/py/buffered_stream_test.py | 15 +++++++++++++++ apitools/base/py/cli.py | 15 +++++++++++++++ apitools/base/py/credentials_lib.py | 15 +++++++++++++++ apitools/base/py/credentials_lib_test.py | 15 +++++++++++++++ apitools/base/py/encoding.py | 15 +++++++++++++++ apitools/base/py/encoding_test.py | 15 +++++++++++++++ apitools/base/py/exceptions.py | 15 +++++++++++++++ apitools/base/py/extra_types.py | 15 +++++++++++++++ apitools/base/py/extra_types_test.py | 15 +++++++++++++++ apitools/base/py/http_wrapper.py | 15 +++++++++++++++ apitools/base/py/http_wrapper_test.py | 15 +++++++++++++++ apitools/base/py/list_pager.py | 15 +++++++++++++++ apitools/base/py/list_pager_test.py | 15 +++++++++++++++ apitools/base/py/stream_slice.py | 15 +++++++++++++++ apitools/base/py/stream_slice_test.py | 15 +++++++++++++++ apitools/base/py/testing/__init__.py | 15 +++++++++++++++ apitools/base/py/testing/mock.py | 15 +++++++++++++++ apitools/base/py/testing/mock_test.py | 15 +++++++++++++++ apitools/base/py/testing/testclient/__init__.py | 15 +++++++++++++++ .../testclient/fusiontables_v1_client.py | 15 +++++++++++++++ .../testclient/fusiontables_v1_messages.py | 15 +++++++++++++++ apitools/base/py/transfer.py | 15 +++++++++++++++ apitools/base/py/transfer_test.py | 15 +++++++++++++++ apitools/base/py/util.py | 15 +++++++++++++++ apitools/base/py/util_test.py | 15 +++++++++++++++ apitools/data/__init__.py | 15 +++++++++++++++ apitools/gen/__init__.py | 15 +++++++++++++++ apitools/gen/client_generation_test.py | 15 +++++++++++++++ apitools/gen/command_registry.py | 15 +++++++++++++++ apitools/gen/extended_descriptor.py | 15 +++++++++++++++ apitools/gen/gen_client.py | 15 +++++++++++++++ apitools/gen/gen_client_lib.py | 15 +++++++++++++++ apitools/gen/gen_client_test.py | 15 +++++++++++++++ apitools/gen/message_registry.py | 15 +++++++++++++++ apitools/gen/service_registry.py | 15 +++++++++++++++ apitools/gen/test_utils.py | 15 +++++++++++++++ apitools/gen/util.py | 15 +++++++++++++++ apitools/gen/util_test.py | 15 +++++++++++++++ apitools/scripts/__init__.py | 15 +++++++++++++++ apitools/scripts/oauth2l.py | 15 +++++++++++++++ apitools/scripts/oauth2l_test.py | 15 +++++++++++++++ ez_setup.py | 15 +++++++++++++++ run_pylint.py | 15 +++++++++++++++ samples/storage_sample/downloads_test.py | 15 +++++++++++++++ samples/storage_sample/generate_clients.sh | 14 ++++++++++++++ samples/storage_sample/storage/__init__.py | 15 +++++++++++++++ samples/storage_sample/storage/storage_v1.py | 15 +++++++++++++++ .../storage_sample/storage/storage_v1_client.py | 15 +++++++++++++++ .../storage/storage_v1_messages.py | 15 +++++++++++++++ samples/storage_sample/uploads_test.py | 15 +++++++++++++++ 60 files changed, 900 insertions(+), 1 deletion(-) diff --git a/apitools/__init__.py b/apitools/__init__.py index 54fa3d5..463cb42 100644 --- a/apitools/__init__.py +++ b/apitools/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Shared __init__.py for apitools.""" from pkgutil import extend_path diff --git a/apitools/base/__init__.py b/apitools/base/__init__.py index 54fa3d5..463cb42 100644 --- a/apitools/base/__init__.py +++ b/apitools/base/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Shared __init__.py for apitools.""" from pkgutil import extend_path diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py index 0bbcf9f..393aa14 100644 --- a/apitools/base/py/__init__.py +++ b/apitools/base/py/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Top-level imports for apitools base files.""" # pylint:disable=wildcard-import diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py index 531a977..9aba591 100644 --- a/apitools/base/py/app2.py +++ b/apitools/base/py/app2.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Appcommands-compatible command class with extra fixins.""" from __future__ import print_function diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index f171dd1..de3a9a3 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Base class for api services.""" import base64 diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 2b03185..96c425b 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import base64 import datetime import sys diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py index 7d6ed05..d6fe67f 100644 --- a/apitools/base/py/base_cli.py +++ b/apitools/base/py/base_cli.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Base script for generated CLI.""" import atexit diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 2cf03a4..71dbf6b 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Library for handling batch HTTP requests for apitools.""" import collections diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py index 7c20171..b966d5a 100644 --- a/apitools/base/py/batch_test.py +++ b/apitools/base/py/batch_test.py @@ -1,4 +1,19 @@ -"""Tests for google3.cloud.bigscience.apitools.base.py.batch.""" +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for apitools.base.py.batch.""" import textwrap diff --git a/apitools/base/py/buffered_stream.py b/apitools/base/py/buffered_stream.py index bda7e65..a170c86 100644 --- a/apitools/base/py/buffered_stream.py +++ b/apitools/base/py/buffered_stream.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Small helper class to provide a small slice of a stream. This class reads ahead to detect if we are at the end of the stream. diff --git a/apitools/base/py/buffered_stream_test.py b/apitools/base/py/buffered_stream_test.py index 4dcf62f..2098fb1 100644 --- a/apitools/base/py/buffered_stream_test.py +++ b/apitools/base/py/buffered_stream_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for buffered_stream.""" import string diff --git a/apitools/base/py/cli.py b/apitools/base/py/cli.py index eccd66b..920cfc5 100644 --- a/apitools/base/py/cli.py +++ b/apitools/base/py/cli.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Top-level import for all CLI-related functionality in apitools. Note that importing this file will ultimately have side-effects, and diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index ba49e0d..dc0fff3 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Common credentials classes and constructors.""" from __future__ import print_function diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index 067b874..f4a32c2 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import re import mock diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index 62bc96b..a639478 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Common code for converting proto to other formats, such as JSON.""" import base64 diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 9d8281b..1ea4cc1 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import base64 import datetime import json diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index 1d73619..e60c1e0 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Exceptions for generated client libraries.""" diff --git a/apitools/base/py/extra_types.py b/apitools/base/py/extra_types.py index de0183a..79a4900 100644 --- a/apitools/base/py/extra_types.py +++ b/apitools/base/py/extra_types.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Extra types understood by apitools.""" import collections diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py index 6106463..d31e195 100644 --- a/apitools/base/py/extra_types_test.py +++ b/apitools/base/py/extra_types_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import datetime import json import math diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 03a094d..297fd0c 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """HTTP wrapper for apitools. This library wraps the underlying http library we use, which is diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index ddb927a..fbe49c3 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for http_wrapper.""" import unittest2 diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py index 8e15f58..817606d 100644 --- a/apitools/base/py/list_pager.py +++ b/apitools/base/py/list_pager.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """A helper function that executes a series of List queries for many APIs.""" from apitools.base.py import encoding diff --git a/apitools/base/py/list_pager_test.py b/apitools/base/py/list_pager_test.py index a98372f..7ea5729 100644 --- a/apitools/base/py/list_pager_test.py +++ b/apitools/base/py/list_pager_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for list_pager.""" import unittest2 diff --git a/apitools/base/py/stream_slice.py b/apitools/base/py/stream_slice.py index bd43daf..8574be8 100644 --- a/apitools/base/py/stream_slice.py +++ b/apitools/base/py/stream_slice.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Small helper class to provide a small slice of a stream.""" from apitools.base.py import exceptions diff --git a/apitools/base/py/stream_slice_test.py b/apitools/base/py/stream_slice_test.py index f9b13b9..4d5cdfb 100644 --- a/apitools/base/py/stream_slice_test.py +++ b/apitools/base/py/stream_slice_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for stream_slice.""" import string diff --git a/apitools/base/py/testing/__init__.py b/apitools/base/py/testing/__init__.py index 27e204d..d47d726 100644 --- a/apitools/base/py/testing/__init__.py +++ b/apitools/base/py/testing/__init__.py @@ -1 +1,16 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Package marker file.""" diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index be09616..6404f81 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """The mock module allows easy mocking of apitools clients. This module allows you to mock out the constructor of a particular apitools diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py index cff0d01..5250c9a 100644 --- a/apitools/base/py/testing/mock_test.py +++ b/apitools/base/py/testing/mock_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for apitools.base.py.testing.mock.""" import unittest2 diff --git a/apitools/base/py/testing/testclient/__init__.py b/apitools/base/py/testing/testclient/__init__.py index 20aa374..ad24da0 100644 --- a/apitools/base/py/testing/testclient/__init__.py +++ b/apitools/base/py/testing/testclient/__init__.py @@ -1,4 +1,19 @@ """Common imports for generated fusiontables client library.""" +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # pylint:disable=wildcard-import from apitools.base.py.testing.testclient.fusiontables_v1_client import * diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_client.py b/apitools/base/py/testing/testclient/fusiontables_v1_client.py index 45db3c3..816fcad 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_client.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_client.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Modified generated client library for fusiontables version v1. This is a hand-customized and pruned version of the fusiontables v1 diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py index 12cd27e..0111f71 100644 --- a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py +++ b/apitools/base/py/testing/testclient/fusiontables_v1_messages.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Generated message classes for fusiontables version v1. API for working with Fusion Tables data. diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index f144582..3bbb5f7 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Upload and download support for apitools.""" from __future__ import print_function diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 8e29f12..53906fd 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for transfer.py.""" import string diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py index 06e01a2..65a1510 100644 --- a/apitools/base/py/util.py +++ b/apitools/base/py/util.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Assorted utilities shared between parts of apitools.""" import collections diff --git a/apitools/base/py/util_test.py b/apitools/base/py/util_test.py index 4deda8f..a06d1a9 100644 --- a/apitools/base/py/util_test.py +++ b/apitools/base/py/util_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for util.py.""" import unittest2 diff --git a/apitools/data/__init__.py b/apitools/data/__init__.py index 54fa3d5..463cb42 100644 --- a/apitools/data/__init__.py +++ b/apitools/data/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Shared __init__.py for apitools.""" from pkgutil import extend_path diff --git a/apitools/gen/__init__.py b/apitools/gen/__init__.py index 54fa3d5..463cb42 100644 --- a/apitools/gen/__init__.py +++ b/apitools/gen/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Shared __init__.py for apitools.""" from pkgutil import extend_path diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 5de7005..385482e 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Test gen_client against all the APIs we use regularly.""" import logging diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index 9c6ac7b..b3c94ef 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Command registry for apitools.""" import logging diff --git a/apitools/gen/extended_descriptor.py b/apitools/gen/extended_descriptor.py index a59bb86..c5d9909 100644 --- a/apitools/gen/extended_descriptor.py +++ b/apitools/gen/extended_descriptor.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Extended protorpc descriptors. This takes existing protorpc Descriptor classes and adds extra diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 5f2ca58..449657a 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Command-line interface to gen_client.""" import argparse diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 1a066db..674a055 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Simple tool for generating a client library. Relevant links: diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 435c42d..896a829 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Test for gen_client module.""" from apitools.gen import gen_client diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 0a98b36..635c163 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Message registry for apitools.""" import collections diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 20bff15..7920a0f 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Service registry for apitools.""" import collections diff --git a/apitools/gen/test_utils.py b/apitools/gen/test_utils.py index 9778823..92934e7 100644 --- a/apitools/gen/test_utils.py +++ b/apitools/gen/test_utils.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Various utilities used in tests.""" import contextlib diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 10881d7..57cb4c0 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Assorted utilities shared between parts of apitools.""" from __future__ import print_function diff --git a/apitools/gen/util_test.py b/apitools/gen/util_test.py index 93e9596..7cb0739 100644 --- a/apitools/gen/util_test.py +++ b/apitools/gen/util_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for util.""" import unittest2 diff --git a/apitools/scripts/__init__.py b/apitools/scripts/__init__.py index 54fa3d5..463cb42 100644 --- a/apitools/scripts/__init__.py +++ b/apitools/scripts/__init__.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Shared __init__.py for apitools.""" from pkgutil import extend_path diff --git a/apitools/scripts/oauth2l.py b/apitools/scripts/oauth2l.py index 44bb9bc..dd13064 100644 --- a/apitools/scripts/oauth2l.py +++ b/apitools/scripts/oauth2l.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Command-line utility for fetching/inspecting credentials. oauth2l (pronounced "oauthtool") is a small utility for fetching diff --git a/apitools/scripts/oauth2l_test.py b/apitools/scripts/oauth2l_test.py index 8f25b35..e8e0edb 100644 --- a/apitools/scripts/oauth2l_test.py +++ b/apitools/scripts/oauth2l_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Tests for oauth2l.""" import json diff --git a/ez_setup.py b/ez_setup.py index 3756829..be314e4 100755 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this diff --git a/run_pylint.py b/run_pylint.py index d6b89d2..f99bf00 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Custom script to run PyLint on apitools codebase. "Inspired" by the similar script in gcloud-python. diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index 72a4821..9ddaa86 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Integration tests for uploading and downloading to GCS. These tests exercise most of the corner cases for upload/download of diff --git a/samples/storage_sample/generate_clients.sh b/samples/storage_sample/generate_clients.sh index 5744480..f1f9b69 100755 --- a/samples/storage_sample/generate_clients.sh +++ b/samples/storage_sample/generate_clients.sh @@ -1,3 +1,17 @@ #!/bin/bash +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. gen_client --discovery_url=storage.v1 --overwrite --outdir=storage --root_package=. client diff --git a/samples/storage_sample/storage/__init__.py b/samples/storage_sample/storage/__init__.py index 6426fac..0c742c1 100644 --- a/samples/storage_sample/storage/__init__.py +++ b/samples/storage_sample/storage/__init__.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Common imports for generated storage client library.""" # pylint:disable=wildcard-import diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage/storage_v1.py index abb2d31..dfde683 100755 --- a/samples/storage_sample/storage/storage_v1.py +++ b/samples/storage_sample/storage/storage_v1.py @@ -1,4 +1,19 @@ #!/usr/bin/env python +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """CLI for storage, version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage/storage_v1_client.py index 01c2317..9593d50 100644 --- a/samples/storage_sample/storage/storage_v1_client.py +++ b/samples/storage_sample/storage/storage_v1_client.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Generated client library for storage version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage/storage_v1_messages.py index dfc2214..c3923d3 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage/storage_v1_messages.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Generated message classes for storage version v1. Lets you store and retrieve potentially-large, immutable data objects. diff --git a/samples/storage_sample/uploads_test.py b/samples/storage_sample/uploads_test.py index 4eb5aba..cfe6aaa 100644 --- a/samples/storage_sample/uploads_test.py +++ b/samples/storage_sample/uploads_test.py @@ -1,3 +1,18 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Integration tests for uploading and downloading to GCS. These tests exercise most of the corner cases for upload/download of -- GitLab From 16b268360dbc8007ebd4633a4ee0cc73763634d1 Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Tue, 27 Oct 2015 16:10:16 -0700 Subject: [PATCH 184/295] actually call testValidateElement corrected also a couple typos in the test itself that prevented it to run. --- apitools/base/protorpclite/messages_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index b802e1e..faad9bd 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -810,7 +810,7 @@ class FieldTest(test_util.TestCase): messages.IntegerField: 10, messages.FloatField: 1.5, messages.BooleanField: False, - messages.BytesField: 'abc', + messages.BytesField: b'abc', messages.StringField: u'abc', } @@ -825,10 +825,10 @@ class FieldTest(test_util.TestCase): # Repeated. field = field_class(1, repeated=True) - self.assertRaises(message.VAlidationError, + self.assertRaises(messages.ValidationError, field.validate_element, []) - self.assertRaises(message.VAlidationError, + self.assertRaises(messages.ValidationError, field.validate_element, ()) field.validate_element(values[field_class]) @@ -842,6 +842,8 @@ class FieldTest(test_util.TestCase): field.validate_element, (values[field_class],)) + self.ActionOnAllFieldClasses(action) + def testReadOnly(self): """Test that objects are all read-only.""" def action(field_class): -- GitLab From b35601457858692ba5e30810ebaf98f51559e1dd Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Tue, 27 Oct 2015 16:58:13 -0700 Subject: [PATCH 185/295] Relax the int vs float constraint As in JSON encoding they are equivalent, we can let them through. --- apitools/base/protorpclite/messages.py | 4 +++- apitools/base/protorpclite/messages_test.py | 26 +++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index 9880c0c..08d985a 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -1321,7 +1321,9 @@ class Field(six.with_metaclass(_FieldMeta, object)): Raises: ValidationError if value is not expected type. """ - if not isinstance(value, self.type): + if not (isinstance(value, self.type) or + (isinstance(value, (float, int)) and + self.type in (float, int))): if value is None: if self.required: raise ValidationError('Required field is missing') diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index faad9bd..1d6cc28 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -747,7 +747,7 @@ class FieldTest(test_util.TestCase): """Test validation of valid values.""" values = { messages.IntegerField: "10", - messages.FloatField: 1, + messages.FloatField: "blah", messages.BooleanField: 0, messages.BytesField: 10.20, messages.StringField: 42, @@ -807,21 +807,23 @@ class FieldTest(test_util.TestCase): def testValidateElement(self): """Test validation of valid values.""" values = { - messages.IntegerField: 10, - messages.FloatField: 1.5, - messages.BooleanField: False, - messages.BytesField: b'abc', - messages.StringField: u'abc', + messages.IntegerField: (10, -1, 0), + messages.FloatField: (1.5, -1.5, 3), # for json it is all a number. + messages.BooleanField: (True, False), + messages.BytesField: (b'abc',), + messages.StringField: (u'abc',), } def action(field_class): # Optional. field = field_class(1) - field.validate_element(values[field_class]) + for value in values[field_class]: + field.validate_element(value) # Required. field = field_class(1, required=True) - field.validate_element(values[field_class]) + for value in values[field_class]: + field.validate_element(value) # Repeated. field = field_class(1, repeated=True) @@ -831,16 +833,16 @@ class FieldTest(test_util.TestCase): self.assertRaises(messages.ValidationError, field.validate_element, ()) - field.validate_element(values[field_class]) - field.validate_element(values[field_class]) + for value in values[field_class]: + field.validate_element(value) # Right value, but repeated. self.assertRaises(messages.ValidationError, field.validate_element, - [values[field_class]]) + list(values[field_class])) # testing list self.assertRaises(messages.ValidationError, field.validate_element, - (values[field_class],)) + values[field_class]) # testing tuple self.ActionOnAllFieldClasses(action) -- GitLab From 8bfe5d7fabb2ed4138a4a11998f4a73b95c03543 Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 28 Oct 2015 08:54:58 -0700 Subject: [PATCH 186/295] simpler test + py2 compatibility. --- apitools/base/protorpclite/messages.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index 08d985a..066c6ed 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -1321,9 +1321,12 @@ class Field(six.with_metaclass(_FieldMeta, object)): Raises: ValidationError if value is not expected type. """ - if not (isinstance(value, self.type) or - (isinstance(value, (float, int)) and - self.type in (float, int))): + if not isinstance(value, self.type): + + # Authorize int values as float. + if isinstance(value, six.integer_types) and self.type == float: + return + if value is None: if self.required: raise ValidationError('Required field is missing') -- GitLab From 4aebe74de43a1ccb9bd71ab17ba9bb8af06a2f9a Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 28 Oct 2015 09:02:34 -0700 Subject: [PATCH 187/295] Made PEP8 happy. --- apitools/base/protorpclite/messages_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index 1d6cc28..626f511 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -808,7 +808,7 @@ class FieldTest(test_util.TestCase): """Test validation of valid values.""" values = { messages.IntegerField: (10, -1, 0), - messages.FloatField: (1.5, -1.5, 3), # for json it is all a number. + messages.FloatField: (1.5, -1.5, 3), # for json it is all a number messages.BooleanField: (True, False), messages.BytesField: (b'abc',), messages.StringField: (u'abc',), -- GitLab From cb72597adff1eb3ddc5d2888fb517e1301b8e01a Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 28 Oct 2015 11:54:09 -0700 Subject: [PATCH 188/295] Force cast from int to float if we expect float. But not in any other cases. --- apitools/base/protorpclite/messages.py | 37 +++++++++++++++------ apitools/base/protorpclite/messages_test.py | 12 +++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index 066c6ed..6639cad 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -1296,7 +1296,7 @@ class Field(six.with_metaclass(_FieldMeta, object)): if self.repeated: value = FieldList(self, value) else: - self.validate(value) + value = self.validate(value) message_instance._Message__tags[self.number] = value def __get__(self, message_instance, message_class): @@ -1318,6 +1318,9 @@ class Field(six.with_metaclass(_FieldMeta, object)): Args: value: Value to validate. + Returns: + The value casted in the expected type. + Raises: ValidationError if value is not expected type. """ @@ -1325,7 +1328,7 @@ class Field(six.with_metaclass(_FieldMeta, object)): # Authorize int values as float. if isinstance(value, six.integer_types) and self.type == float: - return + return float(value) if value is None: if self.required: @@ -1342,6 +1345,7 @@ class Field(six.with_metaclass(_FieldMeta, object)): raise ValidationError( 'Expected type %s for field %s, found %s (type %s)' % (self.type, name, value, type(value))) + return value def __validate(self, value, validate_element): """Internal validation function. @@ -1358,10 +1362,11 @@ class Field(six.with_metaclass(_FieldMeta, object)): """ if not self.repeated: - validate_element(value) + return validate_element(value) else: # Must be a list or tuple, may not be a string. if isinstance(value, (list, tuple)): + result = [] for element in value: if element is None: try: @@ -1374,7 +1379,8 @@ class Field(six.with_metaclass(_FieldMeta, object)): raise ValidationError( 'Repeated values for field %s ' 'may not be None' % name) - validate_element(element) + result.append(validate_element(element)) + return result elif value is not None: try: name = self.name @@ -1384,6 +1390,7 @@ class Field(six.with_metaclass(_FieldMeta, object)): else: raise ValidationError( 'Field %s is repeated. Found: %s' % (name, value)) + return value def validate(self, value): """Validate value assigned to field. @@ -1391,10 +1398,13 @@ class Field(six.with_metaclass(_FieldMeta, object)): Args: value: Value to validate. + Returns: + the value in casted in the correct type. + Raises: ValidationError if value is not expected type. """ - self.__validate(value, self.validate_element) + return self.__validate(value, self.validate_element) def validate_default_element(self, value): """Validate value as assigned to field default field. @@ -1408,11 +1418,14 @@ class Field(six.with_metaclass(_FieldMeta, object)): Args: value: Default value to validate. + Returns: + the value in casted in the correct type. + Raises: ValidationError if value is not expected type. """ - self.validate_element(value) + return self.validate_element(value) def validate_default(self, value): """Validate default value assigned to field. @@ -1420,10 +1433,13 @@ class Field(six.with_metaclass(_FieldMeta, object)): Args: value: Value to validate. + Returns: + the value in casted in the correct type. + Raises: ValidationError if value is not expected type. """ - self.__validate(value, self.validate_default_element) + return self.__validate(value, self.validate_default_element) def message_definition(self): """Get Message definition that contains this Field definition. @@ -1531,7 +1547,8 @@ class StringField(Field): validation_error.field_name = self.name raise validation_error else: - super(StringField, self).validate_element(value) + return super(StringField, self).validate_element(value) + return value class MessageField(Field): @@ -1794,9 +1811,9 @@ class EnumField(Field): # enumerated types. Ignore if type is not yet resolved. if self.__type: self.__type(value) - return + return value - super(EnumField, self).validate_default_element(value) + return super(EnumField, self).validate_default_element(value) @property def type(self): diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index 626f511..1f961de 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -846,6 +846,18 @@ class FieldTest(test_util.TestCase): self.ActionOnAllFieldClasses(action) + def testValidateCastingElement(self): + field = messages.FloatField(1) + self.assertEquals(type(field.validate_element(12)), float) + self.assertEquals(type(field.validate_element(12.0)), float) + field = messages.IntegerField(1) + self.assertEquals(type(field.validate_element(12)), int) + self.assertRaises(messages.ValidationError, + field.validate_element, + 12.0) # should fails from float to int + + + def testReadOnly(self): """Test that objects are all read-only.""" def action(field_class): -- GitLab From 6c2f024acbf056ba92a283858423f51f279e54d1 Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 28 Oct 2015 11:56:21 -0700 Subject: [PATCH 189/295] pep8 --- apitools/base/protorpclite/messages_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index 1f961de..b0c7cd6 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -857,7 +857,6 @@ class FieldTest(test_util.TestCase): 12.0) # should fails from float to int - def testReadOnly(self): """Test that objects are all read-only.""" def action(field_class): -- GitLab From 7794ff923e02e5b0df867104c00347b770821f0f Mon Sep 17 00:00:00 2001 From: Guillaume Binet Date: Wed, 28 Oct 2015 14:07:32 -0700 Subject: [PATCH 190/295] more linting happiness. --- apitools/base/protorpclite/messages_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index b0c7cd6..0414315 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -856,7 +856,6 @@ class FieldTest(test_util.TestCase): field.validate_element, 12.0) # should fails from float to int - def testReadOnly(self): """Test that objects are all read-only.""" def action(field_class): -- GitLab From 22f18655c6a2996651bde41494fa6cc6f93e272a Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 28 Oct 2015 14:40:23 -0700 Subject: [PATCH 191/295] Make `tox -e lint` faster in the common case. This cuts lint checks down from ~28s to ~4s on my machine (with a massive sample size of N=3). --- run_pylint.py | 25 ++++++++++++------------- tox.ini | 6 +++--- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/run_pylint.py b/run_pylint.py index d6b89d2..e081b23 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -96,7 +96,7 @@ def is_production_filename(filename): filename.startswith('regression')) -def get_files_for_linting(allow_limited=True): +def get_files_for_linting(allow_limited=True, diff_base=None): """Gets a list of files in the repository. By default, returns all files via ``git ls-files``. However, in some cases @@ -121,18 +121,15 @@ def get_files_for_linting(allow_limited=True): :returns: Tuple of the diff base using the the list of filenames to be linted. """ - diff_base = None + if os.getenv('TRAVIS') == 'true': + # In travis, don't default to master. + diff_base = None + if (os.getenv('TRAVIS_BRANCH') == 'master' and os.getenv('TRAVIS_PULL_REQUEST') != 'false'): # In the case of a pull request into master, we want to # diff against HEAD in master. diff_base = 'origin/master' - elif os.getenv('TRAVIS') is None: - # Only allow specified remote and branch in local dev. - remote = os.getenv('GCLOUD_REMOTE_FOR_LINT') - branch = os.getenv('GCLOUD_BRANCH_FOR_LINT') - if remote is not None and branch is not None: - diff_base = '%s/%s' % (remote, branch) if diff_base is not None and allow_limited: result = subprocess.check_output(['git', 'diff', '--name-only', @@ -148,7 +145,7 @@ def get_files_for_linting(allow_limited=True): return result.rstrip('\n').split('\n'), diff_base -def get_python_files(all_files=None): +def get_python_files(all_files=None, diff_base=None): """Gets a list of all Python files in the repository that need linting. Relies on :func:`get_files_for_linting()` to determine which files should @@ -167,7 +164,7 @@ def get_python_files(all_files=None): """ using_restricted = False if all_files is None: - all_files, diff_base = get_files_for_linting() + all_files, diff_base = get_files_for_linting(diff_base=diff_base) using_restricted = diff_base is not None library_files = [] @@ -203,10 +200,12 @@ def lint_fileset(filenames, rcfile, description): print 'Skipping %s, no files to lint.' % (description,) -def main(): +def main(argv): """Script entry point. Lints both sets of files.""" + diff_base = argv[1] if len(argv) > 1 else None make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC) - library_files, non_library_files, using_restricted = get_python_files() + library_files, non_library_files, using_restricted = get_python_files( + diff_base=diff_base) try: lint_fileset(library_files, PRODUCTION_RC, 'library code') lint_fileset(non_library_files, TEST_RC, 'test and demo code') @@ -224,4 +223,4 @@ def main(): if __name__ == '__main__': - main() + main(sys.argv) diff --git a/tox.ini b/tox.ini index 8baa1c8..b9010d3 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ deps = nose commands = pip install google-apitools[testing] nosetests [] +passenv = TRAVIS* [testenv:py33] basepython = python3.3 @@ -24,7 +25,7 @@ deps = commands = nosetests [] [pep8] -exclude = samples/storage_sample/storage,samples/storage_sample/testdata,*.egg/,.*/,ez_setup.py +exclude = samples/storage_sample/storage,samples/storage_sample/testdata,*.egg/,*.egg-info/,.*/,ez_setup.py,build verbose = 1 [testenv:lint] @@ -32,7 +33,7 @@ basepython = python2.7 commands = pep8 - python run_pylint.py + python run_pylint.py master deps = pep8 pylint @@ -60,7 +61,6 @@ commands = deps = {[testenv:cover]deps} coveralls -passenv = TRAVIS* [testenv:transfer_coverage] basepython = -- GitLab From 06fc9651d294d31b7a824a16f04a37822f8bc0c6 Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Tue, 3 Nov 2015 13:13:09 -0800 Subject: [PATCH 192/295] Support custom check_response_func in BaseApiClient This lets apis overwrite the default check_response_func used by MakeRequest, e.g. to change retry behavior for specific http error codes. --- apitools/base/py/base_api.py | 21 ++++++++++++++++++--- apitools/base/py/base_api_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index de3a9a3..e3d54ef 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -237,7 +237,8 @@ class BaseApiClient(object): def __init__(self, url, credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, num_retries=5, max_retry_wait=60, credentials_args=None, - default_global_params=None, additional_http_headers=None): + default_global_params=None, additional_http_headers=None, + check_response_func=None): _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) if default_global_params is not None: util.Typecheck(default_global_params, self.params_type) @@ -263,6 +264,7 @@ class BaseApiClient(object): self.__include_fields = None self.additional_http_headers = additional_http_headers or {} + self.__check_response_func = check_response_func # TODO(craigcitro): Finish deprecating these fields. _ = model @@ -375,6 +377,14 @@ class BaseApiClient(object): 'Cannot have negative value for num_retries') self.__num_retries = value + @property + def check_response_func(self): + return self.__check_response_func + + @check_response_func.setter + def check_response_func(self, value): + self.__check_response_func = value + @property def max_retry_wait(self): return self.__max_retry_wait @@ -685,9 +695,14 @@ class BaseApiService(object): http = self.__client.http if upload and upload.bytes_http: http = upload.bytes_http + opts = { + 'retries': self.__client.num_retries, + 'max_retry_wait': self.__client.max_retry_wait, + } + if self.__client.check_response_func: + opts['check_response_func'] = self.__client.check_response_func http_response = http_wrapper.MakeRequest( - http, http_request, retries=self.__client.num_retries, - max_retry_wait=self.__client.max_retry_wait) + http, http_request, **opts) return self.ProcessHttpResponse(method_config, http_response) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index 96c425b..eb8f6dc 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -16,6 +16,7 @@ import base64 import datetime import sys +import contextlib import six from six.moves import urllib_parse @@ -28,6 +29,16 @@ from apitools.base.py import encoding from apitools.base.py import http_wrapper +@contextlib.contextmanager +def mock(module, fn_name, patch): + unpatch = getattr(module, fn_name) + setattr(module, fn_name, patch) + try: + yield + finally: + setattr(module, fn_name, unpatch) + + class SimpleMessage(messages.Message): field = messages.StringField(1) bytes_field = messages.BytesField(2) @@ -134,6 +145,26 @@ class BaseApiTest(unittest2.TestCase): new_request = client.ProcessHttpRequest(http_request) self.assertTrue('Request-Is-Awesome' in new_request.headers) + def testCustomCheckResponse(self): + def check_response(): + pass + + def fakeMakeRequest(*_, **kwargs): + self.assertEqual(check_response, kwargs['check_response_func']) + return http_wrapper.Response( + info={'status': '200'}, content='{"field": "abc"}', + request_url='http://www.google.com') + http_wrapper.MakeRequest = fakeMakeRequest + method_config = base_api.ApiMethodInfo( + request_type_name='SimpleMessage', + response_type_name='SimpleMessage') + client = self.__GetFakeClient() + client.check_response_func = check_response + service = FakeService(client=client) + request = SimpleMessage() + with mock(base_api.http_wrapper, 'MakeRequest', fakeMakeRequest): + service._RunMethod(method_config, request) + def testQueryEncoding(self): method_config = base_api.ApiMethodInfo( request_type_name='MessageWithTime', query_params=['timestamp']) -- GitLab From 44fb5f7bd966aba349aa951a778f5883185e4963 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 6 Nov 2015 11:30:44 -0500 Subject: [PATCH 193/295] Updating message_registry.py to properly use base_package. Currently the base_package parameter is being used inconsistently as follows: - base_package is considered 'apitools.base.py' instead of 'apitools.base'. - protorpclite imports are hardcoded instead of using 'base_package.protorpclite' This commit fixes this usage in message_registry.py. Following commits will fix this problem in other registries. --- apitools/gen/message_registry.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 635c163..5e12d4f 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -89,7 +89,8 @@ class MessageRegistry(object): package=self.__package, description=self.__description) # Add required imports self.__file_descriptor.additional_imports = [ - 'from apitools.base.protorpclite import messages as _messages', + 'from %s.protorpclite import messages as _messages' % + self.__base_files_package, ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() @@ -207,7 +208,7 @@ class MessageRegistry(object): message.enum_mappings.append( extended_descriptor.ExtendedEnumDescriptor.JsonEnumMapping( python_name=enum_value.name, json_name=enum_name)) - self.__AddImport('from %s import encoding' % + self.__AddImport('from %s.py import encoding' % self.__base_files_package) enum_value.number = index enum_value.description = util.CleanDescription( @@ -222,7 +223,7 @@ class MessageRegistry(object): message.name = self.__names.ClassName(schema['id']) message.alias_for = alias_for self.__DeclareDescriptor(message.name) - self.__AddImport('from %s import extra_types' % + self.__AddImport('from %s.py import extra_types' % self.__base_files_package) self.__RegisterDescriptor(message) @@ -245,7 +246,7 @@ class MessageRegistry(object): field_name = 'additionalProperties' message.fields.append(self.__FieldDescriptorFromProperties( field_name, len(properties) + 1, attrs)) - self.__AddImport('from %s import encoding' % self.__base_files_package) + self.__AddImport('from %s.py import encoding' % self.__base_files_package) message.decorators.append( 'encoding.MapUnrecognizedFields(%r)' % field_name) @@ -279,7 +280,7 @@ class MessageRegistry(object): type(message).JsonFieldMapping( python_name=field.name, json_name=name)) self.__AddImport( - 'from %s import encoding' % self.__base_files_package) + 'from %s.py import encoding' % self.__base_files_package) if 'additionalProperties' in schema: self.__AddAdditionalProperties(message, schema, properties) self.__RegisterDescriptor(message) @@ -412,11 +413,11 @@ class MessageRegistry(object): 'apitools.base.protorpclite.message_types.', 'message_types.')): self.__AddImport( - 'from apitools.base.protorpclite import message_types ' - 'as _message_types') + 'from %s.protorpclite import message_types ' + 'as _message_types' % self.__base_files_package) if type_info.type_name.startswith('extra_types.'): self.__AddImport( - 'from %s import extra_types' % self.__base_files_package) + 'from %s.py import extra_types' % self.__base_files_package) return type_info if type_name in self.PRIMITIVE_TYPE_INFO_MAP: @@ -440,7 +441,7 @@ class MessageRegistry(object): else: return self.__GetTypeInfo(items, entry_name_hint) elif type_name == 'any': - self.__AddImport('from %s import extra_types' % + self.__AddImport('from %s.py import extra_types' % self.__base_files_package) return self.PRIMITIVE_TYPE_INFO_MAP['any'] elif type_name == 'object': -- GitLab From 616b422f7672b97a1575f53a5c57eff5638ee33f Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 6 Nov 2015 11:41:56 -0500 Subject: [PATCH 194/295] Undoing changes from previous commit. Decided to go with the less disruptive alternate approach of passing in a separate protorpclite base parameter. --- apitools/gen/message_registry.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 5e12d4f..635c163 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -89,8 +89,7 @@ class MessageRegistry(object): package=self.__package, description=self.__description) # Add required imports self.__file_descriptor.additional_imports = [ - 'from %s.protorpclite import messages as _messages' % - self.__base_files_package, + 'from apitools.base.protorpclite import messages as _messages', ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() @@ -208,7 +207,7 @@ class MessageRegistry(object): message.enum_mappings.append( extended_descriptor.ExtendedEnumDescriptor.JsonEnumMapping( python_name=enum_value.name, json_name=enum_name)) - self.__AddImport('from %s.py import encoding' % + self.__AddImport('from %s import encoding' % self.__base_files_package) enum_value.number = index enum_value.description = util.CleanDescription( @@ -223,7 +222,7 @@ class MessageRegistry(object): message.name = self.__names.ClassName(schema['id']) message.alias_for = alias_for self.__DeclareDescriptor(message.name) - self.__AddImport('from %s.py import extra_types' % + self.__AddImport('from %s import extra_types' % self.__base_files_package) self.__RegisterDescriptor(message) @@ -246,7 +245,7 @@ class MessageRegistry(object): field_name = 'additionalProperties' message.fields.append(self.__FieldDescriptorFromProperties( field_name, len(properties) + 1, attrs)) - self.__AddImport('from %s.py import encoding' % self.__base_files_package) + self.__AddImport('from %s import encoding' % self.__base_files_package) message.decorators.append( 'encoding.MapUnrecognizedFields(%r)' % field_name) @@ -280,7 +279,7 @@ class MessageRegistry(object): type(message).JsonFieldMapping( python_name=field.name, json_name=name)) self.__AddImport( - 'from %s.py import encoding' % self.__base_files_package) + 'from %s import encoding' % self.__base_files_package) if 'additionalProperties' in schema: self.__AddAdditionalProperties(message, schema, properties) self.__RegisterDescriptor(message) @@ -413,11 +412,11 @@ class MessageRegistry(object): 'apitools.base.protorpclite.message_types.', 'message_types.')): self.__AddImport( - 'from %s.protorpclite import message_types ' - 'as _message_types' % self.__base_files_package) + 'from apitools.base.protorpclite import message_types ' + 'as _message_types') if type_info.type_name.startswith('extra_types.'): self.__AddImport( - 'from %s.py import extra_types' % self.__base_files_package) + 'from %s import extra_types' % self.__base_files_package) return type_info if type_name in self.PRIMITIVE_TYPE_INFO_MAP: @@ -441,7 +440,7 @@ class MessageRegistry(object): else: return self.__GetTypeInfo(items, entry_name_hint) elif type_name == 'any': - self.__AddImport('from %s.py import extra_types' % + self.__AddImport('from %s import extra_types' % self.__base_files_package) return self.PRIMITIVE_TYPE_INFO_MAP['any'] elif type_name == 'object': -- GitLab From 59471ffd9a616eb4dc934af7b7163f38f3450ee8 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 6 Nov 2015 11:48:23 -0500 Subject: [PATCH 195/295] Add protorpc_package parameter. --- apitools/gen/gen_client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 449657a..918d69a 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -103,6 +103,7 @@ def _GetCodegenFromFlags(args): return gen_client_lib.DescriptorGenerator( discovery_doc, client_info, names, args.root_package, outdir, base_package=args.base_package, + protorpc_package=args.protorpc_package, generate_cli=args.generate_cli, use_proto2=args.experimental_proto2_output, unelidable_request_methods=args.unelidable_request_methods) @@ -221,6 +222,11 @@ def main(argv=None): '--base_package', default='apitools.base.py', help='Base package path of apitools (defaults to apitools.base.py') + + parser.add_argument( + '--protorpc_package', + default='apitools.base.protorpclite', + help='Base package path of protorpc (defaults to apitools.base.protorpclite') parser.add_argument( '--outdir', -- GitLab From 7be9ba9056250229276584278b3b857b8b827428 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 6 Nov 2015 15:05:20 -0500 Subject: [PATCH 196/295] Pass the protoprc_package parameter through to the appropriate registries and use it in generated import statements. --- apitools/gen/command_registry.py | 8 +++++--- apitools/gen/gen_client.py | 2 +- apitools/gen/gen_client_lib.py | 11 +++++++---- apitools/gen/message_registry.py | 11 ++++++----- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index b3c94ef..ad2aa00 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -116,7 +116,8 @@ class CommandRegistry(object): """Registry for CLI commands.""" def __init__(self, package, version, client_info, message_registry, - root_package, base_files_package, base_url, names): + root_package, base_files_package, protorpc_package, + base_url, names): self.__package = package self.__version = version self.__client_info = client_info @@ -124,6 +125,7 @@ class CommandRegistry(object): self.__message_registry = message_registry self.__root_package = root_package self.__base_files_package = base_files_package + self.__protorpc_package = protorpc_package self.__base_url = base_url self.__command_list = [] self.__global_flags = [] @@ -476,8 +478,8 @@ class CommandRegistry(object): printer('import platform') printer('import sys') printer() - printer('from apitools.base.protorpclite import message_types') - printer('from apitools.base.protorpclite import messages') + printer('from %s import message_types', self.__protorpc_package) + printer('from %s import messages', self.__protorpc_package) printer() appcommands_import = 'from google.apputils import appcommands' printer(appcommands_import) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 918d69a..b8ad3f9 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -222,7 +222,7 @@ def main(argv=None): '--base_package', default='apitools.base.py', help='Base package path of apitools (defaults to apitools.base.py') - + parser.add_argument( '--protorpc_package', default='apitools.base.protorpclite', diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 674a055..742e0eb 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -63,8 +63,8 @@ class DescriptorGenerator(object): """Code generator for a given discovery document.""" def __init__(self, discovery_doc, client_info, names, root_package, outdir, - base_package, generate_cli=False, use_proto2=False, - unelidable_request_methods=None): + base_package, protorpc_package, generate_cli=False, + use_proto2=False, unelidable_request_methods=None): self.__discovery_doc = discovery_doc self.__client_info = client_info self.__outdir = outdir @@ -77,6 +77,7 @@ class DescriptorGenerator(object): self.__generate_cli = generate_cli self.__root_package = root_package self.__base_files_package = base_package + self.__protorpc_package = protorpc_package self.__names = names self.__base_url, self.__base_path = _ComputePaths( self.__package, self.__client_info.url_version, @@ -86,7 +87,8 @@ class DescriptorGenerator(object): # define the services. self.__message_registry = message_registry.MessageRegistry( self.__client_info, self.__names, self.__description, - self.__root_package, self.__base_files_package) + self.__root_package, self.__base_files_package, + self.__protorpc_package) schemas = self.__discovery_doc.get('schemas', {}) for schema_name, schema in schemas.items(): self.__message_registry.AddDescriptorFromSchema( @@ -105,7 +107,8 @@ class DescriptorGenerator(object): self.__command_registry = command_registry.CommandRegistry( self.__package, self.__version, self.__client_info, self.__message_registry, self.__root_package, - self.__base_files_package, self.__base_url, self.__names) + self.__base_files_package, self.__protorpc_package, + self.__base_url, self.__names) self.__command_registry.AddGlobalParameters( self.__message_registry.LookupDescriptorOrDie( 'StandardQueryParameters')) diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 635c163..461b72a 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -77,19 +77,20 @@ class MessageRegistry(object): variant=messages.Variant.MESSAGE), } - def __init__(self, client_info, names, description, - root_package_dir, base_files_package): + def __init__(self, client_info, names, description, root_package_dir, + base_files_package, protorpc_package): self.__names = names self.__client_info = client_info self.__package = client_info.package self.__description = util.CleanDescription(description) self.__root_package_dir = root_package_dir self.__base_files_package = base_files_package + self.__protorpc_package = protorpc_package self.__file_descriptor = extended_descriptor.ExtendedFileDescriptor( package=self.__package, description=self.__description) # Add required imports self.__file_descriptor.additional_imports = [ - 'from apitools.base.protorpclite import messages as _messages', + 'from %s import messages as _messages' % self.__protorpc_package, ] # Map from scoped names (i.e. Foo.Bar) to MessageDescriptors. self.__message_registry = collections.OrderedDict() @@ -412,8 +413,8 @@ class MessageRegistry(object): 'apitools.base.protorpclite.message_types.', 'message_types.')): self.__AddImport( - 'from apitools.base.protorpclite import message_types ' - 'as _message_types') + 'from %s import message_types as _message_types' % + self.__protorpc_package) if type_info.type_name.startswith('extra_types.'): self.__AddImport( 'from %s import extra_types' % self.__base_files_package) -- GitLab From 6fcd8ab39b91e262ad1439c7f0b333ebabec36a2 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Fri, 6 Nov 2015 15:28:11 -0500 Subject: [PATCH 197/295] Shortening long line. --- apitools/gen/gen_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index b8ad3f9..97e9d5d 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -226,7 +226,8 @@ def main(argv=None): parser.add_argument( '--protorpc_package', default='apitools.base.protorpclite', - help='Base package path of protorpc (defaults to apitools.base.protorpclite') + help=('Base package path of protorpc ' + '(defaults to apitools.base.protorpclite')) parser.add_argument( '--outdir', -- GitLab From 7f1e65f2d9f18504831e3aa4b10b342e211a8aec Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Tue, 10 Nov 2015 11:30:30 -0800 Subject: [PATCH 198/295] Update for v0.4.12 release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 89fa5d0..c744470 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.11' +_APITOOLS_VERSION = '0.4.12' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 5dbc311949542e8e462021d79db4f2e39f13fcb3 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 16 Nov 2015 16:16:14 -0800 Subject: [PATCH 199/295] Keep all unknown field keys as strings. Previously, protorpc would keep some keys as ints (for compatibility with JS proto libraries). This doesn't help us, and causes issues, so we drop it. Fixes #73. --- apitools/base/protorpclite/protojson.py | 2 -- apitools/base/protorpclite/protojson_test.py | 5 +++-- apitools/base/py/encoding_test.py | 5 +++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py index 6cb52c8..5f611f4 100644 --- a/apitools/base/protorpclite/protojson.py +++ b/apitools/base/protorpclite/protojson.py @@ -271,8 +271,6 @@ class ProtoJson(object): # Save unknown values. variant = self.__find_variant(value) if variant: - if key.isdigit(): - key = int(key) message.set_unrecognized_field(key, value, variant) else: logging.warning( diff --git a/apitools/base/protorpclite/protojson_test.py b/apitools/base/protorpclite/protojson_test.py index c961368..a349710 100644 --- a/apitools/base/protorpclite/protojson_test.py +++ b/apitools/base/protorpclite/protojson_test.py @@ -298,9 +298,10 @@ class ProtojsonTest(test_util.TestCase, '"456_mixed": 2}') self.assertEquals(decoded.an_integer, 1) self.assertEquals(3, len(decoded.all_unrecognized_fields())) - self.assertTrue(1001 in decoded.all_unrecognized_fields()) + self.assertFalse(1001 in decoded.all_unrecognized_fields()) + self.assertTrue('1001' in decoded.all_unrecognized_fields()) self.assertEquals(('unknown', messages.Variant.STRING), - decoded.get_unrecognized_field_info(1001)) + decoded.get_unrecognized_field_info('1001')) self.assertTrue('-123' in decoded.all_unrecognized_fields()) self.assertEquals(('negative', messages.Variant.STRING), decoded.get_unrecognized_field_info('-123')) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index 1ea4cc1..e6b067a 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -223,6 +223,11 @@ class EncodingTest(unittest2.TestCase): self.assertEqual(1, len(new_msg.additional_properties)) self.assertEqual(2, len(msg.additional_properties)) + def testNumericPropertyName(self): + json_msg = '{"nested": {"123": "def"}}' + msg = encoding.JsonToMessage(HasNestedMessage, json_msg) + self.assertEqual(1, len(msg.nested.additional_properties)) + def testAdditionalMessageProperties(self): json_msg = '{"input": {"index": 0, "name": "output"}}' result = encoding.JsonToMessage( -- GitLab From 48913df20c254f5e1510ebf6e9ed981fa11bf837 Mon Sep 17 00:00:00 2001 From: Travis Hobrla Date: Wed, 18 Nov 2015 17:32:21 -0800 Subject: [PATCH 200/295] Handle transient oauth2client refresh failures in http_wrapper Fixes https://github.com/google/apitools/issues/75 This changes http_wrapper's default retry function to recover gracefully when encountering an HttpAccessTokenRefreshError. Sync oauth2client to >= v1.5.2, since it has this new exception type. --- apitools/base/py/http_wrapper.py | 8 +++++ apitools/base/py/http_wrapper_test.py | 47 +++++++++++++++++++++++++++ setup.py | 2 +- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 297fd0c..0f7c150 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -27,6 +27,7 @@ import socket import time import httplib2 +import oauth2client import six from six.moves import http_client from six.moves.urllib import parse @@ -277,6 +278,13 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): # oauth2client, need to handle it here. logging.debug('Response content was invalid (%s), retrying', retry_args.exc) + elif (isinstance(retry_args.exc, + oauth2client.client.HttpAccessTokenRefreshError) and + (retry_args.exc.status == TOO_MANY_REQUESTS or + retry_args.exc.status >= 500)): + logging.debug( + 'Caught transient credential refresh error (%s), retrying', + retry_args.exc) elif isinstance(retry_args.exc, exceptions.RequestError): logging.debug('Request returned no response, retrying') # API-level failures diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index fbe49c3..a6edf43 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -14,10 +14,29 @@ # limitations under the License. """Tests for http_wrapper.""" +import httplib2 +import oauth2client +import socket import unittest2 +from mock import patch + +from apitools.base.py import exceptions from apitools.base.py import http_wrapper +from six.moves import http_client + + +class _MockHttpRequest(object): + + url = None + + +class _MockHttpResponse(object): + + def __init__(self, status_code): + self.response = {'status': status_code} + class RaisesExceptionOnLen(object): @@ -37,3 +56,31 @@ class HttpWrapperTest(unittest2.TestCase): def testRequestBodyWithLen(self): http_wrapper.Request(body='burrito') + + def testDefaultExceptionHandler(self): + """Ensures exception handles swallows (retries)""" + mock_http_content = 'content'.encode('utf8') + for exception_arg in ( + http_client.BadStatusLine('line'), + http_client.IncompleteRead('partial'), + http_client.ResponseNotReady(), + socket.error(), + socket.gaierror(), + httplib2.ServerNotFoundError(), + ValueError(), + oauth2client.client.HttpAccessTokenRefreshError(status=503), + exceptions.RequestError(), + exceptions.BadStatusCodeError( + {'status': 503}, mock_http_content, 'url'), + exceptions.RetryAfterError( + {'status': 429}, mock_http_content, 'url', 0)): + + retry_args = http_wrapper.ExceptionRetryArgs( + http={'connections': {}}, http_request=_MockHttpRequest(), + exc=exception_arg, num_retries=0, max_retry_wait=0) + + # Disable time.sleep for this handler as it is called with + # a minimum value of 1 second. + with patch('time.sleep', return_value=None): + http_wrapper.HandleExceptionsAndRebuildHttpConnections( + retry_args) diff --git a/setup.py b/setup.py index c744470..7a62f2b 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ except ImportError: # Python version and OS. REQUIRED_PACKAGES = [ 'httplib2>=0.8', - 'oauth2client>=1.4.8', + 'oauth2client>=1.5.2', 'six>=1.9.0', ] -- GitLab From 002457f03006cbd9bd3a07c60939a8075ccfa685 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Mon, 23 Nov 2015 08:12:59 -0800 Subject: [PATCH 201/295] Client libs with apitools pinned to the version used to generate them This change does three things: 1) Client packages will have the version APITOOLS_VERSION.CLIENT_REVISION_DATE (e.g., 0.4.21.20151102) 2) Client packages have the dependency on google-apitools pinned to the version used to generate them. 3) Bumped up version of google-apitools to 0.4.21 to cover the few client packages out there that have an older format for version (0.4.REVISION_DATA). Version 0.4.21 will be newer than versions like 0.4.2015MMDD so more recent packages will be installed. --- apitools/gen/gen_client_lib.py | 11 ++++++++--- setup.py | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 742e0eb..ae09acc 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -30,6 +30,12 @@ from apitools.gen import service_registry from apitools.gen import util +def _ApitoolsVersion(): + """Returns version of the currently installed google-apitools package.""" + import pkg_resources + return pkg_resources.get_distribution('google-apitools').version + + def _StandardQueryParametersSchema(discovery_doc): """Sets up dict of standard query parameters.""" standard_query_schema = { @@ -223,8 +229,7 @@ class DescriptorGenerator(object): printer('import setuptools') printer('REQUIREMENTS = [') with printer.Indent(indent=' '): - # TODO(craigcitro): Have this track apitools' version. - printer('"google-apitools>=0.4.8",') + printer('"google-apitools==%s",', _ApitoolsVersion()) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') @@ -235,7 +240,7 @@ class DescriptorGenerator(object): with printer.Indent(indent=' '): printer('name="google-apitools-%s-%s",', self.__package, self.__version) - printer('version="0.4.%s",', self.__revision) + printer('version="%s.%s",', _ApitoolsVersion(), self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') diff --git a/setup.py b/setup.py index c744470..59274ff 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ except ImportError: REQUIRED_PACKAGES = [ 'httplib2>=0.8', 'oauth2client>=1.4.8', + 'setuptools>=18.5', 'six>=1.9.0', ] @@ -54,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.12' +_APITOOLS_VERSION = '0.4.21' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 832009cfa361e34ca1789662a3401477faa3ae54 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Mon, 23 Nov 2015 14:33:28 -0800 Subject: [PATCH 202/295] Pin version of apitools for >=0.5 keep old behavior for earlier versions. In order to keep compatibility with client packages already out there we keep the behavior for clients built with apitools having version 0.4.N. If apitools is version 0.4.N the clients will have version 0.4.REVISION_DATE and apitools will be a requirement with >=0.4.8, <0.5. If apitools is version >= 0.5 the clients will have version 0.5.N.REVISION_DATE and apitools will be a requirement with 0.5.N (pinned version). --- apitools/gen/gen_client_lib.py | 11 +++++++++-- setup.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index ae09acc..326a3b7 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -206,6 +206,7 @@ class DescriptorGenerator(object): """Write a setup.py for upload to PyPI.""" printer = self._GetPrinter(out) year = datetime.datetime.now().year + apitools_version = _ApitoolsVersion() printer('# Copyright %s Google Inc. All Rights Reserved.' % year) printer('#') printer('# Licensed under the Apache License, Version 2.0 (the' @@ -229,7 +230,10 @@ class DescriptorGenerator(object): printer('import setuptools') printer('REQUIREMENTS = [') with printer.Indent(indent=' '): - printer('"google-apitools==%s",', _ApitoolsVersion()) + if apitools_version.startswith('0.4'): + printer('"google-apitools>=0.4.8,<0.5",') + else: + printer('"google-apitools==%s",', apitools_version) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') @@ -240,7 +244,10 @@ class DescriptorGenerator(object): with printer.Indent(indent=' '): printer('name="google-apitools-%s-%s",', self.__package, self.__version) - printer('version="%s.%s",', _ApitoolsVersion(), self.__revision) + if apitools_version.startswith('0.4'): + printer('version="0.4.%s",', self.__revision) + else: + printer('version="%s.%s",', apitools_version, self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') diff --git a/setup.py b/setup.py index 59274ff..957979c 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.21' +_APITOOLS_VERSION = '0.4.13' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From dd698b124f60e94f24c60ff5e518c2536ff7a9f0 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Mon, 23 Nov 2015 15:14:51 -0800 Subject: [PATCH 203/295] Update version to 0.4.14 We skip 0.4.13 because it was used for a "rollback" of 0.4.12 which introduced incompatible changes with existing client libraries. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 957979c..10505a1 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.13' +_APITOOLS_VERSION = '0.4.14' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 4390b7c96cbff631f89246543b8c29f681a3de4e Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Tue, 24 Nov 2015 07:25:07 -0800 Subject: [PATCH 204/295] Fix lint errors --- apitools/gen/gen_client_lib.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 326a3b7..f109ba9 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -31,9 +31,9 @@ from apitools.gen import util def _ApitoolsVersion(): - """Returns version of the currently installed google-apitools package.""" - import pkg_resources - return pkg_resources.get_distribution('google-apitools').version + """Returns version of the currently installed google-apitools package.""" + import pkg_resources + return pkg_resources.get_distribution('google-apitools').version def _StandardQueryParametersSchema(discovery_doc): @@ -231,9 +231,9 @@ class DescriptorGenerator(object): printer('REQUIREMENTS = [') with printer.Indent(indent=' '): if apitools_version.startswith('0.4'): - printer('"google-apitools>=0.4.8,<0.5",') + printer('"google-apitools>=0.4.8,<0.5",') else: - printer('"google-apitools==%s",', apitools_version) + printer('"google-apitools==%s",', apitools_version) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') @@ -245,9 +245,9 @@ class DescriptorGenerator(object): printer('name="google-apitools-%s-%s",', self.__package, self.__version) if apitools_version.startswith('0.4'): - printer('version="0.4.%s",', self.__revision) + printer('version="0.4.%s",', self.__revision) else: - printer('version="%s.%s",', apitools_version, self.__revision) + printer('version="%s.%s",', apitools_version, self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') -- GitLab From 938dd67b810fc1d51a9db43f9523eced7c317cd4 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Tue, 24 Nov 2015 08:09:55 -0800 Subject: [PATCH 205/295] Add test coverage for the version pinning change Added a --apitools_version command line option so that we can control this from the existing tests layout. Used this to add test cases for version 0.4.x and 0.5.x since 4=>5 is the transition where behavior will change. --- apitools/gen/gen_client.py | 9 ++++++++- apitools/gen/gen_client_lib.py | 22 ++++++++++++++++------ apitools/gen/gen_client_test.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 97e9d5d..79deb5c 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -106,7 +106,8 @@ def _GetCodegenFromFlags(args): protorpc_package=args.protorpc_package, generate_cli=args.generate_cli, use_proto2=args.experimental_proto2_output, - unelidable_request_methods=args.unelidable_request_methods) + unelidable_request_methods=args.unelidable_request_methods, + apitools_version=args.apitools_version) # TODO(craigcitro): Delete this if we don't need this functionality. @@ -296,6 +297,12 @@ def main(argv=None): help=('Full method IDs of methods for which we should NOT try to ' 'elide the request type. (Should be a comma-separated list.')) + parser.add_argument( + '--apitools_version', + default='', dest='apitools_version', + help=('Apitools version used as a requirement in generated clients. ' + 'Defaults to version of apitools used to generate the clients.')) + parser.add_argument( '--experimental_capitalize_enums', default=False, action='store_true', diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index f109ba9..5b2d996 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -70,7 +70,8 @@ class DescriptorGenerator(object): def __init__(self, discovery_doc, client_info, names, root_package, outdir, base_package, protorpc_package, generate_cli=False, - use_proto2=False, unelidable_request_methods=None): + use_proto2=False, unelidable_request_methods=None, + apitools_version=''): self.__discovery_doc = discovery_doc self.__client_info = client_info self.__outdir = outdir @@ -141,6 +142,11 @@ class DescriptorGenerator(object): self.__client_info = self.__client_info._replace( scopes=self.__services_registry.scopes) + # The apitools version that will be used in prerequisites for the + # generated packages. + self.__apitools_version = ( + apitools_version if apitools_version else _ApitoolsVersion()) + @property def client_info(self): return self.__client_info @@ -165,6 +171,10 @@ class DescriptorGenerator(object): def use_proto2(self): return self.__use_proto2 + @property + def apitools_version(self): + return self.__apitools_version + def _GetPrinter(self, out): printer = util.SimplePrettyPrinter(out) return printer @@ -206,7 +216,6 @@ class DescriptorGenerator(object): """Write a setup.py for upload to PyPI.""" printer = self._GetPrinter(out) year = datetime.datetime.now().year - apitools_version = _ApitoolsVersion() printer('# Copyright %s Google Inc. All Rights Reserved.' % year) printer('#') printer('# Licensed under the Apache License, Version 2.0 (the' @@ -230,10 +239,10 @@ class DescriptorGenerator(object): printer('import setuptools') printer('REQUIREMENTS = [') with printer.Indent(indent=' '): - if apitools_version.startswith('0.4'): + if self.apitools_version.startswith('0.4.'): printer('"google-apitools>=0.4.8,<0.5",') else: - printer('"google-apitools==%s",', apitools_version) + printer('"google-apitools==%s",', self.apitools_version) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') @@ -244,10 +253,11 @@ class DescriptorGenerator(object): with printer.Indent(indent=' '): printer('name="google-apitools-%s-%s",', self.__package, self.__version) - if apitools_version.startswith('0.4'): + if self.apitools_version.startswith('0.4.'): printer('version="0.4.%s",', self.__revision) else: - printer('version="%s.%s",', apitools_version, self.__revision) + printer('version="%s.%s",', + self.apitools_version, self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 896a829..0ffb555 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -52,6 +52,38 @@ class ClientGenCliTest(unittest2.TestCase): set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py']), set(os.listdir(tmp_dir_path))) + def testGenClient_SimpleDocWithV4(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--nogenerate_cli', + '--infile', GetDocPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--apitools_version', '0.4.12', + '--root_package', 'google.apis', + 'client' + ]) + self.assertEquals( + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py']), + set(os.listdir(tmp_dir_path))) + + def testGenClient_SimpleDocWithV5(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--nogenerate_cli', + '--infile', GetDocPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--apitools_version', '0.5.0', + '--root_package', 'google.apis', + 'client' + ]) + self.assertEquals( + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py']), + set(os.listdir(tmp_dir_path))) + def testGenPipPackage_SimpleDoc(self): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ -- GitLab From 363745e225be89a5fac9707c724d35908798f0a4 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Wed, 25 Nov 2015 09:54:34 -0500 Subject: [PATCH 206/295] Make sure client.Unmock() undoes client.Mock(). Mocked clients after being unmocked can still hold references to client instance. --- apitools/base/py/testing/mock.py | 5 +++++ apitools/base/py/testing/mock_test.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index 6404f81..09f9b2d 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -285,6 +285,7 @@ class Client(object): self.__real_client = real_client self._request_responses = [] + self.__real_include_fields = None def __enter__(self): return self.Mock() @@ -330,13 +331,17 @@ class Client(object): def Unmock(self): for name, service_class in self.__real_service_classes.items(): setattr(self.__client_class, name, service_class) + delattr(self, service_class._NAME) + self.__real_service_classes = {} if self._request_responses: raise ExpectedRequestsException( [(rq_rs.key, rq_rs.request) for rq_rs in self._request_responses]) + self._request_responses = [] self.__client_class.IncludeFields = self.__real_include_fields + self.__real_include_fields = None def IncludeFields(self, include_fields): if self.__real_client: diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py index 5250c9a..88c8739 100644 --- a/apitools/base/py/testing/mock_test.py +++ b/apitools/base/py/testing/mock_test.py @@ -84,6 +84,14 @@ class MockTest(unittest2.TestCase): client = fusiontables.FusiontablesV1(get_credentials=False) self.assertNotEqual(type(client.column), mocked_service_type) + def testClientUnmock(self): + mock_client = mock.Client(fusiontables.FusiontablesV1) + attributes = set(mock_client.__dict__.keys()) + mock_client = mock_client.Mock() + self.assertTrue(set(mock_client.__dict__.keys()) - attributes) + mock_client.Unmock() + self.assertEqual(attributes, set(mock_client.__dict__.keys())) + class _NestedMessage(messages.Message): nested = messages.StringField(1) -- GitLab From eff5c2e07c28f9a7391bbfba6f69557cf4846055 Mon Sep 17 00:00:00 2001 From: Vilas Jagannath Date: Wed, 2 Dec 2015 17:18:07 -0500 Subject: [PATCH 207/295] Fix --unelidable_request_methods behavior Ensure --unelidable_request_methods accepts a comma separated list of methods. --- apitools/gen/gen_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 79deb5c..8f19dfc 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -202,6 +202,12 @@ def GenerateProto(args): _WriteProtoFiles(codegen) +class _SplitCommaSeparatedList(argparse.Action): + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values.split(',')) + + def main(argv=None): if argv is None: argv = sys.argv @@ -292,7 +298,8 @@ def main(argv=None): parser.set_defaults(generate_cli=True) parser.add_argument( - '--unelidable_request_methods', nargs='*', + '--unelidable_request_methods', + action=_SplitCommaSeparatedList, default=[], help=('Full method IDs of methods for which we should NOT try to ' 'elide the request type. (Should be a comma-separated list.')) -- GitLab From 69736dd7917b28e14208919e1290c8fa6b9c0c3f Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Mon, 7 Dec 2015 18:05:15 -0500 Subject: [PATCH 208/295] Import on demand credentials_lib in base_api. credentials_lib depends on oauth2client which had support for webflow authentication, which brings a lot of dependencies, all not used if user of the client manages its own credentials. --- apitools/base/py/base_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index e3d54ef..7b35956 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -30,7 +30,6 @@ from six.moves import urllib from apitools.base.protorpclite import message_types from apitools.base.protorpclite import messages -from apitools.base.py import credentials_lib from apitools.base.py import encoding from apitools.base.py import exceptions from apitools.base.py import http_wrapper @@ -293,6 +292,8 @@ class BaseApiClient(object): 'user_agent': self._USER_AGENT, } args.update(kwds) + # credentials_lib can be expensive to import so do it only if needed. + from apitools.base.py import credentials_lib # TODO(craigcitro): It's a bit dangerous to pass this # still-half-initialized self into this method, but we might need # to set attributes on it associated with our credentials. -- GitLab From 67a69fa0414e7ba1bef67a88ee1a747b32dcfc58 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Tue, 8 Dec 2015 13:01:44 -0500 Subject: [PATCH 209/295] In client mock explicitly import its dependencies. This helps readability, but also reduces imports which otherwise might not be needed. --- apitools/base/py/testing/mock.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index 09f9b2d..91958a7 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -27,7 +27,9 @@ import difflib import six from apitools.base.protorpclite import messages -import apitools.base.py as apitools_base +from apitools.base.py import base_api +from apitools.base.py import encoding +from apitools.base.py import exceptions class Error(Exception): @@ -72,9 +74,9 @@ class UnexpectedRequestException(Error): expected_key, expected_request = expected_call received_key, received_request = received_call - expected_repr = apitools_base.MessageToRepr( + expected_repr = encoding.MessageToRepr( expected_request, multiline=True) - received_repr = apitools_base.MessageToRepr( + received_repr = encoding.MessageToRepr( received_request, multiline=True) expected_lines = expected_repr.splitlines() @@ -116,7 +118,7 @@ class ExpectedRequestsException(Error): for (key, request) in expected_calls: msg += '{key}({request})\n'.format( key=key, - request=apitools_base.MessageToRepr(request, multiline=True)) + request=encoding.MessageToRepr(request, multiline=True)) super(ExpectedRequestsException, self).__init__(msg) @@ -129,13 +131,13 @@ class _ExpectedRequestResponse(object): self.__request = request if response and exception: - raise apitools_base.ConfigurationValueError( + raise exceptions.ConfigurationValueError( 'Should specify at most one of response and exception') - if response and isinstance(response, apitools_base.Error): - raise apitools_base.ConfigurationValueError( + if response and isinstance(response, exceptions.Error): + raise exceptions.ConfigurationValueError( 'Responses should not be an instance of Error') - if exception and not isinstance(exception, apitools_base.Error): - raise apitools_base.ConfigurationValueError( + if exception and not isinstance(exception, exceptions.Error): + raise exceptions.ConfigurationValueError( 'Exceptions must be instances of Error') self.__response = response @@ -178,7 +180,7 @@ class _ExpectedRequestResponse(object): return self.__response -class _MockedService(apitools_base.BaseApiService): +class _MockedService(base_api.BaseApiService): def __init__(self, key, mocked_client, methods, real_service): super(_MockedService, self).__init__(mocked_client) @@ -246,7 +248,7 @@ class _MockedMethod(object): if response is None and self.__real_method: response = self.__real_method(request) - print(apitools_base.MessageToRepr( + print(encoding.MessageToRepr( response, multiline=True, shortstrings=True)) return response @@ -298,7 +300,7 @@ class Client(object): service_class = getattr(self.__client_class, name) if not isinstance(service_class, type): continue - if not issubclass(service_class, apitools_base.BaseApiService): + if not issubclass(service_class, base_api.BaseApiService): continue self.__real_service_classes[name] = service_class service = service_class(client) -- GitLab From 518feea188556b1569a9ff0d426b5312faff8efc Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 1 Jan 2016 12:42:55 -0500 Subject: [PATCH 210/295] Update lint rules. Fixes #81. --- apitools/base/protorpclite/__init__.py | 19 +++++++ apitools/base/protorpclite/messages.py | 29 +++++----- apitools/base/protorpclite/messages_test.py | 4 ++ apitools/base/protorpclite/test_util.py | 2 +- apitools/base/py/base_api.py | 2 +- apitools/base/py/batch.py | 1 + apitools/base/py/credentials_lib.py | 3 ++ apitools/base/py/encoding_test.py | 1 + apitools/base/py/http_wrapper.py | 3 +- apitools/base/py/http_wrapper_test.py | 6 +-- apitools/base/py/transfer.py | 1 + apitools/gen/client_generation_test.py | 3 +- apitools/gen/gen_client_lib.py | 1 + apitools/gen/gen_client_test.py | 7 +-- apitools/gen/service_registry.py | 4 +- apitools/scripts/oauth2l_test.py | 1 + default.pylintrc | 59 +++++++++++---------- run_pylint.py | 17 +----- samples/storage_sample/downloads_test.py | 1 + tox.ini | 3 +- 20 files changed, 99 insertions(+), 68 deletions(-) diff --git a/apitools/base/protorpclite/__init__.py b/apitools/base/protorpclite/__init__.py index e69de29..224d433 100644 --- a/apitools/base/protorpclite/__init__.py +++ b/apitools/base/protorpclite/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shared __init__.py for apitools.""" + +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py index 6639cad..4a8b03f 100644 --- a/apitools/base/protorpclite/messages.py +++ b/apitools/base/protorpclite/messages.py @@ -15,6 +15,8 @@ # limitations under the License. # +# pylint: disable=too-many-lines + """Stand-alone implementation of in memory protocol messages. Public Classes: @@ -1296,7 +1298,8 @@ class Field(six.with_metaclass(_FieldMeta, object)): if self.repeated: value = FieldList(self, value) else: - value = self.validate(value) + value = ( # pylint: disable=redefined-variable-type + self.validate(value)) message_instance._Message__tags[self.number] = value def __get__(self, message_instance, message_class): @@ -1939,35 +1942,37 @@ def find_definition(name, relative_to=None, importer=__import__): Returns: Message or Enum at the end of name_path, else None. """ - next = relative_to + next_part = relative_to for node in name_path: # Look for attribute first. - attribute = getattr(next, node, None) + attribute = getattr(next_part, node, None) if attribute is not None: - next = attribute + next_part = attribute else: # If module, look for sub-module. - if next is None or isinstance(next, types.ModuleType): - if next is None: + if (next_part is None or + isinstance(next_part, types.ModuleType)): + if next_part is None: module_name = node else: - module_name = '%s.%s' % (next.__name__, node) + module_name = '%s.%s' % (next_part.__name__, node) try: fromitem = module_name.split('.')[-1] - next = importer(module_name, '', '', [str(fromitem)]) + next_part = importer(module_name, '', '', + [str(fromitem)]) except ImportError: return None else: return None - if not isinstance(next, types.ModuleType): - if not (isinstance(next, type) and - issubclass(next, (Message, Enum))): + if not isinstance(next_part, types.ModuleType): + if not (isinstance(next_part, type) and + issubclass(next_part, (Message, Enum))): return None - return next + return next_part while True: found = search_path() diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py index 0414315..04ce871 100644 --- a/apitools/base/protorpclite/messages_test.py +++ b/apitools/base/protorpclite/messages_test.py @@ -37,6 +37,7 @@ from apitools.base.protorpclite import test_util # pylint:disable=redefined-outer-name # pylint:disable=undefined-variable # pylint:disable=unused-variable +# pylint:disable=too-many-lines class ModuleInterfaceTest(test_util.ModuleInterfaceTest, @@ -850,6 +851,7 @@ class FieldTest(test_util.TestCase): field = messages.FloatField(1) self.assertEquals(type(field.validate_element(12)), float) self.assertEquals(type(field.validate_element(12.0)), float) + # pylint: disable=redefined-variable-type field = messages.IntegerField(1) self.assertEquals(type(field.validate_element(12)), int) self.assertRaises(messages.ValidationError, @@ -1657,6 +1659,7 @@ class MessageTest(test_util.TestCase): messages.ValidationError, "Field val is repeated. Found: ", setattr, message, 'val', SubMessage()) + # pylint: disable=redefined-variable-type message.val = [SubMessage()] message_field.validate(message) @@ -1943,6 +1946,7 @@ class FindDefinitionTest(test_util.TestCase): return message_class # pylint:disable=unused-argument + # pylint:disable=redefined-builtin def Importer(self, module, globals='', locals='', fromlist=None): """Importer function. diff --git a/apitools/base/protorpclite/test_util.py b/apitools/base/protorpclite/test_util.py index fad3a53..f6bc2de 100644 --- a/apitools/base/protorpclite/test_util.py +++ b/apitools/base/protorpclite/test_util.py @@ -35,7 +35,7 @@ import socket import types import six -from six.moves import range +from six.moves import range # pylint: disable=redefined-builtin import unittest2 as unittest from apitools.base.protorpclite import message_types diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 7b35956..7dc9ea9 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -205,7 +205,7 @@ class _UrlBuilder(object): # non-ASCII, we may silently fail to encode correctly. We should # figure out who is responsible for owning the object -> str # conversion. - return urllib.parse.urlencode(self.query_params, doseq=True) + return urllib.parse.urlencode(self.query_params, True) @property def url(self): diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py index 71dbf6b..7efe6dd 100644 --- a/apitools/base/py/batch.py +++ b/apitools/base/py/batch.py @@ -443,6 +443,7 @@ class BatchHttpRequest(object): # Disable protected access because namedtuple._replace(...) # is not actually meant to be protected. + # pylint: disable=protected-access self.__request_response_handlers[request_id] = ( self.__request_response_handlers[request_id]._replace( response=response)) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index dc0fff3..31871d1 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -37,6 +37,7 @@ from apitools.base.py import exceptions from apitools.base.py import util try: + # pylint: disable=wrong-import-order import gflags FLAGS = gflags.FLAGS except ImportError: @@ -412,6 +413,7 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): Args: _: (ignored) A function matching httplib2.Http.request's signature. """ + # pylint: disable=import-error from google.appengine.api import app_identity try: token, _ = app_identity.get_access_token(self._scopes) @@ -534,6 +536,7 @@ def _GetServiceAccountCredentials( # pylint: enable=protected-access return credentials if service_account_name is not None: + # pylint: disable=redefined-variable-type credentials = ServiceAccountCredentialsFromFile( service_account_name, service_account_keyfile, scopes, service_account_kwargs={'user_agent': user_agent}) diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py index e6b067a..ad51965 100644 --- a/apitools/base/py/encoding_test.py +++ b/apitools/base/py/encoding_test.py @@ -187,6 +187,7 @@ class EncodingTest(unittest2.TestCase): '{"nested": {"additional_properties": []}}', encoding.MessageToJson( msg, include_fields=['nested.additional_properties'])) + # pylint: disable=redefined-variable-type msg = ExtraNestedMessage(nested=msg) self.assertEqual( '{"nested": {"nested": null}}', diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index 0f7c150..e7925ea 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -240,6 +240,7 @@ def RebuildHttpConnections(http): def RethrowExceptionHandler(*unused_args): + # pylint: disable=misplaced-bare-raise raise @@ -295,7 +296,7 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): logging.debug('Response returned a retry-after header, retrying') retry_after = retry_args.exc.retry_after else: - raise + raise # pylint: disable=misplaced-bare-raise RebuildHttpConnections(retry_args.http) logging.debug('Retrying request to url %s after exception %s', retry_args.http_request.url, retry_args.exc) diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index a6edf43..ff07668 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -14,9 +14,11 @@ # limitations under the License. """Tests for http_wrapper.""" +import socket + import httplib2 import oauth2client -import socket +from six.moves import http_client import unittest2 from mock import patch @@ -24,8 +26,6 @@ from mock import patch from apitools.base.py import exceptions from apitools.base.py import http_wrapper -from six.moves import http_client - class _MockHttpRequest(object): diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 3bbb5f7..397222e 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -988,6 +988,7 @@ class Upload(_Transfer): # https://code.google.com/p/httplib2/issues/detail?id=176 which can # cause httplib2 to skip bytes on 401's for file objects. # Rework this solution to be more general. + # pylint: disable=redefined-variable-type body_stream = body_stream.read(self.chunksize) else: end = min(start + self.chunksize, self.total_size) diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index 385482e..cb0a9c4 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -20,9 +20,10 @@ import os import subprocess import tempfile +import unittest2 + from apitools.gen import test_utils -import unittest2 _API_LIST = [ 'drive.v2', diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 5b2d996..8044761 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -139,6 +139,7 @@ class DescriptorGenerator(object): if api_methods: self.__services_registry.AddServiceFromResource( 'api', {'methods': api_methods}) + # pylint: disable=protected-access self.__client_info = self.__client_info._replace( scopes=self.__services_registry.scopes) diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 0ffb555..25e9e23 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -15,11 +15,12 @@ """Test for gen_client module.""" -from apitools.gen import gen_client -from apitools.gen import test_utils +import os import unittest2 -import os + +from apitools.gen import gen_client +from apitools.gen import test_utils def GetDocPath(name): diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 7920a0f..ded364a 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -223,8 +223,8 @@ class ServiceRegistry(object): printer() printer('MESSAGES_MODULE = messages') printer() - client_info_items = client_info._asdict( - ).items() # pylint:disable=protected-access + # pylint: disable=protected-access + client_info_items = client_info._asdict().items() for attr, val in client_info_items: if attr == 'scopes' and not val: val = ['https://www.googleapis.com/auth/userinfo.email'] diff --git a/apitools/scripts/oauth2l_test.py b/apitools/scripts/oauth2l_test.py index e8e0edb..40e3896 100644 --- a/apitools/scripts/oauth2l_test.py +++ b/apitools/scripts/oauth2l_test.py @@ -30,6 +30,7 @@ import apitools.base.py as apitools_base _OAUTH2L_MAIN_RUN = False if six.PY2: + # pylint: disable=wrong-import-position,wrong-import-order import gflags as flags from google.apputils import appcommands from apitools.scripts import oauth2l diff --git a/default.pylintrc b/default.pylintrc index 3175e39..b4c4577 100644 --- a/default.pylintrc +++ b/default.pylintrc @@ -51,27 +51,11 @@ disable = import-error, locally-disabled, locally-enabled, - maybe-no-member, - method-hidden, - no-init, no-member, no-name-in-module, no-self-use, - redefined-builtin, - redundant-keyword-arg, - similarities, - star-args, super-on-old-class, - too-few-public-methods, - too-many-arguments, - too-many-branches, too-many-function-args, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - too-many-statements, [REPORTS] @@ -111,6 +95,7 @@ reports=no # Minimum lines number of a similarity. # DEFAULT: min-similarity-lines=4 +min-similarity-lines=15 # Ignore comments when computing similarities. # DEFAULT: ignore-comments=yes @@ -120,6 +105,7 @@ reports=no # Ignore imports when computing similarities. # DEFAULT: ignore-imports=no +ignore-imports=yes [VARIABLES] @@ -184,10 +170,15 @@ max-module-lines=1500 [BASIC] -required-attributes= +# Regular expression which should only match function or class names that do +# not require a docstring. +# DEFAULT: no-docstring-rgx=__.*__ no-docstring-rgx=(__.*__|main) +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +# DEFAULT: docstring-min-length=-1 docstring-min-length=10 # Regular expression which should only match correct module names. The @@ -195,13 +186,16 @@ docstring-min-length=10 # guide. module-rgx=^(_?[a-z][a-z0-9_]*)|__init__$ -# Regular expression which should only match correct module level names +# Regular expression matching correct constant names +# DEFAULT: const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ -# Regular expression which should only match correct class attribute +# Regular expression matching correct class attribute names +# DEFAULT: class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ -# Regular expression which should only match correct class names +# Regular expression matching correct class names +# DEFAULT: class-rgx=[A-Z_][a-zA-Z0-9]+$ class-rgx=^_?[A-Z][a-zA-Z0-9]*$ # Regular expression which should only match correct function names. @@ -215,23 +209,28 @@ function-rgx=^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0 # consistent with all naming styles. method-rgx=^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ -# Regular expression which should only match correct instance attribute names +# Regular expression matching correct attribute names +# DEFAULT: attr-rgx=[a-z_][a-z0-9_]{2,30}$ attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ -# Regular expression which should only match correct argument names +# Regular expression matching correct argument names +# DEFAULT: argument-rgx=[a-z_][a-z0-9_]{2,30}$ argument-rgx=^[a-z][a-z0-9_]*$ -# Regular expression which should only match correct variable names +# Regular expression matching correct variable names +# DEFAULT: variable-rgx=[a-z_][a-z0-9_]{2,30}$ variable-rgx=^[a-z][a-z0-9_]*$ -# Regular expression which should only match correct list comprehension / -# generator expression variable names +# Regular expression matching correct inline iteration names +# DEFAULT: inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ inlinevar-rgx=^[a-z][a-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma +# DEFAULT: good-names=i,j,k,ex,Run,_ good-names=main,_ # Bad variable names which should always be refused, separated by a comma +# DEFAULT: bad-names=foo,bar,baz,toto,tutu,tata bad-names= # List of builtins function names that should not be used, separated by a comma @@ -306,7 +305,7 @@ no-docstring-rgx=.* # Maximum number of arguments for function / method # DEFAULT: max-args=5 # RATIONALE: API-mapping -max-args = 10 +max-args = 14 # Argument names that match this expression will be ignored. Default to name # with leading underscore @@ -314,13 +313,15 @@ max-args = 10 # Maximum number of locals for function / method body # DEFAULT: max-locals=15 -max-locals=20 +max-locals=24 # Maximum number of return / yield for function / method body # DEFAULT: max-returns=6 +max-returns=9 # Maximum number of branch for function / method body # DEFAULT: max-branches=12 +max-branches=21 # Maximum number of statements in function / method body # DEFAULT: max-statements=50 @@ -331,7 +332,7 @@ max-locals=20 # Maximum number of attributes for a class (see R0902). # DEFAULT: max-attributes=7 # RATIONALE: API mapping -max-attributes=15 +max-attributes=19 # Minimum number of public methods for a class (see R0903). # DEFAULT: min-public-methods=2 @@ -343,6 +344,8 @@ min-public-methods=0 # RATIONALE: API mapping max-public-methods=40 +[ELIF] +max-nested-blocks=6 [EXCEPTIONS] diff --git a/run_pylint.py b/run_pylint.py index 7d084a2..f8babac 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -42,23 +42,14 @@ IGNORED_FILES = [ PRODUCTION_RC = 'default.pylintrc' TEST_RC = 'reduced.pylintrc' TEST_DISABLED_MESSAGES = [ - 'attribute-defined-outside-init', 'exec-used', - 'import-error', 'invalid-name', 'missing-docstring', - 'no-init', - 'no-self-use', 'protected-access', - 'superfluous-parens', - 'too-few-public-methods', - 'too-many-locals', - 'too-many-public-methods', - 'unbalanced-tuple-unpacking', ] TEST_RC_ADDITIONS = { 'MESSAGES CONTROL': { - 'disable': ', '.join(TEST_DISABLED_MESSAGES), + 'disable': ',\n'.join(TEST_DISABLED_MESSAGES), }, } @@ -86,7 +77,7 @@ def make_test_rc(base_rc_filename, additions_dict, target_filename): curr_val = curr_section.get(opt) if curr_val is None: raise KeyError('Expected to be adding to existing option.') - curr_section[opt] = '%s, %s' % (curr_val, opt_val) + curr_section[opt] = '%s\n%s' % (curr_val, opt_val) with open(target_filename, 'w') as file_obj: test_cfg.write(file_obj) @@ -124,10 +115,6 @@ def get_files_for_linting(allow_limited=True, diff_base=None): One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but this value is not dependable. - To allow faster local ``tox`` runs, the environment variables - ``GCLOUD_REMOTE_FOR_LINT`` and ``GCLOUD_BRANCH_FOR_LINT`` can be set to - specify a remote branch to diff against. - :type allow_limited: boolean :param allow_limited: Boolean indicating if a reduced set of files can be used. diff --git a/samples/storage_sample/downloads_test.py b/samples/storage_sample/downloads_test.py index 9ddaa86..a51cd95 100644 --- a/samples/storage_sample/downloads_test.py +++ b/samples/storage_sample/downloads_test.py @@ -178,6 +178,7 @@ class DownloadsTest(unittest.TestCase): request = storage.StorageObjectsGetRequest( bucket=self._DEFAULT_BUCKET, object=object_name) response = self.__client.objects.Get(request) + # pylint: disable=attribute-defined-outside-init self.__buffer = six.StringIO() download_data = json.dumps({ 'auto_transfer': False, diff --git a/tox.ini b/tox.ini index b9010d3..daf7988 100644 --- a/tox.ini +++ b/tox.ini @@ -32,8 +32,9 @@ verbose = 1 basepython = python2.7 commands = + pip install six google-apitools[testing] pep8 - python run_pylint.py master + python run_pylint.py deps = pep8 pylint -- GitLab From dd2097e74874407c7754e21914059d6dba2ebd4f Mon Sep 17 00:00:00 2001 From: Brad Friedman Date: Mon, 11 Jan 2016 15:23:47 -0800 Subject: [PATCH 211/295] Update encoding.py with some bug fixes --- apitools/base/py/encoding.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py index a639478..698304b 100644 --- a/apitools/base/py/encoding.py +++ b/apitools/base/py/encoding.py @@ -157,7 +157,7 @@ def MessageToRepr(msg, multiline=False, **kwargs): """ - # TODO(user): craigcitro suggests a pretty-printer from apitools/gen. + # TODO(jasmuth): craigcitro suggests a pretty-printer from apitools/gen. indent = kwargs.get('indent', 0) @@ -328,12 +328,15 @@ class _ProtoJsonApiTools(protojson.ProtoJson): if isinstance(message, messages.FieldList): return '[%s]' % (', '.join(self.encode_message(x) for x in message)) - message_type = type(message) - if message_type in _CUSTOM_MESSAGE_CODECS: + + # pylint: disable=unidiomatic-typecheck + if type(message) in _CUSTOM_MESSAGE_CODECS: return _CUSTOM_MESSAGE_CODECS[type(message)].encoder(message) + message = _EncodeUnknownFields(message) result = super(_ProtoJsonApiTools, self).encode_message(message) - return _EncodeCustomFieldNames(message, result) + result = _EncodeCustomFieldNames(message, result) + return json.dumps(json.loads(result), sort_keys=True) def encode_field(self, field, value): """Encode the given value as JSON. @@ -396,7 +399,7 @@ def _DecodeUnknownMessages(message, encoded_message, pair_type): field_type = pair_type.value.type new_values = [] all_field_names = [x.name for x in message.all_fields()] - for name, value_dict in encoded_message.items(): + for name, value_dict in six.iteritems(encoded_message): if name in all_field_names: continue value = PyValueToMessage(field_type, value_dict) @@ -573,8 +576,8 @@ def AddCustomJsonEnumMapping(enum_type, python_name, json_name, Args: enum_type: (messages.Enum) An enum type - python_name: (string) Python name for this value. - json_name: (string) JSON name to be used on the wire. + python_name: (basestring) Python name for this value. + json_name: (basestring) JSON name to be used on the wire. package: (basestring, optional) Package prefix for this enum, if present. We strip this off the enum name in order to generate unique keys. @@ -600,8 +603,8 @@ def AddCustomJsonFieldMapping(message_type, python_name, json_name, Args: message_type: (messages.Message) A message type - python_name: (string) Python name for this value. - json_name: (string) JSON name to be used on the wire. + python_name: (basestring) Python name for this value. + json_name: (basestring) JSON name to be used on the wire. package: (basestring, optional) Package prefix for this message, if present. We strip this off the message name in order to generate unique keys. -- GitLab From 0db9dd00aeac9a0b8913525f49874057cd101a1c Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Wed, 20 Jan 2016 14:10:53 -0800 Subject: [PATCH 212/295] Bump version to 0.4.15 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f867027..aecee66 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.14' +_APITOOLS_VERSION = '0.4.15' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From a057486b6f25ec61ca026970be3de3e97a677e02 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Thu, 11 Feb 2016 23:36:07 -0800 Subject: [PATCH 213/295] Add support for new-style discovery URLs. This just adds a second discovery URL format, which is used by some newer Google APIs. --- apitools/gen/util.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 57cb4c0..e1f7724 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -293,34 +293,37 @@ class SimplePrettyPrinter(object): print('', file=self.__out) -def NormalizeDiscoveryUrl(discovery_url): +def _NormalizeDiscoveryUrls(discovery_url): """Expands a few abbreviations into full discovery urls.""" if discovery_url.startswith('http'): - return discovery_url + return [discovery_url] elif '.' not in discovery_url: raise ValueError('Unrecognized value "%s" for discovery url') api_name, _, api_version = discovery_url.partition('.') - return 'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % ( - api_name, api_version) + return [ + 'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % ( + api_name, api_version), + 'https://%s.googleapis.com/$discovery/rest?version=%s' % ( + api_name, api_version), + ] def FetchDiscoveryDoc(discovery_url, retries=5): """Fetch the discovery document at the given url.""" - discovery_url = NormalizeDiscoveryUrl(discovery_url) + discovery_urls = _NormalizeDiscoveryUrls(discovery_url) discovery_doc = None last_exception = None - for _ in range(retries): - try: - discovery_doc = json.loads( - urllib_request.urlopen(discovery_url).read()) - break - except (urllib_error.HTTPError, - urllib_error.URLError) as last_exception: - logging.warning( - 'Attempting to fetch discovery doc again after "%s"', - last_exception) + for url in discovery_urls: + for _ in range(retries): + try: + discovery_doc = json.loads(urllib_request.urlopen(url).read()) + break + except (urllib_error.HTTPError, urllib_error.URLError) as e: + logging.warning( + 'Attempting to fetch discovery doc again after "%s"', e) + last_exception = e if discovery_doc is None: raise CommunicationError( - 'Could not find discovery doc at url "%s": %s' % ( - discovery_url, last_exception)) + 'Could not find discovery doc at any of %s: %s' % ( + discovery_urls, last_exception)) return discovery_doc -- GitLab From 9cc94e2233858bc9793f32b34e0f79ed56173a42 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 16 Feb 2016 16:50:44 -0800 Subject: [PATCH 214/295] Update oauth2client imports for the new release. We could pin, but this seems more useful, especially given the small surface we use in oauth2client. (Also added a handful of lint disables, since the order it expected was for the try/except imports to come in the middle of the list of imports.) --- apitools/base/py/credentials_lib.py | 36 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 31871d1..c7479e6 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -25,9 +25,6 @@ import threading import httplib2 import oauth2client import oauth2client.client -import oauth2client.gce -import oauth2client.locked_file -import oauth2client.multistore_file import oauth2client.service_account from oauth2client import tools # for gflags declarations from six.moves import http_client @@ -36,8 +33,17 @@ from six.moves import urllib from apitools.base.py import exceptions from apitools.base.py import util +# Note: we try the oauth2client imports two ways, to accomodate layout +# changes in oauth2client 2.0+. We can remove these once we no longer +# support oauth2client < 2.0. +# +# pylint: disable=wrong-import-order,ungrouped-imports +try: + from oauth2client import gce, locked_file, multistore_file +except ImportError: + from oauth2client.contrib import gce, locked_file, multistore_file + try: - # pylint: disable=wrong-import-order import gflags FLAGS = gflags.FLAGS except ImportError: @@ -174,7 +180,7 @@ def _GceMetadataRequest(relative_url, use_metadata_ip=False): return response -class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): +class GceAssertionCredentials(gce.AppAssertionCredentials): """Assertion credentials for GCE instances.""" @@ -231,11 +237,11 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): } with cache_file_lock: if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( + cache_file = locked_file.LockedFile( cache_filename, 'r+b', 'rb') try: - locked_file.open_and_lock() - cached_creds_str = locked_file.file_handle().read() + cache_file.open_and_lock() + cached_creds_str = cache_file.file_handle().read() if cached_creds_str: # Cached credentials metadata dict. cached_creds = json.loads(cached_creds_str) @@ -245,7 +251,7 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): (None, cached_creds['scopes'])): scopes = cached_creds['scopes'] finally: - locked_file.unlock_and_close() + cache_file.unlock_and_close() return scopes def _WriteCacheFile(self, cache_filename, scopes): @@ -260,21 +266,21 @@ class GceAssertionCredentials(oauth2client.gce.AppAssertionCredentials): """ with cache_file_lock: if _EnsureFileExists(cache_filename): - locked_file = oauth2client.locked_file.LockedFile( + cache_file = locked_file.LockedFile( cache_filename, 'r+b', 'rb') try: - locked_file.open_and_lock() - if locked_file.is_locked(): + cache_file.open_and_lock() + if cache_file.is_locked(): creds = { # Credentials metadata dict. 'scopes': sorted(list(scopes)), 'svc_acct_name': self.__service_account_name} - locked_file.file_handle().write( + cache_file.file_handle().write( json.dumps(creds, encoding='ascii')) # If it's not locked, the locking process will # write the same data to the file, so just # continue. finally: - locked_file.unlock_and_close() + cache_file.unlock_and_close() def _ScopesFromMetadataServer(self, scopes): if not util.DetectGce(): @@ -449,7 +455,7 @@ def _GetRunFlowFlags(args=None): # TODO(craigcitro): Switch this from taking a path to taking a stream. def CredentialsFromFile(path, client_info, oauth2client_args=None): """Read credentials from a file.""" - credential_store = oauth2client.multistore_file.get_credential_storage( + credential_store = multistore_file.get_credential_storage( path, client_info['client_id'], client_info['user_agent'], -- GitLab From 187f0275eba6c534865f6d253dd93289361d0048 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 16 Feb 2016 22:42:35 -0800 Subject: [PATCH 215/295] Cleanup some dead test code. It turns out an ancient refactor (merging the gsutil fork) turned some tests into dead code; I noticed that something was awry when the GCE header name was updated but tests didn't fail. The fix was easy (just drop the dead code and add something new), and I bumped up coverage a wee bit in the process. (It'd be easy to keep going, but I hate mocks, and think most of these tests have little value.) Closes #90 (via different means). --- apitools/base/py/credentials_lib_test.py | 44 +++++++++++++----------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py index f4a32c2..3238fca 100644 --- a/apitools/base/py/credentials_lib_test.py +++ b/apitools/base/py/credentials_lib_test.py @@ -13,31 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - import mock import six -from six.moves import http_client import unittest2 from apitools.base.py import credentials_lib from apitools.base.py import util -def CreateUriValidator(uri_regexp, content=''): - def CheckUri(uri, headers=None): - if 'X-Google-Metadata-Request' not in headers: - raise ValueError('Missing required header') - if uri_regexp.match(uri): - message = content - status = http_client.OK - else: - message = 'Expected uri matching pattern %s' % uri_regexp.pattern - status = http_client.BAD_REQUEST - return type('HttpResponse', (object,), {'status': status})(), message - return CheckUri - - class CredentialsLibTest(unittest2.TestCase): def _GetServiceCreds(self, service_account_name=None, scopes=None): @@ -63,13 +46,11 @@ class CredentialsLibTest(unittest2.TestCase): with mock.patch.object(util, 'DetectGce', autospec=True) as mock_detect: mock_detect.return_value = True - validator = CreateUriValidator( - re.compile(r'.*/%s/.*' % service_account_name), - content='{"access_token": "token"}') credentials = credentials_lib.GceAssertionCredentials( scopes, **kwargs) - self.assertIsNone(credentials._refresh(validator)) + self.assertIsNone(credentials._refresh(None)) self.assertEqual(3, opener_mock.call_count) + return credentials def testGceServiceAccounts(self): scopes = ['scope1'] @@ -78,6 +59,27 @@ class CredentialsLibTest(unittest2.TestCase): self._GetServiceCreds(service_account_name='my_service_account', scopes=scopes) + def testGetServiceAccount(self): + # We'd also like to test the metadata calls, which requires + # having some knowledge about how HTTP calls are made (so that + # we can mock them). It's unfortunate, but there's no way + # around it. + creds = self._GetServiceCreds() + opener = mock.MagicMock() + opener.open = mock.MagicMock() + opener.open.return_value = six.StringIO('default/\nanother') + with mock.patch.object(six.moves.urllib.request, 'build_opener', + return_value=opener, + autospec=True) as build_opener: + creds.GetServiceAccount('default') + self.assertEqual(1, build_opener.call_count) + self.assertEqual(1, opener.open.call_count) + req = opener.open.call_args[0][0] + self.assertTrue(req.get_full_url().startswith( + 'http://metadata.google.internal/')) + # The urllib module does weird things with header case. + self.assertEqual('Google', req.get_header('Metadata-flavor')) + class TestGetRunFlowFlags(unittest2.TestCase): -- GitLab From b515171a4353b84d928b81e5599eff043c4f83d0 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 14 Feb 2016 23:23:43 -0800 Subject: [PATCH 216/295] Drop gflags/appcommands in oauth2l, support py3. We'd only used gflags in oauth2l for vestigial reasons (read: I first wrote the code internally), but someone ran into related install trouble this weekend. Swapping out argparse was easy, and as a nice by-product, we pick up python3 support in oauth2l. (As a bonus, there's slightly more documentation for a net drop in LOC. w00t.) --- apitools/base/py/exceptions.py | 4 +- apitools/scripts/oauth2l.py | 331 ++++++++++++++++--------------- apitools/scripts/oauth2l_test.py | 103 ++++------ setup.py | 6 +- 4 files changed, 219 insertions(+), 225 deletions(-) diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py index e60c1e0..a3789b9 100644 --- a/apitools/base/py/exceptions.py +++ b/apitools/base/py/exceptions.py @@ -58,7 +58,9 @@ class HttpError(CommunicationError): self.url = url def __str__(self): - content = self.content.decode('ascii', 'replace') + content = self.content + if isinstance(content, bytes): + content = self.content.decode('ascii', 'replace') return 'HttpError accessing <%s>: response: <%s>, content <%s>' % ( self.url, self.response, content) diff --git a/apitools/scripts/oauth2l.py b/apitools/scripts/oauth2l.py index dd13064..0e768b9 100644 --- a/apitools/scripts/oauth2l.py +++ b/apitools/scripts/oauth2l.py @@ -36,8 +36,9 @@ some sample use: The `header` command is designed to be easy to use with `curl`: - $ curl "$(oauth2l header bigquery)" \ - 'https://www.googleapis.com/bigquery/v2/projects' + $ curl -H "$(oauth2l header bigquery)" \\ + 'https://www.googleapis.com/bigquery/v2/projects' + ... lists all projects ... The token can also be printed in other formats, for easy chaining into other programs: @@ -49,7 +50,9 @@ into other programs: """ -import httplib +from __future__ import print_function + +import argparse import json import logging import os @@ -57,14 +60,11 @@ import pkgutil import sys import textwrap -import gflags as flags -from google.apputils import appcommands import oauth2client.client +from six.moves import http_client import apitools.base.py as apitools_base -from apitools.base.py import cli as apitools_cli -FLAGS = flags.FLAGS # We could use a generated client here, but it's used for precisely # one URL, with one parameter and no worries about URL encoding. Let's # go with simple. @@ -74,23 +74,10 @@ _OAUTH2_TOKENINFO_TEMPLATE = ( ) -flags.DEFINE_string( - 'client_secrets', '', - 'If specified, use the client ID/secret from the named ' - 'file, which should be a client_secrets.json file as downloaded ' - 'from the Developer Console.') -flags.DEFINE_string( - 'credentials_filename', '', - '(optional) Filename for fetching/storing credentials.') -flags.DEFINE_string( - 'service_account_json_keyfile', '', - 'Filename for a JSON service account key downloaded from the Developer ' - 'Console.') - - def GetDefaultClientInfo(): - client_secrets = json.loads(pkgutil.get_data( - 'apitools.data', 'apitools_client_secrets.json'))['installed'] + client_secrets_json = pkgutil.get_data( + 'apitools.data', 'apitools_client_secrets.json').decode('utf8') + client_secrets = json.loads(client_secrets_json)['installed'] return { 'client_id': client_secrets['client_id'], 'client_secret': client_secrets['client_secret'], @@ -98,12 +85,13 @@ def GetDefaultClientInfo(): } -def GetClientInfoFromFlags(): - """Fetch client info from FLAGS.""" - if FLAGS.client_secrets: - client_secrets_path = os.path.expanduser(FLAGS.client_secrets) +def GetClientInfoFromFlags(args): + """Fetch client info from args.""" + if args.client_secrets: + client_secrets_path = os.path.expanduser(args.client_secrets) if not os.path.exists(client_secrets_path): - raise ValueError('Cannot find file: %s' % FLAGS.client_secrets) + raise ValueError( + 'Cannot find file: {0}'.format(args.client_secrets)) with open(client_secrets_path) as client_secrets_file: client_secrets = json.load(client_secrets_file) if 'installed' not in client_secrets: @@ -132,6 +120,12 @@ def _CompactJson(data): return json.dumps(data, sort_keys=True, separators=(',', ':')) +def _AsText(text_or_bytes): + if isinstance(text_or_bytes, bytes): + return text_or_bytes.decode('utf8') + return text_or_bytes + + def _Format(fmt, credentials): """Format credentials according to fmt.""" if fmt == 'bare': @@ -139,9 +133,9 @@ def _Format(fmt, credentials): elif fmt == 'header': return 'Authorization: Bearer %s' % credentials.access_token elif fmt == 'json': - return _PrettyJson(json.loads(credentials.to_json())) + return _PrettyJson(json.loads(_AsText(credentials.to_json()))) elif fmt == 'json_compact': - return _CompactJson(json.loads(credentials.to_json())) + return _CompactJson(json.loads(_AsText(credentials.to_json()))) elif fmt == 'pretty': format_str = textwrap.dedent('\n'.join([ 'Fetched credentials of type:', @@ -151,7 +145,7 @@ def _Format(fmt, credentials): ])) return format_str.format(credentials=credentials, credentials_type=type(credentials)) - raise ValueError('Unknown format: {}'.format(fmt)) + raise ValueError('Unknown format: {0}'.format(fmt)) _FORMATS = set(('bare', 'header', 'json', 'json_compact', 'pretty')) @@ -161,11 +155,11 @@ def _GetTokenScopes(access_token): url = _OAUTH2_TOKENINFO_TEMPLATE.format(access_token=access_token) response = apitools_base.MakeRequest( apitools_base.GetHttp(), apitools_base.Request(url)) - if response.status_code not in [httplib.OK, httplib.BAD_REQUEST]: + if response.status_code not in [http_client.OK, http_client.BAD_REQUEST]: raise apitools_base.HttpError.FromResponse(response) - if response.status_code == httplib.BAD_REQUEST: + if response.status_code == http_client.BAD_REQUEST: return [] - return json.loads(response.content)['scope'].split(' ') + return json.loads(_AsText(response.content))['scope'].split(' ') def _ValidateToken(access_token): @@ -173,20 +167,20 @@ def _ValidateToken(access_token): return bool(_GetTokenScopes(access_token)) -def FetchCredentials(scopes, client_info=None, credentials_filename=None): +def FetchCredentials(args, client_info=None, credentials_filename=None): """Fetch a credential for the given client_info and scopes.""" - client_info = client_info or GetClientInfoFromFlags() - scopes = _ExpandScopes(scopes) + client_info = client_info or GetClientInfoFromFlags(args) + scopes = _ExpandScopes(args.scope) if not scopes: raise ValueError('No scopes provided') - credentials_filename = credentials_filename or FLAGS.credentials_filename + credentials_filename = credentials_filename or args.credentials_filename # TODO(craigcitro): Remove this logging nonsense once we quiet the # spurious logging in oauth2client. old_level = logging.getLogger().level logging.getLogger().setLevel(logging.ERROR) credentials = apitools_base.GetCredentials( 'oauth2l', scopes, credentials_filename=credentials_filename, - service_account_json_keyfile=FLAGS.service_account_json_keyfile, + service_account_json_keyfile=args.service_account_json_keyfile, oauth2client_args='', **client_info) logging.getLogger().setLevel(old_level) if not _ValidateToken(credentials.access_token): @@ -194,131 +188,152 @@ def FetchCredentials(scopes, client_info=None, credentials_filename=None): return credentials -class _Email(apitools_cli.NewCmd): - - """Get user email.""" - - usage = 'email ' - - def RunWithArgs(self, access_token): - """Print the email address for this token, if possible.""" - userinfo = apitools_base.GetUserinfo( - oauth2client.client.AccessTokenCredentials(access_token, - 'oauth2l/1.0')) - user_email = userinfo.get('email') - if user_email: - print user_email - - -class _Fetch(apitools_cli.NewCmd): - - """Fetch credentials.""" - - usage = 'fetch [ ...]' - - def __init__(self, name, flag_values): - super(_Fetch, self).__init__(name, flag_values) - flags.DEFINE_enum( - 'credentials_format', 'pretty', sorted(_FORMATS), - 'Output format for token.', - short_name='f', flag_values=flag_values) - - def RunWithArgs(self, *scopes): - """Fetch a valid access token and display it.""" - credentials = FetchCredentials(scopes) - print _Format(FLAGS.credentials_format.lower(), credentials) - - -class _Header(apitools_cli.NewCmd): - - """Print credentials for a header.""" - - usage = 'header [ ...]' - - def RunWithArgs(self, *scopes): - """Fetch a valid access token and display it formatted for a header.""" - print _Format('header', FetchCredentials(scopes)) +def Email(args): + """Print the email address for this token, if possible.""" + userinfo = apitools_base.GetUserinfo( + oauth2client.client.AccessTokenCredentials(args.access_token, + 'oauth2l/1.0')) + user_email = userinfo.get('email') + if user_email: + print(user_email) -class _Scopes(apitools_cli.NewCmd): +def Fetch(args): + """Fetch a valid access token and display it.""" + credentials = FetchCredentials(args) + print(_Format(args.credentials_format.lower(), credentials)) - """Get the list of scopes for a token.""" - usage = 'scopes ' +def Header(args): + """Fetch an access token and display it formatted as an HTTP header.""" + print(_Format('header', FetchCredentials(args))) - def RunWithArgs(self, access_token): - """Print the list of scopes for a valid token.""" - scopes = _GetTokenScopes(access_token) - if not scopes: - return 1 - for scope in sorted(scopes): - print scope - -class _Userinfo(apitools_cli.NewCmd): - - """Get userinfo.""" - - usage = 'userinfo ' - - def __init__(self, name, flag_values): - super(_Userinfo, self).__init__(name, flag_values) - flags.DEFINE_enum( - 'format', 'json', sorted(('json', 'json_compact')), - 'Output format for userinfo.', - short_name='f', flag_values=flag_values) - - def RunWithArgs(self, access_token): - """Print the userinfo for this token (if we have the right scopes).""" - userinfo = apitools_base.GetUserinfo( - oauth2client.client.AccessTokenCredentials(access_token, - 'oauth2l/1.0')) - if FLAGS.format == 'json': - print _PrettyJson(userinfo) - else: - print _CompactJson(userinfo) - - -class _Validate(apitools_cli.NewCmd): - - """Validate a token.""" - - usage = 'validate ' - - def RunWithArgs(self, access_token): - """Validate an access token. Exits with 0 if valid, 1 otherwise.""" - return 1 - (_ValidateToken(access_token)) - - -def run_main(): # pylint:disable=invalid-name - """Function to be used as setuptools script entry point.""" - # Put the flags for this module somewhere the flags module will look - # for them. - - # pylint:disable=protected-access - new_name = flags._GetMainModule() - sys.modules[new_name] = sys.modules['__main__'] - for flag in FLAGS.FlagsByModuleDict().get(__name__, []): - FLAGS._RegisterFlagByModule(new_name, flag) - for key_flag in FLAGS.KeyFlagsByModuleDict().get(__name__, []): - FLAGS._RegisterKeyFlagForModule(new_name, key_flag) - # pylint:enable=protected-access - - # Now set __main__ appropriately so that appcommands will be - # happy. - sys.modules['__main__'] = sys.modules[__name__] - appcommands.Run() - sys.modules['__main__'] = sys.modules.pop(new_name) - - -def main(unused_argv): - appcommands.AddCmd('email', _Email) - appcommands.AddCmd('fetch', _Fetch) - appcommands.AddCmd('header', _Header) - appcommands.AddCmd('scopes', _Scopes) - appcommands.AddCmd('userinfo', _Userinfo) - appcommands.AddCmd('validate', _Validate) +def Scopes(args): + """Print the list of scopes for a valid token.""" + scopes = _GetTokenScopes(args.access_token) + if not scopes: + return 1 + for scope in sorted(scopes): + print(scope) + + +def Userinfo(args): + """Print the userinfo for this token, if possible.""" + userinfo = apitools_base.GetUserinfo( + oauth2client.client.AccessTokenCredentials(args.access_token, + 'oauth2l/1.0')) + if args.format == 'json': + print(_PrettyJson(userinfo)) + else: + print(_CompactJson(userinfo)) + + +def Validate(args): + """Validate an access token. Exits with 0 if valid, 1 otherwise.""" + return 1 - (_ValidateToken(args.access_token)) + + +def _GetParser(): + + shared_flags = argparse.ArgumentParser(add_help=False) + shared_flags.add_argument( + '--client_secrets', + default='', + help=('If specified, use the client ID/secret from the named ' + 'file, which should be a client_secrets.json file ' + 'downloaded from the Developer Console.')) + shared_flags.add_argument( + '--credentials_filename', + default='', + help='(optional) Filename for fetching/storing credentials.') + shared_flags.add_argument( + '--service_account_json_keyfile', + default='', + help=('Filename for a JSON service account key downloaded from ' + 'the Google Developer Console.')) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest='command') + + # email + email = subparsers.add_parser('email', help=Email.__doc__, + parents=[shared_flags]) + email.set_defaults(func=Email) + email.add_argument( + 'access_token', + help=('Access token to print associated email address for. Must have ' + 'the userinfo.email scope.')) + + # fetch + fetch = subparsers.add_parser('fetch', help=Fetch.__doc__, + parents=[shared_flags]) + fetch.set_defaults(func=Fetch) + fetch.add_argument( + '-f', '--credentials_format', + default='pretty', choices=sorted(_FORMATS), + help='Output format for token.') + fetch.add_argument( + 'scope', + nargs='*', + help='Scope to fetch. May be provided multiple times.') + + # header + header = subparsers.add_parser('header', help=Header.__doc__, + parents=[shared_flags]) + header.set_defaults(func=Header) + header.add_argument( + 'scope', + nargs='*', + help='Scope to header. May be provided multiple times.') + + # scopes + scopes = subparsers.add_parser('scopes', help=Scopes.__doc__, + parents=[shared_flags]) + scopes.set_defaults(func=Scopes) + scopes.add_argument( + 'access_token', + help=('Scopes associated with this token will be printed.')) + + # userinfo + userinfo = subparsers.add_parser('userinfo', help=Userinfo.__doc__, + parents=[shared_flags]) + userinfo.set_defaults(func=Userinfo) + userinfo.add_argument( + '-f', '--format', + default='json', choices=('json', 'json_compact'), + help='Output format for userinfo.') + userinfo.add_argument( + 'access_token', + help=('Access token to print associated email address for. Must have ' + 'the userinfo.email scope.')) + + # validate + validate = subparsers.add_parser('validate', help=Validate.__doc__, + parents=[shared_flags]) + validate.set_defaults(func=Validate) + validate.add_argument( + 'access_token', + help='Access token to validate.') + + return parser + + +def main(argv=None): + argv = argv or sys.argv + # Invoke the newly created parser. + args = _GetParser().parse_args(argv[1:]) + try: + exit_code = args.func(args) + except BaseException as e: + print('Error encountered in {0} operation: {1}'.format( + args.command, e)) + return 1 + return exit_code if __name__ == '__main__': - appcommands.Run() + sys.exit(main(sys.argv)) diff --git a/apitools/scripts/oauth2l_test.py b/apitools/scripts/oauth2l_test.py index 40e3896..5d4472c 100644 --- a/apitools/scripts/oauth2l_test.py +++ b/apitools/scripts/oauth2l_test.py @@ -26,16 +26,10 @@ from six.moves import http_client import unittest2 import apitools.base.py as apitools_base +from apitools.scripts import oauth2l _OAUTH2L_MAIN_RUN = False -if six.PY2: - # pylint: disable=wrong-import-position,wrong-import-order - import gflags as flags - from google.apputils import appcommands - from apitools.scripts import oauth2l - FLAGS = flags.FLAGS - class _FakeResponse(object): @@ -49,35 +43,29 @@ class _FakeResponse(object): self.request_url = 'some-url' -def _GetCommandOutput(t, command_name, command_argv): - global _OAUTH2L_MAIN_RUN # pylint: disable=global-statement - if not _OAUTH2L_MAIN_RUN: - oauth2l.main(None) - _OAUTH2L_MAIN_RUN = True - command = appcommands.GetCommandByName(command_name) - if command is None: - t.fail('Unknown command: %s' % command_name) +def _GetCommandOutput(command_name, command_argv): orig_stdout = sys.stdout + orig_stderr = sys.stderr new_stdout = six.StringIO() + new_stderr = six.StringIO() try: sys.stdout = new_stdout - command.CommandRun([command_name] + command_argv) + sys.stderr = new_stderr + oauth2l.main(['oauth2l', command_name] + command_argv) finally: sys.stdout = orig_stdout - FLAGS.Reset() + sys.stderr = orig_stderr new_stdout.seek(0) return new_stdout.getvalue().rstrip() -@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') -class TestTest(unittest2.TestCase): +class InvalidCommandTest(unittest2.TestCase): def testOutput(self): - self.assertRaises(AssertionError, - _GetCommandOutput, self, 'foo', []) + self.assertRaises(SystemExit, + _GetCommandOutput, 'foo', []) -@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') class Oauth2lFormattingTest(unittest2.TestCase): def setUp(self): @@ -94,7 +82,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'fetch', self._Args('bare')) + output = _GetCommandOutput('fetch', self._Args('bare')) self.assertEqual(self.access_token, output) self.assertEqual(1, mock_credentials.call_count) @@ -102,7 +90,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'fetch', self._Args('header')) + output = _GetCommandOutput('fetch', self._Args('header')) header = 'Authorization: Bearer %s' % self.access_token self.assertEqual(header, output) self.assertEqual(1, mock_credentials.call_count) @@ -111,7 +99,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'header', ['userinfo.email']) + output = _GetCommandOutput('header', ['userinfo.email']) header = 'Authorization: Bearer %s' % self.access_token self.assertEqual(header, output) self.assertEqual(1, mock_credentials.call_count) @@ -120,7 +108,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'fetch', self._Args('json')) + output = _GetCommandOutput('fetch', self._Args('json')) output_lines = [l.strip() for l in output.splitlines()] expected_lines = [ '"_class": "AccessTokenCredentials",', @@ -134,8 +122,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'fetch', - self._Args('json_compact')) + output = _GetCommandOutput('fetch', self._Args('json_compact')) expected_clauses = [ '"_class":"AccessTokenCredentials",', '"access_token":"%s",' % self.access_token, @@ -149,7 +136,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): with mock.patch.object(oauth2l, 'FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: - output = _GetCommandOutput(self, 'fetch', self._Args('pretty')) + output = _GetCommandOutput('fetch', self._Args('pretty')) expecteds = ['oauth2client.client.AccessTokenCredentials', self.access_token] for expected in expecteds: @@ -161,7 +148,6 @@ class Oauth2lFormattingTest(unittest2.TestCase): oauth2l._Format, 'xml', self.credentials) -@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') class TestFetch(unittest2.TestCase): def setUp(self): @@ -172,9 +158,9 @@ class TestFetch(unittest2.TestCase): self.access_token, self.user_agent) def testNoScopes(self): - output = _GetCommandOutput(self, 'fetch', []) + output = _GetCommandOutput('fetch', []) self.assertEqual( - 'Exception raised in fetch operation: No scopes provided', + 'Error encountered in fetch operation: No scopes provided', output) def testScopes(self): @@ -189,7 +175,7 @@ class TestFetch(unittest2.TestCase): return_value=expected_scopes, autospec=True) as mock_get_scopes: output = _GetCommandOutput( - self, 'fetch', ['userinfo.email', 'cloud-platform']) + 'fetch', ['userinfo.email', 'cloud-platform']) self.assertIn(self.access_token, output) self.assertEqual(1, mock_fetch.call_count) args, _ = mock_fetch.call_args @@ -208,8 +194,7 @@ class TestFetch(unittest2.TestCase): with mock.patch.object(self.credentials, 'refresh', return_value=None, autospec=True) as mock_refresh: - output = _GetCommandOutput(self, 'fetch', - ['userinfo.email']) + output = _GetCommandOutput('fetch', ['userinfo.email']) self.assertIn(self.access_token, output) self.assertEqual(1, mock_fetch.call_count) self.assertEqual(1, mock_validate.call_count) @@ -222,7 +207,7 @@ class TestFetch(unittest2.TestCase): with mock.patch.object(oauth2l, '_ValidateToken', return_value=True, autospec=True) as mock_validate: - output = _GetCommandOutput(self, 'fetch', ['userinfo.email']) + output = _GetCommandOutput('fetch', ['userinfo.email']) self.assertIn(self.access_token, output) self.assertEqual(1, mock_fetch.call_count) _, kwargs = mock_fetch.call_args @@ -232,25 +217,20 @@ class TestFetch(unittest2.TestCase): self.assertEqual(1, mock_validate.call_count) def testMissingClientSecrets(self): - try: - FLAGS.client_secrets = '/non/existent/file' - self.assertRaises( - ValueError, - oauth2l.GetClientInfoFromFlags) - finally: - FLAGS.Reset() + args = mock.MagicMock() + args.client_secrets = '/non/existent/file' + self.assertRaises( + ValueError, + oauth2l.GetClientInfoFromFlags, args) def testWrongClientSecretsFormat(self): - client_secrets_path = os.path.join( + args = mock.MagicMock() + args.client_secrets = os.path.join( os.path.dirname(__file__), 'testdata/noninstalled_client_secrets.json') - try: - FLAGS.client_secrets = client_secrets_path - self.assertRaises( - ValueError, - oauth2l.GetClientInfoFromFlags) - finally: - FLAGS.Reset() + self.assertRaises( + ValueError, + oauth2l.GetClientInfoFromFlags, args) def testCustomClientInfo(self): client_secrets_path = os.path.join( @@ -264,7 +244,7 @@ class TestFetch(unittest2.TestCase): fetch_args = [ '--client_secrets=' + client_secrets_path, 'userinfo.email'] - output = _GetCommandOutput(self, 'fetch', fetch_args) + output = _GetCommandOutput('fetch', fetch_args) self.assertIn(self.access_token, output) self.assertEqual(1, mock_fetch.call_count) _, kwargs = mock_fetch.call_args @@ -275,7 +255,6 @@ class TestFetch(unittest2.TestCase): self.assertEqual(1, mock_validate.call_count) -@unittest2.skipIf(six.PY3, 'oauth2l unsupported in python3') class TestOtherCommands(unittest2.TestCase): def setUp(self): @@ -290,7 +269,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'GetUserinfo', return_value=user_info, autospec=True) as mock_get_userinfo: - output = _GetCommandOutput(self, 'email', [self.access_token]) + output = _GetCommandOutput('email', [self.access_token]) self.assertEqual(user_info['email'], output) self.assertEqual(1, mock_get_userinfo.call_count) self.assertEqual(self.access_token, @@ -300,7 +279,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'GetUserinfo', return_value={}, autospec=True) as mock_get_userinfo: - output = _GetCommandOutput(self, 'email', [self.access_token]) + output = _GetCommandOutput('email', [self.access_token]) self.assertEqual('', output) self.assertEqual(1, mock_get_userinfo.call_count) @@ -309,7 +288,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'GetUserinfo', return_value=user_info, autospec=True) as mock_get_userinfo: - output = _GetCommandOutput(self, 'userinfo', [self.access_token]) + output = _GetCommandOutput('userinfo', [self.access_token]) self.assertEqual(json.dumps(user_info, indent=4), output) self.assertEqual(1, mock_get_userinfo.call_count) self.assertEqual(self.access_token, @@ -321,7 +300,7 @@ class TestOtherCommands(unittest2.TestCase): return_value=user_info, autospec=True) as mock_get_userinfo: output = _GetCommandOutput( - self, 'userinfo', ['--format=json_compact', self.access_token]) + 'userinfo', ['--format=json_compact', self.access_token]) self.assertEqual(json.dumps(user_info, separators=(',', ':')), output) self.assertEqual(1, mock_get_userinfo.call_count) @@ -335,7 +314,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'MakeRequest', return_value=response, autospec=True) as mock_make_request: - output = _GetCommandOutput(self, 'scopes', [self.access_token]) + output = _GetCommandOutput('scopes', [self.access_token]) self.assertEqual(sorted(scopes), output.splitlines()) self.assertEqual(1, mock_make_request.call_count) @@ -346,7 +325,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'MakeRequest', return_value=response, autospec=True) as mock_make_request: - output = _GetCommandOutput(self, 'validate', [self.access_token]) + output = _GetCommandOutput('validate', [self.access_token]) self.assertEqual('', output) self.assertEqual(1, mock_make_request.call_count) @@ -355,7 +334,7 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'MakeRequest', return_value=response, autospec=True) as mock_make_request: - output = _GetCommandOutput(self, 'scopes', [self.access_token]) + output = _GetCommandOutput('scopes', [self.access_token]) self.assertEqual('', output) self.assertEqual(1, mock_make_request.call_count) @@ -364,9 +343,9 @@ class TestOtherCommands(unittest2.TestCase): with mock.patch.object(apitools_base, 'MakeRequest', return_value=response, autospec=True) as mock_make_request: - output = _GetCommandOutput(self, 'scopes', [self.access_token]) + output = _GetCommandOutput('scopes', [self.access_token]) self.assertIn(str(http_client.responses[response.status_code]), output) - self.assertIn('Exception raised in scopes operation: HttpError', + self.assertIn('Error encountered in scopes operation: HttpError', output) self.assertEqual(1, mock_make_request.call_count) diff --git a/setup.py b/setup.py index aecee66..f9aff93 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ TESTING_PACKAGES = [ CONSOLE_SCRIPTS = [ 'gen_client = apitools.gen.gen_client:main', - 'oauth2l = apitools.scripts.oauth2l:run_main [cli]', + 'oauth2l = apitools.scripts.oauth2l:main', ] py_version = platform.python_version() @@ -70,9 +70,7 @@ setuptools.setup( author_email='craigcitro@google.com', # Contained modules and scripts. packages=setuptools.find_packages(), - entry_points={ - 'console_scripts': CONSOLE_SCRIPTS, - }, + entry_points={'console_scripts': CONSOLE_SCRIPTS}, install_requires=REQUIRED_PACKAGES, tests_require=REQUIRED_PACKAGES + CLI_PACKAGES + TESTING_PACKAGES, extras_require={ -- GitLab From d046c48ab1f1cf59725e407846bc4dfb0ca5cc88 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Tue, 16 Feb 2016 15:53:32 -0800 Subject: [PATCH 217/295] Code review feedback. --- apitools/scripts/oauth2l.py | 52 ++++++++++++++++---------------- apitools/scripts/oauth2l_test.py | 21 ++++++------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/apitools/scripts/oauth2l.py b/apitools/scripts/oauth2l.py index 0e768b9..faaa859 100644 --- a/apitools/scripts/oauth2l.py +++ b/apitools/scripts/oauth2l.py @@ -85,13 +85,13 @@ def GetDefaultClientInfo(): } -def GetClientInfoFromFlags(args): +def GetClientInfoFromFlags(client_secrets): """Fetch client info from args.""" - if args.client_secrets: - client_secrets_path = os.path.expanduser(args.client_secrets) + if client_secrets: + client_secrets_path = os.path.expanduser(client_secrets) if not os.path.exists(client_secrets_path): raise ValueError( - 'Cannot find file: {0}'.format(args.client_secrets)) + 'Cannot find file: {0}'.format(client_secrets)) with open(client_secrets_path) as client_secrets_file: client_secrets = json.load(client_secrets_file) if 'installed' not in client_secrets: @@ -167,9 +167,9 @@ def _ValidateToken(access_token): return bool(_GetTokenScopes(access_token)) -def FetchCredentials(args, client_info=None, credentials_filename=None): +def _FetchCredentials(args, client_info=None, credentials_filename=None): """Fetch a credential for the given client_info and scopes.""" - client_info = client_info or GetClientInfoFromFlags(args) + client_info = client_info or GetClientInfoFromFlags(args.client_secrets) scopes = _ExpandScopes(args.scope) if not scopes: raise ValueError('No scopes provided') @@ -188,7 +188,7 @@ def FetchCredentials(args, client_info=None, credentials_filename=None): return credentials -def Email(args): +def _Email(args): """Print the email address for this token, if possible.""" userinfo = apitools_base.GetUserinfo( oauth2client.client.AccessTokenCredentials(args.access_token, @@ -198,18 +198,18 @@ def Email(args): print(user_email) -def Fetch(args): +def _Fetch(args): """Fetch a valid access token and display it.""" - credentials = FetchCredentials(args) + credentials = _FetchCredentials(args) print(_Format(args.credentials_format.lower(), credentials)) -def Header(args): +def _Header(args): """Fetch an access token and display it formatted as an HTTP header.""" - print(_Format('header', FetchCredentials(args))) + print(_Format('header', _FetchCredentials(args))) -def Scopes(args): +def _Scopes(args): """Print the list of scopes for a valid token.""" scopes = _GetTokenScopes(args.access_token) if not scopes: @@ -218,7 +218,7 @@ def Scopes(args): print(scope) -def Userinfo(args): +def _Userinfo(args): """Print the userinfo for this token, if possible.""" userinfo = apitools_base.GetUserinfo( oauth2client.client.AccessTokenCredentials(args.access_token, @@ -229,7 +229,7 @@ def Userinfo(args): print(_CompactJson(userinfo)) -def Validate(args): +def _Validate(args): """Validate an access token. Exits with 0 if valid, 1 otherwise.""" return 1 - (_ValidateToken(args.access_token)) @@ -260,18 +260,18 @@ def _GetParser(): subparsers = parser.add_subparsers(dest='command') # email - email = subparsers.add_parser('email', help=Email.__doc__, + email = subparsers.add_parser('email', help=_Email.__doc__, parents=[shared_flags]) - email.set_defaults(func=Email) + email.set_defaults(func=_Email) email.add_argument( 'access_token', help=('Access token to print associated email address for. Must have ' 'the userinfo.email scope.')) # fetch - fetch = subparsers.add_parser('fetch', help=Fetch.__doc__, + fetch = subparsers.add_parser('fetch', help=_Fetch.__doc__, parents=[shared_flags]) - fetch.set_defaults(func=Fetch) + fetch.set_defaults(func=_Fetch) fetch.add_argument( '-f', '--credentials_format', default='pretty', choices=sorted(_FORMATS), @@ -282,26 +282,26 @@ def _GetParser(): help='Scope to fetch. May be provided multiple times.') # header - header = subparsers.add_parser('header', help=Header.__doc__, + header = subparsers.add_parser('header', help=_Header.__doc__, parents=[shared_flags]) - header.set_defaults(func=Header) + header.set_defaults(func=_Header) header.add_argument( 'scope', nargs='*', help='Scope to header. May be provided multiple times.') # scopes - scopes = subparsers.add_parser('scopes', help=Scopes.__doc__, + scopes = subparsers.add_parser('scopes', help=_Scopes.__doc__, parents=[shared_flags]) - scopes.set_defaults(func=Scopes) + scopes.set_defaults(func=_Scopes) scopes.add_argument( 'access_token', help=('Scopes associated with this token will be printed.')) # userinfo - userinfo = subparsers.add_parser('userinfo', help=Userinfo.__doc__, + userinfo = subparsers.add_parser('userinfo', help=_Userinfo.__doc__, parents=[shared_flags]) - userinfo.set_defaults(func=Userinfo) + userinfo.set_defaults(func=_Userinfo) userinfo.add_argument( '-f', '--format', default='json', choices=('json', 'json_compact'), @@ -312,9 +312,9 @@ def _GetParser(): 'the userinfo.email scope.')) # validate - validate = subparsers.add_parser('validate', help=Validate.__doc__, + validate = subparsers.add_parser('validate', help=_Validate.__doc__, parents=[shared_flags]) - validate.set_defaults(func=Validate) + validate.set_defaults(func=_Validate) validate.add_argument( 'access_token', help='Access token to validate.') diff --git a/apitools/scripts/oauth2l_test.py b/apitools/scripts/oauth2l_test.py index 5d4472c..157eb3a 100644 --- a/apitools/scripts/oauth2l_test.py +++ b/apitools/scripts/oauth2l_test.py @@ -79,7 +79,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): return ['--credentials_format=' + credentials_format, 'userinfo.email'] def testFormatBare(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('fetch', self._Args('bare')) @@ -87,7 +87,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): self.assertEqual(1, mock_credentials.call_count) def testFormatHeader(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('fetch', self._Args('header')) @@ -96,7 +96,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): self.assertEqual(1, mock_credentials.call_count) def testHeaderCommand(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('header', ['userinfo.email']) @@ -105,7 +105,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): self.assertEqual(1, mock_credentials.call_count) def testFormatJson(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('fetch', self._Args('json')) @@ -119,7 +119,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): self.assertEqual(1, mock_credentials.call_count) def testFormatJsonCompact(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('fetch', self._Args('json_compact')) @@ -133,7 +133,7 @@ class Oauth2lFormattingTest(unittest2.TestCase): self.assertEqual(1, mock_credentials.call_count) def testFormatPretty(self): - with mock.patch.object(oauth2l, 'FetchCredentials', + with mock.patch.object(oauth2l, '_FetchCredentials', return_value=self.credentials, autospec=True) as mock_credentials: output = _GetCommandOutput('fetch', self._Args('pretty')) @@ -217,20 +217,17 @@ class TestFetch(unittest2.TestCase): self.assertEqual(1, mock_validate.call_count) def testMissingClientSecrets(self): - args = mock.MagicMock() - args.client_secrets = '/non/existent/file' self.assertRaises( ValueError, - oauth2l.GetClientInfoFromFlags, args) + oauth2l.GetClientInfoFromFlags, '/non/existent/file') def testWrongClientSecretsFormat(self): - args = mock.MagicMock() - args.client_secrets = os.path.join( + client_secrets = os.path.join( os.path.dirname(__file__), 'testdata/noninstalled_client_secrets.json') self.assertRaises( ValueError, - oauth2l.GetClientInfoFromFlags, args) + oauth2l.GetClientInfoFromFlags, client_secrets) def testCustomClientInfo(self): client_secrets_path = os.path.join( -- GitLab From 7c39b4031f5cc8e98b8d8c073fd0e4cde9582587 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Thu, 18 Feb 2016 18:51:52 -0800 Subject: [PATCH 218/295] Bump version to 0.5 and update apitools requirements to match that. --- apitools/gen/gen_client_lib.py | 17 ++++++++--------- setup.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 8044761..8812a92 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -21,6 +21,7 @@ Relevant links: """ import datetime +import re from six.moves import urllib_parse @@ -240,10 +241,11 @@ class DescriptorGenerator(object): printer('import setuptools') printer('REQUIREMENTS = [') with printer.Indent(indent=' '): - if self.apitools_version.startswith('0.4.'): - printer('"google-apitools>=0.4.8,<0.5",') - else: - printer('"google-apitools==%s",', self.apitools_version) + match = re.search( + r'^(?P\d+)\.(?P\d+)\..*$', self.apitools_version) + printer('"google-apitools>=%s,~%s.%s",', + self.apitools_version, + match.group('major'), match.group('minor')) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') @@ -254,11 +256,8 @@ class DescriptorGenerator(object): with printer.Indent(indent=' '): printer('name="google-apitools-%s-%s",', self.__package, self.__version) - if self.apitools_version.startswith('0.4.'): - printer('version="0.4.%s",', self.__revision) - else: - printer('version="%s.%s",', - self.apitools_version, self.__revision) + printer('version="%s.%s",', + self.apitools_version, self.__revision) printer('description="Autogenerated apitools library for %s",' % ( self.__package,)) printer('url="https://github.com/google/apitools",') diff --git a/setup.py b/setup.py index f9aff93..2392a6c 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.4.15' +_APITOOLS_VERSION = '0.5.0' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 96a3137aeb9bd66b239057a454b1d6000332f348 Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Thu, 18 Feb 2016 19:00:30 -0800 Subject: [PATCH 219/295] Use the proper compatible release operator. --- apitools/gen/gen_client_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 8812a92..e766a30 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -243,7 +243,7 @@ class DescriptorGenerator(object): with printer.Indent(indent=' '): match = re.search( r'^(?P\d+)\.(?P\d+)\..*$', self.apitools_version) - printer('"google-apitools>=%s,~%s.%s",', + printer('"google-apitools>=%s,~=%s.%s",', self.apitools_version, match.group('major'), match.group('minor')) printer('"httplib2>=0.9",') -- GitLab From a6df4c8bda501c2f3be4c61fa85d9eeb6d30b95b Mon Sep 17 00:00:00 2001 From: Silviu Calinoiu Date: Thu, 18 Feb 2016 21:09:39 -0800 Subject: [PATCH 220/295] Simplify code and lower version to alpha release. --- apitools/gen/gen_client_lib.py | 9 ++++----- setup.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index e766a30..fa0319c 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -21,7 +21,6 @@ Relevant links: """ import datetime -import re from six.moves import urllib_parse @@ -241,11 +240,11 @@ class DescriptorGenerator(object): printer('import setuptools') printer('REQUIREMENTS = [') with printer.Indent(indent=' '): - match = re.search( - r'^(?P\d+)\.(?P\d+)\..*$', self.apitools_version) + parts = self.apitools_version.split('.') + major = parts.pop(0) + minor = parts.pop(0) printer('"google-apitools>=%s,~=%s.%s",', - self.apitools_version, - match.group('major'), match.group('minor')) + self.apitools_version, major, minor) printer('"httplib2>=0.9",') printer('"oauth2client>=1.4.12",') printer(']') diff --git a/setup.py b/setup.py index 2392a6c..1bbdddd 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.0' +_APITOOLS_VERSION = '0.5.0a1' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From abbfcb1732bf749888a14d758429f6c8d356189d Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 21 Feb 2016 21:44:45 -0800 Subject: [PATCH 221/295] Update service_account support for oauth2clientv2. This makes three changes: * Make the code for loading either .json or .p12 credentials work under either oauth2client 2.0.0post1+ or oauth2client < 2.0.0. In particular, this mostly amounts to picking up a few renames from the oauth2client changelog. * Drop `credentials_lib.ServiceAccount()`. From some searching, no one at all is using this function, and supporting it makes the case of .p12 keys significantly more awkward. * Change `credentials_lib.ServiceAccountFromFile()` to handle .json keys, not .p12 keys. I'm fairly certain that apitools itself was the only caller of this code. Tested by hand (via oauth2l). --- apitools/base/py/credentials_lib.py | 91 +++++++++++++++++------------ apitools/gen/util.py | 2 +- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index c7479e6..571414f 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -25,7 +25,7 @@ import threading import httplib2 import oauth2client import oauth2client.client -import oauth2client.service_account +from oauth2client import service_account from oauth2client import tools # for gflags declarations from six.moves import http_client from six.moves import urllib @@ -56,7 +56,6 @@ __all__ = [ 'GceAssertionCredentials', 'GetCredentials', 'GetUserinfo', - 'ServiceAccountCredentials', 'ServiceAccountCredentialsFromFile', ] @@ -130,21 +129,57 @@ def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, raise exceptions.CredentialsError('Could not create valid credentials') -def ServiceAccountCredentialsFromFile( - service_account_name, private_key_filename, scopes, - service_account_kwargs=None): - with open(private_key_filename) as key_file: - return ServiceAccountCredentials( - service_account_name, key_file.read(), scopes, - service_account_kwargs=service_account_kwargs) +def ServiceAccountCredentialsFromFile(filename, scopes, user_agent=None): + """Use the credentials in filename to create a token for scopes.""" + filename = os.path.expanduser(filename) + # We have two options, based on our version of oauth2client. + if oauth2client.__version__ > '1.5.2': + # oauth2client >= 2.0.0 + credentials = ( + service_account.ServiceAccountCredentials.from_json_keyfile_name( + filename, scopes=scopes)) + if credentials is not None: + if user_agent is not None: + credentials.user_agent = user_agent + return credentials + else: + # oauth2client < 2.0.0 + with open(filename) as keyfile: + service_account_info = json.load(keyfile) + account_type = service_account_info.get('type') + if account_type != oauth2client.client.SERVICE_ACCOUNT: + raise exceptions.CredentialsError( + 'Invalid service account credentials: %s' % (filename,)) + # pylint: disable=protected-access + credentials = service_account._ServiceAccountCredentials( + service_account_id=service_account_info['client_id'], + service_account_email=service_account_info['client_email'], + private_key_id=service_account_info['private_key_id'], + private_key_pkcs8_text=service_account_info['private_key'], + scopes=scopes, user_agent=user_agent) + # pylint: enable=protected-access + return credentials -def ServiceAccountCredentials(service_account_name, private_key, scopes, - service_account_kwargs=None): - service_account_kwargs = service_account_kwargs or {} +def ServiceAccountCredentialsFromP12File( + service_account_name, private_key_filename, scopes, user_agent): + """Create a new credential from the named .p12 keyfile.""" + private_key_filename = os.path.expanduser(private_key_filename) scopes = util.NormalizeScopes(scopes) - return oauth2client.client.SignedJwtAssertionCredentials( - service_account_name, private_key, scopes, **service_account_kwargs) + if oauth2client.__version__ > '1.5.2': + # oauth2client >= 2.0.0 + credentials = ( + service_account.ServiceAccountCredentials.from_p12_keyfile( + service_account_name, private_key_filename, scopes=scopes)) + if credentials is not None: + credentials.user_agent = user_agent + return credentials + else: + # oauth2client < 2.0.0 + with open(private_key_filename) as key_file: + return oauth2client.client.SignedJwtAssertionCredentials( + service_account_name, key_file.read(), scopes, + user_agent=user_agent) def _EnsureFileExists(filename): @@ -524,30 +559,14 @@ def _GetServiceAccountCredentials( 'Service account name or keyfile provided without the other') scopes = client_info['scope'].split() user_agent = client_info['user_agent'] + # Use the .json credentials, if provided. if service_account_json_keyfile: - with open(service_account_json_keyfile) as keyfile: - service_account_info = json.load(keyfile) - account_type = service_account_info.get('type') - if account_type != oauth2client.client.SERVICE_ACCOUNT: - raise exceptions.CredentialsError( - 'Invalid service account credentials: %s' % ( - service_account_json_keyfile,)) - # pylint: disable=protected-access - credentials = oauth2client.service_account._ServiceAccountCredentials( - service_account_id=service_account_info['client_id'], - service_account_email=service_account_info['client_email'], - private_key_id=service_account_info['private_key_id'], - private_key_pkcs8_text=service_account_info['private_key'], - scopes=scopes, user_agent=user_agent) - # pylint: enable=protected-access - return credentials + return ServiceAccountCredentialsFromFile( + service_account_json_keyfile, scopes, user_agent=user_agent) + # Fall back to .p12 if there's no .json credentials. if service_account_name is not None: - # pylint: disable=redefined-variable-type - credentials = ServiceAccountCredentialsFromFile( - service_account_name, service_account_keyfile, scopes, - service_account_kwargs={'user_agent': user_agent}) - if credentials is not None: - return credentials + return ServiceAccountCredentialsFromP12File( + service_account_name, service_account_keyfile, scopes, user_agent) @_RegisterCredentialsMethod diff --git a/apitools/gen/util.py b/apitools/gen/util.py index e1f7724..6ceec93 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -319,7 +319,7 @@ def FetchDiscoveryDoc(discovery_url, retries=5): discovery_doc = json.loads(urllib_request.urlopen(url).read()) break except (urllib_error.HTTPError, urllib_error.URLError) as e: - logging.warning( + logging.info( 'Attempting to fetch discovery doc again after "%s"', e) last_exception = e if discovery_doc is None: -- GitLab From 8ecaced20b6a8cc342ff48d5182cc55813b40212 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 22 Feb 2016 22:02:42 -0800 Subject: [PATCH 222/295] Update tox and travis configs to add python 3.5. This just drops python3.3 and adds python3.5. Note that there's a bit of tomfoolery around getting python3.5 to be available. --- .travis.yml | 10 ++++++++++ tox.ini | 10 +++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5bbdd50..9718e67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ env: - TOX_ENV=py26 - TOX_ENV=py27 - TOX_ENV=py34 + - TOX_ENV=py35 - TOX_ENV=pypy - TOX_ENV=lint install: @@ -12,3 +13,12 @@ install: script: tox -e $TOX_ENV after_success: - if [[ "${TOX_ENV}" == "py27" ]]; then tox -e coveralls; fi + +# Tweak for adding python3.5; see +# https://github.com/travis-ci/travis-ci/issues/4794 +addons: + apt: + sources: + - deadsnakes + packages: + - python3.5 diff --git a/tox.ini b/tox.ini index daf7988..44fca5a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pypy,py33,py34,lint,cover +envlist = py26,py27,pypy,py34,py35,lint,cover [testenv] deps = nose @@ -8,16 +8,16 @@ commands = nosetests [] passenv = TRAVIS* -[testenv:py33] -basepython = python3.3 +[testenv:py34] +basepython = python3.4 deps = mock nose unittest2 commands = nosetests [] -[testenv:py34] -basepython = python3.4 +[testenv:py35] +basepython = python3.5 deps = mock nose -- GitLab From 18efc26350b9af2cd0d668e8926bf8f982236a84 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Mon, 22 Feb 2016 22:20:14 -0800 Subject: [PATCH 223/295] Update for v0.5.0 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1bbdddd..2392a6c 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.0a1' +_APITOOLS_VERSION = '0.5.0' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From f268d0ee10ddba2e60a118901922bc225453d0b8 Mon Sep 17 00:00:00 2001 From: Kevin Regan Date: Tue, 8 Mar 2016 15:22:28 -0800 Subject: [PATCH 224/295] Attempt to import newer versions of oauth2client libraries first. We have instances of this library being used in environments where only a portion of the oauth2client library has been updated. This change allows us to find the latest versions of these libraries, when available. --- apitools/base/py/credentials_lib.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 571414f..cf3a8e8 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -39,9 +39,19 @@ from apitools.base.py import util # # pylint: disable=wrong-import-order,ungrouped-imports try: - from oauth2client import gce, locked_file, multistore_file + from oauth2client.contrib import gce except ImportError: - from oauth2client.contrib import gce, locked_file, multistore_file + from oauth2client import gce + +try: + from oauth2client.contrib import locked_file +except ImportError: + from oauth2client import locked_file + +try: + from oauth2client.contrib import multistore_file +except ImportError: + from oauth2client import multistore_file try: import gflags -- GitLab From 8c71deb365824cb631377614211848047f15515b Mon Sep 17 00:00:00 2001 From: Kevin Regan Date: Tue, 8 Mar 2016 16:05:11 -0800 Subject: [PATCH 225/295] The new oauth2client.client.AssertionCredentials has a new abstract method, causing lint to fail. Add a sign_blob definition (that same as in oauth2client.contrib.gce.AppAssertionCredentials so that lint won't complain. --- apitools/base/py/credentials_lib.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index cf3a8e8..a68c044 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -472,6 +472,22 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): raise exceptions.CredentialsError(str(e)) self.access_token = token + def sign_blob(self, blob): + """Cryptographically sign a blob (of bytes). + + This method is provided to support a common interface, but + the actual key used for a Google Compute Engine service account + is not available, so it can't be used to sign content. + + Args: + blob: bytes, Message to be signed. + + Raises: + NotImplementedError, always. + """ + raise NotImplementedError( + 'Compute Engine service accounts cannot sign blobs') + def _GetRunFlowFlags(args=None): # There's one rare situation where gsutil will not have argparse -- GitLab From 0236d836e6ebdcfcf231e2af21f85899a7685b87 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 11 Mar 2016 08:39:08 -0800 Subject: [PATCH 226/295] Cleanup package data, and fix a warning. This does two small-but-independent cleanups: 1. Fixes MANIFEST.in to always pick up the client_secrets.json file. 2. Tweaks a call in credentials_lib to avoid a (useful) warning in oauth2client.contrib.gce. --- MANIFEST.in | 1 + apitools/base/py/credentials_lib.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index fc80ed3..7585bef 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include *.py include *.txt include *.md recursive-include apitools *.py +recursive-include apitools *.json diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index a68c044..6ea4412 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -257,7 +257,10 @@ class GceAssertionCredentials(gce.AppAssertionCredentials): if cache_filename and not cached_scopes: self._WriteCacheFile(cache_filename, scopes) - super(GceAssertionCredentials, self).__init__(scopes, **kwds) + # We check the scopes above, but don't need them again after + # this point; in addition, the parent class spits out a + # warning if we pass them here, so we purposely drop them. + super(GceAssertionCredentials, self).__init__(**kwds) @classmethod def Get(cls, *args, **kwds): -- GitLab From 787435df600e5fbe78cb26c560895020bd1c1483 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 11 Mar 2016 13:21:51 -0800 Subject: [PATCH 227/295] Update for v0.5.1 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2392a6c..50223a6 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.0' +_APITOOLS_VERSION = '0.5.1' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 91a5fd40debcb978ef023e93f52e96078057129c Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 16 Mar 2016 14:21:39 -0700 Subject: [PATCH 228/295] Play nice with older oauth2client versions. This reverts part of a previous change to be compatible with older oauth2client versions. In particular, we don't expect that the `scopes` argument to `gce.AppAssertionCredentials` is optional. --- apitools/base/py/credentials_lib.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 6ea4412..7612f27 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -21,6 +21,7 @@ import datetime import json import os import threading +import warnings import httplib2 import oauth2client @@ -258,9 +259,12 @@ class GceAssertionCredentials(gce.AppAssertionCredentials): self._WriteCacheFile(cache_filename, scopes) # We check the scopes above, but don't need them again after - # this point; in addition, the parent class spits out a - # warning if we pass them here, so we purposely drop them. - super(GceAssertionCredentials, self).__init__(**kwds) + # this point. Newer versions of oauth2client let us drop them + # here, but since we support older versions as well, we just + # catch and squelch the warning. + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + super(GceAssertionCredentials, self).__init__(scopes, **kwds) @classmethod def Get(cls, *args, **kwds): -- GitLab From a4f9f60c8d1b3aeb1afdcaf4e703327e69dfcfb5 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 16 Mar 2016 14:27:51 -0700 Subject: [PATCH 229/295] Add a tox+travis env with an older oauth2client. We can drop this in a few months, when oauth2client2 is safe to assume. --- .travis.yml | 1 + tox.ini | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9718e67..6d9ac9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ sudo: false env: - TOX_ENV=py26 - TOX_ENV=py27 + - TOX_ENV=py27oldoauth2client - TOX_ENV=py34 - TOX_ENV=py35 - TOX_ENV=pypy diff --git a/tox.ini b/tox.ini index 44fca5a..bf28d7e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pypy,py34,py35,lint,cover +envlist = py26,py27,pypy,py34,py35,lint,cover,py27oldoauth2client [testenv] deps = nose @@ -8,6 +8,12 @@ commands = nosetests [] passenv = TRAVIS* +[testenv:py27oldoauth2client] +commands = + pip install oauth2client==1.5.2 + {[testenv]commands} +deps = {[testenv]deps} + [testenv:py34] basepython = python3.4 deps = -- GitLab From 5bc3a0e540b8ca5c6dda473292eb78f40607bcf6 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Wed, 16 Mar 2016 14:39:27 -0700 Subject: [PATCH 230/295] Update for v0.5.2 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 50223a6..d85c20a 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.1' +_APITOOLS_VERSION = '0.5.2' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 740da0070a83f9634e2e854036aa476c5fe9c22d Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 8 Apr 2016 10:43:35 -0400 Subject: [PATCH 231/295] Add diff test to the generator. This will allow to see as part of commit what changes happen to generated code for given change to generator. --- apitools/gen/gen_client_test.py | 29 +- apitools/gen/testdata/gen___init__.py | 11 + apitools/gen/testdata/gen_dns_v1.py | 554 +++++++++++++++++++ apitools/gen/testdata/gen_dns_v1_client.py | 316 +++++++++++ apitools/gen/testdata/gen_dns_v1_messages.py | 425 ++++++++++++++ 5 files changed, 1325 insertions(+), 10 deletions(-) create mode 100644 apitools/gen/testdata/gen___init__.py create mode 100755 apitools/gen/testdata/gen_dns_v1.py create mode 100644 apitools/gen/testdata/gen_dns_v1_client.py create mode 100644 apitools/gen/testdata/gen_dns_v1_messages.py diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 25e9e23..9909f81 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -23,10 +23,15 @@ from apitools.gen import gen_client from apitools.gen import test_utils -def GetDocPath(name): +def GetTestDataPath(name): return os.path.join(os.path.dirname(__file__), 'testdata', name) +def _GetContent(file_path): + with open(file_path) as f: + return f.read() + + @test_utils.RunOnlyOnPython27 class ClientGenCliTest(unittest2.TestCase): @@ -42,23 +47,27 @@ class ClientGenCliTest(unittest2.TestCase): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ gen_client.__file__, - '--nogenerate_cli', - '--infile', GetDocPath('dns_v1.json'), + '--generate_cli', + '--infile', GetTestDataPath('dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', 'client' ]) - self.assertEquals( - set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py']), - set(os.listdir(tmp_dir_path))) + expected_files = (set(['dns_v1.py']) | # CLI files + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py'])) + self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) + for expected_file in expected_files: + self.assertMultiLineEqual( + _GetContent(GetTestDataPath('gen_' + expected_file)), + _GetContent(os.path.join(tmp_dir_path, expected_file))) def testGenClient_SimpleDocWithV4(self): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetDocPath('dns_v1.json'), + '--infile', GetTestDataPath('dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--apitools_version', '0.4.12', @@ -74,7 +83,7 @@ class ClientGenCliTest(unittest2.TestCase): gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetDocPath('dns_v1.json'), + '--infile', GetTestDataPath('dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--apitools_version', '0.5.0', @@ -90,7 +99,7 @@ class ClientGenCliTest(unittest2.TestCase): gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetDocPath('dns_v1.json'), + '--infile', GetTestDataPath('dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', @@ -105,7 +114,7 @@ class ClientGenCliTest(unittest2.TestCase): gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetDocPath('dns_v1.json'), + '--infile', GetTestDataPath('dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', diff --git a/apitools/gen/testdata/gen___init__.py b/apitools/gen/testdata/gen___init__.py new file mode 100644 index 0000000..f8f17ad --- /dev/null +++ b/apitools/gen/testdata/gen___init__.py @@ -0,0 +1,11 @@ +"""Common imports for generated dns client library.""" +# pylint:disable=wildcard-import + +import pkgutil + +from apitools.base.py import * +from google.apis.dns_v1 import * +from google.apis.dns_v1_client import * +from google.apis.dns_v1_messages import * + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/apitools/gen/testdata/gen_dns_v1.py b/apitools/gen/testdata/gen_dns_v1.py new file mode 100755 index 0000000..56e54f3 --- /dev/null +++ b/apitools/gen/testdata/gen_dns_v1.py @@ -0,0 +1,554 @@ +#!/usr/bin/env python +"""CLI for dns, version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. + +import code +import os +import platform +import sys + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages + +from google.apputils import appcommands +import gflags as flags + +import apitools.base.py as apitools_base +from apitools.base.py import cli as apitools_base_cli +import dns_v1_client as client_lib +import dns_v1_messages as messages + + +def _DeclareDnsFlags(): + """Declare global flags in an idempotent way.""" + if 'api_endpoint' in flags.FLAGS: + return + flags.DEFINE_string( + 'api_endpoint', + u'https://www.googleapis.com/dns/v1/', + 'URL of the API endpoint to use.', + short_name='dns_url') + flags.DEFINE_string( + 'history_file', + u'~/.dns.v1.history', + 'File with interactive shell history.') + flags.DEFINE_multistring( + 'add_header', [], + 'Additional http headers (as key=value strings). ' + 'Can be specified multiple times.') + flags.DEFINE_string( + 'service_account_json_keyfile', '', + 'Filename for a JSON service account key downloaded' + ' from the Developer Console.') + flags.DEFINE_enum( + 'alt', + u'json', + [u'json'], + u'Data format for the response.') + flags.DEFINE_string( + 'fields', + None, + u'Selector specifying which fields to include in a partial response.') + flags.DEFINE_string( + 'key', + None, + u'API key. Your API key identifies your project and provides you with ' + u'API access, quota, and reports. Required unless you provide an OAuth ' + u'2.0 token.') + flags.DEFINE_string( + 'oauth_token', + None, + u'OAuth 2.0 token for the current user.') + flags.DEFINE_boolean( + 'prettyPrint', + 'True', + u'Returns response with indentations and line breaks.') + flags.DEFINE_string( + 'quotaUser', + None, + u'Available to use for quota purposes for server-side applications. Can' + u' be any arbitrary string assigned to a user, but should not exceed 40' + u' characters. Overrides userIp if both are provided.') + flags.DEFINE_string( + 'trace', + None, + 'A tracing token of the form "token:" to include in api ' + 'requests.') + flags.DEFINE_string( + 'userIp', + None, + u'IP address of the site where the request originates. Use this if you ' + u'want to enforce per-user limits.') + + +FLAGS = flags.FLAGS +apitools_base_cli.DeclareBaseFlags() +_DeclareDnsFlags() + + +def GetGlobalParamsFromFlags(): + """Return a StandardQueryParameters based on flags.""" + result = messages.StandardQueryParameters() + if FLAGS['alt'].present: + result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt) + if FLAGS['fields'].present: + result.fields = FLAGS.fields.decode('utf8') + if FLAGS['key'].present: + result.key = FLAGS.key.decode('utf8') + if FLAGS['oauth_token'].present: + result.oauth_token = FLAGS.oauth_token.decode('utf8') + if FLAGS['prettyPrint'].present: + result.prettyPrint = FLAGS.prettyPrint + if FLAGS['quotaUser'].present: + result.quotaUser = FLAGS.quotaUser.decode('utf8') + if FLAGS['trace'].present: + result.trace = FLAGS.trace.decode('utf8') + if FLAGS['userIp'].present: + result.userIp = FLAGS.userIp.decode('utf8') + return result + + +def GetClientFromFlags(): + """Return a client object, configured from flags.""" + log_request = FLAGS.log_request or FLAGS.log_request_response + log_response = FLAGS.log_response or FLAGS.log_request_response + api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) + additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header) + credentials_args = { + 'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile) + } + try: + client = client_lib.DnsV1( + api_endpoint, log_request=log_request, + log_response=log_response, + credentials_args=credentials_args, + additional_http_headers=additional_http_headers) + except apitools_base.CredentialsError as e: + print 'Error creating credentials: %s' % e + sys.exit(1) + return client + + +class PyShell(appcommands.Cmd): + + def Run(self, _): + """Run an interactive python shell with the client.""" + client = GetClientFromFlags() + params = GetGlobalParamsFromFlags() + for field in params.all_fields(): + value = params.get_assigned_value(field.name) + if value != field.default: + client.AddGlobalParam(field.name, value) + banner = """ + == dns interactive console == + client: a dns client + apitools_base: base apitools module + messages: the generated messages module + """ + local_vars = { + 'apitools_base': apitools_base, + 'client': client, + 'client_lib': client_lib, + 'messages': messages, + } + if platform.system() == 'Linux': + console = apitools_base_cli.ConsoleWithReadline( + local_vars, histfile=FLAGS.history_file) + else: + console = code.InteractiveConsole(local_vars) + try: + console.interact(banner) + except SystemExit as e: + return e.code + + +class ChangesCreate(apitools_base_cli.NewCmd): + """Command wrapping changes.Create.""" + + usage = """changes_create """ + + def __init__(self, name, fv): + super(ChangesCreate, self).__init__(name, fv) + flags.DEFINE_string( + 'change', + None, + u'A Change resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, project, managedZone): + """Atomically update the ResourceRecordSet collection. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + + Flags: + change: A Change resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsChangesCreateRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + ) + if FLAGS['change'].present: + request.change = apitools_base.JsonToMessage(messages.Change, FLAGS.change) + result = client.changes.Create( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ChangesGet(apitools_base_cli.NewCmd): + """Command wrapping changes.Get.""" + + usage = """changes_get """ + + def __init__(self, name, fv): + super(ChangesGet, self).__init__(name, fv) + + def RunWithArgs(self, project, managedZone, changeId): + """Fetch the representation of an existing Change. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + changeId: The identifier of the requested change, from a previous + ResourceRecordSetsChangeResponse. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsChangesGetRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + changeId=changeId.decode('utf8'), + ) + result = client.changes.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ChangesList(apitools_base_cli.NewCmd): + """Command wrapping changes.List.""" + + usage = """changes_list """ + + def __init__(self, name, fv): + super(ChangesList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Optional. Maximum number of results to be returned. If unspecified,' + u' the server will decide how many results to return.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Optional. A tag returned by a previous list request that was ' + u'truncated. Use this parameter to continue a previous list request.', + flag_values=fv) + flags.DEFINE_enum( + 'sortBy', + u'changeSequence', + [u'changeSequence'], + u'Sorting criterion. The only supported value is change sequence.', + flag_values=fv) + flags.DEFINE_string( + 'sortOrder', + None, + u"Sorting order direction: 'ascending' or 'descending'.", + flag_values=fv) + + def RunWithArgs(self, project, managedZone): + """Enumerate Changes to a ResourceRecordSet collection. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + + Flags: + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + sortBy: Sorting criterion. The only supported value is change sequence. + sortOrder: Sorting order direction: 'ascending' or 'descending'. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsChangesListRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['sortBy'].present: + request.sortBy = messages.DnsChangesListRequest.SortByValueValuesEnum(FLAGS.sortBy) + if FLAGS['sortOrder'].present: + request.sortOrder = FLAGS.sortOrder.decode('utf8') + result = client.changes.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ManagedZonesCreate(apitools_base_cli.NewCmd): + """Command wrapping managedZones.Create.""" + + usage = """managedZones_create """ + + def __init__(self, name, fv): + super(ManagedZonesCreate, self).__init__(name, fv) + flags.DEFINE_string( + 'managedZone', + None, + u'A ManagedZone resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, project): + """Create a new ManagedZone. + + Args: + project: Identifies the project addressed by this request. + + Flags: + managedZone: A ManagedZone resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsManagedZonesCreateRequest( + project=project.decode('utf8'), + ) + if FLAGS['managedZone'].present: + request.managedZone = apitools_base.JsonToMessage(messages.ManagedZone, FLAGS.managedZone) + result = client.managedZones.Create( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ManagedZonesDelete(apitools_base_cli.NewCmd): + """Command wrapping managedZones.Delete.""" + + usage = """managedZones_delete """ + + def __init__(self, name, fv): + super(ManagedZonesDelete, self).__init__(name, fv) + + def RunWithArgs(self, project, managedZone): + """Delete a previously created ManagedZone. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsManagedZonesDeleteRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + ) + result = client.managedZones.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ManagedZonesGet(apitools_base_cli.NewCmd): + """Command wrapping managedZones.Get.""" + + usage = """managedZones_get """ + + def __init__(self, name, fv): + super(ManagedZonesGet, self).__init__(name, fv) + + def RunWithArgs(self, project, managedZone): + """Fetch the representation of an existing ManagedZone. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsManagedZonesGetRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + ) + result = client.managedZones.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ManagedZonesList(apitools_base_cli.NewCmd): + """Command wrapping managedZones.List.""" + + usage = """managedZones_list """ + + def __init__(self, name, fv): + super(ManagedZonesList, self).__init__(name, fv) + flags.DEFINE_string( + 'dnsName', + None, + u'Restricts the list to return only zones with this domain name.', + flag_values=fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Optional. Maximum number of results to be returned. If unspecified,' + u' the server will decide how many results to return.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Optional. A tag returned by a previous list request that was ' + u'truncated. Use this parameter to continue a previous list request.', + flag_values=fv) + + def RunWithArgs(self, project): + """Enumerate ManagedZones that have been created but not yet deleted. + + Args: + project: Identifies the project addressed by this request. + + Flags: + dnsName: Restricts the list to return only zones with this domain name. + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsManagedZonesListRequest( + project=project.decode('utf8'), + ) + if FLAGS['dnsName'].present: + request.dnsName = FLAGS.dnsName.decode('utf8') + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.managedZones.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsGet(apitools_base_cli.NewCmd): + """Command wrapping projects.Get.""" + + usage = """projects_get """ + + def __init__(self, name, fv): + super(ProjectsGet, self).__init__(name, fv) + + def RunWithArgs(self, project): + """Fetch the representation of an existing Project. + + Args: + project: Identifies the project addressed by this request. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsProjectsGetRequest( + project=project.decode('utf8'), + ) + result = client.projects.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ResourceRecordSetsList(apitools_base_cli.NewCmd): + """Command wrapping resourceRecordSets.List.""" + + usage = """resourceRecordSets_list """ + + def __init__(self, name, fv): + super(ResourceRecordSetsList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Optional. Maximum number of results to be returned. If unspecified,' + u' the server will decide how many results to return.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Restricts the list to return only records with this fully qualified' + u' domain name.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Optional. A tag returned by a previous list request that was ' + u'truncated. Use this parameter to continue a previous list request.', + flag_values=fv) + flags.DEFINE_string( + 'type', + None, + u'Restricts the list to return only records of this type. If present,' + u' the "name" parameter must also be present.', + flag_values=fv) + + def RunWithArgs(self, project, managedZone): + """Enumerate ResourceRecordSets that have been created but not yet + deleted. + + Args: + project: Identifies the project addressed by this request. + managedZone: Identifies the managed zone addressed by this request. Can + be the managed zone name or id. + + Flags: + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + name: Restricts the list to return only records with this fully + qualified domain name. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + type: Restricts the list to return only records of this type. If + present, the "name" parameter must also be present. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.DnsResourceRecordSetsListRequest( + project=project.decode('utf8'), + managedZone=managedZone.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['type'].present: + request.type = FLAGS.type.decode('utf8') + result = client.resourceRecordSets.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +def main(_): + appcommands.AddCmd('pyshell', PyShell) + appcommands.AddCmd('changes_create', ChangesCreate) + appcommands.AddCmd('changes_get', ChangesGet) + appcommands.AddCmd('changes_list', ChangesList) + appcommands.AddCmd('managedZones_create', ManagedZonesCreate) + appcommands.AddCmd('managedZones_delete', ManagedZonesDelete) + appcommands.AddCmd('managedZones_get', ManagedZonesGet) + appcommands.AddCmd('managedZones_list', ManagedZonesList) + appcommands.AddCmd('projects_get', ProjectsGet) + appcommands.AddCmd('resourceRecordSets_list', ResourceRecordSetsList) + + apitools_base_cli.SetupLogger() + if hasattr(appcommands, 'SetDefaultCommand'): + appcommands.SetDefaultCommand('pyshell') + + +run_main = apitools_base_cli.run_main + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/gen/testdata/gen_dns_v1_client.py b/apitools/gen/testdata/gen_dns_v1_client.py new file mode 100644 index 0000000..52165cd --- /dev/null +++ b/apitools/gen/testdata/gen_dns_v1_client.py @@ -0,0 +1,316 @@ +"""Generated client library for dns version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. +from apitools.base.py import base_api +from google.apis import dns_v1_messages as messages + + +class DnsV1(base_api.BaseApiClient): + """Generated client library for service dns version v1.""" + + MESSAGES_MODULE = messages + + _PACKAGE = u'dns' + _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/ndev.clouddns.readonly', u'https://www.googleapis.com/auth/ndev.clouddns.readwrite'] + _VERSION = u'v1' + _CLIENT_ID = '1042881264118.apps.googleusercontent.com' + _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _CLIENT_CLASS_NAME = u'DnsV1' + _URL_VERSION = u'v1' + _API_KEY = None + + def __init__(self, url='', credentials=None, + get_credentials=True, http=None, model=None, + log_request=False, log_response=False, + credentials_args=None, default_global_params=None, + additional_http_headers=None): + """Create a new dns handle.""" + url = url or u'https://www.googleapis.com/dns/v1/' + super(DnsV1, self).__init__( + url, credentials=credentials, + get_credentials=get_credentials, http=http, model=model, + log_request=log_request, log_response=log_response, + credentials_args=credentials_args, + default_global_params=default_global_params, + additional_http_headers=additional_http_headers) + self.changes = self.ChangesService(self) + self.managedZones = self.ManagedZonesService(self) + self.projects = self.ProjectsService(self) + self.resourceRecordSets = self.ResourceRecordSetsService(self) + + class ChangesService(base_api.BaseApiService): + """Service class for the changes resource.""" + + _NAME = u'changes' + + def __init__(self, client): + super(DnsV1.ChangesService, self).__init__(client) + self._method_configs = { + 'Create': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dns.changes.create', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes', + request_field=u'change', + request_type_name=u'DnsChangesCreateRequest', + response_type_name=u'Change', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.changes.get', + ordered_params=[u'project', u'managedZone', u'changeId'], + path_params=[u'changeId', u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes/{changeId}', + request_field='', + request_type_name=u'DnsChangesGetRequest', + response_type_name=u'Change', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.changes.list', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[u'maxResults', u'pageToken', u'sortBy', u'sortOrder'], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes', + request_field='', + request_type_name=u'DnsChangesListRequest', + response_type_name=u'ChangesListResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Create(self, request, global_params=None): + """Atomically update the ResourceRecordSet collection. + + Args: + request: (DnsChangesCreateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Change) The response message. + """ + config = self.GetMethodConfig('Create') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Fetch the representation of an existing Change. + + Args: + request: (DnsChangesGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Change) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Enumerate Changes to a ResourceRecordSet collection. + + Args: + request: (DnsChangesListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ChangesListResponse) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + class ManagedZonesService(base_api.BaseApiService): + """Service class for the managedZones resource.""" + + _NAME = u'managedZones' + + def __init__(self, client): + super(DnsV1.ManagedZonesService, self).__init__(client) + self._method_configs = { + 'Create': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dns.managedZones.create', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones', + request_field=u'managedZone', + request_type_name=u'DnsManagedZonesCreateRequest', + response_type_name=u'ManagedZone', + supports_download=False, + ), + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'dns.managedZones.delete', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}', + request_field='', + request_type_name=u'DnsManagedZonesDeleteRequest', + response_type_name=u'DnsManagedZonesDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.managedZones.get', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}', + request_field='', + request_type_name=u'DnsManagedZonesGetRequest', + response_type_name=u'ManagedZone', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.managedZones.list', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[u'dnsName', u'maxResults', u'pageToken'], + relative_path=u'projects/{project}/managedZones', + request_field='', + request_type_name=u'DnsManagedZonesListRequest', + response_type_name=u'ManagedZonesListResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Create(self, request, global_params=None): + """Create a new ManagedZone. + + Args: + request: (DnsManagedZonesCreateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ManagedZone) The response message. + """ + config = self.GetMethodConfig('Create') + return self._RunMethod( + config, request, global_params=global_params) + + def Delete(self, request, global_params=None): + """Delete a previously created ManagedZone. + + Args: + request: (DnsManagedZonesDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (DnsManagedZonesDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Fetch the representation of an existing ManagedZone. + + Args: + request: (DnsManagedZonesGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ManagedZone) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Enumerate ManagedZones that have been created but not yet deleted. + + Args: + request: (DnsManagedZonesListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ManagedZonesListResponse) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + class ProjectsService(base_api.BaseApiService): + """Service class for the projects resource.""" + + _NAME = u'projects' + + def __init__(self, client): + super(DnsV1.ProjectsService, self).__init__(client) + self._method_configs = { + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.projects.get', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[], + relative_path=u'projects/{project}', + request_field='', + request_type_name=u'DnsProjectsGetRequest', + response_type_name=u'Project', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Get(self, request, global_params=None): + """Fetch the representation of an existing Project. + + Args: + request: (DnsProjectsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Project) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + class ResourceRecordSetsService(base_api.BaseApiService): + """Service class for the resourceRecordSets resource.""" + + _NAME = u'resourceRecordSets' + + def __init__(self, client): + super(DnsV1.ResourceRecordSetsService, self).__init__(client) + self._method_configs = { + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.resourceRecordSets.list', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[u'maxResults', u'name', u'pageToken', u'type'], + relative_path=u'projects/{project}/managedZones/{managedZone}/rrsets', + request_field='', + request_type_name=u'DnsResourceRecordSetsListRequest', + response_type_name=u'ResourceRecordSetsListResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def List(self, request, global_params=None): + """Enumerate ResourceRecordSets that have been created but not yet deleted. + + Args: + request: (DnsResourceRecordSetsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ResourceRecordSetsListResponse) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/apitools/gen/testdata/gen_dns_v1_messages.py b/apitools/gen/testdata/gen_dns_v1_messages.py new file mode 100644 index 0000000..ef474c4 --- /dev/null +++ b/apitools/gen/testdata/gen_dns_v1_messages.py @@ -0,0 +1,425 @@ +"""Generated message classes for dns version v1. + +The Google Cloud DNS API provides services for configuring and serving +authoritative DNS records. +""" +# NOTE: This file is autogenerated and should not be edited by hand. + +from apitools.base.protorpclite import messages as _messages + + +package = 'dns' + + +class Change(_messages.Message): + """An atomic update to a collection of ResourceRecordSets. + + Enums: + StatusValueValuesEnum: Status of the operation (output only). + + Fields: + additions: Which ResourceRecordSets to add? + deletions: Which ResourceRecordSets to remove? Must match existing data + exactly. + id: Unique identifier for the resource; defined by the server (output + only). + kind: Identifies what kind of resource this is. Value: the fixed string + "dns#change". + startTime: The time that this operation was started by the server. This is + in RFC3339 text format. + status: Status of the operation (output only). + """ + + class StatusValueValuesEnum(_messages.Enum): + """Status of the operation (output only). + + Values: + done: + pending: + """ + done = 0 + pending = 1 + + additions = _messages.MessageField('ResourceRecordSet', 1, repeated=True) + deletions = _messages.MessageField('ResourceRecordSet', 2, repeated=True) + id = _messages.StringField(3) + kind = _messages.StringField(4, default=u'dns#change') + startTime = _messages.StringField(5) + status = _messages.EnumField('StatusValueValuesEnum', 6) + + +class ChangesListResponse(_messages.Message): + """The response to a request to enumerate Changes to a ResourceRecordSets + collection. + + Fields: + changes: The requested changes. + kind: Type of resource. + nextPageToken: The presence of this field indicates that there exist more + results following your last page of results in pagination order. To + fetch them, make another list request using this value as your + pagination token. In this way you can retrieve the complete contents of + even very large collections one page at a time. However, if the contents + of the collection change between the first and last paginated list + request, the set of all elements returned will be an inconsistent view + of the collection. There is no way to retrieve a "snapshot" of + collections larger than the maximum page size. + """ + + changes = _messages.MessageField('Change', 1, repeated=True) + kind = _messages.StringField(2, default=u'dns#changesListResponse') + nextPageToken = _messages.StringField(3) + + +class DnsChangesCreateRequest(_messages.Message): + """A DnsChangesCreateRequest object. + + Fields: + change: A Change resource to be passed as the request body. + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + project: Identifies the project addressed by this request. + """ + + change = _messages.MessageField('Change', 1) + managedZone = _messages.StringField(2, required=True) + project = _messages.StringField(3, required=True) + + +class DnsChangesGetRequest(_messages.Message): + """A DnsChangesGetRequest object. + + Fields: + changeId: The identifier of the requested change, from a previous + ResourceRecordSetsChangeResponse. + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + project: Identifies the project addressed by this request. + """ + + changeId = _messages.StringField(1, required=True) + managedZone = _messages.StringField(2, required=True) + project = _messages.StringField(3, required=True) + + +class DnsChangesListRequest(_messages.Message): + """A DnsChangesListRequest object. + + Enums: + SortByValueValuesEnum: Sorting criterion. The only supported value is + change sequence. + + Fields: + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + project: Identifies the project addressed by this request. + sortBy: Sorting criterion. The only supported value is change sequence. + sortOrder: Sorting order direction: 'ascending' or 'descending'. + """ + + class SortByValueValuesEnum(_messages.Enum): + """Sorting criterion. The only supported value is change sequence. + + Values: + changeSequence: + """ + changeSequence = 0 + + managedZone = _messages.StringField(1, required=True) + maxResults = _messages.IntegerField(2, variant=_messages.Variant.INT32) + pageToken = _messages.StringField(3) + project = _messages.StringField(4, required=True) + sortBy = _messages.EnumField('SortByValueValuesEnum', 5, default=u'changeSequence') + sortOrder = _messages.StringField(6) + + +class DnsManagedZonesCreateRequest(_messages.Message): + """A DnsManagedZonesCreateRequest object. + + Fields: + managedZone: A ManagedZone resource to be passed as the request body. + project: Identifies the project addressed by this request. + """ + + managedZone = _messages.MessageField('ManagedZone', 1) + project = _messages.StringField(2, required=True) + + +class DnsManagedZonesDeleteRequest(_messages.Message): + """A DnsManagedZonesDeleteRequest object. + + Fields: + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + project: Identifies the project addressed by this request. + """ + + managedZone = _messages.StringField(1, required=True) + project = _messages.StringField(2, required=True) + + +class DnsManagedZonesDeleteResponse(_messages.Message): + """An empty DnsManagedZonesDelete response.""" + + +class DnsManagedZonesGetRequest(_messages.Message): + """A DnsManagedZonesGetRequest object. + + Fields: + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + project: Identifies the project addressed by this request. + """ + + managedZone = _messages.StringField(1, required=True) + project = _messages.StringField(2, required=True) + + +class DnsManagedZonesListRequest(_messages.Message): + """A DnsManagedZonesListRequest object. + + Fields: + dnsName: Restricts the list to return only zones with this domain name. + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + project: Identifies the project addressed by this request. + """ + + dnsName = _messages.StringField(1) + maxResults = _messages.IntegerField(2, variant=_messages.Variant.INT32) + pageToken = _messages.StringField(3) + project = _messages.StringField(4, required=True) + + +class DnsProjectsGetRequest(_messages.Message): + """A DnsProjectsGetRequest object. + + Fields: + project: Identifies the project addressed by this request. + """ + + project = _messages.StringField(1, required=True) + + +class DnsResourceRecordSetsListRequest(_messages.Message): + """A DnsResourceRecordSetsListRequest object. + + Fields: + managedZone: Identifies the managed zone addressed by this request. Can be + the managed zone name or id. + maxResults: Optional. Maximum number of results to be returned. If + unspecified, the server will decide how many results to return. + name: Restricts the list to return only records with this fully qualified + domain name. + pageToken: Optional. A tag returned by a previous list request that was + truncated. Use this parameter to continue a previous list request. + project: Identifies the project addressed by this request. + type: Restricts the list to return only records of this type. If present, + the "name" parameter must also be present. + """ + + managedZone = _messages.StringField(1, required=True) + maxResults = _messages.IntegerField(2, variant=_messages.Variant.INT32) + name = _messages.StringField(3) + pageToken = _messages.StringField(4) + project = _messages.StringField(5, required=True) + type = _messages.StringField(6) + + +class ManagedZone(_messages.Message): + """A zone is a subtree of the DNS namespace under one administrative + responsibility. A ManagedZone is a resource that represents a DNS zone + hosted by the Cloud DNS service. + + Fields: + creationTime: The time that this resource was created on the server. This + is in RFC3339 text format. Output only. + description: A mutable string of at most 1024 characters associated with + this resource for the user's convenience. Has no effect on the managed + zone's function. + dnsName: The DNS name of this managed zone, for instance "example.com.". + id: Unique identifier for the resource; defined by the server (output + only) + kind: Identifies what kind of resource this is. Value: the fixed string + "dns#managedZone". + name: User assigned name for this resource. Must be unique within the + project. The name must be 1-32 characters long, must begin with a + letter, end with a letter or digit, and only contain lowercase letters, + digits or dashes. + nameServerSet: Optionally specifies the NameServerSet for this + ManagedZone. A NameServerSet is a set of DNS name servers that all host + the same ManagedZones. Most users will leave this field unset. + nameServers: Delegate your managed_zone to these virtual name servers; + defined by the server (output only) + """ + + creationTime = _messages.StringField(1) + description = _messages.StringField(2) + dnsName = _messages.StringField(3) + id = _messages.IntegerField(4, variant=_messages.Variant.UINT64) + kind = _messages.StringField(5, default=u'dns#managedZone') + name = _messages.StringField(6) + nameServerSet = _messages.StringField(7) + nameServers = _messages.StringField(8, repeated=True) + + +class ManagedZonesListResponse(_messages.Message): + """A ManagedZonesListResponse object. + + Fields: + kind: Type of resource. + managedZones: The managed zone resources. + nextPageToken: The presence of this field indicates that there exist more + results following your last page of results in pagination order. To + fetch them, make another list request using this value as your page + token. In this way you can retrieve the complete contents of even very + large collections one page at a time. However, if the contents of the + collection change between the first and last paginated list request, the + set of all elements returned will be an inconsistent view of the + collection. There is no way to retrieve a consistent snapshot of a + collection larger than the maximum page size. + """ + + kind = _messages.StringField(1, default=u'dns#managedZonesListResponse') + managedZones = _messages.MessageField('ManagedZone', 2, repeated=True) + nextPageToken = _messages.StringField(3) + + +class Project(_messages.Message): + """A project resource. The project is a top level container for resources + including Cloud DNS ManagedZones. Projects can be created only in the APIs + console. + + Fields: + id: User assigned unique identifier for the resource (output only). + kind: Identifies what kind of resource this is. Value: the fixed string + "dns#project". + number: Unique numeric identifier for the resource; defined by the server + (output only). + quota: Quotas assigned to this project (output only). + """ + + id = _messages.StringField(1) + kind = _messages.StringField(2, default=u'dns#project') + number = _messages.IntegerField(3, variant=_messages.Variant.UINT64) + quota = _messages.MessageField('Quota', 4) + + +class Quota(_messages.Message): + """Limits associated with a Project. + + Fields: + kind: Identifies what kind of resource this is. Value: the fixed string + "dns#quota". + managedZones: Maximum allowed number of managed zones in the project. + resourceRecordsPerRrset: Maximum allowed number of ResourceRecords per + ResourceRecordSet. + rrsetAdditionsPerChange: Maximum allowed number of ResourceRecordSets to + add per ChangesCreateRequest. + rrsetDeletionsPerChange: Maximum allowed number of ResourceRecordSets to + delete per ChangesCreateRequest. + rrsetsPerManagedZone: Maximum allowed number of ResourceRecordSets per + zone in the project. + totalRrdataSizePerChange: Maximum allowed size for total rrdata in one + ChangesCreateRequest in bytes. + """ + + kind = _messages.StringField(1, default=u'dns#quota') + managedZones = _messages.IntegerField(2, variant=_messages.Variant.INT32) + resourceRecordsPerRrset = _messages.IntegerField(3, variant=_messages.Variant.INT32) + rrsetAdditionsPerChange = _messages.IntegerField(4, variant=_messages.Variant.INT32) + rrsetDeletionsPerChange = _messages.IntegerField(5, variant=_messages.Variant.INT32) + rrsetsPerManagedZone = _messages.IntegerField(6, variant=_messages.Variant.INT32) + totalRrdataSizePerChange = _messages.IntegerField(7, variant=_messages.Variant.INT32) + + +class ResourceRecordSet(_messages.Message): + """A unit of data that will be returned by the DNS servers. + + Fields: + kind: Identifies what kind of resource this is. Value: the fixed string + "dns#resourceRecordSet". + name: For example, www.example.com. + rrdatas: As defined in RFC 1035 (section 5) and RFC 1034 (section 3.6.1). + ttl: Number of seconds that this ResourceRecordSet can be cached by + resolvers. + type: The identifier of a supported record type, for example, A, AAAA, MX, + TXT, and so on. + """ + + kind = _messages.StringField(1, default=u'dns#resourceRecordSet') + name = _messages.StringField(2) + rrdatas = _messages.StringField(3, repeated=True) + ttl = _messages.IntegerField(4, variant=_messages.Variant.INT32) + type = _messages.StringField(5) + + +class ResourceRecordSetsListResponse(_messages.Message): + """A ResourceRecordSetsListResponse object. + + Fields: + kind: Type of resource. + nextPageToken: The presence of this field indicates that there exist more + results following your last page of results in pagination order. To + fetch them, make another list request using this value as your + pagination token. In this way you can retrieve the complete contents of + even very large collections one page at a time. However, if the contents + of the collection change between the first and last paginated list + request, the set of all elements returned will be an inconsistent view + of the collection. There is no way to retrieve a consistent snapshot of + a collection larger than the maximum page size. + rrsets: The resource record set resources. + """ + + kind = _messages.StringField(1, default=u'dns#resourceRecordSetsListResponse') + nextPageToken = _messages.StringField(2) + rrsets = _messages.MessageField('ResourceRecordSet', 3, repeated=True) + + +class StandardQueryParameters(_messages.Message): + """Query parameters accepted by all methods. + + Enums: + AltValueValuesEnum: Data format for the response. + + Fields: + alt: Data format for the response. + fields: Selector specifying which fields to include in a partial response. + key: API key. Your API key identifies your project and provides you with + API access, quota, and reports. Required unless you provide an OAuth 2.0 + token. + oauth_token: OAuth 2.0 token for the current user. + prettyPrint: Returns response with indentations and line breaks. + quotaUser: Available to use for quota purposes for server-side + applications. Can be any arbitrary string assigned to a user, but should + not exceed 40 characters. Overrides userIp if both are provided. + trace: A tracing token of the form "token:" to include in api + requests. + userIp: IP address of the site where the request originates. Use this if + you want to enforce per-user limits. + """ + + class AltValueValuesEnum(_messages.Enum): + """Data format for the response. + + Values: + json: Responses with Content-Type of application/json + """ + json = 0 + + alt = _messages.EnumField('AltValueValuesEnum', 1, default=u'json') + fields = _messages.StringField(2) + key = _messages.StringField(3) + oauth_token = _messages.StringField(4) + prettyPrint = _messages.BooleanField(5, default=True) + quotaUser = _messages.StringField(6) + trace = _messages.StringField(7) + userIp = _messages.StringField(8) + + -- GitLab From 1f3af12df3153214808dc6e88f6c87483a2e184a Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 8 Apr 2016 11:35:15 -0400 Subject: [PATCH 232/295] Fix lint issues. --- apitools/gen/gen_client_test.py | 9 +++++---- run_pylint.py | 1 + tox.ini | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 9909f81..b0b7290 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -28,8 +28,8 @@ def GetTestDataPath(name): def _GetContent(file_path): - with open(file_path) as f: - return f.read() + with open(file_path) as f: + return f.read() @test_utils.RunOnlyOnPython27 @@ -54,8 +54,9 @@ class ClientGenCliTest(unittest2.TestCase): '--root_package', 'google.apis', 'client' ]) - expected_files = (set(['dns_v1.py']) | # CLI files - set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py'])) + expected_files = ( + set(['dns_v1.py']) | # CLI files + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py'])) self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) for expected_file in expected_files: self.assertMultiLineEqual( diff --git a/run_pylint.py b/run_pylint.py index f8babac..5ad86bb 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -32,6 +32,7 @@ import sys IGNORED_DIRECTORIES = [ + 'apitools/gen/testdata', 'samples/storage_sample/storage', ] IGNORED_FILES = [ diff --git a/tox.ini b/tox.ini index bf28d7e..249aa41 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ deps = commands = nosetests [] [pep8] -exclude = samples/storage_sample/storage,samples/storage_sample/testdata,*.egg/,*.egg-info/,.*/,ez_setup.py,build +exclude = samples/storage_sample/storage,*/testdata/*,*.egg/,*.egg-info/,.*/,ez_setup.py,build verbose = 1 [testenv:lint] -- GitLab From c4711aec3529de32807f2053ed060739e011dd36 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Sun, 17 Apr 2016 09:41:38 -0400 Subject: [PATCH 233/295] Expose static base url in generated api client. This will allow to access URL attribute without instantiating client. --- apitools/gen/command_registry.py | 6 ++--- apitools/gen/gen_client_lib.py | 19 +------------- apitools/gen/service_registry.py | 29 +++++++++++++--------- apitools/gen/testdata/gen_dns_v1_client.py | 3 ++- apitools/gen/util.py | 25 ++++++++++++++++--- run_pylint.py | 1 + 6 files changed, 45 insertions(+), 38 deletions(-) diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py index ad2aa00..ef0c667 100644 --- a/apitools/gen/command_registry.py +++ b/apitools/gen/command_registry.py @@ -116,8 +116,7 @@ class CommandRegistry(object): """Registry for CLI commands.""" def __init__(self, package, version, client_info, message_registry, - root_package, base_files_package, protorpc_package, - base_url, names): + root_package, base_files_package, protorpc_package, names): self.__package = package self.__version = version self.__client_info = client_info @@ -126,7 +125,6 @@ class CommandRegistry(object): self.__root_package = root_package self.__base_files_package = base_files_package self.__protorpc_package = protorpc_package - self.__base_url = base_url self.__command_list = [] self.__global_flags = [] @@ -305,7 +303,7 @@ class CommandRegistry(object): printer('flags.DEFINE_string(') with printer.Indent(' '): printer("'api_endpoint',") - printer('%r,', self.__base_url) + printer('%r,', self.__client_info.base_url) printer("'URL of the API endpoint to use.',") printer("short_name='%s_url')", self.__package) printer('flags.DEFINE_string(') diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index fa0319c..66fbaf8 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -22,8 +22,6 @@ Relevant links: import datetime -from six.moves import urllib_parse - from apitools.gen import command_registry from apitools.gen import message_registry from apitools.gen import service_registry @@ -54,16 +52,6 @@ def _StandardQueryParametersSchema(discovery_doc): return standard_query_schema -def _ComputePaths(package, version, discovery_doc): - full_path = urllib_parse.urljoin( - discovery_doc['rootUrl'], discovery_doc['servicePath']) - api_path_component = '/'.join((package, version, '')) - if api_path_component not in full_path: - return full_path, '' - prefix, _, suffix = full_path.rpartition(api_path_component) - return prefix + api_path_component, suffix - - class DescriptorGenerator(object): """Code generator for a given discovery document.""" @@ -86,9 +74,6 @@ class DescriptorGenerator(object): self.__base_files_package = base_package self.__protorpc_package = protorpc_package self.__names = names - self.__base_url, self.__base_path = _ComputePaths( - self.__package, self.__client_info.url_version, - self.__discovery_doc) # Order is important here: we need the schemas before we can # define the services. @@ -115,7 +100,7 @@ class DescriptorGenerator(object): self.__package, self.__version, self.__client_info, self.__message_registry, self.__root_package, self.__base_files_package, self.__protorpc_package, - self.__base_url, self.__names) + self.__names) self.__command_registry.AddGlobalParameters( self.__message_registry.LookupDescriptorOrDie( 'StandardQueryParameters')) @@ -124,8 +109,6 @@ class DescriptorGenerator(object): self.__client_info, self.__message_registry, self.__command_registry, - self.__base_url, - self.__base_path, self.__names, self.__root_package, self.__base_files_package, diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index ded364a..863dca5 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -35,8 +35,7 @@ class ServiceRegistry(object): """Registry for service types.""" def __init__(self, client_info, message_registry, command_registry, - base_url, base_path, names, - root_package, base_files_package, + names, root_package, base_files_package, unelidable_request_methods): self.__client_info = client_info self.__package = client_info.package @@ -44,8 +43,6 @@ class ServiceRegistry(object): self.__service_method_info_map = collections.OrderedDict() self.__message_registry = message_registry self.__command_registry = command_registry - self.__base_url = base_url - self.__base_path = base_path self.__root_package = root_package self.__base_files_package = base_files_package self.__unelidable_request_methods = unelidable_request_methods @@ -222,13 +219,20 @@ class ServiceRegistry(object): client_info.package, client_info.version) printer() printer('MESSAGES_MODULE = messages') + printer('BASE_URL = {0!r}'.format(client_info.base_url)) printer() - # pylint: disable=protected-access - client_info_items = client_info._asdict().items() - for attr, val in client_info_items: - if attr == 'scopes' and not val: - val = ['https://www.googleapis.com/auth/userinfo.email'] - printer('_%s = %r' % (attr.upper(), val)) + printer('_PACKAGE = {0!r}'.format(client_info.package)) + printer('_SCOPES = {0!r}'.format( + client_info.scopes or + ['https://www.googleapis.com/auth/userinfo.email'])) + printer('_VERSION = {0!r}'.format(client_info.version)) + printer('_CLIENT_ID = {0!r}'.format(client_info.client_id)) + printer('_CLIENT_SECRET = {0!r}'.format(client_info.client_secret)) + printer('_USER_AGENT = {0!r}'.format(client_info.user_agent)) + printer('_CLIENT_CLASS_NAME = {0!r}'.format( + client_info.client_class_name)) + printer('_URL_VERSION = {0!r}'.format(client_info.url_version)) + printer('_API_KEY = {0!r}'.format(client_info.api_key)) printer() printer("def __init__(self, url='', credentials=None,") with printer.Indent(indent=' '): @@ -238,7 +242,7 @@ class ServiceRegistry(object): printer('additional_http_headers=None):') with printer.Indent(): printer('"""Create a new %s handle."""', client_info.package) - printer('url = url or %r', self.__base_url) + printer('url = url or self.BASE_URL') printer( 'super(%s, self).__init__(', client_info.client_class_name) printer(' url, credentials=credentials,') @@ -368,7 +372,8 @@ class ServiceRegistry(object): request_field): """Compute the base_api.ApiMethodInfo for this method.""" relative_path = self.__names.NormalizeRelativePath( - ''.join((self.__base_path, method_description['path']))) + ''.join((self.__client_info.base_path, + method_description['path']))) method_id = method_description['id'] ordered_params = [] for param_name in method_description.get('parameterOrder', []): diff --git a/apitools/gen/testdata/gen_dns_v1_client.py b/apitools/gen/testdata/gen_dns_v1_client.py index 52165cd..4509b6b 100644 --- a/apitools/gen/testdata/gen_dns_v1_client.py +++ b/apitools/gen/testdata/gen_dns_v1_client.py @@ -8,6 +8,7 @@ class DnsV1(base_api.BaseApiClient): """Generated client library for service dns version v1.""" MESSAGES_MODULE = messages + BASE_URL = u'https://www.googleapis.com/dns/v1/' _PACKAGE = u'dns' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/ndev.clouddns.readonly', u'https://www.googleapis.com/auth/ndev.clouddns.readwrite'] @@ -25,7 +26,7 @@ class DnsV1(base_api.BaseApiClient): credentials_args=None, default_global_params=None, additional_http_headers=None): """Create a new dns handle.""" - url = url or u'https://www.googleapis.com/dns/v1/' + url = url or self.BASE_URL super(DnsV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, diff --git a/apitools/gen/util.py b/apitools/gen/util.py index 6ceec93..146b452 100644 --- a/apitools/gen/util.py +++ b/apitools/gen/util.py @@ -26,6 +26,7 @@ import os import re import six +from six.moves import urllib_parse import six.moves.urllib.error as urllib_error import six.moves.urllib.request as urllib_request @@ -170,9 +171,20 @@ def NormalizeVersion(version): return version.replace('.', '_') +def _ComputePaths(package, version, discovery_doc): + full_path = urllib_parse.urljoin( + discovery_doc['rootUrl'], discovery_doc['servicePath']) + api_path_component = '/'.join((package, version, '')) + if api_path_component not in full_path: + return full_path, '' + prefix, _, suffix = full_path.rpartition(api_path_component) + return prefix + api_path_component, suffix + + class ClientInfo(collections.namedtuple('ClientInfo', ( 'package', 'scopes', 'version', 'client_id', 'client_secret', - 'user_agent', 'client_class_name', 'url_version', 'api_key'))): + 'user_agent', 'client_class_name', 'url_version', 'api_key', + 'base_url', 'base_path'))): """Container for client-related info and names.""" @@ -183,15 +195,22 @@ class ClientInfo(collections.namedtuple('ClientInfo', ( scopes = set( discovery_doc.get('auth', {}).get('oauth2', {}).get('scopes', {})) scopes.update(scope_ls) + package = discovery_doc['name'] + url_version = discovery_doc['version'] + base_url, base_path = _ComputePaths(package, url_version, + discovery_doc) + client_info = { - 'package': discovery_doc['name'], + 'package': package, 'version': NormalizeVersion(discovery_doc['version']), - 'url_version': discovery_doc['version'], + 'url_version': url_version, 'scopes': sorted(list(scopes)), 'client_id': client_id, 'client_secret': client_secret, 'user_agent': user_agent, 'api_key': api_key, + 'base_url': base_url, + 'base_path': base_path, } client_class_name = '%s%s' % ( names.ClassName(client_info['package']), diff --git a/run_pylint.py b/run_pylint.py index 5ad86bb..fd55463 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -34,6 +34,7 @@ import sys IGNORED_DIRECTORIES = [ 'apitools/gen/testdata', 'samples/storage_sample/storage', + 'venv', ] IGNORED_FILES = [ 'ez_setup.py', -- GitLab From ccbe7aa4ec174f7bb2c54973d42f52273ac800cd Mon Sep 17 00:00:00 2001 From: Ivan Naranjo Date: Tue, 3 May 2016 19:33:21 +0100 Subject: [PATCH 234/295] Ensuring that a new url is used after refreshing the credentials. --- apitools/base/py/credentials_lib.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index 7612f27..ba17960 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -570,18 +570,23 @@ def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name aren't available. """ http = http or httplib2.Http() - url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' - query_args = {'access_token': credentials.access_token} - url = '?'.join((url_root, urllib.parse.urlencode(query_args))) + url = _GetUserinfoUrl(credentials) # We ignore communication woes here (i.e. SSL errors, socket # timeout), as handling these should be done in a common location. response, content = http.request(url) if response.status == http_client.BAD_REQUEST: credentials.refresh(http) + url = _GetUserinfoUrl(credentials) response, content = http.request(url) return json.loads(content or '{}') # Save ourselves from an empty reply. +def _GetUserinfoUrl(credentials): + url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' + query_args = {'access_token': credentials.access_token} + return '?'.join((url_root, urllib.parse.urlencode(query_args))) + + @_RegisterCredentialsMethod def _GetServiceAccountCredentials( client_info, service_account_name=None, service_account_keyfile=None, -- GitLab From 3b42c92cc315c58d74fed9449d3ce16d4cc0444a Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Sun, 8 May 2016 23:39:17 -0700 Subject: [PATCH 235/295] Fix an issue with Python3 multipart encodings. As discovered in https://github.com/GoogleCloudPlatform/gcloud-python/issues/1760, we were mangling bytes when encoding them as part of a multipart upload request. The fix is to switch from using `six.StringIO` to `six.BytesIO` in `transfer.py`. The patch here is closely based on https://github.com/GoogleCloudPlatform/gcloud-python/pull/1779. --- apitools/base/py/transfer.py | 16 +++-- apitools/base/py/transfer_test.py | 116 +++++++++++++++++------------- 2 files changed, 77 insertions(+), 55 deletions(-) diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py index 397222e..29bbbcc 100644 --- a/apitools/base/py/transfer.py +++ b/apitools/base/py/transfer.py @@ -758,20 +758,24 @@ class Upload(_Transfer): # NOTE: We encode the body, but can't use # `email.message.Message.as_string` because it prepends # `> ` to `From ` lines. - # NOTE: We must use six.StringIO() instead of io.StringIO() since the - # `email` library uses cStringIO in Py2 and io.StringIO in Py3. - fp = six.StringIO() - g = email_generator.Generator(fp, mangle_from_=False) + fp = six.BytesIO() + if six.PY3: + generator_class = email_generator.BytesGenerator + else: + generator_class = email_generator.Generator + g = generator_class(fp, mangle_from_=False) g.flatten(msg_root, unixfrom=False) http_request.body = fp.getvalue() multipart_boundary = msg_root.get_boundary() http_request.headers['content-type'] = ( 'multipart/related; boundary=%r' % multipart_boundary) + if isinstance(multipart_boundary, six.text_type): + multipart_boundary = multipart_boundary.encode('ascii') body_components = http_request.body.split(multipart_boundary) - headers, _, _ = body_components[-2].partition('\n\n') - body_components[-2] = '\n\n'.join([headers, '\n\n--']) + headers, _, _ = body_components[-2].partition(b'\n\n') + body_components[-2] = b'\n\n'.join([headers, b'\n\n--']) http_request.loggable_body = multipart_boundary.join(body_components) def __ConfigureResumableRequest(self, http_request): diff --git a/apitools/base/py/transfer_test.py b/apitools/base/py/transfer_test.py index 53906fd..a4c43e7 100644 --- a/apitools/base/py/transfer_test.py +++ b/apitools/base/py/transfer_test.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # # Copyright 2015 Google Inc. # @@ -199,52 +200,69 @@ class TransferTest(unittest2.TestCase): self.assertEqual(string.ascii_lowercase + string.ascii_uppercase, download_stream.getvalue()) - def testFromEncoding(self): - # Test a specific corner case in multipart encoding. - - # Python's mime module by default encodes lines that start with - # "From " as ">From ", which we need to make sure we don't run afoul - # of when sending content that isn't intended to be so encoded. This - # test calls out that we get this right. We test for both the - # multipart and non-multipart case. - multipart_body = '{"body_field_one": 7}' - upload_contents = 'line one\nFrom \nline two' - upload_config = base_api.ApiUploadInfo( - accept=['*/*'], - max_size=None, - resumable_multipart=True, - resumable_path=u'/resumable/upload', - simple_multipart=True, - simple_path=u'/upload', - ) - url_builder = base_api._UrlBuilder('http://www.uploads.com') - - # Test multipart: having a body argument in http_request forces - # multipart here. - upload = transfer.Upload.FromStream( - six.StringIO(upload_contents), - 'text/plain', - total_size=len(upload_contents)) - http_request = http_wrapper.Request( - 'http://www.uploads.com', - headers={'content-type': 'text/plain'}, - body=multipart_body) - upload.ConfigureRequest(upload_config, http_request, url_builder) - self.assertEqual(url_builder.query_params['uploadType'], 'multipart') - rewritten_upload_contents = '\n'.join( - http_request.body.split('--')[2].splitlines()[1:]) - self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) - - # Test non-multipart (aka media): no body argument means this is - # sent as media. - upload = transfer.Upload.FromStream( - six.StringIO(upload_contents), - 'text/plain', - total_size=len(upload_contents)) - http_request = http_wrapper.Request( - 'http://www.uploads.com', - headers={'content-type': 'text/plain'}) - upload.ConfigureRequest(upload_config, http_request, url_builder) - self.assertEqual(url_builder.query_params['uploadType'], 'media') - rewritten_upload_contents = http_request.body - self.assertTrue(rewritten_upload_contents.endswith(upload_contents)) + def testMultipartEncoding(self): + # This is really a table test for various issues we've seen in + # the past; see notes below for particular histories. + + test_cases = [ + # Python's mime module by default encodes lines that start + # with "From " as ">From ", which we need to make sure we + # don't run afoul of when sending content that isn't + # intended to be so encoded. This test calls out that we + # get this right. We test for both the multipart and + # non-multipart case. + 'line one\nFrom \nline two', + + # We had originally used a `six.StringIO` to hold the http + # request body in the case of a multipart upload; for + # bytes being uploaded in Python3, however, this causes + # issues like this: + # https://github.com/GoogleCloudPlatform/gcloud-python/issues/1760 + # We test below to ensure that we don't end up mangling + # the body before sending. + u'name,main_ingredient\nRäksmörgås,Räkor\nBaguette,Bröd', + ] + + for upload_contents in test_cases: + multipart_body = '{"body_field_one": 7}' + upload_bytes = upload_contents.encode('ascii', 'backslashreplace') + upload_config = base_api.ApiUploadInfo( + accept=['*/*'], + max_size=None, + resumable_multipart=True, + resumable_path=u'/resumable/upload', + simple_multipart=True, + simple_path=u'/upload', + ) + url_builder = base_api._UrlBuilder('http://www.uploads.com') + + # Test multipart: having a body argument in http_request forces + # multipart here. + upload = transfer.Upload.FromStream( + six.BytesIO(upload_bytes), + 'text/plain', + total_size=len(upload_bytes)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}, + body=multipart_body) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual( + 'multipart', url_builder.query_params['uploadType']) + rewritten_upload_contents = b'\n'.join( + http_request.body.split(b'--')[2].splitlines()[1:]) + self.assertTrue(rewritten_upload_contents.endswith(upload_bytes)) + + # Test non-multipart (aka media): no body argument means this is + # sent as media. + upload = transfer.Upload.FromStream( + six.BytesIO(upload_bytes), + 'text/plain', + total_size=len(upload_bytes)) + http_request = http_wrapper.Request( + 'http://www.uploads.com', + headers={'content-type': 'text/plain'}) + upload.ConfigureRequest(upload_config, http_request, url_builder) + self.assertEqual(url_builder.query_params['uploadType'], 'media') + rewritten_upload_contents = http_request.body + self.assertTrue(rewritten_upload_contents.endswith(upload_bytes)) -- GitLab From 9cef89fe53ce8ee7943309dbd21b8467d65fe56c Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Sun, 8 May 2016 12:01:01 -0400 Subject: [PATCH 236/295] Add option to control how __init__.py is generated. --- apitools/gen/gen_client.py | 11 ++++++++- apitools/gen/gen_client_lib.py | 36 +++++++++++++++++------------ apitools/gen/gen_client_test.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 8f19dfc..48637f6 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -105,6 +105,7 @@ def _GetCodegenFromFlags(args): base_package=args.base_package, protorpc_package=args.protorpc_package, generate_cli=args.generate_cli, + init_wildcards_file=args.init_file == 'wildcards', use_proto2=args.experimental_proto2_output, unelidable_request_methods=args.unelidable_request_methods, apitools_version=args.apitools_version) @@ -167,7 +168,8 @@ def GenerateClient(args): logging.error('Failed to create codegen, exiting.') return 128 _WriteGeneratedFiles(args, codegen) - _WriteInit(codegen) + if args.init_file != 'none': + _WriteInit(codegen) def GeneratePipPackage(args): @@ -297,6 +299,13 @@ def main(argv=None): help='CLI will not be generated.') parser.set_defaults(generate_cli=True) + parser.add_argument( + '--init-file', + choices=['none', 'empty', 'wildcards'], + type=lambda s: s.lower(), + default='wildcards', + help='Controls whether and how to generate package __init__.py file.') + parser.add_argument( '--unelidable_request_methods', action=_SplitCommaSeparatedList, diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py index 66fbaf8..f9abfc6 100644 --- a/apitools/gen/gen_client_lib.py +++ b/apitools/gen/gen_client_lib.py @@ -58,6 +58,7 @@ class DescriptorGenerator(object): def __init__(self, discovery_doc, client_info, names, root_package, outdir, base_package, protorpc_package, generate_cli=False, + init_wildcards_file=True, use_proto2=False, unelidable_request_methods=None, apitools_version=''): self.__discovery_doc = discovery_doc @@ -70,6 +71,7 @@ class DescriptorGenerator(object): self.__version = self.__client_info.version self.__revision = discovery_doc.get('revision', '1') self.__generate_cli = generate_cli + self.__init_wildcards_file = init_wildcards_file self.__root_package = root_package self.__base_files_package = base_package self.__protorpc_package = protorpc_package @@ -166,25 +168,29 @@ class DescriptorGenerator(object): def WriteInit(self, out): """Write a simple __init__.py for the generated client.""" printer = self._GetPrinter(out) - printer('"""Common imports for generated %s client library."""', - self.__client_info.package) - printer('# pylint:disable=wildcard-import') + if self.__init_wildcards_file: + printer('"""Common imports for generated %s client library."""', + self.__client_info.package) + printer('# pylint:disable=wildcard-import') + else: + printer('"""Package marker file."""') printer() printer('import pkgutil') printer() - printer('from %s import *', self.__base_files_package) - if self.__root_package == '.': - import_prefix = '' - else: - import_prefix = '%s.' % self.__root_package - if self.__generate_cli: + if self.__init_wildcards_file: + printer('from %s import *', self.__base_files_package) + if self.__root_package == '.': + import_prefix = '' + else: + import_prefix = '%s.' % self.__root_package + if self.__generate_cli: + printer('from %s%s import *', + import_prefix, self.__client_info.cli_rule_name) printer('from %s%s import *', - import_prefix, self.__client_info.cli_rule_name) - printer('from %s%s import *', - import_prefix, self.__client_info.client_rule_name) - printer('from %s%s import *', - import_prefix, self.__client_info.messages_rule_name) - printer() + import_prefix, self.__client_info.client_rule_name) + printer('from %s%s import *', + import_prefix, self.__client_info.messages_rule_name) + printer() printer('__path__ = pkgutil.extend_path(__path__, __name__)') def WriteIntermediateInit(self, out): diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index b0b7290..1a0f9aa 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -63,6 +63,47 @@ class ClientGenCliTest(unittest2.TestCase): _GetContent(GetTestDataPath('gen_' + expected_file)), _GetContent(os.path.join(tmp_dir_path, expected_file))) + def testGenClient_SimpleDocNoInit(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--generate_cli', + '--init-file', 'none', + '--infile', GetTestDataPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', 'google.apis', + 'client' + ]) + expected_files = ( + set(['dns_v1.py']) | # CLI files + set(['dns_v1_client.py', 'dns_v1_messages.py'])) + self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) + + def testGenClient_SimpleDocEmptyInit(self): + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--generate_cli', + '--init-file', 'empty', + '--infile', GetTestDataPath('dns_v1.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', 'google.apis', + 'client' + ]) + expected_files = ( + set(['dns_v1.py']) | # CLI files + set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py'])) + self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) + init_file = _GetContent(os.path.join(tmp_dir_path, '__init__.py')) + self.assertEqual("""\"""Package marker file.\""" + +import pkgutil + +__path__ = pkgutil.extend_path(__path__, __name__) +""", init_file) + def testGenClient_SimpleDocWithV4(self): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ -- GitLab From 28838100f73673a9ac068e809ac3c7d376500478 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Mon, 9 May 2016 17:20:10 -0400 Subject: [PATCH 237/295] Use parenthesis around boolean for init_wildcard_file parameter. --- apitools/gen/gen_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index 48637f6..f0c830d 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -105,7 +105,7 @@ def _GetCodegenFromFlags(args): base_package=args.base_package, protorpc_package=args.protorpc_package, generate_cli=args.generate_cli, - init_wildcards_file=args.init_file == 'wildcards', + init_wildcards_file=(args.init_file == 'wildcards'), use_proto2=args.experimental_proto2_output, unelidable_request_methods=args.unelidable_request_methods, apitools_version=args.apitools_version) -- GitLab From c47b76556f8047972e72f42ecb0069816c75a134 Mon Sep 17 00:00:00 2001 From: Matt Houglum Date: Tue, 17 May 2016 16:49:28 -0700 Subject: [PATCH 238/295] Allow clients to set retry_func for MakeRequest This change: - Adds an item to ExceptionRetryArgs containing the total number of seconds the user has waited since the first request attempt. - Allows clients to specify a custom retry function for retry logic. My use case for these changes involves using a wrapper function which performs some additional logging if total_wait_sec is high enough, then calls the default retry function. --- apitools/base/py/base_api.py | 15 +++++---------- apitools/base/py/base_api_test.py | 20 ++++++++++++++++++++ apitools/base/py/http_wrapper.py | 12 +++++++----- apitools/base/py/http_wrapper_test.py | 3 ++- default.pylintrc | 1 + 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 7dc9ea9..a668738 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -237,7 +237,7 @@ class BaseApiClient(object): model=None, log_request=False, log_response=False, num_retries=5, max_retry_wait=60, credentials_args=None, default_global_params=None, additional_http_headers=None, - check_response_func=None): + check_response_func=None, retry_func=None): _RequireClassAttrs(self, ('_package', '_scopes', 'messages_module')) if default_global_params is not None: util.Typecheck(default_global_params, self.params_type) @@ -263,7 +263,8 @@ class BaseApiClient(object): self.__include_fields = None self.additional_http_headers = additional_http_headers or {} - self.__check_response_func = check_response_func + self.check_response_func = check_response_func + self.retry_func = retry_func # TODO(craigcitro): Finish deprecating these fields. _ = model @@ -378,14 +379,6 @@ class BaseApiClient(object): 'Cannot have negative value for num_retries') self.__num_retries = value - @property - def check_response_func(self): - return self.__check_response_func - - @check_response_func.setter - def check_response_func(self, value): - self.__check_response_func = value - @property def max_retry_wait(self): return self.__max_retry_wait @@ -702,6 +695,8 @@ class BaseApiService(object): } if self.__client.check_response_func: opts['check_response_func'] = self.__client.check_response_func + if self.__client.retry_func: + opts['retry_func'] = self.__client.retry_func http_response = http_wrapper.MakeRequest( http, http_request, **opts) diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py index eb8f6dc..c41f9d5 100644 --- a/apitools/base/py/base_api_test.py +++ b/apitools/base/py/base_api_test.py @@ -165,6 +165,26 @@ class BaseApiTest(unittest2.TestCase): with mock(base_api.http_wrapper, 'MakeRequest', fakeMakeRequest): service._RunMethod(method_config, request) + def testCustomRetryFunc(self): + def retry_func(): + pass + + def fakeMakeRequest(*_, **kwargs): + self.assertEqual(retry_func, kwargs['retry_func']) + return http_wrapper.Response( + info={'status': '200'}, content='{"field": "abc"}', + request_url='http://www.google.com') + http_wrapper.MakeRequest = fakeMakeRequest + method_config = base_api.ApiMethodInfo( + request_type_name='SimpleMessage', + response_type_name='SimpleMessage') + client = self.__GetFakeClient() + client.retry_func = retry_func + service = FakeService(client=client) + request = SimpleMessage() + with mock(base_api.http_wrapper, 'MakeRequest', fakeMakeRequest): + service._RunMethod(method_config, request) + def testQueryEncoding(self): method_config = base_api.ApiMethodInfo( request_type_name='MessageWithTime', query_params=['timestamp']) diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index e7925ea..7baf09f 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -64,7 +64,7 @@ _REDIRECT_STATUS_CODES = ( # num_retries: Number of retries consumed; used for exponential backoff. ExceptionRetryArgs = collections.namedtuple( 'ExceptionRetryArgs', ['http', 'http_request', 'exc', 'num_retries', - 'max_retry_wait']) + 'max_retry_wait', 'total_wait_sec']) @contextlib.contextmanager @@ -320,8 +320,8 @@ def MakeRequest(http, http_request, retries=7, max_retry_wait=60, max_retry_wait: (int, default 60) Maximum number of seconds to wait when retrying. redirections: (int, default 5) Number of redirects to follow. - retry_func: Function to handle retries on exceptions. Arguments are - (Httplib2.Http, Request, Exception, int num_retries). + retry_func: Function to handle retries on exceptions. Argument is an + ExceptionRetryArgs tuple. check_response_func: Function to validate the HTTP response. Arguments are (Response, response content, url). @@ -333,6 +333,7 @@ def MakeRequest(http, http_request, retries=7, max_retry_wait=60, """ retry = 0 + first_req_time = time.time() while True: try: return _MakeRequestNoRetry( @@ -345,8 +346,9 @@ def MakeRequest(http, http_request, retries=7, max_retry_wait=60, if retry >= retries: raise else: - retry_func(ExceptionRetryArgs( - http, http_request, e, retry, max_retry_wait)) + total_wait_sec = time.time() - first_req_time + retry_func(ExceptionRetryArgs(http, http_request, e, retry, + max_retry_wait, total_wait_sec)) def _MakeRequestNoRetry(http, http_request, redirections=5, diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index ff07668..5df107f 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -77,7 +77,8 @@ class HttpWrapperTest(unittest2.TestCase): retry_args = http_wrapper.ExceptionRetryArgs( http={'connections': {}}, http_request=_MockHttpRequest(), - exc=exception_arg, num_retries=0, max_retry_wait=0) + exc=exception_arg, num_retries=0, max_retry_wait=0, + total_wait_sec=0) # Disable time.sleep for this handler as it is called with # a minimum value of 1 second. diff --git a/default.pylintrc b/default.pylintrc index b4c4577..142fee0 100644 --- a/default.pylintrc +++ b/default.pylintrc @@ -55,6 +55,7 @@ disable = no-name-in-module, no-self-use, super-on-old-class, + too-many-arguments, too-many-function-args, -- GitLab From 2c06cb401ba077926db0f0c242ee2a25df17e8f8 Mon Sep 17 00:00:00 2001 From: Matt Houglum Date: Thu, 19 May 2016 10:27:40 -0700 Subject: [PATCH 239/295] Update for v0.5.3 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d85c20a..66696fd 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.2' +_APITOOLS_VERSION = '0.5.3' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From f65d6d0a857740825502c671348a32f2430f150e Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 6 May 2016 18:00:09 -0400 Subject: [PATCH 240/295] Make generated dns sample a python package so that tests against it can be run. --- apitools/gen/client_generation_test.py | 1 + apitools/gen/gen_client_test.py | 19 +++-- apitools/gen/gen_dns_client_test.py | 84 +++++++++++++++++++ apitools/gen/test_utils.py | 3 + apitools/gen/testdata/dns/__init__.py | 5 ++ apitools/gen/testdata/{ => dns}/dns_v1.json | 0 .../testdata/{gen_dns_v1.py => dns/dns_v1.py} | 0 .../dns_v1_client.py} | 2 +- .../dns_v1_messages.py} | 0 apitools/gen/testdata/gen___init__.py | 11 --- 10 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 apitools/gen/gen_dns_client_test.py create mode 100644 apitools/gen/testdata/dns/__init__.py rename apitools/gen/testdata/{ => dns}/dns_v1.json (100%) rename apitools/gen/testdata/{gen_dns_v1.py => dns/dns_v1.py} (100%) mode change 100755 => 100644 rename apitools/gen/testdata/{gen_dns_v1_client.py => dns/dns_v1_client.py} (99%) rename apitools/gen/testdata/{gen_dns_v1_messages.py => dns/dns_v1_messages.py} (100%) delete mode 100644 apitools/gen/testdata/gen___init__.py diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py index cb0a9c4..ae72ee0 100644 --- a/apitools/gen/client_generation_test.py +++ b/apitools/gen/client_generation_test.py @@ -39,6 +39,7 @@ class ClientGenerationTest(unittest2.TestCase): super(ClientGenerationTest, self).setUp() self.gen_client_binary = 'gen_client' + @test_utils.SkipOnWindows @test_utils.RunOnlyOnPython27 def testGeneration(self): for api in _API_LIST: diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 1a0f9aa..284c589 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -23,8 +23,8 @@ from apitools.gen import gen_client from apitools.gen import test_utils -def GetTestDataPath(name): - return os.path.join(os.path.dirname(__file__), 'testdata', name) +def GetTestDataPath(*path): + return os.path.join(os.path.dirname(__file__), 'testdata', *path) def _GetContent(file_path): @@ -48,10 +48,11 @@ class ClientGenCliTest(unittest2.TestCase): gen_client.main([ gen_client.__file__, '--generate_cli', - '--infile', GetTestDataPath('dns_v1.json'), + '--init-file', 'empty', + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', - '--root_package', 'google.apis', + '--root_package', 'dns', 'client' ]) expected_files = ( @@ -60,7 +61,7 @@ class ClientGenCliTest(unittest2.TestCase): self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) for expected_file in expected_files: self.assertMultiLineEqual( - _GetContent(GetTestDataPath('gen_' + expected_file)), + _GetContent(GetTestDataPath('dns', expected_file)), _GetContent(os.path.join(tmp_dir_path, expected_file))) def testGenClient_SimpleDocNoInit(self): @@ -109,7 +110,7 @@ __path__ = pkgutil.extend_path(__path__, __name__) gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--apitools_version', '0.4.12', @@ -125,7 +126,7 @@ __path__ = pkgutil.extend_path(__path__, __name__) gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--apitools_version', '0.5.0', @@ -141,7 +142,7 @@ __path__ = pkgutil.extend_path(__path__, __name__) gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', @@ -156,7 +157,7 @@ __path__ = pkgutil.extend_path(__path__, __name__) gen_client.main([ gen_client.__file__, '--nogenerate_cli', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', diff --git a/apitools/gen/gen_dns_client_test.py b/apitools/gen/gen_dns_client_test.py new file mode 100644 index 0000000..ba94f39 --- /dev/null +++ b/apitools/gen/gen_dns_client_test.py @@ -0,0 +1,84 @@ +# +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test for generated sample module.""" + +import os +import sys +import unittest2 +import six + +from apitools.base.py import list_pager +from apitools.base.py.testing import mock + +sys.path.append(os.path.join(os.path.dirname(__file__), 'testdata')) + +from dns import dns_v1_client # nopep8 +from dns import dns_v1_messages # nopep8 + + +class DnsGenClientSanityTest(unittest2.TestCase): + + def testBaseUrl(self): + self.assertEquals(u'https://www.googleapis.com/dns/v1/', + dns_v1_client.DnsV1.BASE_URL) + + def testMessagesModule(self): + self.assertEquals(dns_v1_messages, dns_v1_client.DnsV1.MESSAGES_MODULE) + + def testAttributes(self): + inner_classes = set([]) + for key, value in dns_v1_client.DnsV1.__dict__.items(): + if isinstance(value, six.class_types): + inner_classes.add(key) + self.assertEquals(set([ + 'ChangesService', + 'ProjectsService', + 'ManagedZonesService', + 'ResourceRecordSetsService']), inner_classes) + + +class DnsGenClientTest(unittest2.TestCase): + + def setUp(self): + self.mocked_dns_v1 = mock.Client(dns_v1_client.DnsV1) + self.mocked_dns_v1.Mock() + self.addCleanup(self.mocked_dns_v1.Unmock) + + def testRecordSetList(self): + response_record_set = dns_v1_messages.ResourceRecordSet( + kind=u"dns#resourceRecordSet", + name=u"zone.com.", + rrdatas=[u"1.2.3.4"], + ttl=21600, + type=u"A") + self.mocked_dns_v1.resourceRecordSets.List.Expect( + dns_v1_messages.DnsResourceRecordSetsListRequest( + project=u'my-project', + managedZone=u'test_zone_name', + type=u'green', + maxResults=100), + dns_v1_messages.ResourceRecordSetsListResponse( + rrsets=[response_record_set])) + + results = list(list_pager.YieldFromList( + self.mocked_dns_v1.resourceRecordSets, + dns_v1_messages.DnsResourceRecordSetsListRequest( + project='my-project', + managedZone='test_zone_name', + type='green'), + limit=100, field='rrsets')) + + self.assertEquals([response_record_set], results) diff --git a/apitools/gen/test_utils.py b/apitools/gen/test_utils.py index 92934e7..59eea51 100644 --- a/apitools/gen/test_utils.py +++ b/apitools/gen/test_utils.py @@ -28,6 +28,9 @@ import unittest2 RunOnlyOnPython27 = unittest2.skipUnless( sys.version_info[:2] == (2, 7), 'Only runs in Python 2.7') +SkipOnWindows = unittest2.skipIf( + os.name == 'nt', 'Does not run on windows') + @contextlib.contextmanager def TempDir(change_to=False): diff --git a/apitools/gen/testdata/dns/__init__.py b/apitools/gen/testdata/dns/__init__.py new file mode 100644 index 0000000..2816da8 --- /dev/null +++ b/apitools/gen/testdata/dns/__init__.py @@ -0,0 +1,5 @@ +"""Package marker file.""" + +import pkgutil + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/apitools/gen/testdata/dns_v1.json b/apitools/gen/testdata/dns/dns_v1.json similarity index 100% rename from apitools/gen/testdata/dns_v1.json rename to apitools/gen/testdata/dns/dns_v1.json diff --git a/apitools/gen/testdata/gen_dns_v1.py b/apitools/gen/testdata/dns/dns_v1.py old mode 100755 new mode 100644 similarity index 100% rename from apitools/gen/testdata/gen_dns_v1.py rename to apitools/gen/testdata/dns/dns_v1.py diff --git a/apitools/gen/testdata/gen_dns_v1_client.py b/apitools/gen/testdata/dns/dns_v1_client.py similarity index 99% rename from apitools/gen/testdata/gen_dns_v1_client.py rename to apitools/gen/testdata/dns/dns_v1_client.py index 4509b6b..0b16311 100644 --- a/apitools/gen/testdata/gen_dns_v1_client.py +++ b/apitools/gen/testdata/dns/dns_v1_client.py @@ -1,7 +1,7 @@ """Generated client library for dns version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from google.apis import dns_v1_messages as messages +from dns import dns_v1_messages as messages class DnsV1(base_api.BaseApiClient): diff --git a/apitools/gen/testdata/gen_dns_v1_messages.py b/apitools/gen/testdata/dns/dns_v1_messages.py similarity index 100% rename from apitools/gen/testdata/gen_dns_v1_messages.py rename to apitools/gen/testdata/dns/dns_v1_messages.py diff --git a/apitools/gen/testdata/gen___init__.py b/apitools/gen/testdata/gen___init__.py deleted file mode 100644 index f8f17ad..0000000 --- a/apitools/gen/testdata/gen___init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Common imports for generated dns client library.""" -# pylint:disable=wildcard-import - -import pkgutil - -from apitools.base.py import * -from google.apis.dns_v1 import * -from google.apis.dns_v1_client import * -from google.apis.dns_v1_messages import * - -__path__ = pkgutil.extend_path(__path__, __name__) -- GitLab From 61459a77095d537d3f982034164ad17d3e92f6f1 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Tue, 24 May 2016 16:43:12 -0400 Subject: [PATCH 241/295] Add generated client test sample with automic names. --- apitools/gen/gen_client.py | 26 +- apitools/gen/gen_client_test.py | 25 +- apitools/gen/testdata/iam/__init__.py | 5 + apitools/gen/testdata/iam/iam_v1.json | 1220 ++++++++++++++++++ apitools/gen/testdata/iam/iam_v1.py | 921 +++++++++++++ apitools/gen/testdata/iam/iam_v1_client.py | 520 ++++++++ apitools/gen/testdata/iam/iam_v1_messages.py | 964 ++++++++++++++ 7 files changed, 3657 insertions(+), 24 deletions(-) create mode 100644 apitools/gen/testdata/iam/__init__.py create mode 100644 apitools/gen/testdata/iam/iam_v1.json create mode 100644 apitools/gen/testdata/iam/iam_v1.py create mode 100644 apitools/gen/testdata/iam/iam_v1_client.py create mode 100644 apitools/gen/testdata/iam/iam_v1_messages.py diff --git a/apitools/gen/gen_client.py b/apitools/gen/gen_client.py index f0c830d..c36fbc4 100644 --- a/apitools/gen/gen_client.py +++ b/apitools/gen/gen_client.py @@ -39,24 +39,18 @@ def _CopyLocalFile(filename): out.write(src_data) -_DISCOVERY_DOC = None - - def _GetDiscoveryDocFromFlags(args): """Get the discovery doc from flags.""" - global _DISCOVERY_DOC # pylint: disable=global-statement - if _DISCOVERY_DOC is None: - if args.discovery_url: - try: - discovery_doc = util.FetchDiscoveryDoc(args.discovery_url) - except exceptions.CommunicationError: - raise exceptions.GeneratedClientError( - 'Could not fetch discovery doc') - else: - infile = os.path.expanduser(args.infile) or '/dev/stdin' - discovery_doc = json.load(open(infile)) - _DISCOVERY_DOC = discovery_doc - return _DISCOVERY_DOC + if args.discovery_url: + try: + return util.FetchDiscoveryDoc(args.discovery_url) + except exceptions.CommunicationError: + raise exceptions.GeneratedClientError( + 'Could not fetch discovery doc') + + infile = os.path.expanduser(args.infile) or '/dev/stdin' + with open(infile) as f: + return json.load(f) def _GetCodegenFromFlags(args): diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 284c589..7d888af 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -43,34 +43,43 @@ class ClientGenCliTest(unittest2.TestCase): self.assertIn('usage:', err_output) self.assertIn('error: too few arguments', err_output) - def testGenClient_SimpleDoc(self): + def _CheckGeneratedFiles(self, api_name): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ gen_client.__file__, '--generate_cli', '--init-file', 'empty', - '--infile', GetTestDataPath('dns', 'dns_v1.json'), + '--infile', + GetTestDataPath(api_name, api_name + '_v1.json'), '--outdir', tmp_dir_path, '--overwrite', - '--root_package', 'dns', + '--root_package', api_name, 'client' ]) expected_files = ( - set(['dns_v1.py']) | # CLI files - set(['dns_v1_client.py', 'dns_v1_messages.py', '__init__.py'])) + set([api_name + '_v1.py']) | # CLI files + set([api_name + '_v1_client.py', + api_name + '_v1_messages.py', + '__init__.py'])) self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) for expected_file in expected_files: self.assertMultiLineEqual( - _GetContent(GetTestDataPath('dns', expected_file)), + _GetContent(GetTestDataPath(api_name, expected_file)), _GetContent(os.path.join(tmp_dir_path, expected_file))) + def testGenClient_DnsDoc(self): + self._CheckGeneratedFiles('dns') + + def testGenClient_IamDoc(self): + self._CheckGeneratedFiles('iam') + def testGenClient_SimpleDocNoInit(self): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ gen_client.__file__, '--generate_cli', '--init-file', 'none', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', @@ -87,7 +96,7 @@ class ClientGenCliTest(unittest2.TestCase): gen_client.__file__, '--generate_cli', '--init-file', 'empty', - '--infile', GetTestDataPath('dns_v1.json'), + '--infile', GetTestDataPath('dns', 'dns_v1.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', 'google.apis', diff --git a/apitools/gen/testdata/iam/__init__.py b/apitools/gen/testdata/iam/__init__.py new file mode 100644 index 0000000..2816da8 --- /dev/null +++ b/apitools/gen/testdata/iam/__init__.py @@ -0,0 +1,5 @@ +"""Package marker file.""" + +import pkgutil + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/apitools/gen/testdata/iam/iam_v1.json b/apitools/gen/testdata/iam/iam_v1.json new file mode 100644 index 0000000..8e9480e --- /dev/null +++ b/apitools/gen/testdata/iam/iam_v1.json @@ -0,0 +1,1220 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "iam:v1", + "name": "iam", + "canonicalName": "iam", + "version": "v1", + "revision": "0", + "title": "Google Identity and Access Management (IAM) API", + "description": "Manages identity and access control for Google Cloud Platform resources, including the creation of service accounts, which you can use to authenticate to Google and make API calls.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://cloud.google.com/iam/", + "protocol": "rest", + "rootUrl": "https://iam.googleapis.com/", + "servicePath": "", + "baseUrl": "https://iam.googleapis.com/", + "batchPath": "batch", + "version_module": "True", + "parameters": { + "access_token": { + "type": "string", + "description": "OAuth access token.", + "location": "query" + }, + "alt": { + "type": "string", + "description": "Data format for response.", + "default": "json", + "enum": [ + "json", + "media", + "proto" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "location": "query" + }, + "bearer_token": { + "type": "string", + "description": "OAuth bearer token.", + "location": "query" + }, + "callback": { + "type": "string", + "description": "JSONP", + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "pp": { + "type": "boolean", + "description": "Pretty-print response.", + "default": "true", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "location": "query" + }, + "upload_protocol": { + "type": "string", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "location": "query" + }, + "uploadType": { + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query" + }, + "$.xgafv": { + "type": "string", + "description": "V1 error format.", + "enum": [ + "1", + "2" + ], + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + } + } + } + }, + "schemas": { + "ListServiceAccountsResponse": { + "id": "ListServiceAccountsResponse", + "description": "The service account list response.", + "type": "object", + "properties": { + "accounts": { + "description": "The list of matching service accounts.", + "type": "array", + "items": { + "$ref": "ServiceAccount" + } + }, + "nextPageToken": { + "description": "To retrieve the next page of results, set\nListServiceAccountsRequest.page_token\nto this value.", + "type": "string" + } + } + }, + "ServiceAccount": { + "id": "ServiceAccount", + "description": "A service account in the Identity and Access Management API.\n\nTo create a service account, specify the `project_id` and the `account_id`\nfor the account. The `account_id` is unique within the project, and is used\nto generate the service account email address and a stable\n`unique_id`.\n\nAll other methods can identify the service account using the format\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "type": "object", + "properties": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\n\nRequests using `-` as a wildcard for the project will infer the project\nfrom the `account` and the `account` value can be the `email` address or\nthe `unique_id` of the service account.\n\nIn responses the resource name will always be in the format\n`projects\/{project}\/serviceAccounts\/{email}`.", + "type": "string" + }, + "projectId": { + "description": "@OutputOnly The id of the project that owns the service account.", + "type": "string" + }, + "uniqueId": { + "description": "@OutputOnly The unique and stable id of the service account.", + "type": "string" + }, + "email": { + "description": "@OutputOnly The email address of the service account.", + "type": "string" + }, + "displayName": { + "description": "Optional. A user-specified description of the service account. Must be\nfewer than 100 UTF-8 bytes.", + "type": "string" + }, + "etag": { + "description": "Used to perform a consistent read-modify-write.", + "type": "string", + "format": "byte" + }, + "description": { + "description": "Optional. A user-specified opaque description of the service account.", + "type": "string" + }, + "oauth2ClientId": { + "description": "@OutputOnly. The OAuth2 client id for the service account.\nThis is used in conjunction with the OAuth2 clientconfig API to make\nthree legged OAuth2 (3LO) flows to access the data of Google users.", + "type": "string" + } + } + }, + "CreateServiceAccountRequest": { + "id": "CreateServiceAccountRequest", + "description": "The service account create request.", + "type": "object", + "properties": { + "accountId": { + "description": "Required. The account id that is used to generate the service account\nemail address and a stable unique id. It is unique within a project,\nmust be 1-63 characters long, and match the regular expression\n`[a-z]([-a-z0-9]*[a-z0-9])` to comply with RFC1035.", + "type": "string" + }, + "serviceAccount": { + "description": "The ServiceAccount resource to create.\nCurrently, only the following values are user assignable:\n`display_name` .", + "$ref": "ServiceAccount" + } + } + }, + "Empty": { + "id": "Empty", + "description": "A generic empty message that you can re-use to avoid defining duplicated\nempty messages in your APIs. A typical example is to use it as the request\nor the response type of an API method. For instance:\n\n service Foo {\n rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty);\n }\n\nThe JSON representation for `Empty` is empty JSON object `{}`.", + "type": "object", + "properties": { + } + }, + "ListServiceAccountKeysResponse": { + "id": "ListServiceAccountKeysResponse", + "description": "The service account keys list response.", + "type": "object", + "properties": { + "keys": { + "description": "The public keys for the service account.", + "type": "array", + "items": { + "$ref": "ServiceAccountKey" + } + } + } + }, + "ServiceAccountKey": { + "id": "ServiceAccountKey", + "description": "Represents a service account key.\n\nA service account has two sets of key-pairs: user-managed, and\nsystem-managed.\n\nUser-managed key-pairs can be created and deleted by users. Users are\nresponsible for rotating these keys periodically to ensure security of\ntheir service accounts. Users retain the private key of these key-pairs,\nand Google retains ONLY the public key.\n\nSystem-managed key-pairs are managed automatically by Google, and rotated\ndaily without user intervention. The private key never leaves Google's\nservers to maximize security.\n\nPublic keys for all service accounts are also published at the OAuth2\nService Account API.", + "type": "object", + "properties": { + "name": { + "description": "The resource name of the service account key in the following format\n`projects\/{project}\/serviceAccounts\/{account}\/keys\/{key}`.", + "type": "string" + }, + "privateKeyType": { + "description": "The output format for the private key.\nOnly provided in `CreateServiceAccountKey` responses, not\nin `GetServiceAccountKey` or `ListServiceAccountKey` responses.\n\nGoogle never exposes system-managed private keys, and never retains\nuser-managed private keys.", + "enumDescriptions": [ + "Unspecified. Equivalent to `TYPE_GOOGLE_CREDENTIALS_FILE`.", + "PKCS12 format.\nThe password for the PKCS12 file is `notasecret`.\nFor more information, see https:\/\/tools.ietf.org\/html\/rfc7292.", + "Google Credentials File format." + ], + "type": "string", + "enum": [ + "TYPE_UNSPECIFIED", + "TYPE_PKCS12_FILE", + "TYPE_GOOGLE_CREDENTIALS_FILE" + ] + }, + "privateKeyData": { + "description": "The private key data. Only provided in `CreateServiceAccountKey`\nresponses.", + "type": "string", + "format": "byte" + }, + "publicKeyData": { + "description": "The public key data. Only provided in `GetServiceAccountKey` responses.", + "type": "string", + "format": "byte" + }, + "validAfterTime": { + "description": "The key can be used after this timestamp.", + "type": "string", + "format": "google-datetime" + }, + "validBeforeTime": { + "description": "The key can be used before this timestamp.", + "type": "string", + "format": "google-datetime" + } + } + }, + "CreateServiceAccountKeyRequest": { + "id": "CreateServiceAccountKeyRequest", + "description": "The service account key create request.", + "type": "object", + "properties": { + "privateKeyType": { + "description": "The output format of the private key. `GOOGLE_CREDENTIALS_FILE` is the\ndefault output format.", + "enumDescriptions": [ + "Unspecified. Equivalent to `TYPE_GOOGLE_CREDENTIALS_FILE`.", + "PKCS12 format.\nThe password for the PKCS12 file is `notasecret`.\nFor more information, see https:\/\/tools.ietf.org\/html\/rfc7292.", + "Google Credentials File format." + ], + "type": "string", + "enum": [ + "TYPE_UNSPECIFIED", + "TYPE_PKCS12_FILE", + "TYPE_GOOGLE_CREDENTIALS_FILE" + ] + } + } + }, + "SignBlobRequest": { + "id": "SignBlobRequest", + "description": "The service account sign blob request.", + "type": "object", + "properties": { + "bytesToSign": { + "description": "The bytes to sign.", + "type": "string", + "format": "byte" + } + } + }, + "SignBlobResponse": { + "id": "SignBlobResponse", + "description": "The service account sign blob response.", + "type": "object", + "properties": { + "keyId": { + "description": "The id of the key used to sign the blob.", + "type": "string" + }, + "signature": { + "description": "The signed blob.", + "type": "string", + "format": "byte" + } + } + }, + "SignJwtRequest": { + "id": "SignJwtRequest", + "description": "The service account sign JWT request.", + "type": "object", + "properties": { + "payload": { + "description": "The JWT payload to sign, a JSON JWT Claim set.", + "type": "string" + } + } + }, + "SignJwtResponse": { + "id": "SignJwtResponse", + "description": "The service account sign JWT response.", + "type": "object", + "properties": { + "keyId": { + "description": "The id of the key used to sign the JWT.", + "type": "string" + }, + "signedJwt": { + "description": "The signed JWT.", + "type": "string" + } + } + }, + "Policy": { + "id": "Policy", + "description": "Defines an Identity and Access Management (IAM) policy. It is used to\nspecify access control policies for Cloud Platform resources.\n\n\nA `Policy` consists of a list of `bindings`. A `Binding` binds a list of\n`members` to a `role`, where the members can be user accounts, Google groups,\nGoogle domains, and service accounts. A `role` is a named list of permissions\ndefined by IAM.\n\n**Example**\n\n {\n \"bindings\": [\n {\n \"role\": \"roles\/owner\",\n \"members\": [\n \"user:mike@example.com\",\n \"group:admins@example.com\",\n \"domain:google.com\",\n \"serviceAccount:my-other-app@appspot.gserviceaccount.com\",\n ]\n },\n {\n \"role\": \"roles\/viewer\",\n \"members\": [\"user:sean@example.com\"]\n }\n ]\n }\n\nFor a description of IAM and its features, see the\n[IAM developer's guide](https:\/\/cloud.google.com\/iam).", + "type": "object", + "properties": { + "version": { + "description": "Version of the `Policy`. The default version is 0.", + "type": "integer", + "format": "int32" + }, + "bindings": { + "description": "Associates a list of `members` to a `role`.\nMultiple `bindings` must not be specified for the same `role`.\n`bindings` with no members will result in an error.", + "type": "array", + "items": { + "$ref": "Binding" + } + }, + "auditConfigs": { + "description": "Specifies audit logging configs for \"data access\".\n\"data access\": generally refers to data reads\/writes and admin reads.\n\"admin activity\": generally refers to admin writes.\n\nNote: `AuditConfig` doesn't apply to \"admin activity\", which always\nenables audit logging.", + "type": "array", + "items": { + "$ref": "AuditConfig" + } + }, + "rules": { + "description": "If more than one rule is specified, the rules are applied in the following\nmanner:\n- All matching LOG rules are always applied.\n- If any DENY\/DENY_WITH_LOG rule matches, permission is denied.\n Logging will be applied if one or more matching rule requires logging.\n- Otherwise, if any ALLOW\/ALLOW_WITH_LOG rule matches, permission is\n granted.\n Logging will be applied if one or more matching rule requires logging.\n- Otherwise, if no rule applies, permission is denied.", + "type": "array", + "items": { + "$ref": "Rule" + } + }, + "etag": { + "description": "`etag` is used for optimistic concurrency control as a way to help\nprevent simultaneous updates of a policy from overwriting each other.\nIt is strongly suggested that systems make use of the `etag` in the\nread-modify-write cycle to perform policy updates in order to avoid race\nconditions: An `etag` is returned in the response to `getIamPolicy`, and\nsystems are expected to put that etag in the request to `setIamPolicy` to\nensure that their change will be applied to the same version of the policy.\n\nIf no `etag` is provided in the call to `setIamPolicy`, then the existing\npolicy is overwritten blindly.", + "type": "string", + "format": "byte" + }, + "iamOwned": { + + "type": "boolean" + } + } + }, + "Binding": { + "id": "Binding", + "description": "Associates `members` with a `role`.", + "type": "object", + "properties": { + "role": { + "description": "Role that is assigned to `members`.\nFor example, `roles\/viewer`, `roles\/editor`, or `roles\/owner`.\nRequired", + "type": "string" + }, + "members": { + "description": "Specifies the identities requesting access for a Cloud Platform resource.\n`members` can have the following values:\n\n* `allUsers`: A special identifier that represents anyone who is\n on the internet; with or without a Google account.\n\n* `allAuthenticatedUsers`: A special identifier that represents anyone\n who is authenticated with a Google account or a service account.\n\n* `user:{emailid}`: An email address that represents a specific Google\n account. For example, `alice@gmail.com` or `joe@example.com`.\n\n* `serviceAccount:{emailid}`: An email address that represents a service\n account. For example, `my-other-app@appspot.gserviceaccount.com`.\n\n* `group:{emailid}`: An email address that represents a Google group.\n For example, `admins@example.com`.\n\n* `domain:{domain}`: A Google Apps domain name that represents all the\n users of that domain. For example, `google.com` or `example.com`.\n\n\n", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "AuditConfig": { + "id": "AuditConfig", + "description": "Enables \"data access\" audit logging for a service and specifies a list\nof members that are log-exempted.", + "type": "object", + "properties": { + "service": { + "description": "Specifies a service that will be enabled for \"data access\" audit\nlogging.\nFor example, `resourcemanager`, `storage`, `compute`.\n`allServices` is a special value that covers all services.", + "type": "string" + }, + "exemptedMembers": { + "description": "Specifies the identities that are exempted from \"data access\" audit\nlogging for the `service` specified above.\nFollows the same format of Binding.members.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Rule": { + "id": "Rule", + "description": "A rule to be applied in a Policy.", + "type": "object", + "properties": { + "description": { + "description": "Human-readable description of the rule.", + "type": "string" + }, + "permissions": { + "description": "A permission is a string of form '..'\n(e.g., 'storage.buckets.list'). A value of '*' matches all permissions,\nand a verb part of '*' (e.g., 'storage.buckets.*') matches all verbs.", + "type": "array", + "items": { + "type": "string" + } + }, + "action": { + "description": "Required", + "enumDescriptions": [ + "Default no action.", + "Matching 'Entries' grant access.", + "Matching 'Entries' grant access and the caller promises to log\nthe request per the returned log_configs.", + "Matching 'Entries' deny access.", + "Matching 'Entries' deny access and the caller promises to log\nthe request per the returned log_configs.", + "Matching 'Entries' tell IAM.Check callers to generate logs." + ], + "type": "string", + "enum": [ + "NO_ACTION", + "ALLOW", + "ALLOW_WITH_LOG", + "DENY", + "DENY_WITH_LOG", + "LOG" + ] + }, + "in": { + "description": "If one or more 'in' clauses are specified, the rule matches if\nthe PRINCIPAL\/AUTHORITY_SELECTOR is in at least one of these entries.", + "type": "array", + "items": { + "type": "string" + } + }, + "notIn": { + "description": "If one or more 'not_in' clauses are specified, the rule matches\nif the PRINCIPAL\/AUTHORITY_SELECTOR is in none of the entries.\nThe format for in and not_in entries is the same as for members in a\nBinding (see google\/iam\/v1\/policy.proto).", + "type": "array", + "items": { + "type": "string" + } + }, + "conditions": { + "description": "Additional restrictions that must be met", + "type": "array", + "items": { + "$ref": "Condition" + } + }, + "logConfig": { + "description": "The config returned to callers of tech.iam.IAM.CheckPolicy for any entries\nthat match the LOG action.", + "type": "array", + "items": { + "$ref": "LogConfig" + } + } + } + }, + "Condition": { + "id": "Condition", + "description": "A condition to be met.", + "type": "object", + "properties": { + "iam": { + "description": "Trusted attributes supplied by the IAM system.", + "enumDescriptions": [ + "Default non-attribute.", + "Either principal or (if present) authority", + "selector\nAlways the original principal, but making clear" + ], + "type": "string", + "enum": [ + "NO_ATTR", + "AUTHORITY", + "ATTRIBUTION" + ] + }, + "sys": { + "description": "Trusted attributes supplied by any service that owns resources and uses\nthe IAM system for access control.", + "enumDescriptions": [ + "Default non-attribute type", + "Region of the resource", + "Service name", + "Resource name", + "IP address of the caller" + ], + "type": "string", + "enum": [ + "NO_ATTR", + "REGION", + "SERVICE", + "NAME", + "IP" + ] + }, + "svc": { + "description": "Trusted attributes discharged by the service.", + "type": "string" + }, + "op": { + "description": "An operator to apply the subject with.", + "enumDescriptions": [ + "Default no-op.", + "DEPRECATED. Use IN instead.", + "DEPRECATED. Use NOT_IN instead.", + "Set-inclusion check.", + "Set-exclusion check.", + "Subject is discharged" + ], + "type": "string", + "enum": [ + "NO_OP", + "EQUALS", + "NOT_EQUALS", + "IN", + "NOT_IN", + "DISCHARGED" + ] + }, + "value": { + "description": "DEPRECATED. Use 'values' instead.", + "type": "string" + }, + "values": { + "description": "The objects of the condition. This is mutually exclusive with 'value'.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "LogConfig": { + "id": "LogConfig", + "description": "Specifies what kind of log the caller must write\nIncrement a streamz counter with the specified metric and field names.\n\nMetric names should start with a '\/', generally be lowercase-only,\nand end in \"_count\". Field names should not contain an initial slash.\nThe actual exported metric names will have \"\/iam\/policy\" prepended.\n\nField names correspond to IAM request parameters and field values are\ntheir respective values.\n\nAt present the only supported field names are\n - \"iam_principal\", corresponding to IAMContext.principal;\n - \"\" (empty string), resulting in one aggretated counter with no field.\n\nExamples:\n counter { metric: \"\/debug_access_count\" field: \"iam_principal\" }\n ==> increment counter \/iam\/policy\/backend_debug_access_count\n {iam_principal=[value of IAMContext.principal]}\n\nAt this time we do not support:\n* multiple field names (though this may be supported in the future)\n* decrementing the counter\n* incrementing it by anything other than 1", + "type": "object", + "properties": { + "counter": { + "description": "Counter options.", + "$ref": "CounterOptions" + }, + "dataAccess": { + "description": "Data access options.", + "$ref": "DataAccessOptions" + }, + "cloudAudit": { + "description": "Cloud audit options.", + "$ref": "CloudAuditOptions" + } + } + }, + "CounterOptions": { + "id": "CounterOptions", + "description": "Options for counters", + "type": "object", + "properties": { + "metric": { + "description": "The metric to update.", + "type": "string" + }, + "field": { + "description": "The field value to attribute.", + "type": "string" + } + } + }, + "DataAccessOptions": { + "id": "DataAccessOptions", + "description": "Write a Data Access (Gin) log", + "type": "object", + "properties": { + } + }, + "CloudAuditOptions": { + "id": "CloudAuditOptions", + "description": "Write a Cloud Audit log", + "type": "object", + "properties": { + } + }, + "SetIamPolicyRequest": { + "id": "SetIamPolicyRequest", + "description": "Request message for `SetIamPolicy` method.", + "type": "object", + "properties": { + "policy": { + "description": "REQUIRED: The complete policy to be applied to the `resource`. The size of\nthe policy is limited to a few 10s of KB. An empty policy is a\nvalid policy but certain Cloud Platform services (such as Projects)\nmight reject them.", + "$ref": "Policy" + } + } + }, + "TestIamPermissionsRequest": { + "id": "TestIamPermissionsRequest", + "description": "Request message for `TestIamPermissions` method.", + "type": "object", + "properties": { + "permissions": { + "description": "The set of permissions to check for the `resource`. Permissions with\nwildcards (such as '*' or 'storage.*') are not allowed. For more\ninformation see\nIAM Overview.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "TestIamPermissionsResponse": { + "id": "TestIamPermissionsResponse", + "description": "Response message for `TestIamPermissions` method.", + "type": "object", + "properties": { + "permissions": { + "description": "A subset of `TestPermissionsRequest.permissions` that the caller is\nallowed.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "QueryGrantableRolesRequest": { + "id": "QueryGrantableRolesRequest", + "description": "The grantable role query request.", + "type": "object", + "properties": { + "fullResourceName": { + "description": "Required. The full resource name to query from the list of grantable roles.\n\nThe name follows the Google Cloud Platform resource format.\nFor example, a Cloud Platform project with id `my-project` will be named\n`\/\/cloudresourcemanager.googleapis.com\/projects\/my-project`.", + "type": "string" + } + } + }, + "QueryGrantableRolesResponse": { + "id": "QueryGrantableRolesResponse", + "description": "The grantable role query response.", + "type": "object", + "properties": { + "roles": { + "description": "The list of matching roles.", + "type": "array", + "items": { + "$ref": "Role" + } + } + } + }, + "Role": { + "id": "Role", + "description": "A role in the Identity and Access Management API.", + "type": "object", + "properties": { + "name": { + "description": "The name of the role.\n\nExamples of roles names are:\n`roles\/editor`, `roles\/viewer` and `roles\/logging.viewer`.", + "type": "string" + }, + "title": { + "description": "Optional. A human-readable title for the role. Typically this\nis limited to 100 UTF-8 bytes.", + "type": "string" + }, + "description": { + "description": "Optional. A human-readable description for the role.", + "type": "string" + }, + "apiTokens": { + + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "GetPolicyDetailsRequest": { + "id": "GetPolicyDetailsRequest", + "description": "The request to get the current policy and the policies on the inherited\nresources the user has access to.", + "type": "object", + "properties": { + "fullResourcePath": { + "description": "REQUIRED: The full resource path of the current policy being\nrequested, e.g., `\/\/dataflow.googleapis.com\/projects\/..\/jobs\/..`.", + "type": "string" + }, + "pageToken": { + "description": "Optional pagination token returned in an earlier\nGetPolicyDetailsResponse.next_page_token\nresponse.", + "type": "string" + }, + "pageSize": { + "description": "Limit on the number of policies to include in the response.\nFurther accounts can subsequently be obtained by including the\nGetPolicyDetailsResponse.next_page_token\nin a subsequent request.\nIf zero, the default page size 20 will be used.\nMust be given a value in range [0, 100], otherwise an invalid argument\nerror will be returned.", + "type": "integer", + "format": "int32" + } + } + }, + "GetPolicyDetailsResponse": { + "id": "GetPolicyDetailsResponse", + "description": "The response to the `GetPolicyDetailsRequest` containing the current policy and\nthe policies on the inherited resources the user has access to.", + "type": "object", + "properties": { + "policies": { + "description": "The current policy and all the inherited policies the user has\naccess to.", + "type": "array", + "items": { + "$ref": "PolicyDetail" + } + }, + "nextPageToken": { + "description": "To retrieve the next page of results, set\nGetPolicyDetailsRequest.page_token\nto this value.\nIf this value is empty, then there are not any further policies that the\nuser has access to.\nThe lifetime is 60 minutes. An \"Expired pagination token\" error will be\nreturned if exceeded.", + "type": "string" + } + } + }, + "PolicyDetail": { + "id": "PolicyDetail", + "description": "A policy and its full resource path.", + "type": "object", + "properties": { + "policy": { + "description": "The policy of a `resource\/project\/folder`.", + "$ref": "Policy" + }, + "fullResourcePath": { + "description": "The full resource path of the policy\ne.g., `\/\/dataflow.googleapis.com\/projects\/..\/jobs\/..`.\nNote that a resource and its inherited resource have different\n`full_resource_path`.", + "type": "string" + } + } + } + }, + "resources": { + "projects": { + "resources": { + "serviceAccounts": { + "methods": { + "list": { + "id": "iam.projects.serviceAccounts.list", + "path": "v1/{+name}/serviceAccounts", + "flatPath": "v1/projects/{projectsId}/serviceAccounts", + "httpMethod": "GET", + "description": "Lists ServiceAccounts for a project.", + "parameters": { + "name": { + "description": "Required. The resource name of the project associated with the service\naccounts, such as `projects\/my-project-123`.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*$", + "type": "string" + }, + "pageSize": { + "description": "Optional limit on the number of service accounts to include in the\nresponse. Further accounts can subsequently be obtained by including the\nListServiceAccountsResponse.next_page_token\nin a subsequent request.", + "location": "query", + "type": "integer", + "format": "int32" + }, + "pageToken": { + "description": "Optional pagination token returned in an earlier\nListServiceAccountsResponse.next_page_token.", + "location": "query", + "type": "string" + }, + "removeDeletedServiceAccounts": { + "description": "Do not list service accounts deleted from Gaia.\nDO NOT INCLUDE IN EXTERNAL DOCUMENTATION<\/font><\/b>.", + "location": "query", + "type": "boolean" + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "ListServiceAccountsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "get": { + "id": "iam.projects.serviceAccounts.get", + "path": "v1/{+name}", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}", + "httpMethod": "GET", + "description": "Gets a ServiceAccount.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "ServiceAccount" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "create": { + "id": "iam.projects.serviceAccounts.create", + "path": "v1/{+name}/serviceAccounts", + "flatPath": "v1/projects/{projectsId}/serviceAccounts", + "httpMethod": "POST", + "description": "Creates a ServiceAccount\nand returns it.", + "parameters": { + "name": { + "description": "Required. The resource name of the project associated with the service\naccounts, such as `projects\/my-project-123`.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "request": { + "$ref": "CreateServiceAccountRequest" + }, + "response": { + "$ref": "ServiceAccount" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "update": { + "id": "iam.projects.serviceAccounts.update", + "path": "v1/{+name}", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}", + "httpMethod": "PUT", + "description": "Updates a ServiceAccount.\n\nCurrently, only the following fields are updatable:\n`display_name` .\nThe `etag` is mandatory.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\n\nRequests using `-` as a wildcard for the project will infer the project\nfrom the `account` and the `account` value can be the `email` address or\nthe `unique_id` of the service account.\n\nIn responses the resource name will always be in the format\n`projects\/{project}\/serviceAccounts\/{email}`.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "request": { + "$ref": "ServiceAccount" + }, + "response": { + "$ref": "ServiceAccount" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "delete": { + "id": "iam.projects.serviceAccounts.delete", + "path": "v1/{+name}", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}", + "httpMethod": "DELETE", + "description": "Deletes a ServiceAccount.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "signBlob": { + "id": "iam.projects.serviceAccounts.signBlob", + "path": "v1/{+name}:signBlob", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlob", + "httpMethod": "POST", + "description": "Signs a blob using a service account's system-managed private key.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "request": { + "$ref": "SignBlobRequest" + }, + "response": { + "$ref": "SignBlobResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "signJwt": { + "id": "iam.projects.serviceAccounts.signJwt", + "path": "v1/{+name}:signJwt", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwt", + "httpMethod": "POST", + "description": "Signs a JWT using a service account's system-managed private key.\n\nIf no `exp` (expiry) time is contained in the claims, we will\nprovide an expiry of one hour in the future. If an expiry\nof more than one hour in the future is requested, the request\nwill fail.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "request": { + "$ref": "SignJwtRequest" + }, + "response": { + "$ref": "SignJwtResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "getIamPolicy": { + "id": "iam.projects.serviceAccounts.getIamPolicy", + "path": "v1/{+resource}:getIamPolicy", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:getIamPolicy", + "httpMethod": "POST", + "description": "Returns the IAM access control policy for specified IAM resource.", + "parameters": { + "resource": { + "description": "REQUIRED: The resource for which the policy is being requested.\n`resource` is usually specified as a path, such as\n`projects\/*project*\/zones\/*zone*\/disks\/*disk*`.\n\nThe format for the path specified in this value is resource specific and\nis specified in the `getIamPolicy` documentation.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "resource" + ], + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "setIamPolicy": { + "id": "iam.projects.serviceAccounts.setIamPolicy", + "path": "v1/{+resource}:setIamPolicy", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:setIamPolicy", + "httpMethod": "POST", + "description": "Sets the IAM access control policy for the specified IAM resource.", + "parameters": { + "resource": { + "description": "REQUIRED: The resource for which the policy is being specified.\n`resource` is usually specified as a path, such as\n`projects\/*project*\/zones\/*zone*\/disks\/*disk*`.\n\nThe format for the path specified in this value is resource specific and\nis specified in the `setIamPolicy` documentation.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "resource" + ], + "request": { + "$ref": "SetIamPolicyRequest" + }, + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "testIamPermissions": { + "id": "iam.projects.serviceAccounts.testIamPermissions", + "path": "v1/{+resource}:testIamPermissions", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:testIamPermissions", + "httpMethod": "POST", + "description": "Tests the specified permissions against the IAM access control policy\nfor the specified IAM resource.", + "parameters": { + "resource": { + "description": "REQUIRED: The resource for which the policy detail is being requested.\n`resource` is usually specified as a path, such as\n`projects\/*project*\/zones\/*zone*\/disks\/*disk*`.\n\nThe format for the path specified in this value is resource specific and\nis specified in the `testIamPermissions` documentation.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "resource" + ], + "request": { + "$ref": "TestIamPermissionsRequest" + }, + "response": { + "$ref": "TestIamPermissionsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + } + , + "resources": { + "keys": { + "methods": { + "list": { + "id": "iam.projects.serviceAccounts.keys.list", + "path": "v1/{+name}/keys", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys", + "httpMethod": "GET", + "description": "Lists ServiceAccountKeys.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\n\nUsing `-` as a wildcard for the project, will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + }, + "keyTypes": { + "description": "Filters the types of keys the user wants to include in the list\nresponse. Duplicate key types are not allowed. If no key type\nis provided, all keys are returned.", + "location": "query", + "repeated": true, + "type": "string", + "enum": [ + "KEY_TYPE_UNSPECIFIED", + "USER_MANAGED", + "SYSTEM_MANAGED" + ] + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "ListServiceAccountKeysResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "get": { + "id": "iam.projects.serviceAccounts.keys.get", + "path": "v1/{+name}", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}", + "httpMethod": "GET", + "description": "Gets the ServiceAccountKey\nby key id.", + "parameters": { + "name": { + "description": "The resource name of the service account key in the following format:\n`projects\/{project}\/serviceAccounts\/{account}\/keys\/{key}`.\n\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*\/keys\/[^\/]*$", + "type": "string" + }, + "publicKeyType": { + "description": "The output format of the public key requested.\nX509_PEM is the default output format.", + "location": "query", + "type": "string", + "enum": [ + "TYPE_NONE", + "TYPE_X509_PEM_FILE", + "TYPE_RAW_PUBLIC_KEY" + ] + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "ServiceAccountKey" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "create": { + "id": "iam.projects.serviceAccounts.keys.create", + "path": "v1/{+name}/keys", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys", + "httpMethod": "POST", + "description": "Creates a ServiceAccountKey\nand returns it.", + "parameters": { + "name": { + "description": "The resource name of the service account in the following format:\n`projects\/{project}\/serviceAccounts\/{account}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "request": { + "$ref": "CreateServiceAccountKeyRequest" + }, + "response": { + "$ref": "ServiceAccountKey" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + }, + "delete": { + "id": "iam.projects.serviceAccounts.keys.delete", + "path": "v1/{+name}", + "flatPath": "v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}", + "httpMethod": "DELETE", + "description": "Deletes a ServiceAccountKey.", + "parameters": { + "name": { + "description": "The resource name of the service account key in the following format:\n`projects\/{project}\/serviceAccounts\/{account}\/keys\/{key}`.\nUsing `-` as a wildcard for the project will infer the project from\nthe account. The `account` value can be the `email` address or the\n`unique_id` of the service account.", + "location": "path", + "required": true, + "pattern": "^projects\/[^\/]*\/serviceAccounts\/[^\/]*\/keys\/[^\/]*$", + "type": "string" + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "Empty" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + } + } + } + } + } + }, + "roles": { + "methods": { + "queryGrantableRoles": { + "id": "iam.roles.queryGrantableRoles", + "path": "v1/roles:queryGrantableRoles", + "flatPath": "v1/roles:queryGrantableRoles", + "httpMethod": "POST", + "description": "Queries roles that can be granted on a particular resource.", + "parameters": { + }, + "parameterOrder": [ + ], + "request": { + "$ref": "QueryGrantableRolesRequest" + }, + "response": { + "$ref": "QueryGrantableRolesResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + } + }, + "iamPolicies": { + "methods": { + "getPolicyDetails": { + "id": "iam.iamPolicies.getPolicyDetails", + "path": "v1/iamPolicies:getPolicyDetails", + "flatPath": "v1/iamPolicies:getPolicyDetails", + "httpMethod": "POST", + "description": "Returns the current IAM policy and the policies on the inherited resources\nthat the user has access to.", + "parameters": { + }, + "parameterOrder": [ + ], + "request": { + "$ref": "GetPolicyDetailsRequest" + }, + "response": { + "$ref": "GetPolicyDetailsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + } + } + }, + "basePath": "" +} diff --git a/apitools/gen/testdata/iam/iam_v1.py b/apitools/gen/testdata/iam/iam_v1.py new file mode 100644 index 0000000..da9750e --- /dev/null +++ b/apitools/gen/testdata/iam/iam_v1.py @@ -0,0 +1,921 @@ +#!/usr/bin/env python +"""CLI for iam, version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. + +import code +import os +import platform +import sys + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages + +from google.apputils import appcommands +import gflags as flags + +import apitools.base.py as apitools_base +from apitools.base.py import cli as apitools_base_cli +import iam_v1_client as client_lib +import iam_v1_messages as messages + + +def _DeclareIamFlags(): + """Declare global flags in an idempotent way.""" + if 'api_endpoint' in flags.FLAGS: + return + flags.DEFINE_string( + 'api_endpoint', + u'https://iam.googleapis.com/', + 'URL of the API endpoint to use.', + short_name='iam_url') + flags.DEFINE_string( + 'history_file', + u'~/.iam.v1.history', + 'File with interactive shell history.') + flags.DEFINE_multistring( + 'add_header', [], + 'Additional http headers (as key=value strings). ' + 'Can be specified multiple times.') + flags.DEFINE_string( + 'service_account_json_keyfile', '', + 'Filename for a JSON service account key downloaded' + ' from the Developer Console.') + flags.DEFINE_enum( + 'f__xgafv', + u'_1', + [u'_1', u'_2'], + u'V1 error format.') + flags.DEFINE_string( + 'access_token', + None, + u'OAuth access token.') + flags.DEFINE_enum( + 'alt', + u'json', + [u'json', u'media', u'proto'], + u'Data format for response.') + flags.DEFINE_string( + 'bearer_token', + None, + u'OAuth bearer token.') + flags.DEFINE_string( + 'callback', + None, + u'JSONP') + flags.DEFINE_string( + 'fields', + None, + u'Selector specifying which fields to include in a partial response.') + flags.DEFINE_string( + 'key', + None, + u'API key. Your API key identifies your project and provides you with ' + u'API access, quota, and reports. Required unless you provide an OAuth ' + u'2.0 token.') + flags.DEFINE_string( + 'oauth_token', + None, + u'OAuth 2.0 token for the current user.') + flags.DEFINE_boolean( + 'pp', + 'True', + u'Pretty-print response.') + flags.DEFINE_boolean( + 'prettyPrint', + 'True', + u'Returns response with indentations and line breaks.') + flags.DEFINE_string( + 'quotaUser', + None, + u'Available to use for quota purposes for server-side applications. Can' + u' be any arbitrary string assigned to a user, but should not exceed 40' + u' characters.') + flags.DEFINE_string( + 'trace', + None, + 'A tracing token of the form "token:" to include in api ' + 'requests.') + flags.DEFINE_string( + 'uploadType', + None, + u'Legacy upload protocol for media (e.g. "media", "multipart").') + flags.DEFINE_string( + 'upload_protocol', + None, + u'Upload protocol for media (e.g. "raw", "multipart").') + + +FLAGS = flags.FLAGS +apitools_base_cli.DeclareBaseFlags() +_DeclareIamFlags() + + +def GetGlobalParamsFromFlags(): + """Return a StandardQueryParameters based on flags.""" + result = messages.StandardQueryParameters() + if FLAGS['f__xgafv'].present: + result.f__xgafv = messages.StandardQueryParameters.FXgafvValueValuesEnum(FLAGS.f__xgafv) + if FLAGS['access_token'].present: + result.access_token = FLAGS.access_token.decode('utf8') + if FLAGS['alt'].present: + result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt) + if FLAGS['bearer_token'].present: + result.bearer_token = FLAGS.bearer_token.decode('utf8') + if FLAGS['callback'].present: + result.callback = FLAGS.callback.decode('utf8') + if FLAGS['fields'].present: + result.fields = FLAGS.fields.decode('utf8') + if FLAGS['key'].present: + result.key = FLAGS.key.decode('utf8') + if FLAGS['oauth_token'].present: + result.oauth_token = FLAGS.oauth_token.decode('utf8') + if FLAGS['pp'].present: + result.pp = FLAGS.pp + if FLAGS['prettyPrint'].present: + result.prettyPrint = FLAGS.prettyPrint + if FLAGS['quotaUser'].present: + result.quotaUser = FLAGS.quotaUser.decode('utf8') + if FLAGS['trace'].present: + result.trace = FLAGS.trace.decode('utf8') + if FLAGS['uploadType'].present: + result.uploadType = FLAGS.uploadType.decode('utf8') + if FLAGS['upload_protocol'].present: + result.upload_protocol = FLAGS.upload_protocol.decode('utf8') + return result + + +def GetClientFromFlags(): + """Return a client object, configured from flags.""" + log_request = FLAGS.log_request or FLAGS.log_request_response + log_response = FLAGS.log_response or FLAGS.log_request_response + api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) + additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header) + credentials_args = { + 'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile) + } + try: + client = client_lib.IamV1( + api_endpoint, log_request=log_request, + log_response=log_response, + credentials_args=credentials_args, + additional_http_headers=additional_http_headers) + except apitools_base.CredentialsError as e: + print 'Error creating credentials: %s' % e + sys.exit(1) + return client + + +class PyShell(appcommands.Cmd): + + def Run(self, _): + """Run an interactive python shell with the client.""" + client = GetClientFromFlags() + params = GetGlobalParamsFromFlags() + for field in params.all_fields(): + value = params.get_assigned_value(field.name) + if value != field.default: + client.AddGlobalParam(field.name, value) + banner = """ + == iam interactive console == + client: a iam client + apitools_base: base apitools module + messages: the generated messages module + """ + local_vars = { + 'apitools_base': apitools_base, + 'client': client, + 'client_lib': client_lib, + 'messages': messages, + } + if platform.system() == 'Linux': + console = apitools_base_cli.ConsoleWithReadline( + local_vars, histfile=FLAGS.history_file) + else: + console = code.InteractiveConsole(local_vars) + try: + console.interact(banner) + except SystemExit as e: + return e.code + + +class IamPoliciesGetPolicyDetails(apitools_base_cli.NewCmd): + """Command wrapping iamPolicies.GetPolicyDetails.""" + + usage = """iamPolicies_getPolicyDetails""" + + def __init__(self, name, fv): + super(IamPoliciesGetPolicyDetails, self).__init__(name, fv) + flags.DEFINE_string( + 'fullResourcePath', + None, + u'REQUIRED: The full resource path of the current policy being ' + u'requested, e.g., `//dataflow.googleapis.com/projects/../jobs/..`.', + flag_values=fv) + flags.DEFINE_integer( + 'pageSize', + None, + u'Limit on the number of policies to include in the response. Further' + u' accounts can subsequently be obtained by including the ' + u'GetPolicyDetailsResponse.next_page_token in a subsequent request. ' + u'If zero, the default page size 20 will be used. Must be given a ' + u'value in range [0, 100], otherwise an invalid argument error will ' + u'be returned.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Optional pagination token returned in an earlier ' + u'GetPolicyDetailsResponse.next_page_token response.', + flag_values=fv) + + def RunWithArgs(self): + """Returns the current IAM policy and the policies on the inherited + resources that the user has access to. + + Flags: + fullResourcePath: REQUIRED: The full resource path of the current policy + being requested, e.g., + `//dataflow.googleapis.com/projects/../jobs/..`. + pageSize: Limit on the number of policies to include in the response. + Further accounts can subsequently be obtained by including the + GetPolicyDetailsResponse.next_page_token in a subsequent request. If + zero, the default page size 20 will be used. Must be given a value in + range [0, 100], otherwise an invalid argument error will be returned. + pageToken: Optional pagination token returned in an earlier + GetPolicyDetailsResponse.next_page_token response. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.GetPolicyDetailsRequest( + ) + if FLAGS['fullResourcePath'].present: + request.fullResourcePath = FLAGS.fullResourcePath.decode('utf8') + if FLAGS['pageSize'].present: + request.pageSize = FLAGS.pageSize + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.iamPolicies.GetPolicyDetails( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsCreate(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.Create.""" + + usage = """projects_serviceAccounts_create """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsCreate, self).__init__(name, fv) + flags.DEFINE_string( + 'createServiceAccountRequest', + None, + u'A CreateServiceAccountRequest resource to be passed as the request ' + u'body.', + flag_values=fv) + + def RunWithArgs(self, name): + """Creates a ServiceAccount and returns it. + + Args: + name: Required. The resource name of the project associated with the + service accounts, such as `projects/my-project-123`. + + Flags: + createServiceAccountRequest: A CreateServiceAccountRequest resource to + be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsCreateRequest( + name=name.decode('utf8'), + ) + if FLAGS['createServiceAccountRequest'].present: + request.createServiceAccountRequest = apitools_base.JsonToMessage(messages.CreateServiceAccountRequest, FLAGS.createServiceAccountRequest) + result = client.projects_serviceAccounts.Create( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsDelete(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.Delete.""" + + usage = """projects_serviceAccounts_delete """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsDelete, self).__init__(name, fv) + + def RunWithArgs(self, name): + """Deletes a ServiceAccount. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsDeleteRequest( + name=name.decode('utf8'), + ) + result = client.projects_serviceAccounts.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsGet(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.Get.""" + + usage = """projects_serviceAccounts_get """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsGet, self).__init__(name, fv) + + def RunWithArgs(self, name): + """Gets a ServiceAccount. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsGetRequest( + name=name.decode('utf8'), + ) + result = client.projects_serviceAccounts.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsGetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.GetIamPolicy.""" + + usage = """projects_serviceAccounts_getIamPolicy """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsGetIamPolicy, self).__init__(name, fv) + + def RunWithArgs(self, resource): + """Returns the IAM access control policy for specified IAM resource. + + Args: + resource: REQUIRED: The resource for which the policy is being + requested. `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the + path specified in this value is resource specific and is specified in + the `getIamPolicy` documentation. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsGetIamPolicyRequest( + resource=resource.decode('utf8'), + ) + result = client.projects_serviceAccounts.GetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsList(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.List.""" + + usage = """projects_serviceAccounts_list """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsList, self).__init__(name, fv) + flags.DEFINE_integer( + 'pageSize', + None, + u'Optional limit on the number of service accounts to include in the ' + u'response. Further accounts can subsequently be obtained by ' + u'including the ListServiceAccountsResponse.next_page_token in a ' + u'subsequent request.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Optional pagination token returned in an earlier ' + u'ListServiceAccountsResponse.next_page_token.', + flag_values=fv) + flags.DEFINE_boolean( + 'removeDeletedServiceAccounts', + None, + u'Do not list service accounts deleted from Gaia. DO NOT INCLUDE IN EXTERNAL DOCUMENTATION.', + flag_values=fv) + + def RunWithArgs(self, name): + """Lists ServiceAccounts for a project. + + Args: + name: Required. The resource name of the project associated with the + service accounts, such as `projects/my-project-123`. + + Flags: + pageSize: Optional limit on the number of service accounts to include in + the response. Further accounts can subsequently be obtained by + including the ListServiceAccountsResponse.next_page_token in a + subsequent request. + pageToken: Optional pagination token returned in an earlier + ListServiceAccountsResponse.next_page_token. + removeDeletedServiceAccounts: Do not list service accounts deleted from + Gaia. DO NOT INCLUDE IN EXTERNAL + DOCUMENTATION. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsListRequest( + name=name.decode('utf8'), + ) + if FLAGS['pageSize'].present: + request.pageSize = FLAGS.pageSize + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['removeDeletedServiceAccounts'].present: + request.removeDeletedServiceAccounts = FLAGS.removeDeletedServiceAccounts + result = client.projects_serviceAccounts.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsSetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.SetIamPolicy.""" + + usage = """projects_serviceAccounts_setIamPolicy """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsSetIamPolicy, self).__init__(name, fv) + flags.DEFINE_string( + 'setIamPolicyRequest', + None, + u'A SetIamPolicyRequest resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, resource): + """Sets the IAM access control policy for the specified IAM resource. + + Args: + resource: REQUIRED: The resource for which the policy is being + specified. `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the + path specified in this value is resource specific and is specified in + the `setIamPolicy` documentation. + + Flags: + setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the + request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsSetIamPolicyRequest( + resource=resource.decode('utf8'), + ) + if FLAGS['setIamPolicyRequest'].present: + request.setIamPolicyRequest = apitools_base.JsonToMessage(messages.SetIamPolicyRequest, FLAGS.setIamPolicyRequest) + result = client.projects_serviceAccounts.SetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsSignBlob(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.SignBlob.""" + + usage = """projects_serviceAccounts_signBlob """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsSignBlob, self).__init__(name, fv) + flags.DEFINE_string( + 'signBlobRequest', + None, + u'A SignBlobRequest resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, name): + """Signs a blob using a service account's system-managed private key. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + + Flags: + signBlobRequest: A SignBlobRequest resource to be passed as the request + body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsSignBlobRequest( + name=name.decode('utf8'), + ) + if FLAGS['signBlobRequest'].present: + request.signBlobRequest = apitools_base.JsonToMessage(messages.SignBlobRequest, FLAGS.signBlobRequest) + result = client.projects_serviceAccounts.SignBlob( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsSignJwt(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.SignJwt.""" + + usage = """projects_serviceAccounts_signJwt """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsSignJwt, self).__init__(name, fv) + flags.DEFINE_string( + 'signJwtRequest', + None, + u'A SignJwtRequest resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, name): + """Signs a JWT using a service account's system-managed private key. If + no `exp` (expiry) time is contained in the claims, we will provide an + expiry of one hour in the future. If an expiry of more than one hour in + the future is requested, the request will fail. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + + Flags: + signJwtRequest: A SignJwtRequest resource to be passed as the request + body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsSignJwtRequest( + name=name.decode('utf8'), + ) + if FLAGS['signJwtRequest'].present: + request.signJwtRequest = apitools_base.JsonToMessage(messages.SignJwtRequest, FLAGS.signJwtRequest) + result = client.projects_serviceAccounts.SignJwt( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsTestIamPermissions(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.TestIamPermissions.""" + + usage = """projects_serviceAccounts_testIamPermissions """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsTestIamPermissions, self).__init__(name, fv) + flags.DEFINE_string( + 'testIamPermissionsRequest', + None, + u'A TestIamPermissionsRequest resource to be passed as the request ' + u'body.', + flag_values=fv) + + def RunWithArgs(self, resource): + """Tests the specified permissions against the IAM access control policy + for the specified IAM resource. + + Args: + resource: REQUIRED: The resource for which the policy detail is being + requested. `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the + path specified in this value is resource specific and is specified in + the `testIamPermissions` documentation. + + Flags: + testIamPermissionsRequest: A TestIamPermissionsRequest resource to be + passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsTestIamPermissionsRequest( + resource=resource.decode('utf8'), + ) + if FLAGS['testIamPermissionsRequest'].present: + request.testIamPermissionsRequest = apitools_base.JsonToMessage(messages.TestIamPermissionsRequest, FLAGS.testIamPermissionsRequest) + result = client.projects_serviceAccounts.TestIamPermissions( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsUpdate(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts.Update.""" + + usage = """projects_serviceAccounts_update """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'description', + None, + u'Optional. A user-specified opaque description of the service ' + u'account.', + flag_values=fv) + flags.DEFINE_string( + 'displayName', + None, + u'Optional. A user-specified description of the service account. ' + u'Must be fewer than 100 UTF-8 bytes.', + flag_values=fv) + flags.DEFINE_string( + 'email', + None, + u'@OutputOnly The email address of the service account.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'Used to perform a consistent read-modify-write.', + flag_values=fv) + flags.DEFINE_string( + 'oauth2ClientId', + None, + u'@OutputOnly. The OAuth2 client id for the service account. This is ' + u'used in conjunction with the OAuth2 clientconfig API to make three ' + u'legged OAuth2 (3LO) flows to access the data of Google users.', + flag_values=fv) + flags.DEFINE_string( + 'projectId', + None, + u'@OutputOnly The id of the project that owns the service account.', + flag_values=fv) + flags.DEFINE_string( + 'uniqueId', + None, + u'@OutputOnly The unique and stable id of the service account.', + flag_values=fv) + + def RunWithArgs(self, name): + """Updates a ServiceAccount. Currently, only the following fields are + updatable: `display_name` . The `etag` is mandatory. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Requests using `-` as + a wildcard for the project will infer the project from the `account` + and the `account` value can be the `email` address or the `unique_id` + of the service account. In responses the resource name will always be + in the format `projects/{project}/serviceAccounts/{email}`. + + Flags: + description: Optional. A user-specified opaque description of the + service account. + displayName: Optional. A user-specified description of the service + account. Must be fewer than 100 UTF-8 bytes. + email: @OutputOnly The email address of the service account. + etag: Used to perform a consistent read-modify-write. + oauth2ClientId: @OutputOnly. The OAuth2 client id for the service + account. This is used in conjunction with the OAuth2 clientconfig API + to make three legged OAuth2 (3LO) flows to access the data of Google + users. + projectId: @OutputOnly The id of the project that owns the service + account. + uniqueId: @OutputOnly The unique and stable id of the service account. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.ServiceAccount( + name=name.decode('utf8'), + ) + if FLAGS['description'].present: + request.description = FLAGS.description.decode('utf8') + if FLAGS['displayName'].present: + request.displayName = FLAGS.displayName.decode('utf8') + if FLAGS['email'].present: + request.email = FLAGS.email.decode('utf8') + if FLAGS['etag'].present: + request.etag = FLAGS.etag + if FLAGS['oauth2ClientId'].present: + request.oauth2ClientId = FLAGS.oauth2ClientId.decode('utf8') + if FLAGS['projectId'].present: + request.projectId = FLAGS.projectId.decode('utf8') + if FLAGS['uniqueId'].present: + request.uniqueId = FLAGS.uniqueId.decode('utf8') + result = client.projects_serviceAccounts.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsKeysCreate(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts_keys.Create.""" + + usage = """projects_serviceAccounts_keys_create """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsKeysCreate, self).__init__(name, fv) + flags.DEFINE_string( + 'createServiceAccountKeyRequest', + None, + u'A CreateServiceAccountKeyRequest resource to be passed as the ' + u'request body.', + flag_values=fv) + + def RunWithArgs(self, name): + """Creates a ServiceAccountKey and returns it. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + + Flags: + createServiceAccountKeyRequest: A CreateServiceAccountKeyRequest + resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsKeysCreateRequest( + name=name.decode('utf8'), + ) + if FLAGS['createServiceAccountKeyRequest'].present: + request.createServiceAccountKeyRequest = apitools_base.JsonToMessage(messages.CreateServiceAccountKeyRequest, FLAGS.createServiceAccountKeyRequest) + result = client.projects_serviceAccounts_keys.Create( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsKeysDelete(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts_keys.Delete.""" + + usage = """projects_serviceAccounts_keys_delete """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsKeysDelete, self).__init__(name, fv) + + def RunWithArgs(self, name): + """Deletes a ServiceAccountKey. + + Args: + name: The resource name of the service account key in the following + format: `projects/{project}/serviceAccounts/{account}/keys/{key}`. + Using `-` as a wildcard for the project will infer the project from + the account. The `account` value can be the `email` address or the + `unique_id` of the service account. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsKeysDeleteRequest( + name=name.decode('utf8'), + ) + result = client.projects_serviceAccounts_keys.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsKeysGet(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts_keys.Get.""" + + usage = """projects_serviceAccounts_keys_get """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsKeysGet, self).__init__(name, fv) + flags.DEFINE_enum( + 'publicKeyType', + u'TYPE_NONE', + [u'TYPE_NONE', u'TYPE_X509_PEM_FILE', u'TYPE_RAW_PUBLIC_KEY'], + u'The output format of the public key requested. X509_PEM is the ' + u'default output format.', + flag_values=fv) + + def RunWithArgs(self, name): + """Gets the ServiceAccountKey by key id. + + Args: + name: The resource name of the service account key in the following + format: `projects/{project}/serviceAccounts/{account}/keys/{key}`. + Using `-` as a wildcard for the project will infer the project from + the account. The `account` value can be the `email` address or the + `unique_id` of the service account. + + Flags: + publicKeyType: The output format of the public key requested. X509_PEM + is the default output format. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsKeysGetRequest( + name=name.decode('utf8'), + ) + if FLAGS['publicKeyType'].present: + request.publicKeyType = messages.IamProjectsServiceAccountsKeysGetRequest.PublicKeyTypeValueValuesEnum(FLAGS.publicKeyType) + result = client.projects_serviceAccounts_keys.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ProjectsServiceAccountsKeysList(apitools_base_cli.NewCmd): + """Command wrapping projects_serviceAccounts_keys.List.""" + + usage = """projects_serviceAccounts_keys_list """ + + def __init__(self, name, fv): + super(ProjectsServiceAccountsKeysList, self).__init__(name, fv) + flags.DEFINE_enum( + 'keyTypes', + u'KEY_TYPE_UNSPECIFIED', + [u'KEY_TYPE_UNSPECIFIED', u'USER_MANAGED', u'SYSTEM_MANAGED'], + u'Filters the types of keys the user wants to include in the list ' + u'response. Duplicate key types are not allowed. If no key type is ' + u'provided, all keys are returned.', + flag_values=fv) + + def RunWithArgs(self, name): + """Lists ServiceAccountKeys. + + Args: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a + wildcard for the project, will infer the project from the account. The + `account` value can be the `email` address or the `unique_id` of the + service account. + + Flags: + keyTypes: Filters the types of keys the user wants to include in the + list response. Duplicate key types are not allowed. If no key type is + provided, all keys are returned. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.IamProjectsServiceAccountsKeysListRequest( + name=name.decode('utf8'), + ) + if FLAGS['keyTypes'].present: + request.keyTypes = [messages.IamProjectsServiceAccountsKeysListRequest.KeyTypesValueValuesEnum(x) for x in FLAGS.keyTypes] + result = client.projects_serviceAccounts_keys.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class RolesQueryGrantableRoles(apitools_base_cli.NewCmd): + """Command wrapping roles.QueryGrantableRoles.""" + + usage = """roles_queryGrantableRoles""" + + def __init__(self, name, fv): + super(RolesQueryGrantableRoles, self).__init__(name, fv) + flags.DEFINE_string( + 'fullResourceName', + None, + u'Required. The full resource name to query from the list of ' + u'grantable roles. The name follows the Google Cloud Platform ' + u'resource format. For example, a Cloud Platform project with id `my-' + u'project` will be named ' + u'`//cloudresourcemanager.googleapis.com/projects/my-project`.', + flag_values=fv) + + def RunWithArgs(self): + """Queries roles that can be granted on a particular resource. + + Flags: + fullResourceName: Required. The full resource name to query from the + list of grantable roles. The name follows the Google Cloud Platform + resource format. For example, a Cloud Platform project with id `my- + project` will be named `//cloudresourcemanager.googleapis.com/projects + /my-project`. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.QueryGrantableRolesRequest( + ) + if FLAGS['fullResourceName'].present: + request.fullResourceName = FLAGS.fullResourceName.decode('utf8') + result = client.roles.QueryGrantableRoles( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +def main(_): + appcommands.AddCmd('pyshell', PyShell) + appcommands.AddCmd('iamPolicies_getPolicyDetails', IamPoliciesGetPolicyDetails) + appcommands.AddCmd('projects_serviceAccounts_create', ProjectsServiceAccountsCreate) + appcommands.AddCmd('projects_serviceAccounts_delete', ProjectsServiceAccountsDelete) + appcommands.AddCmd('projects_serviceAccounts_get', ProjectsServiceAccountsGet) + appcommands.AddCmd('projects_serviceAccounts_getIamPolicy', ProjectsServiceAccountsGetIamPolicy) + appcommands.AddCmd('projects_serviceAccounts_list', ProjectsServiceAccountsList) + appcommands.AddCmd('projects_serviceAccounts_setIamPolicy', ProjectsServiceAccountsSetIamPolicy) + appcommands.AddCmd('projects_serviceAccounts_signBlob', ProjectsServiceAccountsSignBlob) + appcommands.AddCmd('projects_serviceAccounts_signJwt', ProjectsServiceAccountsSignJwt) + appcommands.AddCmd('projects_serviceAccounts_testIamPermissions', ProjectsServiceAccountsTestIamPermissions) + appcommands.AddCmd('projects_serviceAccounts_update', ProjectsServiceAccountsUpdate) + appcommands.AddCmd('projects_serviceAccounts_keys_create', ProjectsServiceAccountsKeysCreate) + appcommands.AddCmd('projects_serviceAccounts_keys_delete', ProjectsServiceAccountsKeysDelete) + appcommands.AddCmd('projects_serviceAccounts_keys_get', ProjectsServiceAccountsKeysGet) + appcommands.AddCmd('projects_serviceAccounts_keys_list', ProjectsServiceAccountsKeysList) + appcommands.AddCmd('roles_queryGrantableRoles', RolesQueryGrantableRoles) + + apitools_base_cli.SetupLogger() + if hasattr(appcommands, 'SetDefaultCommand'): + appcommands.SetDefaultCommand('pyshell') + + +run_main = apitools_base_cli.run_main + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/gen/testdata/iam/iam_v1_client.py b/apitools/gen/testdata/iam/iam_v1_client.py new file mode 100644 index 0000000..0c10655 --- /dev/null +++ b/apitools/gen/testdata/iam/iam_v1_client.py @@ -0,0 +1,520 @@ +"""Generated client library for iam version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. +from apitools.base.py import base_api +from iam import iam_v1_messages as messages + + +class IamV1(base_api.BaseApiClient): + """Generated client library for service iam version v1.""" + + MESSAGES_MODULE = messages + BASE_URL = u'https://iam.googleapis.com/' + + _PACKAGE = u'iam' + _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform'] + _VERSION = u'v1' + _CLIENT_ID = '1042881264118.apps.googleusercontent.com' + _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _CLIENT_CLASS_NAME = u'IamV1' + _URL_VERSION = u'v1' + _API_KEY = None + + def __init__(self, url='', credentials=None, + get_credentials=True, http=None, model=None, + log_request=False, log_response=False, + credentials_args=None, default_global_params=None, + additional_http_headers=None): + """Create a new iam handle.""" + url = url or self.BASE_URL + super(IamV1, self).__init__( + url, credentials=credentials, + get_credentials=get_credentials, http=http, model=model, + log_request=log_request, log_response=log_response, + credentials_args=credentials_args, + default_global_params=default_global_params, + additional_http_headers=additional_http_headers) + self.iamPolicies = self.IamPoliciesService(self) + self.projects_serviceAccounts_keys = self.ProjectsServiceAccountsKeysService(self) + self.projects_serviceAccounts = self.ProjectsServiceAccountsService(self) + self.projects = self.ProjectsService(self) + self.roles = self.RolesService(self) + + class IamPoliciesService(base_api.BaseApiService): + """Service class for the iamPolicies resource.""" + + _NAME = u'iamPolicies' + + def __init__(self, client): + super(IamV1.IamPoliciesService, self).__init__(client) + self._method_configs = { + 'GetPolicyDetails': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.iamPolicies.getPolicyDetails', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'v1/iamPolicies:getPolicyDetails', + request_field='', + request_type_name=u'GetPolicyDetailsRequest', + response_type_name=u'GetPolicyDetailsResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def GetPolicyDetails(self, request, global_params=None): + """Returns the current IAM policy and the policies on the inherited resources. +that the user has access to. + + Args: + request: (GetPolicyDetailsRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (GetPolicyDetailsResponse) The response message. + """ + config = self.GetMethodConfig('GetPolicyDetails') + return self._RunMethod( + config, request, global_params=global_params) + + class ProjectsServiceAccountsKeysService(base_api.BaseApiService): + """Service class for the projects_serviceAccounts_keys resource.""" + + _NAME = u'projects_serviceAccounts_keys' + + def __init__(self, client): + super(IamV1.ProjectsServiceAccountsKeysService, self).__init__(client) + self._method_configs = { + 'Create': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.keys.create', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}/keys', + request_field=u'createServiceAccountKeyRequest', + request_type_name=u'IamProjectsServiceAccountsKeysCreateRequest', + response_type_name=u'ServiceAccountKey', + supports_download=False, + ), + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'iam.projects.serviceAccounts.keys.delete', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysDeleteRequest', + response_type_name=u'Empty', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.keys.get', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'publicKeyType'], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysGetRequest', + response_type_name=u'ServiceAccountKey', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.keys.list', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'keyTypes'], + relative_path=u'v1/{+name}/keys', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysListRequest', + response_type_name=u'ListServiceAccountKeysResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Create(self, request, global_params=None): + """Creates a ServiceAccountKey. +and returns it. + + Args: + request: (IamProjectsServiceAccountsKeysCreateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ServiceAccountKey) The response message. + """ + config = self.GetMethodConfig('Create') + return self._RunMethod( + config, request, global_params=global_params) + + def Delete(self, request, global_params=None): + """Deletes a ServiceAccountKey. + + Args: + request: (IamProjectsServiceAccountsKeysDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Empty) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Gets the ServiceAccountKey. +by key id. + + Args: + request: (IamProjectsServiceAccountsKeysGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ServiceAccountKey) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Lists ServiceAccountKeys. + + Args: + request: (IamProjectsServiceAccountsKeysListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ListServiceAccountKeysResponse) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + class ProjectsServiceAccountsService(base_api.BaseApiService): + """Service class for the projects_serviceAccounts resource.""" + + _NAME = u'projects_serviceAccounts' + + def __init__(self, client): + super(IamV1.ProjectsServiceAccountsService, self).__init__(client) + self._method_configs = { + 'Create': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.create', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}/serviceAccounts', + request_field=u'createServiceAccountRequest', + request_type_name=u'IamProjectsServiceAccountsCreateRequest', + response_type_name=u'ServiceAccount', + supports_download=False, + ), + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'iam.projects.serviceAccounts.delete', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsDeleteRequest', + response_type_name=u'Empty', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.get', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsGetRequest', + response_type_name=u'ServiceAccount', + supports_download=False, + ), + 'GetIamPolicy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.getIamPolicy', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:getIamPolicy', + request_field='', + request_type_name=u'IamProjectsServiceAccountsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.list', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'pageSize', u'pageToken', u'removeDeletedServiceAccounts'], + relative_path=u'v1/{+name}/serviceAccounts', + request_field='', + request_type_name=u'IamProjectsServiceAccountsListRequest', + response_type_name=u'ListServiceAccountsResponse', + supports_download=False, + ), + 'SetIamPolicy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.setIamPolicy', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:setIamPolicy', + request_field=u'setIamPolicyRequest', + request_type_name=u'IamProjectsServiceAccountsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), + 'SignBlob': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.signBlob', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}:signBlob', + request_field=u'signBlobRequest', + request_type_name=u'IamProjectsServiceAccountsSignBlobRequest', + response_type_name=u'SignBlobResponse', + supports_download=False, + ), + 'SignJwt': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.signJwt', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}:signJwt', + request_field=u'signJwtRequest', + request_type_name=u'IamProjectsServiceAccountsSignJwtRequest', + response_type_name=u'SignJwtResponse', + supports_download=False, + ), + 'TestIamPermissions': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.testIamPermissions', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:testIamPermissions', + request_field=u'testIamPermissionsRequest', + request_type_name=u'IamProjectsServiceAccountsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'iam.projects.serviceAccounts.update', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'ServiceAccount', + response_type_name=u'ServiceAccount', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Create(self, request, global_params=None): + """Creates a ServiceAccount. +and returns it. + + Args: + request: (IamProjectsServiceAccountsCreateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ServiceAccount) The response message. + """ + config = self.GetMethodConfig('Create') + return self._RunMethod( + config, request, global_params=global_params) + + def Delete(self, request, global_params=None): + """Deletes a ServiceAccount. + + Args: + request: (IamProjectsServiceAccountsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Empty) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Gets a ServiceAccount. + + Args: + request: (IamProjectsServiceAccountsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ServiceAccount) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def GetIamPolicy(self, request, global_params=None): + """Returns the IAM access control policy for specified IAM resource. + + Args: + request: (IamProjectsServiceAccountsGetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('GetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Lists ServiceAccounts for a project. + + Args: + request: (IamProjectsServiceAccountsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ListServiceAccountsResponse) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def SetIamPolicy(self, request, global_params=None): + """Sets the IAM access control policy for the specified IAM resource. + + Args: + request: (IamProjectsServiceAccountsSetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('SetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + + def SignBlob(self, request, global_params=None): + """Signs a blob using a service account's system-managed private key. + + Args: + request: (IamProjectsServiceAccountsSignBlobRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (SignBlobResponse) The response message. + """ + config = self.GetMethodConfig('SignBlob') + return self._RunMethod( + config, request, global_params=global_params) + + def SignJwt(self, request, global_params=None): + """Signs a JWT using a service account's system-managed private key. + +If no `exp` (expiry) time is contained in the claims, we will +provide an expiry of one hour in the future. If an expiry +of more than one hour in the future is requested, the request +will fail. + + Args: + request: (IamProjectsServiceAccountsSignJwtRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (SignJwtResponse) The response message. + """ + config = self.GetMethodConfig('SignJwt') + return self._RunMethod( + config, request, global_params=global_params) + + def TestIamPermissions(self, request, global_params=None): + """Tests the specified permissions against the IAM access control policy. +for the specified IAM resource. + + Args: + request: (IamProjectsServiceAccountsTestIamPermissionsRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TestIamPermissionsResponse) The response message. + """ + config = self.GetMethodConfig('TestIamPermissions') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates a ServiceAccount. + +Currently, only the following fields are updatable: +`display_name` . +The `etag` is mandatory. + + Args: + request: (ServiceAccount) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ServiceAccount) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class ProjectsService(base_api.BaseApiService): + """Service class for the projects resource.""" + + _NAME = u'projects' + + def __init__(self, client): + super(IamV1.ProjectsService, self).__init__(client) + self._method_configs = { + } + + self._upload_configs = { + } + + class RolesService(base_api.BaseApiService): + """Service class for the roles resource.""" + + _NAME = u'roles' + + def __init__(self, client): + super(IamV1.RolesService, self).__init__(client) + self._method_configs = { + 'QueryGrantableRoles': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.roles.queryGrantableRoles', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'v1/roles:queryGrantableRoles', + request_field='', + request_type_name=u'QueryGrantableRolesRequest', + response_type_name=u'QueryGrantableRolesResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def QueryGrantableRoles(self, request, global_params=None): + """Queries roles that can be granted on a particular resource. + + Args: + request: (QueryGrantableRolesRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (QueryGrantableRolesResponse) The response message. + """ + config = self.GetMethodConfig('QueryGrantableRoles') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/apitools/gen/testdata/iam/iam_v1_messages.py b/apitools/gen/testdata/iam/iam_v1_messages.py new file mode 100644 index 0000000..1db85b0 --- /dev/null +++ b/apitools/gen/testdata/iam/iam_v1_messages.py @@ -0,0 +1,964 @@ +"""Generated message classes for iam version v1. + +Manages identity and access control for Google Cloud Platform resources, +including the creation of service accounts, which you can use to authenticate +to Google and make API calls. +""" +# NOTE: This file is autogenerated and should not be edited by hand. + +from apitools.base.protorpclite import messages as _messages +from apitools.base.py import encoding + + +package = 'iam' + + +class AuditConfig(_messages.Message): + """Enables "data access" audit logging for a service and specifies a list of + members that are log-exempted. + + Fields: + exemptedMembers: Specifies the identities that are exempted from "data + access" audit logging for the `service` specified above. Follows the + same format of Binding.members. + service: Specifies a service that will be enabled for "data access" audit + logging. For example, `resourcemanager`, `storage`, `compute`. + `allServices` is a special value that covers all services. + """ + + exemptedMembers = _messages.StringField(1, repeated=True) + service = _messages.StringField(2) + + +class Binding(_messages.Message): + """Associates `members` with a `role`. + + Fields: + members: Specifies the identities requesting access for a Cloud Platform + resource. `members` can have the following values: * `allUsers`: A + special identifier that represents anyone who is on the internet; + with or without a Google account. * `allAuthenticatedUsers`: A special + identifier that represents anyone who is authenticated with a Google + account or a service account. * `user:{emailid}`: An email address that + represents a specific Google account. For example, `alice@gmail.com` + or `joe@example.com`. * `serviceAccount:{emailid}`: An email address + that represents a service account. For example, `my-other- + app@appspot.gserviceaccount.com`. * `group:{emailid}`: An email address + that represents a Google group. For example, `admins@example.com`. * + `domain:{domain}`: A Google Apps domain name that represents all the + users of that domain. For example, `google.com` or `example.com`. + role: Role that is assigned to `members`. For example, `roles/viewer`, + `roles/editor`, or `roles/owner`. Required + """ + + members = _messages.StringField(1, repeated=True) + role = _messages.StringField(2) + + +class CloudAuditOptions(_messages.Message): + """Write a Cloud Audit log""" + + +class Condition(_messages.Message): + """A condition to be met. + + Enums: + IamValueValuesEnum: Trusted attributes supplied by the IAM system. + OpValueValuesEnum: An operator to apply the subject with. + SysValueValuesEnum: Trusted attributes supplied by any service that owns + resources and uses the IAM system for access control. + + Fields: + iam: Trusted attributes supplied by the IAM system. + op: An operator to apply the subject with. + svc: Trusted attributes discharged by the service. + sys: Trusted attributes supplied by any service that owns resources and + uses the IAM system for access control. + value: DEPRECATED. Use 'values' instead. + values: The objects of the condition. This is mutually exclusive with + 'value'. + """ + + class IamValueValuesEnum(_messages.Enum): + """Trusted attributes supplied by the IAM system. + + Values: + NO_ATTR: Default non-attribute. + AUTHORITY: Either principal or (if present) authority + ATTRIBUTION: selector Always the original principal, but making clear + """ + NO_ATTR = 0 + AUTHORITY = 1 + ATTRIBUTION = 2 + + class OpValueValuesEnum(_messages.Enum): + """An operator to apply the subject with. + + Values: + NO_OP: Default no-op. + EQUALS: DEPRECATED. Use IN instead. + NOT_EQUALS: DEPRECATED. Use NOT_IN instead. + IN: Set-inclusion check. + NOT_IN: Set-exclusion check. + DISCHARGED: Subject is discharged + """ + NO_OP = 0 + EQUALS = 1 + NOT_EQUALS = 2 + IN = 3 + NOT_IN = 4 + DISCHARGED = 5 + + class SysValueValuesEnum(_messages.Enum): + """Trusted attributes supplied by any service that owns resources and uses + the IAM system for access control. + + Values: + NO_ATTR: Default non-attribute type + REGION: Region of the resource + SERVICE: Service name + NAME: Resource name + IP: IP address of the caller + """ + NO_ATTR = 0 + REGION = 1 + SERVICE = 2 + NAME = 3 + IP = 4 + + iam = _messages.EnumField('IamValueValuesEnum', 1) + op = _messages.EnumField('OpValueValuesEnum', 2) + svc = _messages.StringField(3) + sys = _messages.EnumField('SysValueValuesEnum', 4) + value = _messages.StringField(5) + values = _messages.StringField(6, repeated=True) + + +class CounterOptions(_messages.Message): + """Options for counters + + Fields: + field: The field value to attribute. + metric: The metric to update. + """ + + field = _messages.StringField(1) + metric = _messages.StringField(2) + + +class CreateServiceAccountKeyRequest(_messages.Message): + """The service account key create request. + + Enums: + PrivateKeyTypeValueValuesEnum: The output format of the private key. + `GOOGLE_CREDENTIALS_FILE` is the default output format. + + Fields: + privateKeyType: The output format of the private key. + `GOOGLE_CREDENTIALS_FILE` is the default output format. + """ + + class PrivateKeyTypeValueValuesEnum(_messages.Enum): + """The output format of the private key. `GOOGLE_CREDENTIALS_FILE` is the + default output format. + + Values: + TYPE_UNSPECIFIED: Unspecified. Equivalent to + `TYPE_GOOGLE_CREDENTIALS_FILE`. + TYPE_PKCS12_FILE: PKCS12 format. The password for the PKCS12 file is + `notasecret`. For more information, see + https://tools.ietf.org/html/rfc7292. + TYPE_GOOGLE_CREDENTIALS_FILE: Google Credentials File format. + """ + TYPE_UNSPECIFIED = 0 + TYPE_PKCS12_FILE = 1 + TYPE_GOOGLE_CREDENTIALS_FILE = 2 + + privateKeyType = _messages.EnumField('PrivateKeyTypeValueValuesEnum', 1) + + +class CreateServiceAccountRequest(_messages.Message): + """The service account create request. + + Fields: + accountId: Required. The account id that is used to generate the service + account email address and a stable unique id. It is unique within a + project, must be 1-63 characters long, and match the regular expression + `[a-z]([-a-z0-9]*[a-z0-9])` to comply with RFC1035. + serviceAccount: The ServiceAccount resource to create. Currently, only the + following values are user assignable: `display_name` . + """ + + accountId = _messages.StringField(1) + serviceAccount = _messages.MessageField('ServiceAccount', 2) + + +class DataAccessOptions(_messages.Message): + """Write a Data Access (Gin) log""" + + +class Empty(_messages.Message): + """A generic empty message that you can re-use to avoid defining duplicated + empty messages in your APIs. A typical example is to use it as the request + or the response type of an API method. For instance: service Foo { + rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); } The + JSON representation for `Empty` is empty JSON object `{}`. + """ + + + +class GetPolicyDetailsRequest(_messages.Message): + """The request to get the current policy and the policies on the inherited + resources the user has access to. + + Fields: + fullResourcePath: REQUIRED: The full resource path of the current policy + being requested, e.g., `//dataflow.googleapis.com/projects/../jobs/..`. + pageSize: Limit on the number of policies to include in the response. + Further accounts can subsequently be obtained by including the + GetPolicyDetailsResponse.next_page_token in a subsequent request. If + zero, the default page size 20 will be used. Must be given a value in + range [0, 100], otherwise an invalid argument error will be returned. + pageToken: Optional pagination token returned in an earlier + GetPolicyDetailsResponse.next_page_token response. + """ + + fullResourcePath = _messages.StringField(1) + pageSize = _messages.IntegerField(2, variant=_messages.Variant.INT32) + pageToken = _messages.StringField(3) + + +class GetPolicyDetailsResponse(_messages.Message): + """The response to the `GetPolicyDetailsRequest` containing the current + policy and the policies on the inherited resources the user has access to. + + Fields: + nextPageToken: To retrieve the next page of results, set + GetPolicyDetailsRequest.page_token to this value. If this value is + empty, then there are not any further policies that the user has access + to. The lifetime is 60 minutes. An "Expired pagination token" error will + be returned if exceeded. + policies: The current policy and all the inherited policies the user has + access to. + """ + + nextPageToken = _messages.StringField(1) + policies = _messages.MessageField('PolicyDetail', 2, repeated=True) + + +class IamProjectsServiceAccountsCreateRequest(_messages.Message): + """A IamProjectsServiceAccountsCreateRequest object. + + Fields: + createServiceAccountRequest: A CreateServiceAccountRequest resource to be + passed as the request body. + name: Required. The resource name of the project associated with the + service accounts, such as `projects/my-project-123`. + """ + + createServiceAccountRequest = _messages.MessageField('CreateServiceAccountRequest', 1) + name = _messages.StringField(2, required=True) + + +class IamProjectsServiceAccountsDeleteRequest(_messages.Message): + """A IamProjectsServiceAccountsDeleteRequest object. + + Fields: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + """ + + name = _messages.StringField(1, required=True) + + +class IamProjectsServiceAccountsGetIamPolicyRequest(_messages.Message): + """A IamProjectsServiceAccountsGetIamPolicyRequest object. + + Fields: + resource: REQUIRED: The resource for which the policy is being requested. + `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the path + specified in this value is resource specific and is specified in the + `getIamPolicy` documentation. + """ + + resource = _messages.StringField(1, required=True) + + +class IamProjectsServiceAccountsGetRequest(_messages.Message): + """A IamProjectsServiceAccountsGetRequest object. + + Fields: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + """ + + name = _messages.StringField(1, required=True) + + +class IamProjectsServiceAccountsKeysCreateRequest(_messages.Message): + """A IamProjectsServiceAccountsKeysCreateRequest object. + + Fields: + createServiceAccountKeyRequest: A CreateServiceAccountKeyRequest resource + to be passed as the request body. + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + """ + + createServiceAccountKeyRequest = _messages.MessageField('CreateServiceAccountKeyRequest', 1) + name = _messages.StringField(2, required=True) + + +class IamProjectsServiceAccountsKeysDeleteRequest(_messages.Message): + """A IamProjectsServiceAccountsKeysDeleteRequest object. + + Fields: + name: The resource name of the service account key in the following + format: `projects/{project}/serviceAccounts/{account}/keys/{key}`. Using + `-` as a wildcard for the project will infer the project from the + account. The `account` value can be the `email` address or the + `unique_id` of the service account. + """ + + name = _messages.StringField(1, required=True) + + +class IamProjectsServiceAccountsKeysGetRequest(_messages.Message): + """A IamProjectsServiceAccountsKeysGetRequest object. + + Enums: + PublicKeyTypeValueValuesEnum: The output format of the public key + requested. X509_PEM is the default output format. + + Fields: + name: The resource name of the service account key in the following + format: `projects/{project}/serviceAccounts/{account}/keys/{key}`. + Using `-` as a wildcard for the project will infer the project from the + account. The `account` value can be the `email` address or the + `unique_id` of the service account. + publicKeyType: The output format of the public key requested. X509_PEM is + the default output format. + """ + + class PublicKeyTypeValueValuesEnum(_messages.Enum): + """The output format of the public key requested. X509_PEM is the default + output format. + + Values: + TYPE_NONE: + TYPE_X509_PEM_FILE: + TYPE_RAW_PUBLIC_KEY: + """ + TYPE_NONE = 0 + TYPE_X509_PEM_FILE = 1 + TYPE_RAW_PUBLIC_KEY = 2 + + name = _messages.StringField(1, required=True) + publicKeyType = _messages.EnumField('PublicKeyTypeValueValuesEnum', 2) + + +class IamProjectsServiceAccountsKeysListRequest(_messages.Message): + """A IamProjectsServiceAccountsKeysListRequest object. + + Enums: + KeyTypesValueValuesEnum: Filters the types of keys the user wants to + include in the list response. Duplicate key types are not allowed. If no + key type is provided, all keys are returned. + + Fields: + keyTypes: Filters the types of keys the user wants to include in the list + response. Duplicate key types are not allowed. If no key type is + provided, all keys are returned. + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project, will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + """ + + class KeyTypesValueValuesEnum(_messages.Enum): + """Filters the types of keys the user wants to include in the list + response. Duplicate key types are not allowed. If no key type is provided, + all keys are returned. + + Values: + KEY_TYPE_UNSPECIFIED: + USER_MANAGED: + SYSTEM_MANAGED: + """ + KEY_TYPE_UNSPECIFIED = 0 + USER_MANAGED = 1 + SYSTEM_MANAGED = 2 + + keyTypes = _messages.EnumField('KeyTypesValueValuesEnum', 1, repeated=True) + name = _messages.StringField(2, required=True) + + +class IamProjectsServiceAccountsListRequest(_messages.Message): + """A IamProjectsServiceAccountsListRequest object. + + Fields: + name: Required. The resource name of the project associated with the + service accounts, such as `projects/my-project-123`. + pageSize: Optional limit on the number of service accounts to include in + the response. Further accounts can subsequently be obtained by including + the ListServiceAccountsResponse.next_page_token in a subsequent request. + pageToken: Optional pagination token returned in an earlier + ListServiceAccountsResponse.next_page_token. + removeDeletedServiceAccounts: Do not list service accounts deleted from + Gaia. DO NOT INCLUDE IN EXTERNAL + DOCUMENTATION. + """ + + name = _messages.StringField(1, required=True) + pageSize = _messages.IntegerField(2, variant=_messages.Variant.INT32) + pageToken = _messages.StringField(3) + removeDeletedServiceAccounts = _messages.BooleanField(4) + + +class IamProjectsServiceAccountsSetIamPolicyRequest(_messages.Message): + """A IamProjectsServiceAccountsSetIamPolicyRequest object. + + Fields: + resource: REQUIRED: The resource for which the policy is being specified. + `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the path + specified in this value is resource specific and is specified in the + `setIamPolicy` documentation. + setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the + request body. + """ + + resource = _messages.StringField(1, required=True) + setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) + + +class IamProjectsServiceAccountsSignBlobRequest(_messages.Message): + """A IamProjectsServiceAccountsSignBlobRequest object. + + Fields: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + signBlobRequest: A SignBlobRequest resource to be passed as the request + body. + """ + + name = _messages.StringField(1, required=True) + signBlobRequest = _messages.MessageField('SignBlobRequest', 2) + + +class IamProjectsServiceAccountsSignJwtRequest(_messages.Message): + """A IamProjectsServiceAccountsSignJwtRequest object. + + Fields: + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard + for the project will infer the project from the account. The `account` + value can be the `email` address or the `unique_id` of the service + account. + signJwtRequest: A SignJwtRequest resource to be passed as the request + body. + """ + + name = _messages.StringField(1, required=True) + signJwtRequest = _messages.MessageField('SignJwtRequest', 2) + + +class IamProjectsServiceAccountsTestIamPermissionsRequest(_messages.Message): + """A IamProjectsServiceAccountsTestIamPermissionsRequest object. + + Fields: + resource: REQUIRED: The resource for which the policy detail is being + requested. `resource` is usually specified as a path, such as + `projects/*project*/zones/*zone*/disks/*disk*`. The format for the path + specified in this value is resource specific and is specified in the + `testIamPermissions` documentation. + testIamPermissionsRequest: A TestIamPermissionsRequest resource to be + passed as the request body. + """ + + resource = _messages.StringField(1, required=True) + testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) + + +class ListServiceAccountKeysResponse(_messages.Message): + """The service account keys list response. + + Fields: + keys: The public keys for the service account. + """ + + keys = _messages.MessageField('ServiceAccountKey', 1, repeated=True) + + +class ListServiceAccountsResponse(_messages.Message): + """The service account list response. + + Fields: + accounts: The list of matching service accounts. + nextPageToken: To retrieve the next page of results, set + ListServiceAccountsRequest.page_token to this value. + """ + + accounts = _messages.MessageField('ServiceAccount', 1, repeated=True) + nextPageToken = _messages.StringField(2) + + +class LogConfig(_messages.Message): + """Specifies what kind of log the caller must write Increment a streamz + counter with the specified metric and field names. Metric names should + start with a '/', generally be lowercase-only, and end in "_count". Field + names should not contain an initial slash. The actual exported metric names + will have "/iam/policy" prepended. Field names correspond to IAM request + parameters and field values are their respective values. At present the + only supported field names are - "iam_principal", corresponding to + IAMContext.principal; - "" (empty string), resulting in one aggretated + counter with no field. Examples: counter { metric: "/debug_access_count" + field: "iam_principal" } ==> increment counter + /iam/policy/backend_debug_access_count + {iam_principal=[value of IAMContext.principal]} At this time we do not + support: * multiple field names (though this may be supported in the future) + * decrementing the counter * incrementing it by anything other than 1 + + Fields: + cloudAudit: Cloud audit options. + counter: Counter options. + dataAccess: Data access options. + """ + + cloudAudit = _messages.MessageField('CloudAuditOptions', 1) + counter = _messages.MessageField('CounterOptions', 2) + dataAccess = _messages.MessageField('DataAccessOptions', 3) + + +class Policy(_messages.Message): + """Defines an Identity and Access Management (IAM) policy. It is used to + specify access control policies for Cloud Platform resources. A `Policy` + consists of a list of `bindings`. A `Binding` binds a list of `members` to a + `role`, where the members can be user accounts, Google groups, Google + domains, and service accounts. A `role` is a named list of permissions + defined by IAM. **Example** { "bindings": [ { + "role": "roles/owner", "members": [ + "user:mike@example.com", "group:admins@example.com", + "domain:google.com", "serviceAccount:my-other- + app@appspot.gserviceaccount.com", ] }, { + "role": "roles/viewer", "members": ["user:sean@example.com"] + } ] } For a description of IAM and its features, see the [IAM + developer's guide](https://cloud.google.com/iam). + + Fields: + auditConfigs: Specifies audit logging configs for "data access". "data + access": generally refers to data reads/writes and admin reads. "admin + activity": generally refers to admin writes. Note: `AuditConfig` + doesn't apply to "admin activity", which always enables audit logging. + bindings: Associates a list of `members` to a `role`. Multiple `bindings` + must not be specified for the same `role`. `bindings` with no members + will result in an error. + etag: `etag` is used for optimistic concurrency control as a way to help + prevent simultaneous updates of a policy from overwriting each other. It + is strongly suggested that systems make use of the `etag` in the read- + modify-write cycle to perform policy updates in order to avoid race + conditions: An `etag` is returned in the response to `getIamPolicy`, and + systems are expected to put that etag in the request to `setIamPolicy` + to ensure that their change will be applied to the same version of the + policy. If no `etag` is provided in the call to `setIamPolicy`, then + the existing policy is overwritten blindly. + iamOwned: A boolean attribute. + rules: If more than one rule is specified, the rules are applied in the + following manner: - All matching LOG rules are always applied. - If any + DENY/DENY_WITH_LOG rule matches, permission is denied. Logging will be + applied if one or more matching rule requires logging. - Otherwise, if + any ALLOW/ALLOW_WITH_LOG rule matches, permission is granted. + Logging will be applied if one or more matching rule requires logging. - + Otherwise, if no rule applies, permission is denied. + version: Version of the `Policy`. The default version is 0. + """ + + auditConfigs = _messages.MessageField('AuditConfig', 1, repeated=True) + bindings = _messages.MessageField('Binding', 2, repeated=True) + etag = _messages.BytesField(3) + iamOwned = _messages.BooleanField(4) + rules = _messages.MessageField('Rule', 5, repeated=True) + version = _messages.IntegerField(6, variant=_messages.Variant.INT32) + + +class PolicyDetail(_messages.Message): + """A policy and its full resource path. + + Fields: + fullResourcePath: The full resource path of the policy e.g., + `//dataflow.googleapis.com/projects/../jobs/..`. Note that a resource + and its inherited resource have different `full_resource_path`. + policy: The policy of a `resource/project/folder`. + """ + + fullResourcePath = _messages.StringField(1) + policy = _messages.MessageField('Policy', 2) + + +class QueryGrantableRolesRequest(_messages.Message): + """The grantable role query request. + + Fields: + fullResourceName: Required. The full resource name to query from the list + of grantable roles. The name follows the Google Cloud Platform resource + format. For example, a Cloud Platform project with id `my-project` will + be named `//cloudresourcemanager.googleapis.com/projects/my-project`. + """ + + fullResourceName = _messages.StringField(1) + + +class QueryGrantableRolesResponse(_messages.Message): + """The grantable role query response. + + Fields: + roles: The list of matching roles. + """ + + roles = _messages.MessageField('Role', 1, repeated=True) + + +class Role(_messages.Message): + """A role in the Identity and Access Management API. + + Fields: + apiTokens: A string attribute. + description: Optional. A human-readable description for the role. + name: The name of the role. Examples of roles names are: `roles/editor`, + `roles/viewer` and `roles/logging.viewer`. + title: Optional. A human-readable title for the role. Typically this is + limited to 100 UTF-8 bytes. + """ + + apiTokens = _messages.StringField(1, repeated=True) + description = _messages.StringField(2) + name = _messages.StringField(3) + title = _messages.StringField(4) + + +class Rule(_messages.Message): + """A rule to be applied in a Policy. + + Enums: + ActionValueValuesEnum: Required + + Fields: + action: Required + conditions: Additional restrictions that must be met + description: Human-readable description of the rule. + in_: If one or more 'in' clauses are specified, the rule matches if the + PRINCIPAL/AUTHORITY_SELECTOR is in at least one of these entries. + logConfig: The config returned to callers of tech.iam.IAM.CheckPolicy for + any entries that match the LOG action. + notIn: If one or more 'not_in' clauses are specified, the rule matches if + the PRINCIPAL/AUTHORITY_SELECTOR is in none of the entries. The format + for in and not_in entries is the same as for members in a Binding (see + google/iam/v1/policy.proto). + permissions: A permission is a string of form '..' (e.g., 'storage.buckets.list'). A value of '*' matches all + permissions, and a verb part of '*' (e.g., 'storage.buckets.*') matches + all verbs. + """ + + class ActionValueValuesEnum(_messages.Enum): + """Required + + Values: + NO_ACTION: Default no action. + ALLOW: Matching 'Entries' grant access. + ALLOW_WITH_LOG: Matching 'Entries' grant access and the caller promises + to log the request per the returned log_configs. + DENY: Matching 'Entries' deny access. + DENY_WITH_LOG: Matching 'Entries' deny access and the caller promises to + log the request per the returned log_configs. + LOG: Matching 'Entries' tell IAM.Check callers to generate logs. + """ + NO_ACTION = 0 + ALLOW = 1 + ALLOW_WITH_LOG = 2 + DENY = 3 + DENY_WITH_LOG = 4 + LOG = 5 + + action = _messages.EnumField('ActionValueValuesEnum', 1) + conditions = _messages.MessageField('Condition', 2, repeated=True) + description = _messages.StringField(3) + in_ = _messages.StringField(4, repeated=True) + logConfig = _messages.MessageField('LogConfig', 5, repeated=True) + notIn = _messages.StringField(6, repeated=True) + permissions = _messages.StringField(7, repeated=True) + + +class ServiceAccount(_messages.Message): + """A service account in the Identity and Access Management API. To create a + service account, specify the `project_id` and the `account_id` for the + account. The `account_id` is unique within the project, and is used to + generate the service account email address and a stable `unique_id`. All + other methods can identify the service account using the format + `projects/{project}/serviceAccounts/{account}`. Using `-` as a wildcard for + the project will infer the project from the account. The `account` value can + be the `email` address or the `unique_id` of the service account. + + Fields: + description: Optional. A user-specified opaque description of the service + account. + displayName: Optional. A user-specified description of the service + account. Must be fewer than 100 UTF-8 bytes. + email: @OutputOnly The email address of the service account. + etag: Used to perform a consistent read-modify-write. + name: The resource name of the service account in the following format: + `projects/{project}/serviceAccounts/{account}`. Requests using `-` as a + wildcard for the project will infer the project from the `account` and + the `account` value can be the `email` address or the `unique_id` of the + service account. In responses the resource name will always be in the + format `projects/{project}/serviceAccounts/{email}`. + oauth2ClientId: @OutputOnly. The OAuth2 client id for the service account. + This is used in conjunction with the OAuth2 clientconfig API to make + three legged OAuth2 (3LO) flows to access the data of Google users. + projectId: @OutputOnly The id of the project that owns the service + account. + uniqueId: @OutputOnly The unique and stable id of the service account. + """ + + description = _messages.StringField(1) + displayName = _messages.StringField(2) + email = _messages.StringField(3) + etag = _messages.BytesField(4) + name = _messages.StringField(5) + oauth2ClientId = _messages.StringField(6) + projectId = _messages.StringField(7) + uniqueId = _messages.StringField(8) + + +class ServiceAccountKey(_messages.Message): + """Represents a service account key. A service account has two sets of key- + pairs: user-managed, and system-managed. User-managed key-pairs can be + created and deleted by users. Users are responsible for rotating these keys + periodically to ensure security of their service accounts. Users retain the + private key of these key-pairs, and Google retains ONLY the public key. + System-managed key-pairs are managed automatically by Google, and rotated + daily without user intervention. The private key never leaves Google's + servers to maximize security. Public keys for all service accounts are also + published at the OAuth2 Service Account API. + + Enums: + PrivateKeyTypeValueValuesEnum: The output format for the private key. Only + provided in `CreateServiceAccountKey` responses, not in + `GetServiceAccountKey` or `ListServiceAccountKey` responses. Google + never exposes system-managed private keys, and never retains user- + managed private keys. + + Fields: + name: The resource name of the service account key in the following format + `projects/{project}/serviceAccounts/{account}/keys/{key}`. + privateKeyData: The private key data. Only provided in + `CreateServiceAccountKey` responses. + privateKeyType: The output format for the private key. Only provided in + `CreateServiceAccountKey` responses, not in `GetServiceAccountKey` or + `ListServiceAccountKey` responses. Google never exposes system-managed + private keys, and never retains user-managed private keys. + publicKeyData: The public key data. Only provided in + `GetServiceAccountKey` responses. + validAfterTime: The key can be used after this timestamp. + validBeforeTime: The key can be used before this timestamp. + """ + + class PrivateKeyTypeValueValuesEnum(_messages.Enum): + """The output format for the private key. Only provided in + `CreateServiceAccountKey` responses, not in `GetServiceAccountKey` or + `ListServiceAccountKey` responses. Google never exposes system-managed + private keys, and never retains user-managed private keys. + + Values: + TYPE_UNSPECIFIED: Unspecified. Equivalent to + `TYPE_GOOGLE_CREDENTIALS_FILE`. + TYPE_PKCS12_FILE: PKCS12 format. The password for the PKCS12 file is + `notasecret`. For more information, see + https://tools.ietf.org/html/rfc7292. + TYPE_GOOGLE_CREDENTIALS_FILE: Google Credentials File format. + """ + TYPE_UNSPECIFIED = 0 + TYPE_PKCS12_FILE = 1 + TYPE_GOOGLE_CREDENTIALS_FILE = 2 + + name = _messages.StringField(1) + privateKeyData = _messages.BytesField(2) + privateKeyType = _messages.EnumField('PrivateKeyTypeValueValuesEnum', 3) + publicKeyData = _messages.BytesField(4) + validAfterTime = _messages.StringField(5) + validBeforeTime = _messages.StringField(6) + + +class SetIamPolicyRequest(_messages.Message): + """Request message for `SetIamPolicy` method. + + Fields: + policy: REQUIRED: The complete policy to be applied to the `resource`. The + size of the policy is limited to a few 10s of KB. An empty policy is a + valid policy but certain Cloud Platform services (such as Projects) + might reject them. + """ + + policy = _messages.MessageField('Policy', 1) + + +class SignBlobRequest(_messages.Message): + """The service account sign blob request. + + Fields: + bytesToSign: The bytes to sign. + """ + + bytesToSign = _messages.BytesField(1) + + +class SignBlobResponse(_messages.Message): + """The service account sign blob response. + + Fields: + keyId: The id of the key used to sign the blob. + signature: The signed blob. + """ + + keyId = _messages.StringField(1) + signature = _messages.BytesField(2) + + +class SignJwtRequest(_messages.Message): + """The service account sign JWT request. + + Fields: + payload: The JWT payload to sign, a JSON JWT Claim set. + """ + + payload = _messages.StringField(1) + + +class SignJwtResponse(_messages.Message): + """The service account sign JWT response. + + Fields: + keyId: The id of the key used to sign the JWT. + signedJwt: The signed JWT. + """ + + keyId = _messages.StringField(1) + signedJwt = _messages.StringField(2) + + +class StandardQueryParameters(_messages.Message): + """Query parameters accepted by all methods. + + Enums: + FXgafvValueValuesEnum: V1 error format. + AltValueValuesEnum: Data format for response. + + Fields: + f__xgafv: V1 error format. + access_token: OAuth access token. + alt: Data format for response. + bearer_token: OAuth bearer token. + callback: JSONP + fields: Selector specifying which fields to include in a partial response. + key: API key. Your API key identifies your project and provides you with + API access, quota, and reports. Required unless you provide an OAuth 2.0 + token. + oauth_token: OAuth 2.0 token for the current user. + pp: Pretty-print response. + prettyPrint: Returns response with indentations and line breaks. + quotaUser: Available to use for quota purposes for server-side + applications. Can be any arbitrary string assigned to a user, but should + not exceed 40 characters. + trace: A tracing token of the form "token:" to include in api + requests. + uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). + upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). + """ + + class AltValueValuesEnum(_messages.Enum): + """Data format for response. + + Values: + json: Responses with Content-Type of application/json + media: Media download with context-dependent Content-Type + proto: Responses with Content-Type of application/x-protobuf + """ + json = 0 + media = 1 + proto = 2 + + class FXgafvValueValuesEnum(_messages.Enum): + """V1 error format. + + Values: + _1: v1 error format + _2: v2 error format + """ + _1 = 0 + _2 = 1 + + f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1) + access_token = _messages.StringField(2) + alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json') + bearer_token = _messages.StringField(4) + callback = _messages.StringField(5) + fields = _messages.StringField(6) + key = _messages.StringField(7) + oauth_token = _messages.StringField(8) + pp = _messages.BooleanField(9, default=True) + prettyPrint = _messages.BooleanField(10, default=True) + quotaUser = _messages.StringField(11) + trace = _messages.StringField(12) + uploadType = _messages.StringField(13) + upload_protocol = _messages.StringField(14) + + +class TestIamPermissionsRequest(_messages.Message): + """Request message for `TestIamPermissions` method. + + Fields: + permissions: The set of permissions to check for the `resource`. + Permissions with wildcards (such as '*' or 'storage.*') are not allowed. + For more information see IAM Overview. + """ + + permissions = _messages.StringField(1, repeated=True) + + +class TestIamPermissionsResponse(_messages.Message): + """Response message for `TestIamPermissions` method. + + Fields: + permissions: A subset of `TestPermissionsRequest.permissions` that the + caller is allowed. + """ + + permissions = _messages.StringField(1, repeated=True) + + +encoding.AddCustomJsonFieldMapping( + Rule, 'in_', 'in', + package=u'iam') +encoding.AddCustomJsonFieldMapping( + StandardQueryParameters, 'f__xgafv', '$.xgafv', + package=u'iam') +encoding.AddCustomJsonEnumMapping( + StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1', + package=u'iam') +encoding.AddCustomJsonEnumMapping( + StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2', + package=u'iam') -- GitLab From 6e81406d56799ed2e676c894502433adfe37f8f8 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Wed, 25 May 2016 10:02:19 -0400 Subject: [PATCH 242/295] Parameterize generated client test with version. --- apitools/gen/gen_client_test.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index 7d888af..e3e7f50 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -43,23 +43,24 @@ class ClientGenCliTest(unittest2.TestCase): self.assertIn('usage:', err_output) self.assertIn('error: too few arguments', err_output) - def _CheckGeneratedFiles(self, api_name): + def _CheckGeneratedFiles(self, api_name, api_version): + prefix = api_name + '_' + api_version with test_utils.TempDir() as tmp_dir_path: gen_client.main([ gen_client.__file__, '--generate_cli', '--init-file', 'empty', '--infile', - GetTestDataPath(api_name, api_name + '_v1.json'), + GetTestDataPath(api_name, prefix + '.json'), '--outdir', tmp_dir_path, '--overwrite', '--root_package', api_name, 'client' ]) expected_files = ( - set([api_name + '_v1.py']) | # CLI files - set([api_name + '_v1_client.py', - api_name + '_v1_messages.py', + set([prefix + '.py']) | # CLI files + set([prefix + '_client.py', + prefix + '_messages.py', '__init__.py'])) self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) for expected_file in expected_files: @@ -68,10 +69,10 @@ class ClientGenCliTest(unittest2.TestCase): _GetContent(os.path.join(tmp_dir_path, expected_file))) def testGenClient_DnsDoc(self): - self._CheckGeneratedFiles('dns') + self._CheckGeneratedFiles('dns', 'v1') def testGenClient_IamDoc(self): - self._CheckGeneratedFiles('iam') + self._CheckGeneratedFiles('iam', 'v1') def testGenClient_SimpleDocNoInit(self): with test_utils.TempDir() as tmp_dir_path: -- GitLab From 9edd3a729d2d3547151192e3bef8ab74bb0fe8d8 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Tue, 31 May 2016 12:00:19 -0400 Subject: [PATCH 243/295] Consolidate gen test samples. --- apitools/gen/gen_client_test.py | 31 - run_pylint.py | 4 +- .../{storage_sample/storage => }/__init__.py | 15 +- samples/dns_sample/dns_v1.json | 707 ++++ .../dns_sample/dns_v1}/__init__.py | 0 .../dns_sample/dns_v1}/dns_v1.py | 0 .../dns_sample/dns_v1}/dns_v1_client.py | 0 .../dns_sample/dns_v1}/dns_v1_messages.py | 0 .../dns_sample}/gen_dns_client_test.py | 0 .../iam => samples/iam_sample}/iam_v1.json | 0 .../iam_sample/iam_v1}/__init__.py | 0 .../iam_sample/iam_v1}/iam_v1.py | 0 .../iam_sample/iam_v1}/iam_v1_client.py | 0 .../iam_sample/iam_v1}/iam_v1_messages.py | 0 samples/regenerate_samples.py | 50 + samples/storage_sample/storage_v1.json | 3338 +++++++++++++++++ samples/storage_sample/storage_v1/__init__.py | 5 + .../{storage => storage_v1}/storage_v1.py | 436 ++- .../storage_v1_client.py | 284 +- .../storage_v1_messages.py | 435 ++- samples/uptodate_check_test.py | 68 + tox.ini | 2 +- 22 files changed, 5238 insertions(+), 137 deletions(-) rename samples/{storage_sample/storage => }/__init__.py (63%) create mode 100644 samples/dns_sample/dns_v1.json rename {apitools/gen/testdata/dns => samples/dns_sample/dns_v1}/__init__.py (100%) rename {apitools/gen/testdata/dns => samples/dns_sample/dns_v1}/dns_v1.py (100%) rename {apitools/gen/testdata/dns => samples/dns_sample/dns_v1}/dns_v1_client.py (100%) rename {apitools/gen/testdata/dns => samples/dns_sample/dns_v1}/dns_v1_messages.py (100%) rename {apitools/gen => samples/dns_sample}/gen_dns_client_test.py (100%) rename {apitools/gen/testdata/iam => samples/iam_sample}/iam_v1.json (100%) rename {apitools/gen/testdata/iam => samples/iam_sample/iam_v1}/__init__.py (100%) rename {apitools/gen/testdata/iam => samples/iam_sample/iam_v1}/iam_v1.py (100%) rename {apitools/gen/testdata/iam => samples/iam_sample/iam_v1}/iam_v1_client.py (100%) rename {apitools/gen/testdata/iam => samples/iam_sample/iam_v1}/iam_v1_messages.py (100%) create mode 100644 samples/regenerate_samples.py create mode 100644 samples/storage_sample/storage_v1.json create mode 100644 samples/storage_sample/storage_v1/__init__.py rename samples/storage_sample/{storage => storage_v1}/storage_v1.py (89%) mode change 100755 => 100644 rename samples/storage_sample/{storage => storage_v1}/storage_v1_client.py (80%) rename samples/storage_sample/{storage => storage_v1}/storage_v1_messages.py (81%) create mode 100644 samples/uptodate_check_test.py diff --git a/apitools/gen/gen_client_test.py b/apitools/gen/gen_client_test.py index e3e7f50..3be3e7a 100644 --- a/apitools/gen/gen_client_test.py +++ b/apitools/gen/gen_client_test.py @@ -43,37 +43,6 @@ class ClientGenCliTest(unittest2.TestCase): self.assertIn('usage:', err_output) self.assertIn('error: too few arguments', err_output) - def _CheckGeneratedFiles(self, api_name, api_version): - prefix = api_name + '_' + api_version - with test_utils.TempDir() as tmp_dir_path: - gen_client.main([ - gen_client.__file__, - '--generate_cli', - '--init-file', 'empty', - '--infile', - GetTestDataPath(api_name, prefix + '.json'), - '--outdir', tmp_dir_path, - '--overwrite', - '--root_package', api_name, - 'client' - ]) - expected_files = ( - set([prefix + '.py']) | # CLI files - set([prefix + '_client.py', - prefix + '_messages.py', - '__init__.py'])) - self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) - for expected_file in expected_files: - self.assertMultiLineEqual( - _GetContent(GetTestDataPath(api_name, expected_file)), - _GetContent(os.path.join(tmp_dir_path, expected_file))) - - def testGenClient_DnsDoc(self): - self._CheckGeneratedFiles('dns', 'v1') - - def testGenClient_IamDoc(self): - self._CheckGeneratedFiles('iam', 'v1') - def testGenClient_SimpleDocNoInit(self): with test_utils.TempDir() as tmp_dir_path: gen_client.main([ diff --git a/run_pylint.py b/run_pylint.py index fd55463..3f0a315 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -33,7 +33,9 @@ import sys IGNORED_DIRECTORIES = [ 'apitools/gen/testdata', - 'samples/storage_sample/storage', + 'samples/dns_sample/dns_v1', + 'samples/iam_sample/iam_v1', + 'samples/storage_sample/storage_v1', 'venv', ] IGNORED_FILES = [ diff --git a/samples/storage_sample/storage/__init__.py b/samples/__init__.py similarity index 63% rename from samples/storage_sample/storage/__init__.py rename to samples/__init__.py index 0c742c1..58e0d91 100644 --- a/samples/storage_sample/storage/__init__.py +++ b/samples/__init__.py @@ -1,5 +1,4 @@ -# -# Copyright 2015 Google Inc. +# Copyright 2016 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,15 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -"""Common imports for generated storage client library.""" -# pylint:disable=wildcard-import - -import pkgutil - -from apitools.base.py import * -from storage_v1 import * -from storage_v1_client import * -from storage_v1_messages import * - -__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/samples/dns_sample/dns_v1.json b/samples/dns_sample/dns_v1.json new file mode 100644 index 0000000..77c1553 --- /dev/null +++ b/samples/dns_sample/dns_v1.json @@ -0,0 +1,707 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "dns:v1", + "name": "dns", + "version": "v1", + "revision": "20150807", + "title": "Google Cloud DNS API", + "description": "The Google Cloud DNS API provides services for configuring and serving authoritative DNS records.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/cloud-dns", + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/dns/v1/projects/", + "basePath": "/dns/v1/projects/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "dns/v1/projects/", + "batchPath": "batch", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/cloud-platform.read-only": { + "description": "MESSAGE UNDER CONSTRUCTION View your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readonly": { + "description": "View your DNS records hosted by Google Cloud DNS" + }, + "https://www.googleapis.com/auth/ndev.clouddns.readwrite": { + "description": "View and manage your DNS records hosted by Google Cloud DNS" + } + } + } + }, + "schemas": { + "Change": { + "id": "Change", + "type": "object", + "description": "An atomic update to a collection of ResourceRecordSets.", + "properties": { + "additions": { + "type": "array", + "description": "Which ResourceRecordSets to add?", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "deletions": { + "type": "array", + "description": "Which ResourceRecordSets to remove? Must match existing data exactly.", + "items": { + "$ref": "ResourceRecordSet" + } + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#change\".", + "default": "dns#change" + }, + "startTime": { + "type": "string", + "description": "The time that this operation was started by the server. This is in RFC3339 text format." + }, + "status": { + "type": "string", + "description": "Status of the operation (output only).", + "enum": [ + "done", + "pending" + ], + "enumDescriptions": [ + "", + "" + ] + } + } + }, + "ChangesListResponse": { + "id": "ChangesListResponse", + "type": "object", + "description": "The response to a request to enumerate Changes to a ResourceRecordSets collection.", + "properties": { + "changes": { + "type": "array", + "description": "The requested changes.", + "items": { + "$ref": "Change" + } + }, + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#changesListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a \"snapshot\" of collections larger than the maximum page size." + } + } + }, + "ManagedZone": { + "id": "ManagedZone", + "type": "object", + "description": "A zone is a subtree of the DNS namespace under one administrative responsibility. A ManagedZone is a resource that represents a DNS zone hosted by the Cloud DNS service.", + "properties": { + "creationTime": { + "type": "string", + "description": "The time that this resource was created on the server. This is in RFC3339 text format. Output only." + }, + "description": { + "type": "string", + "description": "A mutable string of at most 1024 characters associated with this resource for the user's convenience. Has no effect on the managed zone's function." + }, + "dnsName": { + "type": "string", + "description": "The DNS name of this managed zone, for instance \"example.com.\"." + }, + "id": { + "type": "string", + "description": "Unique identifier for the resource; defined by the server (output only)", + "format": "uint64" + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#managedZone\".", + "default": "dns#managedZone" + }, + "name": { + "type": "string", + "description": "User assigned name for this resource. Must be unique within the project. The name must be 1-32 characters long, must begin with a letter, end with a letter or digit, and only contain lowercase letters, digits or dashes." + }, + "nameServerSet": { + "type": "string", + "description": "Optionally specifies the NameServerSet for this ManagedZone. A NameServerSet is a set of DNS name servers that all host the same ManagedZones. Most users will leave this field unset." + }, + "nameServers": { + "type": "array", + "description": "Delegate your managed_zone to these virtual name servers; defined by the server (output only)", + "items": { + "type": "string" + } + } + } + }, + "ManagedZonesListResponse": { + "id": "ManagedZonesListResponse", + "type": "object", + "properties": { + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#managedZonesListResponse" + }, + "managedZones": { + "type": "array", + "description": "The managed zone resources.", + "items": { + "$ref": "ManagedZone" + } + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your page token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + } + } + }, + "Project": { + "id": "Project", + "type": "object", + "description": "A project resource. The project is a top level container for resources including Cloud DNS ManagedZones. Projects can be created only in the APIs console.", + "properties": { + "id": { + "type": "string", + "description": "User assigned unique identifier for the resource (output only)." + }, + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#project\".", + "default": "dns#project" + }, + "number": { + "type": "string", + "description": "Unique numeric identifier for the resource; defined by the server (output only).", + "format": "uint64" + }, + "quota": { + "$ref": "Quota", + "description": "Quotas assigned to this project (output only)." + } + } + }, + "Quota": { + "id": "Quota", + "type": "object", + "description": "Limits associated with a Project.", + "properties": { + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#quota\".", + "default": "dns#quota" + }, + "managedZones": { + "type": "integer", + "description": "Maximum allowed number of managed zones in the project.", + "format": "int32" + }, + "resourceRecordsPerRrset": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecords per ResourceRecordSet.", + "format": "int32" + }, + "rrsetAdditionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to add per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetDeletionsPerChange": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets to delete per ChangesCreateRequest.", + "format": "int32" + }, + "rrsetsPerManagedZone": { + "type": "integer", + "description": "Maximum allowed number of ResourceRecordSets per zone in the project.", + "format": "int32" + }, + "totalRrdataSizePerChange": { + "type": "integer", + "description": "Maximum allowed size for total rrdata in one ChangesCreateRequest in bytes.", + "format": "int32" + } + } + }, + "ResourceRecordSet": { + "id": "ResourceRecordSet", + "type": "object", + "description": "A unit of data that will be returned by the DNS servers.", + "properties": { + "kind": { + "type": "string", + "description": "Identifies what kind of resource this is. Value: the fixed string \"dns#resourceRecordSet\".", + "default": "dns#resourceRecordSet" + }, + "name": { + "type": "string", + "description": "For example, www.example.com." + }, + "rrdatas": { + "type": "array", + "description": "As defined in RFC 1035 (section 5) and RFC 1034 (section 3.6.1).", + "items": { + "type": "string" + } + }, + "ttl": { + "type": "integer", + "description": "Number of seconds that this ResourceRecordSet can be cached by resolvers.", + "format": "int32" + }, + "type": { + "type": "string", + "description": "The identifier of a supported record type, for example, A, AAAA, MX, TXT, and so on." + } + } + }, + "ResourceRecordSetsListResponse": { + "id": "ResourceRecordSetsListResponse", + "type": "object", + "properties": { + "kind": { + "type": "string", + "description": "Type of resource.", + "default": "dns#resourceRecordSetsListResponse" + }, + "nextPageToken": { + "type": "string", + "description": "The presence of this field indicates that there exist more results following your last page of results in pagination order. To fetch them, make another list request using this value as your pagination token.\n\nIn this way you can retrieve the complete contents of even very large collections one page at a time. However, if the contents of the collection change between the first and last paginated list request, the set of all elements returned will be an inconsistent view of the collection. There is no way to retrieve a consistent snapshot of a collection larger than the maximum page size." + }, + "rrsets": { + "type": "array", + "description": "The resource record set resources.", + "items": { + "$ref": "ResourceRecordSet" + } + } + } + } + }, + "resources": { + "changes": { + "methods": { + "create": { + "id": "dns.changes.create", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "POST", + "description": "Atomically update the ResourceRecordSet collection.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "request": { + "$ref": "Change" + }, + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.changes.get", + "path": "{project}/managedZones/{managedZone}/changes/{changeId}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Change.", + "parameters": { + "changeId": { + "type": "string", + "description": "The identifier of the requested change, from a previous ResourceRecordSetsChangeResponse.", + "required": true, + "location": "path" + }, + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone", + "changeId" + ], + "response": { + "$ref": "Change" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.changes.list", + "path": "{project}/managedZones/{managedZone}/changes", + "httpMethod": "GET", + "description": "Enumerate Changes to a ResourceRecordSet collection.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "sortBy": { + "type": "string", + "description": "Sorting criterion. The only supported value is change sequence.", + "default": "changeSequence", + "enum": [ + "changeSequence" + ], + "enumDescriptions": [ + "" + ], + "location": "query" + }, + "sortOrder": { + "type": "string", + "description": "Sorting order direction: 'ascending' or 'descending'.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ChangesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "managedZones": { + "methods": { + "create": { + "id": "dns.managedZones.create", + "path": "{project}/managedZones", + "httpMethod": "POST", + "description": "Create a new ManagedZone.", + "parameters": { + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "request": { + "$ref": "ManagedZone" + }, + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "delete": { + "id": "dns.managedZones.delete", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "DELETE", + "description": "Delete a previously created ManagedZone.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "get": { + "id": "dns.managedZones.get", + "path": "{project}/managedZones/{managedZone}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing ManagedZone.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ManagedZone" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + }, + "list": { + "id": "dns.managedZones.list", + "path": "{project}/managedZones", + "httpMethod": "GET", + "description": "Enumerate ManagedZones that have been created but not yet deleted.", + "parameters": { + "dnsName": { + "type": "string", + "description": "Restricts the list to return only zones with this domain name.", + "location": "query" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "ManagedZonesListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "projects": { + "methods": { + "get": { + "id": "dns.projects.get", + "path": "{project}", + "httpMethod": "GET", + "description": "Fetch the representation of an existing Project.", + "parameters": { + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "Project" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + }, + "resourceRecordSets": { + "methods": { + "list": { + "id": "dns.resourceRecordSets.list", + "path": "{project}/managedZones/{managedZone}/rrsets", + "httpMethod": "GET", + "description": "Enumerate ResourceRecordSets that have been created but not yet deleted.", + "parameters": { + "managedZone": { + "type": "string", + "description": "Identifies the managed zone addressed by this request. Can be the managed zone name or id.", + "required": true, + "location": "path" + }, + "maxResults": { + "type": "integer", + "description": "Optional. Maximum number of results to be returned. If unspecified, the server will decide how many results to return.", + "format": "int32", + "location": "query" + }, + "name": { + "type": "string", + "description": "Restricts the list to return only records with this fully qualified domain name.", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Optional. A tag returned by a previous list request that was truncated. Use this parameter to continue a previous list request.", + "location": "query" + }, + "project": { + "type": "string", + "description": "Identifies the project addressed by this request.", + "required": true, + "location": "path" + }, + "type": { + "type": "string", + "description": "Restricts the list to return only records of this type. If present, the \"name\" parameter must also be present.", + "location": "query" + } + }, + "parameterOrder": [ + "project", + "managedZone" + ], + "response": { + "$ref": "ResourceRecordSetsListResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/ndev.clouddns.readonly", + "https://www.googleapis.com/auth/ndev.clouddns.readwrite" + ] + } + } + } + } +} diff --git a/apitools/gen/testdata/dns/__init__.py b/samples/dns_sample/dns_v1/__init__.py similarity index 100% rename from apitools/gen/testdata/dns/__init__.py rename to samples/dns_sample/dns_v1/__init__.py diff --git a/apitools/gen/testdata/dns/dns_v1.py b/samples/dns_sample/dns_v1/dns_v1.py similarity index 100% rename from apitools/gen/testdata/dns/dns_v1.py rename to samples/dns_sample/dns_v1/dns_v1.py diff --git a/apitools/gen/testdata/dns/dns_v1_client.py b/samples/dns_sample/dns_v1/dns_v1_client.py similarity index 100% rename from apitools/gen/testdata/dns/dns_v1_client.py rename to samples/dns_sample/dns_v1/dns_v1_client.py diff --git a/apitools/gen/testdata/dns/dns_v1_messages.py b/samples/dns_sample/dns_v1/dns_v1_messages.py similarity index 100% rename from apitools/gen/testdata/dns/dns_v1_messages.py rename to samples/dns_sample/dns_v1/dns_v1_messages.py diff --git a/apitools/gen/gen_dns_client_test.py b/samples/dns_sample/gen_dns_client_test.py similarity index 100% rename from apitools/gen/gen_dns_client_test.py rename to samples/dns_sample/gen_dns_client_test.py diff --git a/apitools/gen/testdata/iam/iam_v1.json b/samples/iam_sample/iam_v1.json similarity index 100% rename from apitools/gen/testdata/iam/iam_v1.json rename to samples/iam_sample/iam_v1.json diff --git a/apitools/gen/testdata/iam/__init__.py b/samples/iam_sample/iam_v1/__init__.py similarity index 100% rename from apitools/gen/testdata/iam/__init__.py rename to samples/iam_sample/iam_v1/__init__.py diff --git a/apitools/gen/testdata/iam/iam_v1.py b/samples/iam_sample/iam_v1/iam_v1.py similarity index 100% rename from apitools/gen/testdata/iam/iam_v1.py rename to samples/iam_sample/iam_v1/iam_v1.py diff --git a/apitools/gen/testdata/iam/iam_v1_client.py b/samples/iam_sample/iam_v1/iam_v1_client.py similarity index 100% rename from apitools/gen/testdata/iam/iam_v1_client.py rename to samples/iam_sample/iam_v1/iam_v1_client.py diff --git a/apitools/gen/testdata/iam/iam_v1_messages.py b/samples/iam_sample/iam_v1/iam_v1_messages.py similarity index 100% rename from apitools/gen/testdata/iam/iam_v1_messages.py rename to samples/iam_sample/iam_v1/iam_v1_messages.py diff --git a/samples/regenerate_samples.py b/samples/regenerate_samples.py new file mode 100644 index 0000000..c9f768a --- /dev/null +++ b/samples/regenerate_samples.py @@ -0,0 +1,50 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to regenerate samples with latest client generator.""" + +import os +import subprocess + +_GEN_CLIENT_BINARY = 'gen_client' + +_SAMPLES = [ + 'dns_sample/dns_v1.json', + 'iam_sample/iam_v1.json', + 'storage_sample/storage_v1.json', +] + + +def _Generate(samples): + for sample in samples: + sample_dir, sample_doc = os.path.split(sample) + name, ext = os.path.splitext(sample_doc) + if ext != '.json': + raise RuntimeError('Expected .json discovery doc [{0}]' + .format(sample)) + api_name, _ = name.split('_') + args = [ + _GEN_CLIENT_BINARY, + '--infile', sample, + '--init-file', 'empty', + '--outdir={0}'.format(os.path.join(sample_dir, name)), + '--overwrite', + '--root_package', api_name, + 'client', + ] + subprocess.check_call(args) + + +if __name__ == '__main__': + _Generate(_SAMPLES) diff --git a/samples/storage_sample/storage_v1.json b/samples/storage_sample/storage_v1.json new file mode 100644 index 0000000..ff0b909 --- /dev/null +++ b/samples/storage_sample/storage_v1.json @@ -0,0 +1,3338 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "storage:v1", + "name": "storage", + "version": "v1", + "revision": "20160525", + "title": "Cloud Storage JSON API", + "description": "Stores and retrieves potentially large, immutable data objects.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "https://www.google.com/images/icons/product/cloud_storage-16.png", + "x32": "https://www.google.com/images/icons/product/cloud_storage-32.png" + }, + "documentationLink": "https://developers.google.com/storage/docs/json_api/", + "labels": [ + "graduated" + ], + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/storage/v1/", + "basePath": "/storage/v1/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "storage/v1/", + "batchPath": "batch", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/cloud-platform.read-only": { + "description": "View your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/devstorage.full_control": { + "description": "Manage your data and permissions in Google Cloud Storage" + }, + "https://www.googleapis.com/auth/devstorage.read_only": { + "description": "View your data in Google Cloud Storage" + }, + "https://www.googleapis.com/auth/devstorage.read_write": { + "description": "Manage your data in Google Cloud Storage" + } + } + } + }, + "schemas": { + "Bucket": { + "id": "Bucket", + "type": "object", + "description": "A bucket.", + "properties": { + "acl": { + "type": "array", + "description": "Access controls on the bucket.", + "items": { + "$ref": "BucketAccessControl" + }, + "annotations": { + "required": [ + "storage.buckets.update" + ] + } + }, + "cors": { + "type": "array", + "description": "The bucket's Cross-Origin Resource Sharing (CORS) configuration.", + "items": { + "type": "object", + "properties": { + "maxAgeSeconds": { + "type": "integer", + "description": "The value, in seconds, to return in the Access-Control-Max-Age header used in preflight responses.", + "format": "int32" + }, + "method": { + "type": "array", + "description": "The list of HTTP methods on which to include CORS response headers, (GET, OPTIONS, POST, etc) Note: \"*\" is permitted in the list of methods, and means \"any method\".", + "items": { + "type": "string" + } + }, + "origin": { + "type": "array", + "description": "The list of Origins eligible to receive CORS response headers. Note: \"*\" is permitted in the list of origins, and means \"any Origin\".", + "items": { + "type": "string" + } + }, + "responseHeader": { + "type": "array", + "description": "The list of HTTP headers other than the simple response headers to give permission for the user-agent to share across domains.", + "items": { + "type": "string" + } + } + } + } + }, + "defaultObjectAcl": { + "type": "array", + "description": "Default access controls to apply to new objects when no ACL is provided.", + "items": { + "$ref": "ObjectAccessControl" + } + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for the bucket." + }, + "id": { + "type": "string", + "description": "The ID of the bucket." + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For buckets, this is always storage#bucket.", + "default": "storage#bucket" + }, + "lifecycle": { + "type": "object", + "description": "The bucket's lifecycle configuration. See lifecycle management for more information.", + "properties": { + "rule": { + "type": "array", + "description": "A lifecycle management rule, which is made of an action to take and the condition(s) under which the action will be taken.", + "items": { + "type": "object", + "properties": { + "action": { + "type": "object", + "description": "The action to take.", + "properties": { + "type": { + "type": "string", + "description": "Type of the action. Currently, only Delete is supported." + } + } + }, + "condition": { + "type": "object", + "description": "The condition(s) under which the action will be taken.", + "properties": { + "age": { + "type": "integer", + "description": "Age of an object (in days). This condition is satisfied when an object reaches the specified age.", + "format": "int32" + }, + "createdBefore": { + "type": "string", + "description": "A date in RFC 3339 format with only the date part (for instance, \"2013-01-15\"). This condition is satisfied when an object is created before midnight of the specified date in UTC.", + "format": "date" + }, + "isLive": { + "type": "boolean", + "description": "Relevant only for versioned objects. If the value is true, this condition matches live objects; if the value is false, it matches archived objects." + }, + "numNewerVersions": { + "type": "integer", + "description": "Relevant only for versioned objects. If the value is N, this condition is satisfied when there are at least N versions (including the live version) newer than this version of the object.", + "format": "int32" + } + } + } + } + } + } + } + }, + "location": { + "type": "string", + "description": "The location of the bucket. Object data for objects in the bucket resides in physical storage within this region. Defaults to US. See the developer's guide for the authoritative list." + }, + "logging": { + "type": "object", + "description": "The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs.", + "properties": { + "logBucket": { + "type": "string", + "description": "The destination bucket where the current bucket's logs should be placed." + }, + "logObjectPrefix": { + "type": "string", + "description": "A prefix for log object names." + } + } + }, + "metageneration": { + "type": "string", + "description": "The metadata generation of this bucket.", + "format": "int64" + }, + "name": { + "type": "string", + "description": "The name of the bucket.", + "annotations": { + "required": [ + "storage.buckets.insert" + ] + } + }, + "owner": { + "type": "object", + "description": "The owner of the bucket. This is always the project team's owner group.", + "properties": { + "entity": { + "type": "string", + "description": "The entity, in the form project-owner-projectId." + }, + "entityId": { + "type": "string", + "description": "The ID for the entity." + } + } + }, + "projectNumber": { + "type": "string", + "description": "The project number of the project the bucket belongs to.", + "format": "uint64" + }, + "selfLink": { + "type": "string", + "description": "The URI of this bucket." + }, + "storageClass": { + "type": "string", + "description": "The bucket's storage class. This defines how objects in the bucket are stored and determines the SLA and the cost of storage. Values include STANDARD, NEARLINE and DURABLE_REDUCED_AVAILABILITY. Defaults to STANDARD. For more information, see storage classes." + }, + "timeCreated": { + "type": "string", + "description": "The creation time of the bucket in RFC 3339 format.", + "format": "date-time" + }, + "updated": { + "type": "string", + "description": "The modification time of the bucket in RFC 3339 format.", + "format": "date-time" + }, + "versioning": { + "type": "object", + "description": "The bucket's versioning configuration.", + "properties": { + "enabled": { + "type": "boolean", + "description": "While set to true, versioning is fully enabled for this bucket." + } + } + }, + "website": { + "type": "object", + "description": "The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information.", + "properties": { + "mainPageSuffix": { + "type": "string", + "description": "If the requested object path is missing, the service will ensure the path has a trailing '/', append this suffix, and attempt to retrieve the resulting object. This allows the creation of index.html objects to represent directory pages." + }, + "notFoundPage": { + "type": "string", + "description": "If the requested object path is missing, and any mainPageSuffix object is missing, if applicable, the service will return the named object from this bucket as the content for a 404 Not Found result." + } + } + } + } + }, + "BucketAccessControl": { + "id": "BucketAccessControl", + "type": "object", + "description": "An access-control entry.", + "properties": { + "bucket": { + "type": "string", + "description": "The name of the bucket." + }, + "domain": { + "type": "string", + "description": "The domain associated with the entity, if any." + }, + "email": { + "type": "string", + "description": "The email address associated with the entity, if any." + }, + "entity": { + "type": "string", + "description": "The entity holding the permission, in one of the following forms: \n- user-userId \n- user-email \n- group-groupId \n- group-email \n- domain-domain \n- project-team-projectId \n- allUsers \n- allAuthenticatedUsers Examples: \n- The user liz@example.com would be user-liz@example.com. \n- The group example@googlegroups.com would be group-example@googlegroups.com. \n- To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com.", + "annotations": { + "required": [ + "storage.bucketAccessControls.insert" + ] + } + }, + "entityId": { + "type": "string", + "description": "The ID for the entity, if any." + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for the access-control entry." + }, + "id": { + "type": "string", + "description": "The ID of the access-control entry." + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For bucket access control entries, this is always storage#bucketAccessControl.", + "default": "storage#bucketAccessControl" + }, + "projectTeam": { + "type": "object", + "description": "The project team associated with the entity, if any.", + "properties": { + "projectNumber": { + "type": "string", + "description": "The project number." + }, + "team": { + "type": "string", + "description": "The team. Can be owners, editors, or viewers." + } + } + }, + "role": { + "type": "string", + "description": "The access permission for the entity. Can be READER, WRITER, or OWNER.", + "annotations": { + "required": [ + "storage.bucketAccessControls.insert" + ] + } + }, + "selfLink": { + "type": "string", + "description": "The link to this access-control entry." + } + } + }, + "BucketAccessControls": { + "id": "BucketAccessControls", + "type": "object", + "description": "An access-control list.", + "properties": { + "items": { + "type": "array", + "description": "The list of items.", + "items": { + "$ref": "BucketAccessControl" + } + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For lists of bucket access control entries, this is always storage#bucketAccessControls.", + "default": "storage#bucketAccessControls" + } + } + }, + "Buckets": { + "id": "Buckets", + "type": "object", + "description": "A list of buckets.", + "properties": { + "items": { + "type": "array", + "description": "The list of items.", + "items": { + "$ref": "Bucket" + } + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For lists of buckets, this is always storage#buckets.", + "default": "storage#buckets" + }, + "nextPageToken": { + "type": "string", + "description": "The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results." + } + } + }, + "Channel": { + "id": "Channel", + "type": "object", + "description": "An notification channel used to watch for resource changes.", + "properties": { + "address": { + "type": "string", + "description": "The address where notifications are delivered for this channel." + }, + "expiration": { + "type": "string", + "description": "Date and time of notification channel expiration, expressed as a Unix timestamp, in milliseconds. Optional.", + "format": "int64" + }, + "id": { + "type": "string", + "description": "A UUID or similar unique string that identifies this channel." + }, + "kind": { + "type": "string", + "description": "Identifies this as a notification channel used to watch for changes to a resource. Value: the fixed string \"api#channel\".", + "default": "api#channel" + }, + "params": { + "type": "object", + "description": "Additional parameters controlling delivery channel behavior. Optional.", + "additionalProperties": { + "type": "string", + "description": "Declares a new parameter by name." + } + }, + "payload": { + "type": "boolean", + "description": "A Boolean value to indicate whether payload is wanted. Optional." + }, + "resourceId": { + "type": "string", + "description": "An opaque ID that identifies the resource being watched on this channel. Stable across different API versions." + }, + "resourceUri": { + "type": "string", + "description": "A version-specific identifier for the watched resource." + }, + "token": { + "type": "string", + "description": "An arbitrary string delivered to the target address with each notification delivered over this channel. Optional." + }, + "type": { + "type": "string", + "description": "The type of delivery mechanism used for this channel." + } + } + }, + "ComposeRequest": { + "id": "ComposeRequest", + "type": "object", + "description": "A Compose request.", + "properties": { + "destination": { + "$ref": "Object", + "description": "Properties of the resulting object." + }, + "kind": { + "type": "string", + "description": "The kind of item this is.", + "default": "storage#composeRequest" + }, + "sourceObjects": { + "type": "array", + "description": "The list of source objects that will be concatenated into a single object.", + "items": { + "type": "object", + "properties": { + "generation": { + "type": "string", + "description": "The generation of this object to use as the source.", + "format": "int64" + }, + "name": { + "type": "string", + "description": "The source object's name. The source object's bucket is implicitly the destination bucket.", + "annotations": { + "required": [ + "storage.objects.compose" + ] + } + }, + "objectPreconditions": { + "type": "object", + "description": "Conditions that must be met for this operation to execute.", + "properties": { + "ifGenerationMatch": { + "type": "string", + "description": "Only perform the composition if the generation of the source object that would be used matches this value. If this value and a generation are both specified, they must be the same value or the call will fail.", + "format": "int64" + } + } + } + } + }, + "annotations": { + "required": [ + "storage.objects.compose" + ] + } + } + } + }, + "Notification": { + "id": "Notification", + "type": "object", + "description": "A subscription to receive Google PubSub notifications.", + "properties": { + "bucket": { + "type": "string", + "description": "The name of the bucket this subscription is particular to.", + "annotations": { + "required": [ + "storage.notifications.insert" + ] + } + }, + "custom_attributes": { + "type": "object", + "description": "An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription.", + "additionalProperties": { + "type": "string" + } + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for this subscription notification." + }, + "event_types": { + "type": "array", + "description": "If present, only send notifications about listed event types. If empty, sent notifications for all event types.", + "items": { + "type": "string" + } + }, + "id": { + "type": "string", + "description": "The ID of the notification." + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For notifications, this is always storage#notification.", + "default": "storage#notification" + }, + "object_metadata_format": { + "type": "string", + "description": "If payload_content is OBJECT_METADATA, controls the format of that metadata. Otherwise, must not be set.", + "default": "JSON_API_V1" + }, + "object_name_prefix": { + "type": "string", + "description": "If present, only apply this notification configuration to object names that begin with this prefix." + }, + "payload_content": { + "type": "string", + "description": "The desired content of the Payload. Defaults to OBJECT_METADATA.", + "default": "OBJECT_METADATA" + }, + "selfLink": { + "type": "string", + "description": "The canonical URL of this notification." + }, + "topic": { + "type": "string", + "description": "The Cloud PubSub topic to which this subscription publishes. Formatted as: '//pubsub.googleapis.com/projects/{project-identifier}/topics/{my-topic}'", + "annotations": { + "required": [ + "storage.notifications.insert" + ] + } + } + } + }, + "Notifications": { + "id": "Notifications", + "type": "object", + "description": "A list of notification subscriptions.", + "properties": { + "items": { + "type": "array", + "description": "The list of items.", + "items": { + "$ref": "Notification" + } + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For lists of notifications, this is always storage#notifications.", + "default": "storage#notifications" + } + } + }, + "Object": { + "id": "Object", + "type": "object", + "description": "An object.", + "properties": { + "acl": { + "type": "array", + "description": "Access controls on the object.", + "items": { + "$ref": "ObjectAccessControl" + }, + "annotations": { + "required": [ + "storage.objects.update" + ] + } + }, + "bucket": { + "type": "string", + "description": "The name of the bucket containing this object." + }, + "cacheControl": { + "type": "string", + "description": "Cache-Control directive for the object data." + }, + "componentCount": { + "type": "integer", + "description": "Number of underlying components that make up this object. Components are accumulated by compose operations.", + "format": "int32" + }, + "contentDisposition": { + "type": "string", + "description": "Content-Disposition of the object data." + }, + "contentEncoding": { + "type": "string", + "description": "Content-Encoding of the object data." + }, + "contentLanguage": { + "type": "string", + "description": "Content-Language of the object data." + }, + "contentType": { + "type": "string", + "description": "Content-Type of the object data. If contentType is not specified, object downloads will be served as application/octet-stream." + }, + "crc32c": { + "type": "string", + "description": "CRC32c checksum, as described in RFC 4960, Appendix B; encoded using base64 in big-endian byte order. For more information about using the CRC32c checksum, see Hashes and ETags: Best Practices." + }, + "customerEncryption": { + "type": "object", + "description": "Metadata of customer-supplied encryption key, if the object is encrypted by such a key.", + "properties": { + "encryptionAlgorithm": { + "type": "string", + "description": "The encryption algorithm." + }, + "keySha256": { + "type": "string", + "description": "SHA256 hash value of the encryption key." + } + } + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for the object." + }, + "generation": { + "type": "string", + "description": "The content generation of this object. Used for object versioning.", + "format": "int64" + }, + "id": { + "type": "string", + "description": "The ID of the object." + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For objects, this is always storage#object.", + "default": "storage#object" + }, + "md5Hash": { + "type": "string", + "description": "MD5 hash of the data; encoded using base64. For more information about using the MD5 hash, see Hashes and ETags: Best Practices." + }, + "mediaLink": { + "type": "string", + "description": "Media download link." + }, + "metadata": { + "type": "object", + "description": "User-provided metadata, in key/value pairs.", + "additionalProperties": { + "type": "string", + "description": "An individual metadata entry." + } + }, + "metageneration": { + "type": "string", + "description": "The version of the metadata for this object at this generation. Used for preconditions and for detecting changes in metadata. A metageneration number is only meaningful in the context of a particular generation of a particular object.", + "format": "int64" + }, + "name": { + "type": "string", + "description": "The name of this object. Required if not specified by URL parameter." + }, + "owner": { + "type": "object", + "description": "The owner of the object. This will always be the uploader of the object.", + "properties": { + "entity": { + "type": "string", + "description": "The entity, in the form user-userId." + }, + "entityId": { + "type": "string", + "description": "The ID for the entity." + } + } + }, + "selfLink": { + "type": "string", + "description": "The link to this object." + }, + "size": { + "type": "string", + "description": "Content-Length of the data in bytes.", + "format": "uint64" + }, + "storageClass": { + "type": "string", + "description": "Storage class of the object." + }, + "timeCreated": { + "type": "string", + "description": "The creation time of the object in RFC 3339 format.", + "format": "date-time" + }, + "timeDeleted": { + "type": "string", + "description": "The deletion time of the object in RFC 3339 format. Will be returned if and only if this version of the object has been deleted.", + "format": "date-time" + }, + "updated": { + "type": "string", + "description": "The modification time of the object metadata in RFC 3339 format.", + "format": "date-time" + } + } + }, + "ObjectAccessControl": { + "id": "ObjectAccessControl", + "type": "object", + "description": "An access-control entry.", + "properties": { + "bucket": { + "type": "string", + "description": "The name of the bucket." + }, + "domain": { + "type": "string", + "description": "The domain associated with the entity, if any." + }, + "email": { + "type": "string", + "description": "The email address associated with the entity, if any." + }, + "entity": { + "type": "string", + "description": "The entity holding the permission, in one of the following forms: \n- user-userId \n- user-email \n- group-groupId \n- group-email \n- domain-domain \n- project-team-projectId \n- allUsers \n- allAuthenticatedUsers Examples: \n- The user liz@example.com would be user-liz@example.com. \n- The group example@googlegroups.com would be group-example@googlegroups.com. \n- To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com." + }, + "entityId": { + "type": "string", + "description": "The ID for the entity, if any." + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for the access-control entry." + }, + "generation": { + "type": "string", + "description": "The content generation of the object.", + "format": "int64" + }, + "id": { + "type": "string", + "description": "The ID of the access-control entry." + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For object access control entries, this is always storage#objectAccessControl.", + "default": "storage#objectAccessControl" + }, + "object": { + "type": "string", + "description": "The name of the object." + }, + "projectTeam": { + "type": "object", + "description": "The project team associated with the entity, if any.", + "properties": { + "projectNumber": { + "type": "string", + "description": "The project number." + }, + "team": { + "type": "string", + "description": "The team. Can be owners, editors, or viewers." + } + } + }, + "role": { + "type": "string", + "description": "The access permission for the entity. Can be READER or OWNER." + }, + "selfLink": { + "type": "string", + "description": "The link to this access-control entry." + } + } + }, + "ObjectAccessControls": { + "id": "ObjectAccessControls", + "type": "object", + "description": "An access-control list.", + "properties": { + "items": { + "type": "array", + "description": "The list of items.", + "items": { + "type": "any" + } + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For lists of object access control entries, this is always storage#objectAccessControls.", + "default": "storage#objectAccessControls" + } + } + }, + "Objects": { + "id": "Objects", + "type": "object", + "description": "A list of objects.", + "properties": { + "items": { + "type": "array", + "description": "The list of items.", + "items": { + "$ref": "Object" + } + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For lists of objects, this is always storage#objects.", + "default": "storage#objects" + }, + "nextPageToken": { + "type": "string", + "description": "The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results." + }, + "prefixes": { + "type": "array", + "description": "The list of prefixes of objects matching-but-not-listed up to and including the requested delimiter.", + "items": { + "type": "string" + } + } + } + }, + "Policy": { + "id": "Policy", + "type": "object", + "description": "A bucket/object IAM policy.", + "properties": { + "bindings": { + "type": "array", + "description": "An association between a role, which comes with a set of permissions, and members who may assume that role.", + "items": { + "type": "object", + "properties": { + "members": { + "type": "array", + "description": "A collection of identifiers for members who may assume the provided role. Recognized identifiers are as follows: \n- allUsers — A special identifier that represents anyone on the internet; with or without a Google account. \n- allAuthenticatedUsers — A special identifier that represents anyone who is authenticated with a Google account or a service account. \n- user:emailid — An email address that represents a specific account. For example, user:alice@gmail.com or user:joe@example.com. \n- serviceAccount:emailid — An email address that represents a service account. For example, serviceAccount:my-other-app@appspot.gserviceaccount.com . \n- group:emailid — An email address that represents a Google group. For example, group:admins@example.com. \n- domain:domain — A Google Apps domain name that represents all the users of that domain. For example, domain:google.com or domain:example.com. \n- projectOwner:projectid — Owners of the given project. For example, projectOwner:my-example-project \n- projectEditor:projectid — Editors of the given project. For example, projectEditor:my-example-project \n- projectViewer:projectid — Viewers of the given project. For example, projectViewer:my-example-project", + "items": { + "type": "string" + }, + "annotations": { + "required": [ + "storage.buckets.setIamPolicy", + "storage.objects.setIamPolicy" + ] + } + }, + "role": { + "type": "string", + "description": "The role to which members belong. Two types of roles are supported: new IAM roles, which grant permissions that do not map directly to those provided by ACLs, and legacy IAM roles, which do map directly to ACL permissions. All roles are of the format roles/storage.specificRole.\nThe new IAM roles are: \n- roles/storage.admin — Full control of Google Cloud Storage resources. \n- roles/storage.objectViewer — Read-Only access to Google Cloud Storage objects. \n- roles/storage.objectCreator — Access to create objects in Google Cloud Storage. \n- roles/storage.objectAdmin — Full control of Google Cloud Storage objects. The legacy IAM roles are: \n- roles/storage.legacyObjectReader — Read-only access to objects without listing. Equivalent to an ACL entry on an object with the READER role. \n- roles/storage.legacyObjectOwner — Read/write access to existing objects without listing. Equivalent to an ACL entry on an object with the OWNER role. \n- roles/storage.legacyBucketReader — Read access to buckets with object listing. Equivalent to an ACL entry on a bucket with the READER role. \n- roles/storage.legacyBucketWriter — Read access to buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the WRITER role. \n- roles/storage.legacyBucketOwner — Read and write access to existing buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the OWNER role.", + "annotations": { + "required": [ + "storage.buckets.setIamPolicy", + "storage.objects.setIamPolicy" + ] + } + } + } + }, + "annotations": { + "required": [ + "storage.buckets.setIamPolicy", + "storage.objects.setIamPolicy" + ] + } + }, + "etag": { + "type": "string", + "description": "HTTP 1.1 Entity tag for the policy.", + "format": "byte" + }, + "kind": { + "type": "string", + "description": "The kind of item this is. For policies, this is always storage#policy. This field is ignored on input.", + "default": "storage#policy" + }, + "resourceId": { + "type": "string", + "description": "The ID of the resource to which this policy belongs. Will be of the form buckets/bucket for buckets, and buckets/bucket/objects/object for objects. A specific generation may be specified by appending #generationNumber to the end of the object name, e.g. buckets/my-bucket/objects/data.txt#17. The current generation can be denoted with #0. This field is ignored on input." + } + } + }, + "RewriteResponse": { + "id": "RewriteResponse", + "type": "object", + "description": "A rewrite response.", + "properties": { + "done": { + "type": "boolean", + "description": "true if the copy is finished; otherwise, false if the copy is in progress. This property is always present in the response." + }, + "kind": { + "type": "string", + "description": "The kind of item this is.", + "default": "storage#rewriteResponse" + }, + "objectSize": { + "type": "string", + "description": "The total size of the object being copied in bytes. This property is always present in the response.", + "format": "uint64" + }, + "resource": { + "$ref": "Object", + "description": "A resource containing the metadata for the copied-to object. This property is present in the response only when copying completes." + }, + "rewriteToken": { + "type": "string", + "description": "A token to use in subsequent requests to continue copying data. This token is present in the response only when there is more data to copy." + }, + "totalBytesRewritten": { + "type": "string", + "description": "The total bytes written so far, which can be used to provide a waiting user with a progress indicator. This property is always present in the response.", + "format": "uint64" + } + } + }, + "TestIamPermissionsResponse": { + "id": "TestIamPermissionsResponse", + "type": "object", + "description": "A storage.(buckets|objects).testIamPermissions response.", + "properties": { + "kind": { + "type": "string", + "description": "The kind of item this is.", + "default": "storage#testIamPermissionsResponse" + }, + "permissions": { + "type": "array", + "description": "The permissions held by the caller. Permissions are always of the format storage.resource.capability, where resource is one of buckets or objects. The supported permissions are as follows: \n- storage.buckets.delete — Delete bucket. \n- storage.buckets.get — Read bucket metadata. \n- storage.buckets.getIamPolicy — Read bucket IAM policy. \n- storage.buckets.create — Create bucket. \n- storage.buckets.list — List buckets. \n- storage.buckets.setIamPolicy — Update bucket IAM policy. \n- storage.buckets.update — Update bucket metadata. \n- storage.objects.delete — Delete object. \n- storage.objects.get — Read object data and metadata. \n- storage.objects.getIamPolicy — Read object IAM policy. \n- storage.objects.create — Create object. \n- storage.objects.list — List objects. \n- storage.objects.setIamPolicy — Update object IAM policy. \n- storage.objects.update — Update object metadata.", + "items": { + "type": "string" + } + } + } + } + }, + "resources": { + "bucketAccessControls": { + "methods": { + "delete": { + "id": "storage.bucketAccessControls.delete", + "path": "b/{bucket}/acl/{entity}", + "httpMethod": "DELETE", + "description": "Permanently deletes the ACL entry for the specified entity on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "get": { + "id": "storage.bucketAccessControls.get", + "path": "b/{bucket}/acl/{entity}", + "httpMethod": "GET", + "description": "Returns the ACL entry for the specified entity on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "response": { + "$ref": "BucketAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "insert": { + "id": "storage.bucketAccessControls.insert", + "path": "b/{bucket}/acl", + "httpMethod": "POST", + "description": "Creates a new ACL entry on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "BucketAccessControl" + }, + "response": { + "$ref": "BucketAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "list": { + "id": "storage.bucketAccessControls.list", + "path": "b/{bucket}/acl", + "httpMethod": "GET", + "description": "Retrieves ACL entries on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "BucketAccessControls" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "patch": { + "id": "storage.bucketAccessControls.patch", + "path": "b/{bucket}/acl/{entity}", + "httpMethod": "PATCH", + "description": "Updates an ACL entry on the specified bucket. This method supports patch semantics.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "request": { + "$ref": "BucketAccessControl" + }, + "response": { + "$ref": "BucketAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "update": { + "id": "storage.bucketAccessControls.update", + "path": "b/{bucket}/acl/{entity}", + "httpMethod": "PUT", + "description": "Updates an ACL entry on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "request": { + "$ref": "BucketAccessControl" + }, + "response": { + "$ref": "BucketAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + } + } + }, + "buckets": { + "methods": { + "delete": { + "id": "storage.buckets.delete", + "path": "b/{bucket}", + "httpMethod": "DELETE", + "description": "Permanently deletes an empty bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "If set, only deletes the bucket if its metageneration matches this value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "If set, only deletes the bucket if its metageneration does not match this value.", + "format": "int64", + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "get": { + "id": "storage.buckets.get", + "path": "b/{bucket}", + "httpMethod": "GET", + "description": "Returns metadata for the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit owner, acl and defaultObjectAcl properties." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "Bucket" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "getIamPolicy": { + "id": "storage.buckets.getIamPolicy", + "path": "b/{bucket}/iam", + "httpMethod": "GET", + "description": "Returns an IAM policy for the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "insert": { + "id": "storage.buckets.insert", + "path": "b", + "httpMethod": "POST", + "description": "Creates a new bucket.", + "parameters": { + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this bucket.", + "enum": [ + "authenticatedRead", + "private", + "projectPrivate", + "publicRead", + "publicReadWrite" + ], + "enumDescriptions": [ + "Project team owners get OWNER access, and allAuthenticatedUsers get READER access.", + "Project team owners get OWNER access.", + "Project team members get access according to their roles.", + "Project team owners get OWNER access, and allUsers get READER access.", + "Project team owners get OWNER access, and allUsers get WRITER access." + ], + "location": "query" + }, + "predefinedDefaultObjectAcl": { + "type": "string", + "description": "Apply a predefined set of default object access controls to this bucket.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "project": { + "type": "string", + "description": "A valid API project identifier.", + "required": true, + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit owner, acl and defaultObjectAcl properties." + ], + "location": "query" + } + }, + "parameterOrder": [ + "project" + ], + "request": { + "$ref": "Bucket" + }, + "response": { + "$ref": "Bucket" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "list": { + "id": "storage.buckets.list", + "path": "b", + "httpMethod": "GET", + "description": "Retrieves a list of buckets for a given project.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of buckets to return.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "A previously-returned page token representing part of the larger set of results to view.", + "location": "query" + }, + "prefix": { + "type": "string", + "description": "Filter results to buckets whose names begin with this prefix.", + "location": "query" + }, + "project": { + "type": "string", + "description": "A valid API project identifier.", + "required": true, + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit owner, acl and defaultObjectAcl properties." + ], + "location": "query" + } + }, + "parameterOrder": [ + "project" + ], + "response": { + "$ref": "Buckets" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "patch": { + "id": "storage.buckets.patch", + "path": "b/{bucket}", + "httpMethod": "PATCH", + "description": "Updates a bucket. This method supports patch semantics.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this bucket.", + "enum": [ + "authenticatedRead", + "private", + "projectPrivate", + "publicRead", + "publicReadWrite" + ], + "enumDescriptions": [ + "Project team owners get OWNER access, and allAuthenticatedUsers get READER access.", + "Project team owners get OWNER access.", + "Project team members get access according to their roles.", + "Project team owners get OWNER access, and allUsers get READER access.", + "Project team owners get OWNER access, and allUsers get WRITER access." + ], + "location": "query" + }, + "predefinedDefaultObjectAcl": { + "type": "string", + "description": "Apply a predefined set of default object access controls to this bucket.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit owner, acl and defaultObjectAcl properties." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "Bucket" + }, + "response": { + "$ref": "Bucket" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "setIamPolicy": { + "id": "storage.buckets.setIamPolicy", + "path": "b/{bucket}/iam", + "httpMethod": "PUT", + "description": "Updates an IAM policy for the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "Policy" + }, + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "testIamPermissions": { + "id": "storage.buckets.testIamPermissions", + "path": "b/{bucket}/iam/testPermissions", + "httpMethod": "GET", + "description": "Tests a set of permissions on the given bucket to see which, if any, are held by the caller.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "permissions": { + "type": "string", + "description": "Permissions to test.", + "required": true, + "repeated": true, + "location": "query" + } + }, + "parameterOrder": [ + "bucket", + "permissions" + ], + "response": { + "$ref": "TestIamPermissionsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "update": { + "id": "storage.buckets.update", + "path": "b/{bucket}", + "httpMethod": "PUT", + "description": "Updates a bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this bucket.", + "enum": [ + "authenticatedRead", + "private", + "projectPrivate", + "publicRead", + "publicReadWrite" + ], + "enumDescriptions": [ + "Project team owners get OWNER access, and allAuthenticatedUsers get READER access.", + "Project team owners get OWNER access.", + "Project team members get access according to their roles.", + "Project team owners get OWNER access, and allUsers get READER access.", + "Project team owners get OWNER access, and allUsers get WRITER access." + ], + "location": "query" + }, + "predefinedDefaultObjectAcl": { + "type": "string", + "description": "Apply a predefined set of default object access controls to this bucket.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit owner, acl and defaultObjectAcl properties." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "Bucket" + }, + "response": { + "$ref": "Bucket" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + } + } + }, + "channels": { + "methods": { + "stop": { + "id": "storage.channels.stop", + "path": "channels/stop", + "httpMethod": "POST", + "description": "Stop watching resources through this channel", + "request": { + "$ref": "Channel", + "parameterName": "resource" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + } + } + }, + "defaultObjectAccessControls": { + "methods": { + "delete": { + "id": "storage.defaultObjectAccessControls.delete", + "path": "b/{bucket}/defaultObjectAcl/{entity}", + "httpMethod": "DELETE", + "description": "Permanently deletes the default object ACL entry for the specified entity on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "get": { + "id": "storage.defaultObjectAccessControls.get", + "path": "b/{bucket}/defaultObjectAcl/{entity}", + "httpMethod": "GET", + "description": "Returns the default object ACL entry for the specified entity on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "insert": { + "id": "storage.defaultObjectAccessControls.insert", + "path": "b/{bucket}/defaultObjectAcl", + "httpMethod": "POST", + "description": "Creates a new default object ACL entry on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "list": { + "id": "storage.defaultObjectAccessControls.list", + "path": "b/{bucket}/defaultObjectAcl", + "httpMethod": "GET", + "description": "Retrieves default object ACL entries on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "If present, only return default ACL listing if the bucket's current metageneration matches this value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "If present, only return default ACL listing if the bucket's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "ObjectAccessControls" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "patch": { + "id": "storage.defaultObjectAccessControls.patch", + "path": "b/{bucket}/defaultObjectAcl/{entity}", + "httpMethod": "PATCH", + "description": "Updates a default object ACL entry on the specified bucket. This method supports patch semantics.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "update": { + "id": "storage.defaultObjectAccessControls.update", + "path": "b/{bucket}/defaultObjectAcl/{entity}", + "httpMethod": "PUT", + "description": "Updates a default object ACL entry on the specified bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "entity" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + } + } + }, + "notifications": { + "methods": { + "delete": { + "id": "storage.notifications.delete", + "path": "notifications/{notification}", + "httpMethod": "DELETE", + "description": "Permanently deletes a notification subscription.", + "parameters": { + "notification": { + "type": "string", + "description": "ID of the notification to delete.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "notification" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "get": { + "id": "storage.notifications.get", + "path": "notifications/{notification}", + "httpMethod": "GET", + "description": "View a notification configuration.", + "parameters": { + "notification": { + "type": "string", + "description": "Notification ID", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "notification" + ], + "response": { + "$ref": "Notification" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "insert": { + "id": "storage.notifications.insert", + "path": "notifications", + "httpMethod": "POST", + "description": "Creates a notification subscription for a given bucket.", + "request": { + "$ref": "Notification" + }, + "response": { + "$ref": "Notification" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "list": { + "id": "storage.notifications.list", + "path": "notifications", + "httpMethod": "GET", + "description": "Retrieves a list of notification subscriptions for a given bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a GCS bucket.", + "required": true, + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "Notifications" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + } + } + }, + "objectAccessControls": { + "methods": { + "delete": { + "id": "storage.objectAccessControls.delete", + "path": "b/{bucket}/o/{object}/acl/{entity}", + "httpMethod": "DELETE", + "description": "Permanently deletes the ACL entry for the specified entity on the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object", + "entity" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "get": { + "id": "storage.objectAccessControls.get", + "path": "b/{bucket}/o/{object}/acl/{entity}", + "httpMethod": "GET", + "description": "Returns the ACL entry for the specified entity on the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object", + "entity" + ], + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "insert": { + "id": "storage.objectAccessControls.insert", + "path": "b/{bucket}/o/{object}/acl", + "httpMethod": "POST", + "description": "Creates a new ACL entry on the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "list": { + "id": "storage.objectAccessControls.list", + "path": "b/{bucket}/o/{object}/acl", + "httpMethod": "GET", + "description": "Retrieves ACL entries on the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "response": { + "$ref": "ObjectAccessControls" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "patch": { + "id": "storage.objectAccessControls.patch", + "path": "b/{bucket}/o/{object}/acl/{entity}", + "httpMethod": "PATCH", + "description": "Updates an ACL entry on the specified object. This method supports patch semantics.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object", + "entity" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "update": { + "id": "storage.objectAccessControls.update", + "path": "b/{bucket}/o/{object}/acl/{entity}", + "httpMethod": "PUT", + "description": "Updates an ACL entry on the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of a bucket.", + "required": true, + "location": "path" + }, + "entity": { + "type": "string", + "description": "The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object", + "entity" + ], + "request": { + "$ref": "ObjectAccessControl" + }, + "response": { + "$ref": "ObjectAccessControl" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + } + } + }, + "objects": { + "methods": { + "compose": { + "id": "storage.objects.compose", + "path": "b/{destinationBucket}/o/{destinationObject}/compose", + "httpMethod": "POST", + "description": "Concatenates a list of existing objects into a new object in the same bucket.", + "parameters": { + "destinationBucket": { + "type": "string", + "description": "Name of the bucket in which to store the new object.", + "required": true, + "location": "path" + }, + "destinationObject": { + "type": "string", + "description": "Name of the new object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "destinationPredefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to the destination object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + } + }, + "parameterOrder": [ + "destinationBucket", + "destinationObject" + ], + "request": { + "$ref": "ComposeRequest" + }, + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsMediaDownload": true + }, + "copy": { + "id": "storage.objects.copy", + "path": "b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}", + "httpMethod": "POST", + "description": "Copies a source object to a destination object. Optionally overrides metadata.", + "parameters": { + "destinationBucket": { + "type": "string", + "description": "Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any.For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "destinationObject": { + "type": "string", + "description": "Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any.", + "required": true, + "location": "path" + }, + "destinationPredefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to the destination object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + }, + "sourceBucket": { + "type": "string", + "description": "Name of the bucket in which to find the source object.", + "required": true, + "location": "path" + }, + "sourceGeneration": { + "type": "string", + "description": "If present, selects a specific revision of the source object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "sourceObject": { + "type": "string", + "description": "Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "sourceBucket", + "sourceObject", + "destinationBucket", + "destinationObject" + ], + "request": { + "$ref": "Object" + }, + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsMediaDownload": true + }, + "delete": { + "id": "storage.objects.delete", + "path": "b/{bucket}/o/{object}", + "httpMethod": "DELETE", + "description": "Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, permanently deletes a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "get": { + "id": "storage.objects.get", + "path": "b/{bucket}/o/{object}", + "httpMethod": "GET", + "description": "Retrieves an object or its metadata.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsMediaDownload": true + }, + "getIamPolicy": { + "id": "storage.objects.getIamPolicy", + "path": "b/{bucket}/o/{object}/iam", + "httpMethod": "GET", + "description": "Returns an IAM policy for the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "insert": { + "id": "storage.objects.insert", + "path": "b/{bucket}/o", + "httpMethod": "POST", + "description": "Stores a new object and metadata.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any.", + "required": true, + "location": "path" + }, + "contentEncoding": { + "type": "string", + "description": "If set, sets the contentEncoding property of the final object to this value. Setting this parameter is equivalent to setting the contentEncoding metadata property. This can be useful when uploading an object with uploadType=media to indicate the encoding of the content being uploaded.", + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "name": { + "type": "string", + "description": "Name of the object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "location": "query" + }, + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "Object" + }, + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsMediaDownload": true, + "supportsMediaUpload": true, + "mediaUpload": { + "accept": [ + "*/*" + ], + "protocols": { + "simple": { + "multipart": true, + "path": "/upload/storage/v1/b/{bucket}/o" + }, + "resumable": { + "multipart": true, + "path": "/resumable/upload/storage/v1/b/{bucket}/o" + } + } + } + }, + "list": { + "id": "storage.objects.list", + "path": "b/{bucket}/o", + "httpMethod": "GET", + "description": "Retrieves a list of objects matching the criteria.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which to look for objects.", + "required": true, + "location": "path" + }, + "delimiter": { + "type": "string", + "description": "Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted.", + "location": "query" + }, + "maxResults": { + "type": "integer", + "description": "Maximum number of items plus prefixes to return. As duplicate prefixes are omitted, fewer total results may be returned than requested. The default value of this parameter is 1,000 items.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "A previously-returned page token representing part of the larger set of results to view.", + "location": "query" + }, + "prefix": { + "type": "string", + "description": "Filter results to objects whose names begin with this prefix.", + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + }, + "versions": { + "type": "boolean", + "description": "If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning.", + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "response": { + "$ref": "Objects" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsSubscription": true + }, + "patch": { + "id": "storage.objects.patch", + "path": "b/{bucket}/o/{object}", + "httpMethod": "PATCH", + "description": "Updates an object's metadata. This method supports patch semantics.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "request": { + "$ref": "Object" + }, + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ] + }, + "rewrite": { + "id": "storage.objects.rewrite", + "path": "b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}", + "httpMethod": "POST", + "description": "Rewrites a source object to a destination object. Optionally overrides metadata.", + "parameters": { + "destinationBucket": { + "type": "string", + "description": "Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any.", + "required": true, + "location": "path" + }, + "destinationObject": { + "type": "string", + "description": "Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "destinationPredefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to the destination object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the destination object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifSourceMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the source object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "maxBytesRewrittenPerCall": { + "type": "string", + "description": "The maximum number of bytes that will be rewritten per rewrite request. Most callers shouldn't need to specify this parameter - it is primarily in place to support testing. If specified the value must be an integral multiple of 1 MiB (1048576). Also, this only applies to requests where the source and destination span locations and/or storage classes. Finally, this value must not change across rewrite calls else you'll get an error that the rewriteToken is invalid.", + "format": "int64", + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + }, + "rewriteToken": { + "type": "string", + "description": "Include this field (from the previous rewrite response) on each rewrite request after the first one, until the rewrite response 'done' flag is true. Calls that provide a rewriteToken can omit all other request fields, but if included those fields must match the values provided in the first rewrite request.", + "location": "query" + }, + "sourceBucket": { + "type": "string", + "description": "Name of the bucket in which to find the source object.", + "required": true, + "location": "path" + }, + "sourceGeneration": { + "type": "string", + "description": "If present, selects a specific revision of the source object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "sourceObject": { + "type": "string", + "description": "Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "sourceBucket", + "sourceObject", + "destinationBucket", + "destinationObject" + ], + "request": { + "$ref": "Object" + }, + "response": { + "$ref": "RewriteResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "setIamPolicy": { + "id": "storage.objects.setIamPolicy", + "path": "b/{bucket}/o/{object}/iam", + "httpMethod": "PUT", + "description": "Updates an IAM policy for the specified object.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "request": { + "$ref": "Policy" + }, + "response": { + "$ref": "Policy" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "testIamPermissions": { + "id": "storage.objects.testIamPermissions", + "path": "b/{bucket}/o/{object}/iam/testPermissions", + "httpMethod": "GET", + "description": "Tests a set of permissions on the given object to see which, if any, are held by the caller.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "permissions": { + "type": "string", + "description": "Permissions to test.", + "required": true, + "repeated": true, + "location": "query" + } + }, + "parameterOrder": [ + "bucket", + "object", + "permissions" + ], + "response": { + "$ref": "TestIamPermissionsResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ] + }, + "update": { + "id": "storage.objects.update", + "path": "b/{bucket}/o/{object}", + "httpMethod": "PUT", + "description": "Updates an object's metadata.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which the object resides.", + "required": true, + "location": "path" + }, + "generation": { + "type": "string", + "description": "If present, selects a specific revision of this object (as opposed to the latest version, the default).", + "format": "int64", + "location": "query" + }, + "ifGenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation matches the given value.", + "format": "int64", + "location": "query" + }, + "ifGenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current generation does not match the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration matches the given value.", + "format": "int64", + "location": "query" + }, + "ifMetagenerationNotMatch": { + "type": "string", + "description": "Makes the operation conditional on whether the object's current metageneration does not match the given value.", + "format": "int64", + "location": "query" + }, + "object": { + "type": "string", + "description": "Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts.", + "required": true, + "location": "path" + }, + "predefinedAcl": { + "type": "string", + "description": "Apply a predefined set of access controls to this object.", + "enum": [ + "authenticatedRead", + "bucketOwnerFullControl", + "bucketOwnerRead", + "private", + "projectPrivate", + "publicRead" + ], + "enumDescriptions": [ + "Object owner gets OWNER access, and allAuthenticatedUsers get READER access.", + "Object owner gets OWNER access, and project team owners get OWNER access.", + "Object owner gets OWNER access, and project team owners get READER access.", + "Object owner gets OWNER access.", + "Object owner gets OWNER access, and project team members get access according to their roles.", + "Object owner gets OWNER access, and allUsers get READER access." + ], + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to full.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + } + }, + "parameterOrder": [ + "bucket", + "object" + ], + "request": { + "$ref": "Object" + }, + "response": { + "$ref": "Object" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/devstorage.full_control" + ], + "supportsMediaDownload": true + }, + "watchAll": { + "id": "storage.objects.watchAll", + "path": "b/{bucket}/o/watch", + "httpMethod": "POST", + "description": "Watch for changes on all objects in a bucket.", + "parameters": { + "bucket": { + "type": "string", + "description": "Name of the bucket in which to look for objects.", + "required": true, + "location": "path" + }, + "delimiter": { + "type": "string", + "description": "Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted.", + "location": "query" + }, + "maxResults": { + "type": "integer", + "description": "Maximum number of items plus prefixes to return. As duplicate prefixes are omitted, fewer total results may be returned than requested. The default value of this parameter is 1,000 items.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "A previously-returned page token representing part of the larger set of results to view.", + "location": "query" + }, + "prefix": { + "type": "string", + "description": "Filter results to objects whose names begin with this prefix.", + "location": "query" + }, + "projection": { + "type": "string", + "description": "Set of properties to return. Defaults to noAcl.", + "enum": [ + "full", + "noAcl" + ], + "enumDescriptions": [ + "Include all properties.", + "Omit the owner, acl property." + ], + "location": "query" + }, + "versions": { + "type": "boolean", + "description": "If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning.", + "location": "query" + } + }, + "parameterOrder": [ + "bucket" + ], + "request": { + "$ref": "Channel", + "parameterName": "resource" + }, + "response": { + "$ref": "Channel" + }, + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only", + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/devstorage.read_write" + ], + "supportsSubscription": true + } + } + } + } +} diff --git a/samples/storage_sample/storage_v1/__init__.py b/samples/storage_sample/storage_v1/__init__.py new file mode 100644 index 0000000..2816da8 --- /dev/null +++ b/samples/storage_sample/storage_v1/__init__.py @@ -0,0 +1,5 @@ +"""Package marker file.""" + +import pkgutil + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/samples/storage_sample/storage/storage_v1.py b/samples/storage_sample/storage_v1/storage_v1.py old mode 100755 new mode 100644 similarity index 89% rename from samples/storage_sample/storage/storage_v1.py rename to samples/storage_sample/storage_v1/storage_v1.py index dfde683..d7cff48 --- a/samples/storage_sample/storage/storage_v1.py +++ b/samples/storage_sample/storage_v1/storage_v1.py @@ -1,19 +1,4 @@ #!/usr/bin/env python -# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """CLI for storage, version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. @@ -708,6 +693,30 @@ class BucketsGet(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) +class BucketsGetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping buckets.GetIamPolicy.""" + + usage = """buckets_getIamPolicy """ + + def __init__(self, name, fv): + super(BucketsGetIamPolicy, self).__init__(name, fv) + + def RunWithArgs(self, bucket): + """Returns an IAM policy for the specified bucket. + + Args: + bucket: Name of a bucket. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsGetIamPolicyRequest( + bucket=bucket.decode('utf8'), + ) + result = client.buckets.GetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + class BucketsInsert(apitools_base_cli.NewCmd): """Command wrapping buckets.Insert.""" @@ -921,6 +930,67 @@ class BucketsPatch(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) +class BucketsSetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping buckets.SetIamPolicy.""" + + usage = """buckets_setIamPolicy """ + + def __init__(self, name, fv): + super(BucketsSetIamPolicy, self).__init__(name, fv) + flags.DEFINE_string( + 'policy', + None, + u'A Policy resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, bucket): + """Updates an IAM policy for the specified bucket. + + Args: + bucket: Name of a bucket. + + Flags: + policy: A Policy resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsSetIamPolicyRequest( + bucket=bucket.decode('utf8'), + ) + if FLAGS['policy'].present: + request.policy = apitools_base.JsonToMessage(messages.Policy, FLAGS.policy) + result = client.buckets.SetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class BucketsTestIamPermissions(apitools_base_cli.NewCmd): + """Command wrapping buckets.TestIamPermissions.""" + + usage = """buckets_testIamPermissions """ + + def __init__(self, name, fv): + super(BucketsTestIamPermissions, self).__init__(name, fv) + + def RunWithArgs(self, bucket, permissions): + """Tests a set of permissions on the given bucket to see which, if any, + are held by the caller. + + Args: + bucket: Name of a bucket. + permissions: Permissions to test. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageBucketsTestIamPermissionsRequest( + bucket=bucket.decode('utf8'), + permissions=permissions.decode('utf8'), + ) + result = client.buckets.TestIamPermissions( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + class BucketsUpdate(apitools_base_cli.NewCmd): """Command wrapping buckets.Update.""" @@ -1612,6 +1682,204 @@ class DefaultObjectAccessControlsUpdate(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) +class NotificationsDelete(apitools_base_cli.NewCmd): + """Command wrapping notifications.Delete.""" + + usage = """notifications_delete """ + + def __init__(self, name, fv): + super(NotificationsDelete, self).__init__(name, fv) + + def RunWithArgs(self, notification): + """Permanently deletes a notification subscription. + + Args: + notification: ID of the notification to delete. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageNotificationsDeleteRequest( + notification=notification.decode('utf8'), + ) + result = client.notifications.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class NotificationsGet(apitools_base_cli.NewCmd): + """Command wrapping notifications.Get.""" + + usage = """notifications_get """ + + def __init__(self, name, fv): + super(NotificationsGet, self).__init__(name, fv) + + def RunWithArgs(self, notification): + """View a notification configuration. + + Args: + notification: Notification ID + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageNotificationsGetRequest( + notification=notification.decode('utf8'), + ) + result = client.notifications.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class NotificationsInsert(apitools_base_cli.NewCmd): + """Command wrapping notifications.Insert.""" + + usage = """notifications_insert""" + + def __init__(self, name, fv): + super(NotificationsInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'bucket', + None, + u'The name of the bucket this subscription is particular to.', + flag_values=fv) + flags.DEFINE_string( + 'custom_attributes', + None, + u'An optional list of additional attributes to attach to each Cloud ' + u'PubSub message published for this notification subscription.', + flag_values=fv) + flags.DEFINE_string( + 'etag', + None, + u'HTTP 1.1 Entity tag for this subscription notification.', + flag_values=fv) + flags.DEFINE_string( + 'event_types', + None, + u'If present, only send notifications about listed event types. If ' + u'empty, sent notifications for all event types.', + flag_values=fv) + flags.DEFINE_string( + 'id', + None, + u'The ID of the notification.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'storage#notification', + u'The kind of item this is. For notifications, this is always ' + u'storage#notification.', + flag_values=fv) + flags.DEFINE_string( + 'object_metadata_format', + u'JSON_API_V1', + u'If payload_content is OBJECT_METADATA, controls the format of that ' + u'metadata. Otherwise, must not be set.', + flag_values=fv) + flags.DEFINE_string( + 'object_name_prefix', + None, + u'If present, only apply this notification configuration to object ' + u'names that begin with this prefix.', + flag_values=fv) + flags.DEFINE_string( + 'payload_content', + u'OBJECT_METADATA', + u'The desired content of the Payload. Defaults to OBJECT_METADATA.', + flag_values=fv) + flags.DEFINE_string( + 'selfLink', + None, + u'The canonical URL of this notification.', + flag_values=fv) + flags.DEFINE_string( + 'topic', + None, + u'The Cloud PubSub topic to which this subscription publishes. ' + u"Formatted as: '//pubsub.googleapis.com/projects/{project-" + u"identifier}/topics/{my-topic}'", + flag_values=fv) + + def RunWithArgs(self): + """Creates a notification subscription for a given bucket. + + Flags: + bucket: The name of the bucket this subscription is particular to. + custom_attributes: An optional list of additional attributes to attach + to each Cloud PubSub message published for this notification + subscription. + etag: HTTP 1.1 Entity tag for this subscription notification. + event_types: If present, only send notifications about listed event + types. If empty, sent notifications for all event types. + id: The ID of the notification. + kind: The kind of item this is. For notifications, this is always + storage#notification. + object_metadata_format: If payload_content is OBJECT_METADATA, controls + the format of that metadata. Otherwise, must not be set. + object_name_prefix: If present, only apply this notification + configuration to object names that begin with this prefix. + payload_content: The desired content of the Payload. Defaults to + OBJECT_METADATA. + selfLink: The canonical URL of this notification. + topic: The Cloud PubSub topic to which this subscription publishes. + Formatted as: '//pubsub.googleapis.com/projects/{project- + identifier}/topics/{my-topic}' + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Notification( + ) + if FLAGS['bucket'].present: + request.bucket = FLAGS.bucket.decode('utf8') + if FLAGS['custom_attributes'].present: + request.custom_attributes = apitools_base.JsonToMessage(messages.Notification.CustomAttributesValue, FLAGS.custom_attributes) + if FLAGS['etag'].present: + request.etag = FLAGS.etag.decode('utf8') + if FLAGS['event_types'].present: + request.event_types = [x.decode('utf8') for x in FLAGS.event_types] + if FLAGS['id'].present: + request.id = FLAGS.id.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['object_metadata_format'].present: + request.object_metadata_format = FLAGS.object_metadata_format.decode('utf8') + if FLAGS['object_name_prefix'].present: + request.object_name_prefix = FLAGS.object_name_prefix.decode('utf8') + if FLAGS['payload_content'].present: + request.payload_content = FLAGS.payload_content.decode('utf8') + if FLAGS['selfLink'].present: + request.selfLink = FLAGS.selfLink.decode('utf8') + if FLAGS['topic'].present: + request.topic = FLAGS.topic.decode('utf8') + result = client.notifications.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class NotificationsList(apitools_base_cli.NewCmd): + """Command wrapping notifications.List.""" + + usage = """notifications_list """ + + def __init__(self, name, fv): + super(NotificationsList, self).__init__(name, fv) + + def RunWithArgs(self, bucket): + """Retrieves a list of notification subscriptions for a given bucket. + + Args: + bucket: Name of a GCS bucket. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageNotificationsListRequest( + bucket=bucket.decode('utf8'), + ) + result = client.notifications.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + class ObjectAccessControlsDelete(apitools_base_cli.NewCmd): """Command wrapping objectAccessControls.Delete.""" @@ -2343,6 +2611,45 @@ class ObjectsGet(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) +class ObjectsGetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping objects.GetIamPolicy.""" + + usage = """objects_getIamPolicy """ + + def __init__(self, name, fv): + super(ObjectsGetIamPolicy, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Returns an IAM policy for the specified object. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsGetIamPolicyRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objects.GetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + class ObjectsInsert(apitools_base_cli.NewCmd): """Command wrapping objects.Insert.""" @@ -2902,6 +3209,95 @@ class ObjectsRewrite(apitools_base_cli.NewCmd): print apitools_base_cli.FormatOutput(result) +class ObjectsSetIamPolicy(apitools_base_cli.NewCmd): + """Command wrapping objects.SetIamPolicy.""" + + usage = """objects_setIamPolicy """ + + def __init__(self, name, fv): + super(ObjectsSetIamPolicy, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + flags.DEFINE_string( + 'policy', + None, + u'A Policy resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, bucket, object): + """Updates an IAM policy for the specified object. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + policy: A Policy resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsSetIamPolicyRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + if FLAGS['policy'].present: + request.policy = apitools_base.JsonToMessage(messages.Policy, FLAGS.policy) + result = client.objects.SetIamPolicy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ObjectsTestIamPermissions(apitools_base_cli.NewCmd): + """Command wrapping objects.TestIamPermissions.""" + + usage = """objects_testIamPermissions """ + + def __init__(self, name, fv): + super(ObjectsTestIamPermissions, self).__init__(name, fv) + flags.DEFINE_string( + 'generation', + None, + u'If present, selects a specific revision of this object (as opposed ' + u'to the latest version, the default).', + flag_values=fv) + + def RunWithArgs(self, bucket, object, permissions): + """Tests a set of permissions on the given object to see which, if any, + are held by the caller. + + Args: + bucket: Name of the bucket in which the object resides. + object: Name of the object. For information about how to URL encode + object names to be path safe, see Encoding URI Path Parts. + permissions: Permissions to test. + + Flags: + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StorageObjectsTestIamPermissionsRequest( + bucket=bucket.decode('utf8'), + object=object.decode('utf8'), + permissions=permissions.decode('utf8'), + ) + if FLAGS['generation'].present: + request.generation = int(FLAGS.generation) + result = client.objects.TestIamPermissions( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + class ObjectsUpdate(apitools_base_cli.NewCmd): """Command wrapping objects.Update.""" @@ -3133,9 +3529,12 @@ def main(_): appcommands.AddCmd('bucketAccessControls_update', BucketAccessControlsUpdate) appcommands.AddCmd('buckets_delete', BucketsDelete) appcommands.AddCmd('buckets_get', BucketsGet) + appcommands.AddCmd('buckets_getIamPolicy', BucketsGetIamPolicy) appcommands.AddCmd('buckets_insert', BucketsInsert) appcommands.AddCmd('buckets_list', BucketsList) appcommands.AddCmd('buckets_patch', BucketsPatch) + appcommands.AddCmd('buckets_setIamPolicy', BucketsSetIamPolicy) + appcommands.AddCmd('buckets_testIamPermissions', BucketsTestIamPermissions) appcommands.AddCmd('buckets_update', BucketsUpdate) appcommands.AddCmd('channels_stop', ChannelsStop) appcommands.AddCmd('defaultObjectAccessControls_delete', DefaultObjectAccessControlsDelete) @@ -3144,6 +3543,10 @@ def main(_): appcommands.AddCmd('defaultObjectAccessControls_list', DefaultObjectAccessControlsList) appcommands.AddCmd('defaultObjectAccessControls_patch', DefaultObjectAccessControlsPatch) appcommands.AddCmd('defaultObjectAccessControls_update', DefaultObjectAccessControlsUpdate) + appcommands.AddCmd('notifications_delete', NotificationsDelete) + appcommands.AddCmd('notifications_get', NotificationsGet) + appcommands.AddCmd('notifications_insert', NotificationsInsert) + appcommands.AddCmd('notifications_list', NotificationsList) appcommands.AddCmd('objectAccessControls_delete', ObjectAccessControlsDelete) appcommands.AddCmd('objectAccessControls_get', ObjectAccessControlsGet) appcommands.AddCmd('objectAccessControls_insert', ObjectAccessControlsInsert) @@ -3154,10 +3557,13 @@ def main(_): appcommands.AddCmd('objects_copy', ObjectsCopy) appcommands.AddCmd('objects_delete', ObjectsDelete) appcommands.AddCmd('objects_get', ObjectsGet) + appcommands.AddCmd('objects_getIamPolicy', ObjectsGetIamPolicy) appcommands.AddCmd('objects_insert', ObjectsInsert) appcommands.AddCmd('objects_list', ObjectsList) appcommands.AddCmd('objects_patch', ObjectsPatch) appcommands.AddCmd('objects_rewrite', ObjectsRewrite) + appcommands.AddCmd('objects_setIamPolicy', ObjectsSetIamPolicy) + appcommands.AddCmd('objects_testIamPermissions', ObjectsTestIamPermissions) appcommands.AddCmd('objects_update', ObjectsUpdate) appcommands.AddCmd('objects_watchAll', ObjectsWatchAll) diff --git a/samples/storage_sample/storage/storage_v1_client.py b/samples/storage_sample/storage_v1/storage_v1_client.py similarity index 80% rename from samples/storage_sample/storage/storage_v1_client.py rename to samples/storage_sample/storage_v1/storage_v1_client.py index 9593d50..b1bd6a9 100644 --- a/samples/storage_sample/storage/storage_v1_client.py +++ b/samples/storage_sample/storage_v1/storage_v1_client.py @@ -1,28 +1,14 @@ -# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Generated client library for storage version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -import storage_v1_messages as messages +from storage import storage_v1_messages as messages class StorageV1(base_api.BaseApiClient): """Generated client library for service storage version v1.""" MESSAGES_MODULE = messages + BASE_URL = u'https://www.googleapis.com/storage/v1/' _PACKAGE = u'storage' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] @@ -40,7 +26,7 @@ class StorageV1(base_api.BaseApiClient): credentials_args=None, default_global_params=None, additional_http_headers=None): """Create a new storage handle.""" - url = url or u'https://www.googleapis.com/storage/v1/' + url = url or self.BASE_URL super(StorageV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, @@ -52,6 +38,7 @@ class StorageV1(base_api.BaseApiClient): self.buckets = self.BucketsService(self) self.channels = self.ChannelsService(self) self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) + self.notifications = self.NotificationsService(self) self.objectAccessControls = self.ObjectAccessControlsService(self) self.objects = self.ObjectsService(self) @@ -250,6 +237,18 @@ class StorageV1(base_api.BaseApiClient): response_type_name=u'Bucket', supports_download=False, ), + 'GetIamPolicy': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.getIamPolicy', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/iam', + request_field='', + request_type_name=u'StorageBucketsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.buckets.insert', @@ -286,6 +285,30 @@ class StorageV1(base_api.BaseApiClient): response_type_name=u'Bucket', supports_download=False, ), + 'SetIamPolicy': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.buckets.setIamPolicy', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/iam', + request_field=u'policy', + request_type_name=u'StorageBucketsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), + 'TestIamPermissions': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.testIamPermissions', + ordered_params=[u'bucket', u'permissions'], + path_params=[u'bucket'], + query_params=[u'permissions'], + relative_path=u'b/{bucket}/iam/testPermissions', + request_field='', + request_type_name=u'StorageBucketsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.buckets.update', @@ -329,6 +352,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + def GetIamPolicy(self, request, global_params=None): + """Returns an IAM policy for the specified bucket. + + Args: + request: (StorageBucketsGetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('GetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + def Insert(self, request, global_params=None): """Creates a new bucket. @@ -368,6 +404,32 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + def SetIamPolicy(self, request, global_params=None): + """Updates an IAM policy for the specified bucket. + + Args: + request: (StorageBucketsSetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('SetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + + def TestIamPermissions(self, request, global_params=None): + """Tests a set of permissions on the given bucket to see which, if any, are held by the caller. + + Args: + request: (StorageBucketsTestIamPermissionsRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TestIamPermissionsResponse) The response message. + """ + config = self.GetMethodConfig('TestIamPermissions') + return self._RunMethod( + config, request, global_params=global_params) + def Update(self, request, global_params=None): """Updates a bucket. @@ -582,6 +644,119 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + class NotificationsService(base_api.BaseApiService): + """Service class for the notifications resource.""" + + _NAME = u'notifications' + + def __init__(self, client): + super(StorageV1.NotificationsService, self).__init__(client) + self._method_configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.notifications.delete', + ordered_params=[u'notification'], + path_params=[u'notification'], + query_params=[], + relative_path=u'notifications/{notification}', + request_field='', + request_type_name=u'StorageNotificationsDeleteRequest', + response_type_name=u'StorageNotificationsDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.notifications.get', + ordered_params=[u'notification'], + path_params=[u'notification'], + query_params=[], + relative_path=u'notifications/{notification}', + request_field='', + request_type_name=u'StorageNotificationsGetRequest', + response_type_name=u'Notification', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.notifications.insert', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'notifications', + request_field='', + request_type_name=u'Notification', + response_type_name=u'Notification', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.notifications.list', + ordered_params=[u'bucket'], + path_params=[], + query_params=[u'bucket'], + relative_path=u'notifications', + request_field='', + request_type_name=u'StorageNotificationsListRequest', + response_type_name=u'Notifications', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Delete(self, request, global_params=None): + """Permanently deletes a notification subscription. + + Args: + request: (StorageNotificationsDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StorageNotificationsDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """View a notification configuration. + + Args: + request: (StorageNotificationsGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Notification) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a notification subscription for a given bucket. + + Args: + request: (Notification) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Notification) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of notification subscriptions for a given bucket. + + Args: + request: (StorageNotificationsListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Notifications) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + class ObjectAccessControlsService(base_api.BaseApiService): """Service class for the objectAccessControls resource.""" @@ -801,6 +976,18 @@ class StorageV1(base_api.BaseApiClient): response_type_name=u'Object', supports_download=True, ), + 'GetIamPolicy': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.getIamPolicy', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/iam', + request_field='', + request_type_name=u'StorageObjectsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), 'Insert': base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.insert', @@ -849,6 +1036,30 @@ class StorageV1(base_api.BaseApiClient): response_type_name=u'RewriteResponse', supports_download=False, ), + 'SetIamPolicy': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objects.setIamPolicy', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/iam', + request_field=u'policy', + request_type_name=u'StorageObjectsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ), + 'TestIamPermissions': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.testIamPermissions', + ordered_params=[u'bucket', u'object', u'permissions'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'permissions'], + relative_path=u'b/{bucket}/o/{object}/iam/testPermissions', + request_field='', + request_type_name=u'StorageObjectsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ), 'Update': base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.objects.update', @@ -947,6 +1158,19 @@ class StorageV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + def GetIamPolicy(self, request, global_params=None): + """Returns an IAM policy for the specified object. + + Args: + request: (StorageObjectsGetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('GetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + def Insert(self, request, global_params=None, upload=None, download=None): """Stores a new object and metadata. @@ -1006,6 +1230,32 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + def SetIamPolicy(self, request, global_params=None): + """Updates an IAM policy for the specified object. + + Args: + request: (StorageObjectsSetIamPolicyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Policy) The response message. + """ + config = self.GetMethodConfig('SetIamPolicy') + return self._RunMethod( + config, request, global_params=global_params) + + def TestIamPermissions(self, request, global_params=None): + """Tests a set of permissions on the given object to see which, if any, are held by the caller. + + Args: + request: (StorageObjectsTestIamPermissionsRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TestIamPermissionsResponse) The response message. + """ + config = self.GetMethodConfig('TestIamPermissions') + return self._RunMethod( + config, request, global_params=global_params) + def Update(self, request, global_params=None, download=None): """Updates an object's metadata. diff --git a/samples/storage_sample/storage/storage_v1_messages.py b/samples/storage_sample/storage_v1/storage_v1_messages.py similarity index 81% rename from samples/storage_sample/storage/storage_v1_messages.py rename to samples/storage_sample/storage_v1/storage_v1_messages.py index c3923d3..f392334 100644 --- a/samples/storage_sample/storage/storage_v1_messages.py +++ b/samples/storage_sample/storage_v1/storage_v1_messages.py @@ -1,21 +1,6 @@ -# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Generated message classes for storage version v1. -Lets you store and retrieve potentially-large, immutable data objects. +Stores and retrieves potentially large, immutable data objects. """ # NOTE: This file is autogenerated and should not be edited by hand. @@ -41,7 +26,9 @@ class Bucket(_messages.Message): OwnerValue: The owner of the bucket. This is always the project team's owner group. VersioningValue: The bucket's versioning configuration. - WebsiteValue: The bucket's website configuration. + WebsiteValue: The bucket's website configuration, controlling how the + service behaves when accessing bucket contents as a web site. See the + Static Website Examples for more information. Fields: acl: Access controls on the bucket. @@ -69,9 +56,12 @@ class Bucket(_messages.Message): bucket are stored and determines the SLA and the cost of storage. Values include STANDARD, NEARLINE and DURABLE_REDUCED_AVAILABILITY. Defaults to STANDARD. For more information, see storage classes. - timeCreated: Creation time of the bucket in RFC 3339 format. + timeCreated: The creation time of the bucket in RFC 3339 format. + updated: The modification time of the bucket in RFC 3339 format. versioning: The bucket's versioning configuration. - website: The bucket's website configuration. + website: The bucket's website configuration, controlling how the service + behaves when accessing bucket contents as a web site. See the Static + Website Examples for more information. """ class CorsValueListEntry(_messages.Message): @@ -189,13 +179,19 @@ class Bucket(_messages.Message): enabled = _messages.BooleanField(1) class WebsiteValue(_messages.Message): - """The bucket's website configuration. + """The bucket's website configuration, controlling how the service behaves + when accessing bucket contents as a web site. See the Static Website + Examples for more information. Fields: - mainPageSuffix: Behaves as the bucket's directory index where missing - objects are treated as potential directories. - notFoundPage: The custom object to return when a requested resource is - not found. + mainPageSuffix: If the requested object path is missing, the service + will ensure the path has a trailing '/', append this suffix, and + attempt to retrieve the resulting object. This allows the creation of + index.html objects to represent directory pages. + notFoundPage: If the requested object path is missing, and any + mainPageSuffix object is missing, if applicable, the service will + return the named object from this bucket as the content for a 404 Not + Found result. """ mainPageSuffix = _messages.StringField(1) @@ -217,8 +213,9 @@ class Bucket(_messages.Message): selfLink = _messages.StringField(14) storageClass = _messages.StringField(15) timeCreated = _message_types.DateTimeField(16) - versioning = _messages.MessageField('VersioningValue', 17) - website = _messages.MessageField('WebsiteValue', 18) + updated = _message_types.DateTimeField(17) + versioning = _messages.MessageField('VersioningValue', 18) + website = _messages.MessageField('WebsiteValue', 19) class BucketAccessControl(_messages.Message): @@ -413,10 +410,95 @@ class ComposeRequest(_messages.Message): sourceObjects = _messages.MessageField('SourceObjectsValueListEntry', 3, repeated=True) +class Notification(_messages.Message): + """A subscription to receive Google PubSub notifications. + + Messages: + CustomAttributesValue: An optional list of additional attributes to attach + to each Cloud PubSub message published for this notification + subscription. + + Fields: + bucket: The name of the bucket this subscription is particular to. + custom_attributes: An optional list of additional attributes to attach to + each Cloud PubSub message published for this notification subscription. + etag: HTTP 1.1 Entity tag for this subscription notification. + event_types: If present, only send notifications about listed event types. + If empty, sent notifications for all event types. + id: The ID of the notification. + kind: The kind of item this is. For notifications, this is always + storage#notification. + object_metadata_format: If payload_content is OBJECT_METADATA, controls + the format of that metadata. Otherwise, must not be set. + object_name_prefix: If present, only apply this notification configuration + to object names that begin with this prefix. + payload_content: The desired content of the Payload. Defaults to + OBJECT_METADATA. + selfLink: The canonical URL of this notification. + topic: The Cloud PubSub topic to which this subscription publishes. + Formatted as: '//pubsub.googleapis.com/projects/{project- + identifier}/topics/{my-topic}' + """ + + @encoding.MapUnrecognizedFields('additionalProperties') + class CustomAttributesValue(_messages.Message): + """An optional list of additional attributes to attach to each Cloud + PubSub message published for this notification subscription. + + Messages: + AdditionalProperty: An additional property for a CustomAttributesValue + object. + + Fields: + additionalProperties: Additional properties of type + CustomAttributesValue + """ + + class AdditionalProperty(_messages.Message): + """An additional property for a CustomAttributesValue object. + + Fields: + key: Name of the additional property. + value: A string attribute. + """ + + key = _messages.StringField(1) + value = _messages.StringField(2) + + additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) + + bucket = _messages.StringField(1) + custom_attributes = _messages.MessageField('CustomAttributesValue', 2) + etag = _messages.StringField(3) + event_types = _messages.StringField(4, repeated=True) + id = _messages.StringField(5) + kind = _messages.StringField(6, default=u'storage#notification') + object_metadata_format = _messages.StringField(7, default=u'JSON_API_V1') + object_name_prefix = _messages.StringField(8) + payload_content = _messages.StringField(9, default=u'OBJECT_METADATA') + selfLink = _messages.StringField(10) + topic = _messages.StringField(11) + + +class Notifications(_messages.Message): + """A list of notification subscriptions. + + Fields: + items: The list of items. + kind: The kind of item this is. For lists of notifications, this is always + storage#notifications. + """ + + items = _messages.MessageField('Notification', 1, repeated=True) + kind = _messages.StringField(2, default=u'storage#notifications') + + class Object(_messages.Message): """An object. Messages: + CustomerEncryptionValue: Metadata of customer-supplied encryption key, if + the object is encrypted by such a key. MetadataValue: User-provided metadata, in key/value pairs. OwnerValue: The owner of the object. This will always be the uploader of the object. @@ -430,10 +512,13 @@ class Object(_messages.Message): contentDisposition: Content-Disposition of the object data. contentEncoding: Content-Encoding of the object data. contentLanguage: Content-Language of the object data. - contentType: Content-Type of the object data. + contentType: Content-Type of the object data. If contentType is not + specified, object downloads will be served as application/octet-stream. crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded using base64 in big-endian byte order. For more information about using the CRC32c checksum, see Hashes and ETags: Best Practices. + customerEncryption: Metadata of customer-supplied encryption key, if the + object is encrypted by such a key. etag: HTTP 1.1 Entity tag for the object. generation: The content generation of this object. Used for object versioning. @@ -454,13 +539,24 @@ class Object(_messages.Message): selfLink: The link to this object. size: Content-Length of the data in bytes. storageClass: Storage class of the object. + timeCreated: The creation time of the object in RFC 3339 format. timeDeleted: The deletion time of the object in RFC 3339 format. Will be returned if and only if this version of the object has been deleted. - updated: The creation or modification time of the object in RFC 3339 - format. For buckets with versioning enabled, changing an object's - metadata does not change this property. + updated: The modification time of the object metadata in RFC 3339 format. """ + class CustomerEncryptionValue(_messages.Message): + """Metadata of customer-supplied encryption key, if the object is + encrypted by such a key. + + Fields: + encryptionAlgorithm: The encryption algorithm. + keySha256: SHA256 hash value of the encryption key. + """ + + encryptionAlgorithm = _messages.StringField(1) + keySha256 = _messages.StringField(2) + @encoding.MapUnrecognizedFields('additionalProperties') class MetadataValue(_messages.Message): """User-provided metadata, in key/value pairs. @@ -506,21 +602,23 @@ class Object(_messages.Message): contentLanguage = _messages.StringField(7) contentType = _messages.StringField(8) crc32c = _messages.StringField(9) - etag = _messages.StringField(10) - generation = _messages.IntegerField(11) - id = _messages.StringField(12) - kind = _messages.StringField(13, default=u'storage#object') - md5Hash = _messages.StringField(14) - mediaLink = _messages.StringField(15) - metadata = _messages.MessageField('MetadataValue', 16) - metageneration = _messages.IntegerField(17) - name = _messages.StringField(18) - owner = _messages.MessageField('OwnerValue', 19) - selfLink = _messages.StringField(20) - size = _messages.IntegerField(21, variant=_messages.Variant.UINT64) - storageClass = _messages.StringField(22) - timeDeleted = _message_types.DateTimeField(23) - updated = _message_types.DateTimeField(24) + customerEncryption = _messages.MessageField('CustomerEncryptionValue', 10) + etag = _messages.StringField(11) + generation = _messages.IntegerField(12) + id = _messages.StringField(13) + kind = _messages.StringField(14, default=u'storage#object') + md5Hash = _messages.StringField(15) + mediaLink = _messages.StringField(16) + metadata = _messages.MessageField('MetadataValue', 17) + metageneration = _messages.IntegerField(18) + name = _messages.StringField(19) + owner = _messages.MessageField('OwnerValue', 20) + selfLink = _messages.StringField(21) + size = _messages.IntegerField(22, variant=_messages.Variant.UINT64) + storageClass = _messages.StringField(23) + timeCreated = _message_types.DateTimeField(24) + timeDeleted = _message_types.DateTimeField(25) + updated = _message_types.DateTimeField(26) class ObjectAccessControl(_messages.Message): @@ -611,6 +709,82 @@ class Objects(_messages.Message): prefixes = _messages.StringField(4, repeated=True) +class Policy(_messages.Message): + """A bucket/object IAM policy. + + Messages: + BindingsValueListEntry: A BindingsValueListEntry object. + + Fields: + bindings: An association between a role, which comes with a set of + permissions, and members who may assume that role. + etag: HTTP 1.1 Entity tag for the policy. + kind: The kind of item this is. For policies, this is always + storage#policy. This field is ignored on input. + resourceId: The ID of the resource to which this policy belongs. Will be + of the form buckets/bucket for buckets, and + buckets/bucket/objects/object for objects. A specific generation may be + specified by appending #generationNumber to the end of the object name, + e.g. buckets/my-bucket/objects/data.txt#17. The current generation can + be denoted with #0. This field is ignored on input. + """ + + class BindingsValueListEntry(_messages.Message): + """A BindingsValueListEntry object. + + Fields: + members: A collection of identifiers for members who may assume the + provided role. Recognized identifiers are as follows: - allUsers \u2014 A + special identifier that represents anyone on the internet; with or + without a Google account. - allAuthenticatedUsers \u2014 A special + identifier that represents anyone who is authenticated with a Google + account or a service account. - user:emailid \u2014 An email address that + represents a specific account. For example, user:alice@gmail.com or + user:joe@example.com. - serviceAccount:emailid \u2014 An email address + that represents a service account. For example, serviceAccount:my- + other-app@appspot.gserviceaccount.com . - group:emailid \u2014 An email + address that represents a Google group. For example, + group:admins@example.com. - domain:domain \u2014 A Google Apps domain + name that represents all the users of that domain. For example, + domain:google.com or domain:example.com. - projectOwner:projectid \u2014 + Owners of the given project. For example, projectOwner:my-example- + project - projectEditor:projectid \u2014 Editors of the given project. + For example, projectEditor:my-example-project - + projectViewer:projectid \u2014 Viewers of the given project. For example, + projectViewer:my-example-project + role: The role to which members belong. Two types of roles are + supported: new IAM roles, which grant permissions that do not map + directly to those provided by ACLs, and legacy IAM roles, which do map + directly to ACL permissions. All roles are of the format + roles/storage.specificRole. The new IAM roles are: - + roles/storage.admin \u2014 Full control of Google Cloud Storage resources. + - roles/storage.objectViewer \u2014 Read-Only access to Google Cloud + Storage objects. - roles/storage.objectCreator \u2014 Access to create + objects in Google Cloud Storage. - roles/storage.objectAdmin \u2014 Full + control of Google Cloud Storage objects. The legacy IAM roles are: + - roles/storage.legacyObjectReader \u2014 Read-only access to objects + without listing. Equivalent to an ACL entry on an object with the + READER role. - roles/storage.legacyObjectOwner \u2014 Read/write access + to existing objects without listing. Equivalent to an ACL entry on an + object with the OWNER role. - roles/storage.legacyBucketReader \u2014 + Read access to buckets with object listing. Equivalent to an ACL entry + on a bucket with the READER role. - roles/storage.legacyBucketWriter + \u2014 Read access to buckets with object listing/creation/deletion. + Equivalent to an ACL entry on a bucket with the WRITER role. - + roles/storage.legacyBucketOwner \u2014 Read and write access to existing + buckets with object listing/creation/deletion. Equivalent to an ACL + entry on a bucket with the OWNER role. + """ + + members = _messages.StringField(1, repeated=True) + role = _messages.StringField(2) + + bindings = _messages.MessageField('BindingsValueListEntry', 1, repeated=True) + etag = _messages.BytesField(2) + kind = _messages.StringField(3, default=u'storage#policy') + resourceId = _messages.StringField(4) + + class RewriteResponse(_messages.Message): """A rewrite response. @@ -741,6 +915,16 @@ class StorageBucketsDeleteResponse(_messages.Message): """An empty StorageBucketsDelete response.""" +class StorageBucketsGetIamPolicyRequest(_messages.Message): + """A StorageBucketsGetIamPolicyRequest object. + + Fields: + bucket: Name of a bucket. + """ + + bucket = _messages.StringField(1, required=True) + + class StorageBucketsGetRequest(_messages.Message): """A StorageBucketsGetRequest object. @@ -762,7 +946,7 @@ class StorageBucketsGetRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit acl and defaultObjectAcl properties. + noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 @@ -847,7 +1031,7 @@ class StorageBucketsInsertRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit acl and defaultObjectAcl properties. + noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 @@ -879,7 +1063,7 @@ class StorageBucketsListRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit acl and defaultObjectAcl properties. + noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 @@ -964,7 +1148,7 @@ class StorageBucketsPatchRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit acl and defaultObjectAcl properties. + noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 @@ -978,6 +1162,30 @@ class StorageBucketsPatchRequest(_messages.Message): projection = _messages.EnumField('ProjectionValueValuesEnum', 7) +class StorageBucketsSetIamPolicyRequest(_messages.Message): + """A StorageBucketsSetIamPolicyRequest object. + + Fields: + bucket: Name of a bucket. + policy: A Policy resource to be passed as the request body. + """ + + bucket = _messages.StringField(1, required=True) + policy = _messages.MessageField('Policy', 2) + + +class StorageBucketsTestIamPermissionsRequest(_messages.Message): + """A StorageBucketsTestIamPermissionsRequest object. + + Fields: + bucket: Name of a bucket. + permissions: Permissions to test. + """ + + bucket = _messages.StringField(1, required=True) + permissions = _messages.StringField(2, required=True) + + class StorageBucketsUpdateRequest(_messages.Message): """A StorageBucketsUpdateRequest object. @@ -1051,7 +1259,7 @@ class StorageBucketsUpdateRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit acl and defaultObjectAcl properties. + noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 @@ -1117,6 +1325,40 @@ class StorageDefaultObjectAccessControlsListRequest(_messages.Message): ifMetagenerationNotMatch = _messages.IntegerField(3) +class StorageNotificationsDeleteRequest(_messages.Message): + """A StorageNotificationsDeleteRequest object. + + Fields: + notification: ID of the notification to delete. + """ + + notification = _messages.StringField(1, required=True) + + +class StorageNotificationsDeleteResponse(_messages.Message): + """An empty StorageNotificationsDelete response.""" + + +class StorageNotificationsGetRequest(_messages.Message): + """A StorageNotificationsGetRequest object. + + Fields: + notification: Notification ID + """ + + notification = _messages.StringField(1, required=True) + + +class StorageNotificationsListRequest(_messages.Message): + """A StorageNotificationsListRequest object. + + Fields: + bucket: Name of a GCS bucket. + """ + + bucket = _messages.StringField(1, required=True) + + class StorageObjectAccessControlsDeleteRequest(_messages.Message): """A StorageObjectAccessControlsDeleteRequest object. @@ -1371,7 +1613,7 @@ class StorageObjectsCopyRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1426,6 +1668,22 @@ class StorageObjectsDeleteResponse(_messages.Message): """An empty StorageObjectsDelete response.""" +class StorageObjectsGetIamPolicyRequest(_messages.Message): + """A StorageObjectsGetIamPolicyRequest object. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. + """ + + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + object = _messages.StringField(3, required=True) + + class StorageObjectsGetRequest(_messages.Message): """A StorageObjectsGetRequest object. @@ -1454,7 +1712,7 @@ class StorageObjectsGetRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1534,7 +1792,7 @@ class StorageObjectsInsertRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1580,7 +1838,7 @@ class StorageObjectsListRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1649,7 +1907,7 @@ class StorageObjectsPatchRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1755,7 +2013,7 @@ class StorageObjectsRewriteRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1780,6 +2038,42 @@ class StorageObjectsRewriteRequest(_messages.Message): sourceObject = _messages.StringField(18, required=True) +class StorageObjectsSetIamPolicyRequest(_messages.Message): + """A StorageObjectsSetIamPolicyRequest object. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. + policy: A Policy resource to be passed as the request body. + """ + + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + object = _messages.StringField(3, required=True) + policy = _messages.MessageField('Policy', 4) + + +class StorageObjectsTestIamPermissionsRequest(_messages.Message): + """A StorageObjectsTestIamPermissionsRequest object. + + Fields: + bucket: Name of the bucket in which the object resides. + generation: If present, selects a specific revision of this object (as + opposed to the latest version, the default). + object: Name of the object. For information about how to URL encode object + names to be path safe, see Encoding URI Path Parts. + permissions: Permissions to test. + """ + + bucket = _messages.StringField(1, required=True) + generation = _messages.IntegerField(2) + object = _messages.StringField(3, required=True) + permissions = _messages.StringField(4, required=True) + + class StorageObjectsUpdateRequest(_messages.Message): """A StorageObjectsUpdateRequest object. @@ -1835,7 +2129,7 @@ class StorageObjectsUpdateRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1882,7 +2176,7 @@ class StorageObjectsWatchAllRequest(_messages.Message): Values: full: Include all properties. - noAcl: Omit the acl property. + noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 @@ -1897,3 +2191,28 @@ class StorageObjectsWatchAllRequest(_messages.Message): versions = _messages.BooleanField(8) +class TestIamPermissionsResponse(_messages.Message): + """A storage.(buckets|objects).testIamPermissions response. + + Fields: + kind: The kind of item this is. + permissions: The permissions held by the caller. Permissions are always of + the format storage.resource.capability, where resource is one of buckets + or objects. The supported permissions are as follows: - + storage.buckets.delete \u2014 Delete bucket. - storage.buckets.get \u2014 Read + bucket metadata. - storage.buckets.getIamPolicy \u2014 Read bucket IAM + policy. - storage.buckets.create \u2014 Create bucket. - + storage.buckets.list \u2014 List buckets. - storage.buckets.setIamPolicy \u2014 + Update bucket IAM policy. - storage.buckets.update \u2014 Update bucket + metadata. - storage.objects.delete \u2014 Delete object. - + storage.objects.get \u2014 Read object data and metadata. - + storage.objects.getIamPolicy \u2014 Read object IAM policy. - + storage.objects.create \u2014 Create object. - storage.objects.list \u2014 List + objects. - storage.objects.setIamPolicy \u2014 Update object IAM policy. + - storage.objects.update \u2014 Update object metadata. + """ + + kind = _messages.StringField(1, default=u'storage#testIamPermissionsResponse') + permissions = _messages.StringField(2, repeated=True) + + diff --git a/samples/uptodate_check_test.py b/samples/uptodate_check_test.py new file mode 100644 index 0000000..2829b3f --- /dev/null +++ b/samples/uptodate_check_test.py @@ -0,0 +1,68 @@ +# Copyright 2015 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import unittest2 + +from apitools.gen import gen_client +from apitools.gen import test_utils + + +def GetSampleClientPath(api_name, *path): + return os.path.join(os.path.dirname(__file__), api_name + '_sample', *path) + + +def _GetContent(file_path): + with open(file_path) as f: + return f.read() + + +@test_utils.RunOnlyOnPython27 +class ClientGenCliTest(unittest2.TestCase): + + def _CheckGeneratedFiles(self, api_name, api_version): + prefix = api_name + '_' + api_version + with test_utils.TempDir() as tmp_dir_path: + gen_client.main([ + gen_client.__file__, + '--generate_cli', + '--init-file', 'empty', + '--infile', + GetSampleClientPath(api_name, prefix + '.json'), + '--outdir', tmp_dir_path, + '--overwrite', + '--root_package', api_name, + 'client' + ]) + expected_files = ( + set([prefix + '.py']) | # CLI files + set([prefix + '_client.py', + prefix + '_messages.py', + '__init__.py'])) + self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) + for expected_file in expected_files: + self.assertMultiLineEqual( + _GetContent(GetSampleClientPath( + api_name, prefix, expected_file)), + _GetContent(os.path.join(tmp_dir_path, expected_file))) + + def testGenClient_DnsDoc(self): + self._CheckGeneratedFiles('dns', 'v1') + + def testGenClient_IamDoc(self): + self._CheckGeneratedFiles('iam', 'v1') + + def testGenClient_StorageDoc(self): + self._CheckGeneratedFiles('storage', 'v1') diff --git a/tox.ini b/tox.ini index 249aa41..dbdeb78 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ deps = commands = nosetests [] [pep8] -exclude = samples/storage_sample/storage,*/testdata/*,*.egg/,*.egg-info/,.*/,ez_setup.py,build +exclude = samples/*_sample/*/*,*/testdata/*,*.egg/,*.egg-info/,.*/,ez_setup.py,build verbose = 1 [testenv:lint] -- GitLab From 80d4c3ade761f7dbb1edda6aa3605f581a15a653 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 3 Jun 2016 09:25:42 -0700 Subject: [PATCH 244/295] Fix an ADC-related bug, and make an install tweak. When getting the application default credentials from a file, we need to make sure we call create_scoped(), otherwise we'll get back invalid access tokens. While I was here, I dropped the hook to add an oauth2l script, since it's now its own package on PyPI. I'll drop the code in a separate PR. --- apitools/base/py/credentials_lib.py | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py index ba17960..a3819bc 100644 --- a/apitools/base/py/credentials_lib.py +++ b/apitools/base/py/credentials_lib.py @@ -642,5 +642,5 @@ def _GetApplicationDefaultCredentials( # ADC will work. cp = 'https://www.googleapis.com/auth/cloud-platform' if not isinstance(credentials, gc) or cp in scopes: - return credentials + return credentials.create_scoped(scopes) return None diff --git a/setup.py b/setup.py index 66696fd..2ae9e6b 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ TESTING_PACKAGES = [ CONSOLE_SCRIPTS = [ 'gen_client = apitools.gen.gen_client:main', - 'oauth2l = apitools.scripts.oauth2l:main', ] py_version = platform.python_version() -- GitLab From 0d7fe64da7312e4d18293e47e80e13be50e995b6 Mon Sep 17 00:00:00 2001 From: Craig Citro Date: Fri, 3 Jun 2016 10:16:39 -0700 Subject: [PATCH 245/295] Update for v0.5.4 release. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ae9e6b..a014347 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ py_version = platform.python_version() if py_version < '2.7': REQUIRED_PACKAGES.append('argparse>=1.2.1') -_APITOOLS_VERSION = '0.5.3' +_APITOOLS_VERSION = '0.5.4' with open('README.rst') as fileobj: README = fileobj.read() -- GitLab From 159b6f57dae796e70e9a57bfbb2305b57db1bedb Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 3 Jun 2016 13:45:05 -0400 Subject: [PATCH 246/295] Expose flat_path property in generated client services. --- apitools/base/py/base_api.py | 24 +++--- apitools/gen/service_registry.py | 11 ++- samples/dns_sample/__init__.py | 13 ++++ samples/dns_sample/dns_v1/dns_v1_client.py | 2 +- samples/dns_sample/gen_dns_client_test.py | 14 ++-- samples/iam_sample/__init__.py | 13 ++++ samples/iam_sample/iam_client_test.py | 77 +++++++++++++++++++ samples/iam_sample/iam_v1/iam_v1_client.py | 16 +++- samples/regenerate_samples.py | 5 +- .../storage_v1/storage_v1_client.py | 2 +- samples/uptodate_check_test.py | 13 +++- 11 files changed, 164 insertions(+), 26 deletions(-) create mode 100644 samples/dns_sample/__init__.py create mode 100644 samples/iam_sample/__init__.py create mode 100644 samples/iam_sample/iam_client_test.py diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index a668738..2bdd1ec 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -82,6 +82,7 @@ class ApiMethodInfo(messages.Message): Fields: relative_path: Relative path for this method. + flat_path: Expanded version (if any) of relative_path. method_id: ID for this method. http_method: HTTP verb to use for this method. path_params: (repeated) path parameters for this method. @@ -102,17 +103,18 @@ class ApiMethodInfo(messages.Message): """ relative_path = messages.StringField(1) - method_id = messages.StringField(2) - http_method = messages.StringField(3) - path_params = messages.StringField(4, repeated=True) - query_params = messages.StringField(5, repeated=True) - ordered_params = messages.StringField(6, repeated=True) - description = messages.StringField(7) - request_type_name = messages.StringField(8) - response_type_name = messages.StringField(9) - request_field = messages.StringField(10, default='') - upload_config = messages.MessageField(ApiUploadInfo, 11) - supports_download = messages.BooleanField(12, default=False) + flat_path = messages.StringField(2) + method_id = messages.StringField(3) + http_method = messages.StringField(4) + path_params = messages.StringField(5, repeated=True) + query_params = messages.StringField(6, repeated=True) + ordered_params = messages.StringField(7, repeated=True) + description = messages.StringField(8) + request_type_name = messages.StringField(9) + response_type_name = messages.StringField(10) + request_field = messages.StringField(11, default='') + upload_config = messages.MessageField(ApiUploadInfo, 12) + supports_download = messages.BooleanField(13, default=False) REQUEST_IS_BODY = '' diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index 863dca5..f9c0cd3 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -114,8 +114,9 @@ class ServiceRegistry(object): for attr in attrs: if attr in ('upload_config', 'description'): continue - printer( - '%s=%r,', attr, getattr(method_info, attr)) + value = getattr(method_info, attr) + if value is not None: + printer('%s=%r,', attr, value) printer('),') printer('}') printer() @@ -393,6 +394,12 @@ class ServiceRegistry(object): response_type_name=self.__names.ClassName(response), request_field=request_field, ) + flat_path = method_description.get('flatPath', None) + if flat_path is not None: + flat_path = self.__names.NormalizeRelativePath( + self.__client_info.base_path + flat_path) + if flat_path != relative_path: + method_info.flat_path = flat_path if method_description.get('supportsMediaUpload', False): method_info.upload_config = self.__ComputeUploadConfig( method_description.get('mediaUpload'), method_id) diff --git a/samples/dns_sample/__init__.py b/samples/dns_sample/__init__.py new file mode 100644 index 0000000..58e0d91 --- /dev/null +++ b/samples/dns_sample/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/dns_sample/dns_v1/dns_v1_client.py b/samples/dns_sample/dns_v1/dns_v1_client.py index 0b16311..42f5359 100644 --- a/samples/dns_sample/dns_v1/dns_v1_client.py +++ b/samples/dns_sample/dns_v1/dns_v1_client.py @@ -1,7 +1,7 @@ """Generated client library for dns version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from dns import dns_v1_messages as messages +from samples.dns_sample.dns_v1 import dns_v1_messages as messages class DnsV1(base_api.BaseApiClient): diff --git a/samples/dns_sample/gen_dns_client_test.py b/samples/dns_sample/gen_dns_client_test.py index ba94f39..dff6812 100644 --- a/samples/dns_sample/gen_dns_client_test.py +++ b/samples/dns_sample/gen_dns_client_test.py @@ -15,18 +15,14 @@ """Test for generated sample module.""" -import os -import sys import unittest2 import six from apitools.base.py import list_pager from apitools.base.py.testing import mock -sys.path.append(os.path.join(os.path.dirname(__file__), 'testdata')) - -from dns import dns_v1_client # nopep8 -from dns import dns_v1_messages # nopep8 +from samples.dns_sample.dns_v1 import dns_v1_client +from samples.dns_sample.dns_v1 import dns_v1_messages class DnsGenClientSanityTest(unittest2.TestCase): @@ -57,6 +53,12 @@ class DnsGenClientTest(unittest2.TestCase): self.mocked_dns_v1.Mock() self.addCleanup(self.mocked_dns_v1.Unmock) + def testFlatPath(self): + get_method_config = self.mocked_dns_v1.projects.GetMethodConfig('Get') + self.assertIsNone(get_method_config.flat_path) + self.assertEquals('projects/{project}', + get_method_config.relative_path) + def testRecordSetList(self): response_record_set = dns_v1_messages.ResourceRecordSet( kind=u"dns#resourceRecordSet", diff --git a/samples/iam_sample/__init__.py b/samples/iam_sample/__init__.py new file mode 100644 index 0000000..58e0d91 --- /dev/null +++ b/samples/iam_sample/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/iam_sample/iam_client_test.py b/samples/iam_sample/iam_client_test.py new file mode 100644 index 0000000..39d25a4 --- /dev/null +++ b/samples/iam_sample/iam_client_test.py @@ -0,0 +1,77 @@ +# +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test for generated sample module.""" + +import unittest2 +import six + +from apitools.base.py.testing import mock + +from samples.iam_sample.iam_v1 import iam_v1_client # nopep8 +from samples.iam_sample.iam_v1 import iam_v1_messages # nopep8 + + +class DnsGenClientSanityTest(unittest2.TestCase): + + def testBaseUrl(self): + self.assertEquals(u'https://iam.googleapis.com/', + iam_v1_client.IamV1.BASE_URL) + + def testMessagesModule(self): + self.assertEquals(iam_v1_messages, iam_v1_client.IamV1.MESSAGES_MODULE) + + def testAttributes(self): + inner_classes = set([]) + for key, value in iam_v1_client.IamV1.__dict__.items(): + if isinstance(value, six.class_types): + inner_classes.add(key) + self.assertEquals(set([ + 'IamPoliciesService', + 'ProjectsService', + 'ProjectsServiceAccountsKeysService', + 'ProjectsServiceAccountsService', + 'RolesService']), inner_classes) + + +class IamGenClientTest(unittest2.TestCase): + + def setUp(self): + self.mocked_iam_v1 = mock.Client(iam_v1_client.IamV1) + self.mocked_iam_v1.Mock() + self.addCleanup(self.mocked_iam_v1.Unmock) + + def testFlatPath(self): + get_method_config = (self.mocked_iam_v1.projects_serviceAccounts_keys + .GetMethodConfig('Get')) + self.assertEquals('v1/projects/{projectsId}/serviceAccounts' + '/{serviceAccountsId}/keys/{keysId}', + get_method_config.flat_path) + self.assertEquals('v1/{+name}', get_method_config.relative_path) + + def testServiceAccountsKeysList(self): + response_key = iam_v1_messages.ServiceAccountKey( + name=u'test-key') + self.mocked_iam_v1.projects_serviceAccounts_keys.List.Expect( + iam_v1_messages.IamProjectsServiceAccountsKeysListRequest( + name=u'test-service-account.'), + iam_v1_messages.ListServiceAccountKeysResponse( + keys=[response_key])) + + result = self.mocked_iam_v1.projects_serviceAccounts_keys.List( + iam_v1_messages.IamProjectsServiceAccountsKeysListRequest( + name=u'test-service-account.')) + + self.assertEquals([response_key], result.keys) diff --git a/samples/iam_sample/iam_v1/iam_v1_client.py b/samples/iam_sample/iam_v1/iam_v1_client.py index 0c10655..88f153b 100644 --- a/samples/iam_sample/iam_v1/iam_v1_client.py +++ b/samples/iam_sample/iam_v1/iam_v1_client.py @@ -1,7 +1,7 @@ """Generated client library for iam version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from iam import iam_v1_messages as messages +from samples.iam_sample.iam_v1 import iam_v1_messages as messages class IamV1(base_api.BaseApiClient): @@ -88,6 +88,7 @@ that the user has access to. super(IamV1.ProjectsServiceAccountsKeysService, self).__init__(client) self._method_configs = { 'Create': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.keys.create', ordered_params=[u'name'], @@ -100,6 +101,7 @@ that the user has access to. supports_download=False, ), 'Delete': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', http_method=u'DELETE', method_id=u'iam.projects.serviceAccounts.keys.delete', ordered_params=[u'name'], @@ -112,6 +114,7 @@ that the user has access to. supports_download=False, ), 'Get': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', http_method=u'GET', method_id=u'iam.projects.serviceAccounts.keys.get', ordered_params=[u'name'], @@ -124,6 +127,7 @@ that the user has access to. supports_download=False, ), 'List': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', http_method=u'GET', method_id=u'iam.projects.serviceAccounts.keys.list', ordered_params=[u'name'], @@ -203,6 +207,7 @@ by key id. super(IamV1.ProjectsServiceAccountsService, self).__init__(client) self._method_configs = { 'Create': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.create', ordered_params=[u'name'], @@ -215,6 +220,7 @@ by key id. supports_download=False, ), 'Delete': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', http_method=u'DELETE', method_id=u'iam.projects.serviceAccounts.delete', ordered_params=[u'name'], @@ -227,6 +233,7 @@ by key id. supports_download=False, ), 'Get': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', http_method=u'GET', method_id=u'iam.projects.serviceAccounts.get', ordered_params=[u'name'], @@ -239,6 +246,7 @@ by key id. supports_download=False, ), 'GetIamPolicy': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:getIamPolicy', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.getIamPolicy', ordered_params=[u'resource'], @@ -251,6 +259,7 @@ by key id. supports_download=False, ), 'List': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts', http_method=u'GET', method_id=u'iam.projects.serviceAccounts.list', ordered_params=[u'name'], @@ -263,6 +272,7 @@ by key id. supports_download=False, ), 'SetIamPolicy': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:setIamPolicy', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.setIamPolicy', ordered_params=[u'resource'], @@ -275,6 +285,7 @@ by key id. supports_download=False, ), 'SignBlob': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlob', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.signBlob', ordered_params=[u'name'], @@ -287,6 +298,7 @@ by key id. supports_download=False, ), 'SignJwt': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwt', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.signJwt', ordered_params=[u'name'], @@ -299,6 +311,7 @@ by key id. supports_download=False, ), 'TestIamPermissions': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:testIamPermissions', http_method=u'POST', method_id=u'iam.projects.serviceAccounts.testIamPermissions', ordered_params=[u'resource'], @@ -311,6 +324,7 @@ by key id. supports_download=False, ), 'Update': base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', http_method=u'PUT', method_id=u'iam.projects.serviceAccounts.update', ordered_params=[u'name'], diff --git a/samples/regenerate_samples.py b/samples/regenerate_samples.py index c9f768a..5a65bff 100644 --- a/samples/regenerate_samples.py +++ b/samples/regenerate_samples.py @@ -33,14 +33,15 @@ def _Generate(samples): if ext != '.json': raise RuntimeError('Expected .json discovery doc [{0}]' .format(sample)) - api_name, _ = name.split('_') + api_name, api_version = name.split('_') args = [ _GEN_CLIENT_BINARY, '--infile', sample, '--init-file', 'empty', '--outdir={0}'.format(os.path.join(sample_dir, name)), '--overwrite', - '--root_package', api_name, + '--root_package', + 'samples.{0}_sample.{0}_{1}'.format(api_name, api_version), 'client', ] subprocess.check_call(args) diff --git a/samples/storage_sample/storage_v1/storage_v1_client.py b/samples/storage_sample/storage_v1/storage_v1_client.py index b1bd6a9..854f4ea 100644 --- a/samples/storage_sample/storage_v1/storage_v1_client.py +++ b/samples/storage_sample/storage_v1/storage_v1_client.py @@ -1,7 +1,7 @@ """Generated client library for storage version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from storage import storage_v1_messages as messages +from samples.storage_sample.storage_v1 import storage_v1_messages as messages class StorageV1(base_api.BaseApiClient): diff --git a/samples/uptodate_check_test.py b/samples/uptodate_check_test.py index 2829b3f..d584b71 100644 --- a/samples/uptodate_check_test.py +++ b/samples/uptodate_check_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import difflib import unittest2 @@ -32,6 +33,13 @@ def _GetContent(file_path): @test_utils.RunOnlyOnPython27 class ClientGenCliTest(unittest2.TestCase): + def AssertDiffEqual(self, expected, actual): + """Like unittest.assertEqual with a diff in the exception message.""" + if expected != actual: + unified_diff = difflib.unified_diff( + expected.splitlines(), actual.splitlines()) + raise AssertionError('\n'.join(unified_diff)) + def _CheckGeneratedFiles(self, api_name, api_version): prefix = api_name + '_' + api_version with test_utils.TempDir() as tmp_dir_path: @@ -43,7 +51,8 @@ class ClientGenCliTest(unittest2.TestCase): GetSampleClientPath(api_name, prefix + '.json'), '--outdir', tmp_dir_path, '--overwrite', - '--root_package', api_name, + '--root_package', + 'samples.{0}_sample.{0}_{1}'.format(api_name, api_version), 'client' ]) expected_files = ( @@ -53,7 +62,7 @@ class ClientGenCliTest(unittest2.TestCase): '__init__.py'])) self.assertEquals(expected_files, set(os.listdir(tmp_dir_path))) for expected_file in expected_files: - self.assertMultiLineEqual( + self.AssertDiffEqual( _GetContent(GetSampleClientPath( api_name, prefix, expected_file)), _GetContent(os.path.join(tmp_dir_path, expected_file))) -- GitLab From 96948d34eef5a184f364c52d808d57c354575b57 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Thu, 23 Jun 2016 14:55:06 -0400 Subject: [PATCH 247/295] Move fusionteables sample with test to samples directory. --- .../fusiontables_sample}/__init__.py | 9 +- .../fusiontables_sample/fusiontables_v1.json | 1826 +++++++++++++++++ .../fusiontables_v1/__init__.py | 5 + .../fusiontables_v1/fusiontables_v1.py | 1797 ++++++++++++++++ .../fusiontables_v1_client.py | 0 .../fusiontables_v1_messages.py | 0 samples/regenerate_samples.py | 1 + 7 files changed, 3630 insertions(+), 8 deletions(-) rename {apitools/base/py/testing/testclient => samples/fusiontables_sample}/__init__.py (66%) create mode 100644 samples/fusiontables_sample/fusiontables_v1.json create mode 100644 samples/fusiontables_sample/fusiontables_v1/__init__.py create mode 100644 samples/fusiontables_sample/fusiontables_v1/fusiontables_v1.py rename {apitools/base/py/testing/testclient => samples/fusiontables_sample/fusiontables_v1}/fusiontables_v1_client.py (100%) rename {apitools/base/py/testing/testclient => samples/fusiontables_sample/fusiontables_v1}/fusiontables_v1_messages.py (100%) diff --git a/apitools/base/py/testing/testclient/__init__.py b/samples/fusiontables_sample/__init__.py similarity index 66% rename from apitools/base/py/testing/testclient/__init__.py rename to samples/fusiontables_sample/__init__.py index ad24da0..58e0d91 100644 --- a/apitools/base/py/testing/testclient/__init__.py +++ b/samples/fusiontables_sample/__init__.py @@ -1,6 +1,4 @@ -"""Common imports for generated fusiontables client library.""" -# -# Copyright 2015 Google Inc. +# Copyright 2016 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,8 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# pylint:disable=wildcard-import - -from apitools.base.py.testing.testclient.fusiontables_v1_client import * -from apitools.base.py.testing.testclient.fusiontables_v1_messages import * diff --git a/samples/fusiontables_sample/fusiontables_v1.json b/samples/fusiontables_sample/fusiontables_v1.json new file mode 100644 index 0000000..297c671 --- /dev/null +++ b/samples/fusiontables_sample/fusiontables_v1.json @@ -0,0 +1,1826 @@ +{ + "kind": "discovery#restDescription", + "etag": "\"C5oy1hgQsABtYOYIOXWcR3BgYqU/-xDlQ3Z80n_rfxYaz7dDf-mP00c\"", + "discoveryVersion": "v1", + "id": "fusiontables:v1", + "name": "fusiontables", + "version": "v1", + "revision": "20160526", + "title": "Fusion Tables API", + "description": "API for working with Fusion Tables data.", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://developers.google.com/fusiontables", + "protocol": "rest", + "baseUrl": "https://www.googleapis.com/fusiontables/v1/", + "basePath": "/fusiontables/v1/", + "rootUrl": "https://www.googleapis.com/", + "servicePath": "fusiontables/v1/", + "batchPath": "batch", + "parameters": { + "alt": { + "type": "string", + "description": "Data format for the response.", + "default": "json", + "enum": [ + "csv", + "json" + ], + "enumDescriptions": [ + "Responses with Content-Type of text/csv", + "Responses with Content-Type of application/json" + ], + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.", + "location": "query" + }, + "userIp": { + "type": "string", + "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.", + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/fusiontables": { + "description": "Manage your Fusion Tables" + }, + "https://www.googleapis.com/auth/fusiontables.readonly": { + "description": "View your Fusion Tables" + } + } + } + }, + "schemas": { + "Bucket": { + "id": "Bucket", + "type": "object", + "description": "Specifies the minimum and maximum values, the color, opacity, icon and weight of a bucket within a StyleSetting.", + "properties": { + "color": { + "type": "string", + "description": "Color of line or the interior of a polygon in #RRGGBB format." + }, + "icon": { + "type": "string", + "description": "Icon name used for a point." + }, + "max": { + "type": "number", + "description": "Maximum value in the selected column for a row to be styled according to the bucket color, opacity, icon, or weight.", + "format": "double" + }, + "min": { + "type": "number", + "description": "Minimum value in the selected column for a row to be styled according to the bucket color, opacity, icon, or weight.", + "format": "double" + }, + "opacity": { + "type": "number", + "description": "Opacity of the color: 0.0 (transparent) to 1.0 (opaque).", + "format": "double" + }, + "weight": { + "type": "integer", + "description": "Width of a line (in pixels).", + "format": "int32" + } + } + }, + "Column": { + "id": "Column", + "type": "object", + "description": "Specifies the id, name and type of a column in a table.", + "properties": { + "baseColumn": { + "type": "object", + "description": "Optional identifier of the base column. If present, this column is derived from the specified base column.", + "properties": { + "columnId": { + "type": "integer", + "description": "The id of the column in the base table from which this column is derived.", + "format": "int32" + }, + "tableIndex": { + "type": "integer", + "description": "Offset to the entry in the list of base tables in the table definition.", + "format": "int32" + } + } + }, + "columnId": { + "type": "integer", + "description": "Identifier for the column.", + "format": "int32" + }, + "description": { + "type": "string", + "description": "Optional column description." + }, + "graph_predicate": { + "type": "string", + "description": "Optional column predicate. Used to map table to graph data model (subject,predicate,object) See http://www.w3.org/TR/2014/REC-rdf11-concepts-20140225/#data-model" + }, + "kind": { + "type": "string", + "description": "Type name: a template for an individual column.", + "default": "fusiontables#column" + }, + "name": { + "type": "string", + "description": "Required name of the column.", + "annotations": { + "required": [ + "fusiontables.column.insert" + ] + } + }, + "type": { + "type": "string", + "description": "Required type of the column.", + "annotations": { + "required": [ + "fusiontables.column.insert" + ] + } + } + } + }, + "ColumnList": { + "id": "ColumnList", + "type": "object", + "description": "Represents a list of columns in a table.", + "properties": { + "items": { + "type": "array", + "description": "List of all requested columns.", + "items": { + "$ref": "Column" + } + }, + "kind": { + "type": "string", + "description": "Type name: a list of all columns.", + "default": "fusiontables#columnList" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result. No token is displayed if there are no more pages left." + }, + "totalItems": { + "type": "integer", + "description": "Total number of columns for the table.", + "format": "int32" + } + } + }, + "Geometry": { + "id": "Geometry", + "type": "object", + "description": "Represents a Geometry object.", + "properties": { + "geometries": { + "type": "array", + "description": "The list of geometries in this geometry collection.", + "items": { + "type": "any" + } + }, + "geometry": { + "type": "any" + }, + "type": { + "type": "string", + "description": "Type: A collection of geometries.", + "default": "GeometryCollection" + } + } + }, + "Import": { + "id": "Import", + "type": "object", + "description": "Represents an import request.", + "properties": { + "kind": { + "type": "string", + "description": "Type name: a template for an import request.", + "default": "fusiontables#import" + }, + "numRowsReceived": { + "type": "string", + "description": "The number of rows received from the import request.", + "format": "int64" + } + } + }, + "Line": { + "id": "Line", + "type": "object", + "description": "Represents a line geometry.", + "properties": { + "coordinates": { + "type": "array", + "description": "The coordinates that define the line.", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + } + }, + "type": { + "type": "string", + "description": "Type: A line geometry.", + "default": "LineString" + } + } + }, + "LineStyle": { + "id": "LineStyle", + "type": "object", + "description": "Represents a LineStyle within a StyleSetting", + "properties": { + "strokeColor": { + "type": "string", + "description": "Color of the line in #RRGGBB format." + }, + "strokeColorStyler": { + "$ref": "StyleFunction", + "description": "Column-value, gradient or buckets styler that is used to determine the line color and opacity." + }, + "strokeOpacity": { + "type": "number", + "description": "Opacity of the line : 0.0 (transparent) to 1.0 (opaque).", + "format": "double" + }, + "strokeWeight": { + "type": "integer", + "description": "Width of the line in pixels.", + "format": "int32" + }, + "strokeWeightStyler": { + "$ref": "StyleFunction", + "description": "Column-value or bucket styler that is used to determine the width of the line." + } + } + }, + "Point": { + "id": "Point", + "type": "object", + "description": "Represents a point object.", + "properties": { + "coordinates": { + "type": "array", + "description": "The coordinates that define the point.", + "items": { + "type": "number", + "format": "double" + } + }, + "type": { + "type": "string", + "description": "Point: A point geometry.", + "default": "Point" + } + } + }, + "PointStyle": { + "id": "PointStyle", + "type": "object", + "description": "Represents a PointStyle within a StyleSetting", + "properties": { + "iconName": { + "type": "string", + "description": "Name of the icon. Use values defined in http://www.google.com/fusiontables/DataSource?dsrcid=308519" + }, + "iconStyler": { + "$ref": "StyleFunction", + "description": "Column or a bucket value from which the icon name is to be determined." + } + } + }, + "Polygon": { + "id": "Polygon", + "type": "object", + "description": "Represents a polygon object.", + "properties": { + "coordinates": { + "type": "array", + "description": "The coordinates that define the polygon.", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + } + } + }, + "type": { + "type": "string", + "description": "Type: A polygon geometry.", + "default": "Polygon" + } + } + }, + "PolygonStyle": { + "id": "PolygonStyle", + "type": "object", + "description": "Represents a PolygonStyle within a StyleSetting", + "properties": { + "fillColor": { + "type": "string", + "description": "Color of the interior of the polygon in #RRGGBB format." + }, + "fillColorStyler": { + "$ref": "StyleFunction", + "description": "Column-value, gradient, or bucket styler that is used to determine the interior color and opacity of the polygon." + }, + "fillOpacity": { + "type": "number", + "description": "Opacity of the interior of the polygon: 0.0 (transparent) to 1.0 (opaque).", + "format": "double" + }, + "strokeColor": { + "type": "string", + "description": "Color of the polygon border in #RRGGBB format." + }, + "strokeColorStyler": { + "$ref": "StyleFunction", + "description": "Column-value, gradient or buckets styler that is used to determine the border color and opacity." + }, + "strokeOpacity": { + "type": "number", + "description": "Opacity of the polygon border: 0.0 (transparent) to 1.0 (opaque).", + "format": "double" + }, + "strokeWeight": { + "type": "integer", + "description": "Width of the polyon border in pixels.", + "format": "int32" + }, + "strokeWeightStyler": { + "$ref": "StyleFunction", + "description": "Column-value or bucket styler that is used to determine the width of the polygon border." + } + } + }, + "Sqlresponse": { + "id": "Sqlresponse", + "type": "object", + "description": "Represents a response to an sql statement.", + "properties": { + "columns": { + "type": "array", + "description": "Columns in the table.", + "items": { + "type": "string" + } + }, + "kind": { + "type": "string", + "description": "Type name: a template for an individual table.", + "default": "fusiontables#sqlresponse" + }, + "rows": { + "type": "array", + "description": "The rows in the table. For each cell we print out whatever cell value (e.g., numeric, string) exists. Thus it is important that each cell contains only one value.", + "items": { + "type": "array", + "items": { + "type": "any" + } + } + } + } + }, + "StyleFunction": { + "id": "StyleFunction", + "type": "object", + "description": "Represents a StyleFunction within a StyleSetting", + "properties": { + "buckets": { + "type": "array", + "description": "Bucket function that assigns a style based on the range a column value falls into.", + "items": { + "$ref": "Bucket" + } + }, + "columnName": { + "type": "string", + "description": "Name of the column whose value is used in the style.", + "annotations": { + "required": [ + "fusiontables.style.insert" + ] + } + }, + "gradient": { + "type": "object", + "description": "Gradient function that interpolates a range of colors based on column value.", + "properties": { + "colors": { + "type": "array", + "description": "Array with two or more colors.", + "items": { + "type": "object", + "properties": { + "color": { + "type": "string", + "description": "Color in #RRGGBB format." + }, + "opacity": { + "type": "number", + "description": "Opacity of the color: 0.0 (transparent) to 1.0 (opaque).", + "format": "double" + } + } + } + }, + "max": { + "type": "number", + "description": "Higher-end of the interpolation range: rows with this value will be assigned to colors[n-1].", + "format": "double" + }, + "min": { + "type": "number", + "description": "Lower-end of the interpolation range: rows with this value will be assigned to colors[0].", + "format": "double" + } + } + }, + "kind": { + "type": "string", + "description": "Stylers can be one of three kinds: \"fusiontables#fromColumn\" if the column value is to be used as is, i.e., the column values can have colors in #RRGGBBAA format or integer line widths or icon names; \"fusiontables#gradient\" if the styling of the row is to be based on applying the gradient function on the column value; or \"fusiontables#buckets\" if the styling is to based on the bucket into which the the column value falls." + } + } + }, + "StyleSetting": { + "id": "StyleSetting", + "type": "object", + "description": "Represents a complete StyleSettings object. The primary key is a combination of the tableId and a styleId.", + "properties": { + "kind": { + "type": "string", + "description": "Type name: an individual style setting. A StyleSetting contains the style defintions for points, lines, and polygons in a table. Since a table can have any one or all of them, a style definition can have point, line and polygon style definitions.", + "default": "fusiontables#styleSetting" + }, + "markerOptions": { + "$ref": "PointStyle", + "description": "Style definition for points in the table." + }, + "name": { + "type": "string", + "description": "Optional name for the style setting." + }, + "polygonOptions": { + "$ref": "PolygonStyle", + "description": "Style definition for polygons in the table." + }, + "polylineOptions": { + "$ref": "LineStyle", + "description": "Style definition for lines in the table." + }, + "styleId": { + "type": "integer", + "description": "Identifier for the style setting (unique only within tables).", + "format": "int32" + }, + "tableId": { + "type": "string", + "description": "Identifier for the table." + } + } + }, + "StyleSettingList": { + "id": "StyleSettingList", + "type": "object", + "description": "Represents a list of styles for a given table.", + "properties": { + "items": { + "type": "array", + "description": "All requested style settings.", + "items": { + "$ref": "StyleSetting" + } + }, + "kind": { + "type": "string", + "description": "Type name: in this case, a list of style settings.", + "default": "fusiontables#styleSettingList" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result. No token is displayed if there are no more pages left." + }, + "totalItems": { + "type": "integer", + "description": "Total number of styles for the table.", + "format": "int32" + } + } + }, + "Table": { + "id": "Table", + "type": "object", + "description": "Represents a table. Specifies the name, whether it is exportable, description, attribution, and attribution link.", + "properties": { + "attribution": { + "type": "string", + "description": "Optional attribution assigned to the table." + }, + "attributionLink": { + "type": "string", + "description": "Optional link for attribution." + }, + "baseTableIds": { + "type": "array", + "description": "Optional base table identifier if this table is a view or merged table.", + "items": { + "type": "string" + } + }, + "columns": { + "type": "array", + "description": "Columns in the table.", + "items": { + "$ref": "Column" + }, + "annotations": { + "required": [ + "fusiontables.table.insert", + "fusiontables.table.update" + ] + } + }, + "description": { + "type": "string", + "description": "Optional description assigned to the table." + }, + "isExportable": { + "type": "boolean", + "description": "Variable for whether table is exportable.", + "annotations": { + "required": [ + "fusiontables.table.insert", + "fusiontables.table.update" + ] + } + }, + "kind": { + "type": "string", + "description": "Type name: a template for an individual table.", + "default": "fusiontables#table" + }, + "name": { + "type": "string", + "description": "Name assigned to a table.", + "annotations": { + "required": [ + "fusiontables.table.insert", + "fusiontables.table.update" + ] + } + }, + "sql": { + "type": "string", + "description": "Optional sql that encodes the table definition for derived tables." + }, + "tableId": { + "type": "string", + "description": "Encrypted unique alphanumeric identifier for the table." + } + } + }, + "TableList": { + "id": "TableList", + "type": "object", + "description": "Represents a list of tables.", + "properties": { + "items": { + "type": "array", + "description": "List of all requested tables.", + "items": { + "$ref": "Table" + } + }, + "kind": { + "type": "string", + "description": "Type name: a list of all tables.", + "default": "fusiontables#tableList" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result. No token is displayed if there are no more pages left." + } + } + }, + "Task": { + "id": "Task", + "type": "object", + "description": "Specifies the identifier, name, and type of a task in a table.", + "properties": { + "kind": { + "type": "string", + "description": "Type of the resource. This is always \"fusiontables#task\".", + "default": "fusiontables#task" + }, + "progress": { + "type": "string", + "description": "An indication of task progress." + }, + "started": { + "type": "boolean", + "description": "false while the table is busy with some other task. true if this background task is currently running." + }, + "taskId": { + "type": "string", + "description": "Identifier for the task.", + "format": "int64" + }, + "type": { + "type": "string", + "description": "Type of background task. One of DELETE_ROWS Deletes one or more rows from the table. ADD_ROWS \"Adds one or more rows to a table. Includes importing data into a new table and importing more rows into an existing table. ADD_COLUMN Adds a new column to the table. CHANGE_TYPE Changes the type of a column." + } + } + }, + "TaskList": { + "id": "TaskList", + "type": "object", + "description": "Represents a list of tasks for a table.", + "properties": { + "items": { + "type": "array", + "description": "List of all requested tasks.", + "items": { + "$ref": "Task" + } + }, + "kind": { + "type": "string", + "description": "Type of the resource. This is always \"fusiontables#taskList\".", + "default": "fusiontables#taskList" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result. No token is displayed if there are no more pages left." + }, + "totalItems": { + "type": "integer", + "description": "Total number of tasks for the table.", + "format": "int32" + } + } + }, + "Template": { + "id": "Template", + "type": "object", + "description": "Represents the contents of InfoWindow templates.", + "properties": { + "automaticColumnNames": { + "type": "array", + "description": "List of columns from which the template is to be automatically constructed. Only one of body or automaticColumns can be specified.", + "items": { + "type": "string" + } + }, + "body": { + "type": "string", + "description": "Body of the template. It contains HTML with {column_name} to insert values from a particular column. The body is sanitized to remove certain tags, e.g., script. Only one of body or automaticColumns can be specified." + }, + "kind": { + "type": "string", + "description": "Type name: a template for the info window contents. The template can either include an HTML body or a list of columns from which the template is computed automatically.", + "default": "fusiontables#template" + }, + "name": { + "type": "string", + "description": "Optional name assigned to a template." + }, + "tableId": { + "type": "string", + "description": "Identifier for the table for which the template is defined." + }, + "templateId": { + "type": "integer", + "description": "Identifier for the template, unique within the context of a particular table.", + "format": "int32" + } + } + }, + "TemplateList": { + "id": "TemplateList", + "type": "object", + "description": "Represents a list of templates for a given table.", + "properties": { + "items": { + "type": "array", + "description": "List of all requested templates.", + "items": { + "$ref": "Template" + } + }, + "kind": { + "type": "string", + "description": "Type name: a list of all templates.", + "default": "fusiontables#templateList" + }, + "nextPageToken": { + "type": "string", + "description": "Token used to access the next page of this result. No token is displayed if there are no more pages left." + }, + "totalItems": { + "type": "integer", + "description": "Total number of templates for the table.", + "format": "int32" + } + } + } + }, + "resources": { + "column": { + "methods": { + "delete": { + "id": "fusiontables.column.delete", + "path": "tables/{tableId}/columns/{columnId}", + "httpMethod": "DELETE", + "description": "Deletes the column.", + "parameters": { + "columnId": { + "type": "string", + "description": "Name or identifier for the column being deleted.", + "required": true, + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table from which the column is being deleted.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "columnId" + ], + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "get": { + "id": "fusiontables.column.get", + "path": "tables/{tableId}/columns/{columnId}", + "httpMethod": "GET", + "description": "Retrieves a specific column by its id.", + "parameters": { + "columnId": { + "type": "string", + "description": "Name or identifier for the column that is being requested.", + "required": true, + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table to which the column belongs.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "columnId" + ], + "response": { + "$ref": "Column" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "insert": { + "id": "fusiontables.column.insert", + "path": "tables/{tableId}/columns", + "httpMethod": "POST", + "description": "Adds a new column to the table.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table for which a new column is being added.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "request": { + "$ref": "Column" + }, + "response": { + "$ref": "Column" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "list": { + "id": "fusiontables.column.list", + "path": "tables/{tableId}/columns", + "httpMethod": "GET", + "description": "Retrieves a list of columns.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of columns to return. Optional. Default is 5.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Continuation token specifying which result page to return. Optional.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "Table whose columns are being listed.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "ColumnList" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "patch": { + "id": "fusiontables.column.patch", + "path": "tables/{tableId}/columns/{columnId}", + "httpMethod": "PATCH", + "description": "Updates the name or type of an existing column. This method supports patch semantics.", + "parameters": { + "columnId": { + "type": "string", + "description": "Name or identifier for the column that is being updated.", + "required": true, + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table for which the column is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "columnId" + ], + "request": { + "$ref": "Column" + }, + "response": { + "$ref": "Column" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "update": { + "id": "fusiontables.column.update", + "path": "tables/{tableId}/columns/{columnId}", + "httpMethod": "PUT", + "description": "Updates the name or type of an existing column.", + "parameters": { + "columnId": { + "type": "string", + "description": "Name or identifier for the column that is being updated.", + "required": true, + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table for which the column is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "columnId" + ], + "request": { + "$ref": "Column" + }, + "response": { + "$ref": "Column" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + } + } + }, + "query": { + "methods": { + "sql": { + "id": "fusiontables.query.sql", + "path": "query", + "httpMethod": "POST", + "description": "Executes an SQL SELECT/INSERT/UPDATE/DELETE/SHOW/DESCRIBE/CREATE statement.", + "parameters": { + "hdrs": { + "type": "boolean", + "description": "Should column names be included (in the first row)?. Default is true.", + "location": "query" + }, + "sql": { + "type": "string", + "description": "An SQL SELECT/SHOW/DESCRIBE/INSERT/UPDATE/DELETE/CREATE statement.", + "required": true, + "location": "query" + }, + "typed": { + "type": "boolean", + "description": "Should typed values be returned in the (JSON) response -- numbers for numeric values and parsed geometries for KML values? Default is true.", + "location": "query" + } + }, + "parameterOrder": [ + "sql" + ], + "response": { + "$ref": "Sqlresponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ], + "supportsMediaDownload": true, + "useMediaDownloadService": true + }, + "sqlGet": { + "id": "fusiontables.query.sqlGet", + "path": "query", + "httpMethod": "GET", + "description": "Executes an SQL SELECT/SHOW/DESCRIBE statement.", + "parameters": { + "hdrs": { + "type": "boolean", + "description": "Should column names be included (in the first row)?. Default is true.", + "location": "query" + }, + "sql": { + "type": "string", + "description": "An SQL SELECT/SHOW/DESCRIBE statement.", + "required": true, + "location": "query" + }, + "typed": { + "type": "boolean", + "description": "Should typed values be returned in the (JSON) response -- numbers for numeric values and parsed geometries for KML values? Default is true.", + "location": "query" + } + }, + "parameterOrder": [ + "sql" + ], + "response": { + "$ref": "Sqlresponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ], + "supportsMediaDownload": true, + "useMediaDownloadService": true + } + } + }, + "style": { + "methods": { + "delete": { + "id": "fusiontables.style.delete", + "path": "tables/{tableId}/styles/{styleId}", + "httpMethod": "DELETE", + "description": "Deletes a style.", + "parameters": { + "styleId": { + "type": "integer", + "description": "Identifier (within a table) for the style being deleted", + "required": true, + "format": "int32", + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table from which the style is being deleted", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "styleId" + ], + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "get": { + "id": "fusiontables.style.get", + "path": "tables/{tableId}/styles/{styleId}", + "httpMethod": "GET", + "description": "Gets a specific style.", + "parameters": { + "styleId": { + "type": "integer", + "description": "Identifier (integer) for a specific style in a table", + "required": true, + "format": "int32", + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table to which the requested style belongs", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "styleId" + ], + "response": { + "$ref": "StyleSetting" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "insert": { + "id": "fusiontables.style.insert", + "path": "tables/{tableId}/styles", + "httpMethod": "POST", + "description": "Adds a new style for the table.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table for which a new style is being added", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "request": { + "$ref": "StyleSetting" + }, + "response": { + "$ref": "StyleSetting" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "list": { + "id": "fusiontables.style.list", + "path": "tables/{tableId}/styles", + "httpMethod": "GET", + "description": "Retrieves a list of styles.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of styles to return. Optional. Default is 5.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Continuation token specifying which result page to return. Optional.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "Table whose styles are being listed", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "StyleSettingList" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "patch": { + "id": "fusiontables.style.patch", + "path": "tables/{tableId}/styles/{styleId}", + "httpMethod": "PATCH", + "description": "Updates an existing style. This method supports patch semantics.", + "parameters": { + "styleId": { + "type": "integer", + "description": "Identifier (within a table) for the style being updated.", + "required": true, + "format": "int32", + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table whose style is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "styleId" + ], + "request": { + "$ref": "StyleSetting" + }, + "response": { + "$ref": "StyleSetting" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "update": { + "id": "fusiontables.style.update", + "path": "tables/{tableId}/styles/{styleId}", + "httpMethod": "PUT", + "description": "Updates an existing style.", + "parameters": { + "styleId": { + "type": "integer", + "description": "Identifier (within a table) for the style being updated.", + "required": true, + "format": "int32", + "location": "path" + }, + "tableId": { + "type": "string", + "description": "Table whose style is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "styleId" + ], + "request": { + "$ref": "StyleSetting" + }, + "response": { + "$ref": "StyleSetting" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + } + } + }, + "table": { + "methods": { + "copy": { + "id": "fusiontables.table.copy", + "path": "tables/{tableId}/copy", + "httpMethod": "POST", + "description": "Copies a table.", + "parameters": { + "copyPresentation": { + "type": "boolean", + "description": "Whether to also copy tabs, styles, and templates. Default is false.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "ID of the table that is being copied.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "delete": { + "id": "fusiontables.table.delete", + "path": "tables/{tableId}", + "httpMethod": "DELETE", + "description": "Deletes a table.", + "parameters": { + "tableId": { + "type": "string", + "description": "ID of the table that is being deleted.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "get": { + "id": "fusiontables.table.get", + "path": "tables/{tableId}", + "httpMethod": "GET", + "description": "Retrieves a specific table by its id.", + "parameters": { + "tableId": { + "type": "string", + "description": "Identifier(ID) for the table being requested.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "importRows": { + "id": "fusiontables.table.importRows", + "path": "tables/{tableId}/import", + "httpMethod": "POST", + "description": "Import more rows into a table.", + "parameters": { + "delimiter": { + "type": "string", + "description": "The delimiter used to separate cell values. This can only consist of a single character. Default is ','.", + "location": "query" + }, + "encoding": { + "type": "string", + "description": "The encoding of the content. Default is UTF-8. Use 'auto-detect' if you are unsure of the encoding.", + "location": "query" + }, + "endLine": { + "type": "integer", + "description": "The index of the last line from which to start importing, exclusive. Thus, the number of imported lines is endLine - startLine. If this parameter is not provided, the file will be imported until the last line of the file. If endLine is negative, then the imported content will exclude the last endLine lines. That is, if endline is negative, no line will be imported whose index is greater than N + endLine where N is the number of lines in the file, and the number of imported lines will be N + endLine - startLine.", + "format": "int32", + "location": "query" + }, + "isStrict": { + "type": "boolean", + "description": "Whether the CSV must have the same number of values for each row. If false, rows with fewer values will be padded with empty values. Default is true.", + "location": "query" + }, + "startLine": { + "type": "integer", + "description": "The index of the first line from which to start importing, inclusive. Default is 0.", + "format": "int32", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "The table into which new rows are being imported.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "Import" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ], + "supportsMediaUpload": true, + "mediaUpload": { + "accept": [ + "application/octet-stream" + ], + "maxSize": "250MB", + "protocols": { + "simple": { + "multipart": true, + "path": "/upload/fusiontables/v1/tables/{tableId}/import" + }, + "resumable": { + "multipart": true, + "path": "/resumable/upload/fusiontables/v1/tables/{tableId}/import" + } + } + } + }, + "importTable": { + "id": "fusiontables.table.importTable", + "path": "tables/import", + "httpMethod": "POST", + "description": "Import a new table.", + "parameters": { + "delimiter": { + "type": "string", + "description": "The delimiter used to separate cell values. This can only consist of a single character. Default is ','.", + "location": "query" + }, + "encoding": { + "type": "string", + "description": "The encoding of the content. Default is UTF-8. Use 'auto-detect' if you are unsure of the encoding.", + "location": "query" + }, + "name": { + "type": "string", + "description": "The name to be assigned to the new table.", + "required": true, + "location": "query" + } + }, + "parameterOrder": [ + "name" + ], + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ], + "supportsMediaUpload": true, + "mediaUpload": { + "accept": [ + "application/octet-stream" + ], + "maxSize": "250MB", + "protocols": { + "simple": { + "multipart": true, + "path": "/upload/fusiontables/v1/tables/import" + }, + "resumable": { + "multipart": true, + "path": "/resumable/upload/fusiontables/v1/tables/import" + } + } + } + }, + "insert": { + "id": "fusiontables.table.insert", + "path": "tables", + "httpMethod": "POST", + "description": "Creates a new table.", + "request": { + "$ref": "Table" + }, + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "list": { + "id": "fusiontables.table.list", + "path": "tables", + "httpMethod": "GET", + "description": "Retrieves a list of tables a user owns.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of styles to return. Optional. Default is 5.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Continuation token specifying which result page to return. Optional.", + "location": "query" + } + }, + "response": { + "$ref": "TableList" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "patch": { + "id": "fusiontables.table.patch", + "path": "tables/{tableId}", + "httpMethod": "PATCH", + "description": "Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated. This method supports patch semantics.", + "parameters": { + "replaceViewDefinition": { + "type": "boolean", + "description": "Should the view definition also be updated? The specified view definition replaces the existing one. Only a view can be updated with a new definition.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "ID of the table that is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "request": { + "$ref": "Table" + }, + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "update": { + "id": "fusiontables.table.update", + "path": "tables/{tableId}", + "httpMethod": "PUT", + "description": "Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated.", + "parameters": { + "replaceViewDefinition": { + "type": "boolean", + "description": "Should the view definition also be updated? The specified view definition replaces the existing one. Only a view can be updated with a new definition.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "ID of the table that is being updated.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "request": { + "$ref": "Table" + }, + "response": { + "$ref": "Table" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + } + } + }, + "task": { + "methods": { + "delete": { + "id": "fusiontables.task.delete", + "path": "tables/{tableId}/tasks/{taskId}", + "httpMethod": "DELETE", + "description": "Deletes the task, unless already started.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table from which the task is being deleted.", + "required": true, + "location": "path" + }, + "taskId": { + "type": "string", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "taskId" + ], + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "get": { + "id": "fusiontables.task.get", + "path": "tables/{tableId}/tasks/{taskId}", + "httpMethod": "GET", + "description": "Retrieves a specific task by its id.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table to which the task belongs.", + "required": true, + "location": "path" + }, + "taskId": { + "type": "string", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "taskId" + ], + "response": { + "$ref": "Task" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "list": { + "id": "fusiontables.task.list", + "path": "tables/{tableId}/tasks", + "httpMethod": "GET", + "description": "Retrieves a list of tasks.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of columns to return. Optional. Default is 5.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "location": "query" + }, + "startIndex": { + "type": "integer", + "format": "uint32", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "Table whose tasks are being listed.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "TaskList" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + } + } + }, + "template": { + "methods": { + "delete": { + "id": "fusiontables.template.delete", + "path": "tables/{tableId}/templates/{templateId}", + "httpMethod": "DELETE", + "description": "Deletes a template", + "parameters": { + "tableId": { + "type": "string", + "description": "Table from which the template is being deleted", + "required": true, + "location": "path" + }, + "templateId": { + "type": "integer", + "description": "Identifier for the template which is being deleted", + "required": true, + "format": "int32", + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "templateId" + ], + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "get": { + "id": "fusiontables.template.get", + "path": "tables/{tableId}/templates/{templateId}", + "httpMethod": "GET", + "description": "Retrieves a specific template by its id", + "parameters": { + "tableId": { + "type": "string", + "description": "Table to which the template belongs", + "required": true, + "location": "path" + }, + "templateId": { + "type": "integer", + "description": "Identifier for the template that is being requested", + "required": true, + "format": "int32", + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "templateId" + ], + "response": { + "$ref": "Template" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "insert": { + "id": "fusiontables.template.insert", + "path": "tables/{tableId}/templates", + "httpMethod": "POST", + "description": "Creates a new template for the table.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table for which a new template is being created", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "request": { + "$ref": "Template" + }, + "response": { + "$ref": "Template" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "list": { + "id": "fusiontables.template.list", + "path": "tables/{tableId}/templates", + "httpMethod": "GET", + "description": "Retrieves a list of templates.", + "parameters": { + "maxResults": { + "type": "integer", + "description": "Maximum number of templates to return. Optional. Default is 5.", + "format": "uint32", + "minimum": "0", + "location": "query" + }, + "pageToken": { + "type": "string", + "description": "Continuation token specifying which results page to return. Optional.", + "location": "query" + }, + "tableId": { + "type": "string", + "description": "Identifier for the table whose templates are being requested", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "tableId" + ], + "response": { + "$ref": "TemplateList" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables", + "https://www.googleapis.com/auth/fusiontables.readonly" + ] + }, + "patch": { + "id": "fusiontables.template.patch", + "path": "tables/{tableId}/templates/{templateId}", + "httpMethod": "PATCH", + "description": "Updates an existing template. This method supports patch semantics.", + "parameters": { + "tableId": { + "type": "string", + "description": "Table to which the updated template belongs", + "required": true, + "location": "path" + }, + "templateId": { + "type": "integer", + "description": "Identifier for the template that is being updated", + "required": true, + "format": "int32", + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "templateId" + ], + "request": { + "$ref": "Template" + }, + "response": { + "$ref": "Template" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + }, + "update": { + "id": "fusiontables.template.update", + "path": "tables/{tableId}/templates/{templateId}", + "httpMethod": "PUT", + "description": "Updates an existing template", + "parameters": { + "tableId": { + "type": "string", + "description": "Table to which the updated template belongs", + "required": true, + "location": "path" + }, + "templateId": { + "type": "integer", + "description": "Identifier for the template that is being updated", + "required": true, + "format": "int32", + "location": "path" + } + }, + "parameterOrder": [ + "tableId", + "templateId" + ], + "request": { + "$ref": "Template" + }, + "response": { + "$ref": "Template" + }, + "scopes": [ + "https://www.googleapis.com/auth/fusiontables" + ] + } + } + } + } +} diff --git a/samples/fusiontables_sample/fusiontables_v1/__init__.py b/samples/fusiontables_sample/fusiontables_v1/__init__.py new file mode 100644 index 0000000..2816da8 --- /dev/null +++ b/samples/fusiontables_sample/fusiontables_v1/__init__.py @@ -0,0 +1,5 @@ +"""Package marker file.""" + +import pkgutil + +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1.py new file mode 100644 index 0000000..8bb2c8a --- /dev/null +++ b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1.py @@ -0,0 +1,1797 @@ +#!/usr/bin/env python +"""CLI for fusiontables, version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. + +import code +import os +import platform +import sys + +from apitools.base.protorpclite import message_types +from apitools.base.protorpclite import messages + +from google.apputils import appcommands +import gflags as flags + +import apitools.base.py as apitools_base +from apitools.base.py import cli as apitools_base_cli +import fusiontables_v1_client as client_lib +import fusiontables_v1_messages as messages + + +def _DeclareFusiontablesFlags(): + """Declare global flags in an idempotent way.""" + if 'api_endpoint' in flags.FLAGS: + return + flags.DEFINE_string( + 'api_endpoint', + u'https://www.googleapis.com/fusiontables/v1/', + 'URL of the API endpoint to use.', + short_name='fusiontables_url') + flags.DEFINE_string( + 'history_file', + u'~/.fusiontables.v1.history', + 'File with interactive shell history.') + flags.DEFINE_multistring( + 'add_header', [], + 'Additional http headers (as key=value strings). ' + 'Can be specified multiple times.') + flags.DEFINE_string( + 'service_account_json_keyfile', '', + 'Filename for a JSON service account key downloaded' + ' from the Developer Console.') + flags.DEFINE_enum( + 'alt', + u'json', + [u'csv', u'json'], + u'Data format for the response.') + flags.DEFINE_string( + 'fields', + None, + u'Selector specifying which fields to include in a partial response.') + flags.DEFINE_string( + 'key', + None, + u'API key. Your API key identifies your project and provides you with ' + u'API access, quota, and reports. Required unless you provide an OAuth ' + u'2.0 token.') + flags.DEFINE_string( + 'oauth_token', + None, + u'OAuth 2.0 token for the current user.') + flags.DEFINE_boolean( + 'prettyPrint', + 'True', + u'Returns response with indentations and line breaks.') + flags.DEFINE_string( + 'quotaUser', + None, + u'Available to use for quota purposes for server-side applications. Can' + u' be any arbitrary string assigned to a user, but should not exceed 40' + u' characters. Overrides userIp if both are provided.') + flags.DEFINE_string( + 'trace', + None, + 'A tracing token of the form "token:" to include in api ' + 'requests.') + flags.DEFINE_string( + 'userIp', + None, + u'IP address of the site where the request originates. Use this if you ' + u'want to enforce per-user limits.') + + +FLAGS = flags.FLAGS +apitools_base_cli.DeclareBaseFlags() +_DeclareFusiontablesFlags() + + +def GetGlobalParamsFromFlags(): + """Return a StandardQueryParameters based on flags.""" + result = messages.StandardQueryParameters() + if FLAGS['alt'].present: + result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt) + if FLAGS['fields'].present: + result.fields = FLAGS.fields.decode('utf8') + if FLAGS['key'].present: + result.key = FLAGS.key.decode('utf8') + if FLAGS['oauth_token'].present: + result.oauth_token = FLAGS.oauth_token.decode('utf8') + if FLAGS['prettyPrint'].present: + result.prettyPrint = FLAGS.prettyPrint + if FLAGS['quotaUser'].present: + result.quotaUser = FLAGS.quotaUser.decode('utf8') + if FLAGS['trace'].present: + result.trace = FLAGS.trace.decode('utf8') + if FLAGS['userIp'].present: + result.userIp = FLAGS.userIp.decode('utf8') + return result + + +def GetClientFromFlags(): + """Return a client object, configured from flags.""" + log_request = FLAGS.log_request or FLAGS.log_request_response + log_response = FLAGS.log_response or FLAGS.log_request_response + api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint) + additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header) + credentials_args = { + 'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile) + } + try: + client = client_lib.FusiontablesV1( + api_endpoint, log_request=log_request, + log_response=log_response, + credentials_args=credentials_args, + additional_http_headers=additional_http_headers) + except apitools_base.CredentialsError as e: + print 'Error creating credentials: %s' % e + sys.exit(1) + return client + + +class PyShell(appcommands.Cmd): + + def Run(self, _): + """Run an interactive python shell with the client.""" + client = GetClientFromFlags() + params = GetGlobalParamsFromFlags() + for field in params.all_fields(): + value = params.get_assigned_value(field.name) + if value != field.default: + client.AddGlobalParam(field.name, value) + banner = """ + == fusiontables interactive console == + client: a fusiontables client + apitools_base: base apitools module + messages: the generated messages module + """ + local_vars = { + 'apitools_base': apitools_base, + 'client': client, + 'client_lib': client_lib, + 'messages': messages, + } + if platform.system() == 'Linux': + console = apitools_base_cli.ConsoleWithReadline( + local_vars, histfile=FLAGS.history_file) + else: + console = code.InteractiveConsole(local_vars) + try: + console.interact(banner) + except SystemExit as e: + return e.code + + +class ColumnDelete(apitools_base_cli.NewCmd): + """Command wrapping column.Delete.""" + + usage = """column_delete """ + + def __init__(self, name, fv): + super(ColumnDelete, self).__init__(name, fv) + + def RunWithArgs(self, tableId, columnId): + """Deletes the column. + + Args: + tableId: Table from which the column is being deleted. + columnId: Name or identifier for the column being deleted. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnDeleteRequest( + tableId=tableId.decode('utf8'), + columnId=columnId.decode('utf8'), + ) + result = client.column.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ColumnGet(apitools_base_cli.NewCmd): + """Command wrapping column.Get.""" + + usage = """column_get """ + + def __init__(self, name, fv): + super(ColumnGet, self).__init__(name, fv) + + def RunWithArgs(self, tableId, columnId): + """Retrieves a specific column by its id. + + Args: + tableId: Table to which the column belongs. + columnId: Name or identifier for the column that is being requested. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnGetRequest( + tableId=tableId.decode('utf8'), + columnId=columnId.decode('utf8'), + ) + result = client.column.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ColumnInsert(apitools_base_cli.NewCmd): + """Command wrapping column.Insert.""" + + usage = """column_insert """ + + def __init__(self, name, fv): + super(ColumnInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'column', + None, + u'A Column resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Adds a new column to the table. + + Args: + tableId: Table for which a new column is being added. + + Flags: + column: A Column resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnInsertRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['column'].present: + request.column = apitools_base.JsonToMessage(messages.Column, FLAGS.column) + result = client.column.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ColumnList(apitools_base_cli.NewCmd): + """Command wrapping column.List.""" + + usage = """column_list """ + + def __init__(self, name, fv): + super(ColumnList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of columns to return. Optional. Default is 5.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Continuation token specifying which result page to return. ' + u'Optional.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Retrieves a list of columns. + + Args: + tableId: Table whose columns are being listed. + + Flags: + maxResults: Maximum number of columns to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnListRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.column.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ColumnPatch(apitools_base_cli.NewCmd): + """Command wrapping column.Patch.""" + + usage = """column_patch """ + + def __init__(self, name, fv): + super(ColumnPatch, self).__init__(name, fv) + flags.DEFINE_string( + 'column', + None, + u'A Column resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, tableId, columnId): + """Updates the name or type of an existing column. This method supports + patch semantics. + + Args: + tableId: Table for which the column is being updated. + columnId: Name or identifier for the column that is being updated. + + Flags: + column: A Column resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnPatchRequest( + tableId=tableId.decode('utf8'), + columnId=columnId.decode('utf8'), + ) + if FLAGS['column'].present: + request.column = apitools_base.JsonToMessage(messages.Column, FLAGS.column) + result = client.column.Patch( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class ColumnUpdate(apitools_base_cli.NewCmd): + """Command wrapping column.Update.""" + + usage = """column_update """ + + def __init__(self, name, fv): + super(ColumnUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'column', + None, + u'A Column resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, tableId, columnId): + """Updates the name or type of an existing column. + + Args: + tableId: Table for which the column is being updated. + columnId: Name or identifier for the column that is being updated. + + Flags: + column: A Column resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesColumnUpdateRequest( + tableId=tableId.decode('utf8'), + columnId=columnId.decode('utf8'), + ) + if FLAGS['column'].present: + request.column = apitools_base.JsonToMessage(messages.Column, FLAGS.column) + result = client.column.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class QuerySql(apitools_base_cli.NewCmd): + """Command wrapping query.Sql.""" + + usage = """query_sql """ + + def __init__(self, name, fv): + super(QuerySql, self).__init__(name, fv) + flags.DEFINE_boolean( + 'hdrs', + None, + u'Should column names be included (in the first row)?. Default is ' + u'true.', + flag_values=fv) + flags.DEFINE_boolean( + 'typed', + None, + u'Should typed values be returned in the (JSON) response -- numbers ' + u'for numeric values and parsed geometries for KML values? Default is' + u' true.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, sql): + """Executes an SQL SELECT/INSERT/UPDATE/DELETE/SHOW/DESCRIBE/CREATE + statement. + + Args: + sql: An SQL SELECT/SHOW/DESCRIBE/INSERT/UPDATE/DELETE/CREATE statement. + + Flags: + hdrs: Should column names be included (in the first row)?. Default is + true. + typed: Should typed values be returned in the (JSON) response -- numbers + for numeric values and parsed geometries for KML values? Default is + true. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesQuerySqlRequest( + sql=sql.decode('utf8'), + ) + if FLAGS['hdrs'].present: + request.hdrs = FLAGS.hdrs + if FLAGS['typed'].present: + request.typed = FLAGS.typed + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.query.Sql( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) + + +class QuerySqlGet(apitools_base_cli.NewCmd): + """Command wrapping query.SqlGet.""" + + usage = """query_sqlGet """ + + def __init__(self, name, fv): + super(QuerySqlGet, self).__init__(name, fv) + flags.DEFINE_boolean( + 'hdrs', + None, + u'Should column names be included (in the first row)?. Default is ' + u'true.', + flag_values=fv) + flags.DEFINE_boolean( + 'typed', + None, + u'Should typed values be returned in the (JSON) response -- numbers ' + u'for numeric values and parsed geometries for KML values? Default is' + u' true.', + flag_values=fv) + flags.DEFINE_string( + 'download_filename', + '', + 'Filename to use for download.', + flag_values=fv) + flags.DEFINE_boolean( + 'overwrite', + 'False', + 'If True, overwrite the existing file when downloading.', + flag_values=fv) + + def RunWithArgs(self, sql): + """Executes an SQL SELECT/SHOW/DESCRIBE statement. + + Args: + sql: An SQL SELECT/SHOW/DESCRIBE statement. + + Flags: + hdrs: Should column names be included (in the first row)?. Default is + true. + typed: Should typed values be returned in the (JSON) response -- numbers + for numeric values and parsed geometries for KML values? Default is + true. + download_filename: Filename to use for download. + overwrite: If True, overwrite the existing file when downloading. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesQuerySqlGetRequest( + sql=sql.decode('utf8'), + ) + if FLAGS['hdrs'].present: + request.hdrs = FLAGS.hdrs + if FLAGS['typed'].present: + request.typed = FLAGS.typed + download = None + if FLAGS.download_filename: + download = apitools_base.Download.FromFile(FLAGS.download_filename, overwrite=FLAGS.overwrite, + progress_callback=apitools_base.DownloadProgressPrinter, + finish_callback=apitools_base.DownloadCompletePrinter) + result = client.query.SqlGet( + request, global_params=global_params, download=download) + print apitools_base_cli.FormatOutput(result) + + +class StyleDelete(apitools_base_cli.NewCmd): + """Command wrapping style.Delete.""" + + usage = """style_delete """ + + def __init__(self, name, fv): + super(StyleDelete, self).__init__(name, fv) + + def RunWithArgs(self, tableId, styleId): + """Deletes a style. + + Args: + tableId: Table from which the style is being deleted + styleId: Identifier (within a table) for the style being deleted + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesStyleDeleteRequest( + tableId=tableId.decode('utf8'), + styleId=styleId, + ) + result = client.style.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class StyleGet(apitools_base_cli.NewCmd): + """Command wrapping style.Get.""" + + usage = """style_get """ + + def __init__(self, name, fv): + super(StyleGet, self).__init__(name, fv) + + def RunWithArgs(self, tableId, styleId): + """Gets a specific style. + + Args: + tableId: Table to which the requested style belongs + styleId: Identifier (integer) for a specific style in a table + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesStyleGetRequest( + tableId=tableId.decode('utf8'), + styleId=styleId, + ) + result = client.style.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class StyleInsert(apitools_base_cli.NewCmd): + """Command wrapping style.Insert.""" + + usage = """style_insert """ + + def __init__(self, name, fv): + super(StyleInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#styleSetting', + u'Type name: an individual style setting. A StyleSetting contains the' + u' style defintions for points, lines, and polygons in a table. Since' + u' a table can have any one or all of them, a style definition can ' + u'have point, line and polygon style definitions.', + flag_values=fv) + flags.DEFINE_string( + 'markerOptions', + None, + u'Style definition for points in the table.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name for the style setting.', + flag_values=fv) + flags.DEFINE_string( + 'polygonOptions', + None, + u'Style definition for polygons in the table.', + flag_values=fv) + flags.DEFINE_string( + 'polylineOptions', + None, + u'Style definition for lines in the table.', + flag_values=fv) + flags.DEFINE_integer( + 'styleId', + None, + u'Identifier for the style setting (unique only within tables).', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Adds a new style for the table. + + Args: + tableId: Identifier for the table. + + Flags: + kind: Type name: an individual style setting. A StyleSetting contains + the style defintions for points, lines, and polygons in a table. Since + a table can have any one or all of them, a style definition can have + point, line and polygon style definitions. + markerOptions: Style definition for points in the table. + name: Optional name for the style setting. + polygonOptions: Style definition for polygons in the table. + polylineOptions: Style definition for lines in the table. + styleId: Identifier for the style setting (unique only within tables). + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StyleSetting( + tableId=tableId.decode('utf8'), + ) + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['markerOptions'].present: + request.markerOptions = apitools_base.JsonToMessage(messages.PointStyle, FLAGS.markerOptions) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['polygonOptions'].present: + request.polygonOptions = apitools_base.JsonToMessage(messages.PolygonStyle, FLAGS.polygonOptions) + if FLAGS['polylineOptions'].present: + request.polylineOptions = apitools_base.JsonToMessage(messages.LineStyle, FLAGS.polylineOptions) + if FLAGS['styleId'].present: + request.styleId = FLAGS.styleId + result = client.style.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class StyleList(apitools_base_cli.NewCmd): + """Command wrapping style.List.""" + + usage = """style_list """ + + def __init__(self, name, fv): + super(StyleList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of styles to return. Optional. Default is 5.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Continuation token specifying which result page to return. ' + u'Optional.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Retrieves a list of styles. + + Args: + tableId: Table whose styles are being listed + + Flags: + maxResults: Maximum number of styles to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesStyleListRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.style.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class StylePatch(apitools_base_cli.NewCmd): + """Command wrapping style.Patch.""" + + usage = """style_patch """ + + def __init__(self, name, fv): + super(StylePatch, self).__init__(name, fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#styleSetting', + u'Type name: an individual style setting. A StyleSetting contains the' + u' style defintions for points, lines, and polygons in a table. Since' + u' a table can have any one or all of them, a style definition can ' + u'have point, line and polygon style definitions.', + flag_values=fv) + flags.DEFINE_string( + 'markerOptions', + None, + u'Style definition for points in the table.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name for the style setting.', + flag_values=fv) + flags.DEFINE_string( + 'polygonOptions', + None, + u'Style definition for polygons in the table.', + flag_values=fv) + flags.DEFINE_string( + 'polylineOptions', + None, + u'Style definition for lines in the table.', + flag_values=fv) + + def RunWithArgs(self, tableId, styleId): + """Updates an existing style. This method supports patch semantics. + + Args: + tableId: Identifier for the table. + styleId: Identifier for the style setting (unique only within tables). + + Flags: + kind: Type name: an individual style setting. A StyleSetting contains + the style defintions for points, lines, and polygons in a table. Since + a table can have any one or all of them, a style definition can have + point, line and polygon style definitions. + markerOptions: Style definition for points in the table. + name: Optional name for the style setting. + polygonOptions: Style definition for polygons in the table. + polylineOptions: Style definition for lines in the table. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StyleSetting( + tableId=tableId.decode('utf8'), + styleId=styleId, + ) + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['markerOptions'].present: + request.markerOptions = apitools_base.JsonToMessage(messages.PointStyle, FLAGS.markerOptions) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['polygonOptions'].present: + request.polygonOptions = apitools_base.JsonToMessage(messages.PolygonStyle, FLAGS.polygonOptions) + if FLAGS['polylineOptions'].present: + request.polylineOptions = apitools_base.JsonToMessage(messages.LineStyle, FLAGS.polylineOptions) + result = client.style.Patch( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class StyleUpdate(apitools_base_cli.NewCmd): + """Command wrapping style.Update.""" + + usage = """style_update """ + + def __init__(self, name, fv): + super(StyleUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#styleSetting', + u'Type name: an individual style setting. A StyleSetting contains the' + u' style defintions for points, lines, and polygons in a table. Since' + u' a table can have any one or all of them, a style definition can ' + u'have point, line and polygon style definitions.', + flag_values=fv) + flags.DEFINE_string( + 'markerOptions', + None, + u'Style definition for points in the table.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name for the style setting.', + flag_values=fv) + flags.DEFINE_string( + 'polygonOptions', + None, + u'Style definition for polygons in the table.', + flag_values=fv) + flags.DEFINE_string( + 'polylineOptions', + None, + u'Style definition for lines in the table.', + flag_values=fv) + + def RunWithArgs(self, tableId, styleId): + """Updates an existing style. + + Args: + tableId: Identifier for the table. + styleId: Identifier for the style setting (unique only within tables). + + Flags: + kind: Type name: an individual style setting. A StyleSetting contains + the style defintions for points, lines, and polygons in a table. Since + a table can have any one or all of them, a style definition can have + point, line and polygon style definitions. + markerOptions: Style definition for points in the table. + name: Optional name for the style setting. + polygonOptions: Style definition for polygons in the table. + polylineOptions: Style definition for lines in the table. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.StyleSetting( + tableId=tableId.decode('utf8'), + styleId=styleId, + ) + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['markerOptions'].present: + request.markerOptions = apitools_base.JsonToMessage(messages.PointStyle, FLAGS.markerOptions) + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['polygonOptions'].present: + request.polygonOptions = apitools_base.JsonToMessage(messages.PolygonStyle, FLAGS.polygonOptions) + if FLAGS['polylineOptions'].present: + request.polylineOptions = apitools_base.JsonToMessage(messages.LineStyle, FLAGS.polylineOptions) + result = client.style.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableCopy(apitools_base_cli.NewCmd): + """Command wrapping table.Copy.""" + + usage = """table_copy """ + + def __init__(self, name, fv): + super(TableCopy, self).__init__(name, fv) + flags.DEFINE_boolean( + 'copyPresentation', + None, + u'Whether to also copy tabs, styles, and templates. Default is false.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Copies a table. + + Args: + tableId: ID of the table that is being copied. + + Flags: + copyPresentation: Whether to also copy tabs, styles, and templates. + Default is false. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableCopyRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['copyPresentation'].present: + request.copyPresentation = FLAGS.copyPresentation + result = client.table.Copy( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableDelete(apitools_base_cli.NewCmd): + """Command wrapping table.Delete.""" + + usage = """table_delete """ + + def __init__(self, name, fv): + super(TableDelete, self).__init__(name, fv) + + def RunWithArgs(self, tableId): + """Deletes a table. + + Args: + tableId: ID of the table that is being deleted. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableDeleteRequest( + tableId=tableId.decode('utf8'), + ) + result = client.table.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableGet(apitools_base_cli.NewCmd): + """Command wrapping table.Get.""" + + usage = """table_get """ + + def __init__(self, name, fv): + super(TableGet, self).__init__(name, fv) + + def RunWithArgs(self, tableId): + """Retrieves a specific table by its id. + + Args: + tableId: Identifier(ID) for the table being requested. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableGetRequest( + tableId=tableId.decode('utf8'), + ) + result = client.table.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableImportRows(apitools_base_cli.NewCmd): + """Command wrapping table.ImportRows.""" + + usage = """table_importRows """ + + def __init__(self, name, fv): + super(TableImportRows, self).__init__(name, fv) + flags.DEFINE_string( + 'delimiter', + None, + u'The delimiter used to separate cell values. This can only consist ' + u"of a single character. Default is ','.", + flag_values=fv) + flags.DEFINE_string( + 'encoding', + None, + u"The encoding of the content. Default is UTF-8. Use 'auto-detect' if" + u' you are unsure of the encoding.', + flag_values=fv) + flags.DEFINE_integer( + 'endLine', + None, + u'The index of the last line from which to start importing, ' + u'exclusive. Thus, the number of imported lines is endLine - ' + u'startLine. If this parameter is not provided, the file will be ' + u'imported until the last line of the file. If endLine is negative, ' + u'then the imported content will exclude the last endLine lines. That' + u' is, if endline is negative, no line will be imported whose index ' + u'is greater than N + endLine where N is the number of lines in the ' + u'file, and the number of imported lines will be N + endLine - ' + u'startLine.', + flag_values=fv) + flags.DEFINE_boolean( + 'isStrict', + None, + u'Whether the CSV must have the same number of values for each row. ' + u'If false, rows with fewer values will be padded with empty values. ' + u'Default is true.', + flag_values=fv) + flags.DEFINE_integer( + 'startLine', + None, + u'The index of the first line from which to start importing, ' + u'inclusive. Default is 0.', + flag_values=fv) + flags.DEFINE_string( + 'upload_filename', + '', + 'Filename to use for upload.', + flag_values=fv) + flags.DEFINE_string( + 'upload_mime_type', + '', + 'MIME type to use for the upload. Only needed if the extension on ' + '--upload_filename does not determine the correct (or any) MIME ' + 'type.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Import more rows into a table. + + Args: + tableId: The table into which new rows are being imported. + + Flags: + delimiter: The delimiter used to separate cell values. This can only + consist of a single character. Default is ','. + encoding: The encoding of the content. Default is UTF-8. Use 'auto- + detect' if you are unsure of the encoding. + endLine: The index of the last line from which to start importing, + exclusive. Thus, the number of imported lines is endLine - startLine. + If this parameter is not provided, the file will be imported until the + last line of the file. If endLine is negative, then the imported + content will exclude the last endLine lines. That is, if endline is + negative, no line will be imported whose index is greater than N + + endLine where N is the number of lines in the file, and the number of + imported lines will be N + endLine - startLine. + isStrict: Whether the CSV must have the same number of values for each + row. If false, rows with fewer values will be padded with empty + values. Default is true. + startLine: The index of the first line from which to start importing, + inclusive. Default is 0. + upload_filename: Filename to use for upload. + upload_mime_type: MIME type to use for the upload. Only needed if the + extension on --upload_filename does not determine the correct (or any) + MIME type. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableImportRowsRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['encoding'].present: + request.encoding = FLAGS.encoding.decode('utf8') + if FLAGS['endLine'].present: + request.endLine = FLAGS.endLine + if FLAGS['isStrict'].present: + request.isStrict = FLAGS.isStrict + if FLAGS['startLine'].present: + request.startLine = FLAGS.startLine + upload = None + if FLAGS.upload_filename: + upload = apitools_base.Upload.FromFile( + FLAGS.upload_filename, FLAGS.upload_mime_type, + progress_callback=apitools_base.UploadProgressPrinter, + finish_callback=apitools_base.UploadCompletePrinter) + result = client.table.ImportRows( + request, global_params=global_params, upload=upload) + print apitools_base_cli.FormatOutput(result) + + +class TableImportTable(apitools_base_cli.NewCmd): + """Command wrapping table.ImportTable.""" + + usage = """table_importTable """ + + def __init__(self, name, fv): + super(TableImportTable, self).__init__(name, fv) + flags.DEFINE_string( + 'delimiter', + None, + u'The delimiter used to separate cell values. This can only consist ' + u"of a single character. Default is ','.", + flag_values=fv) + flags.DEFINE_string( + 'encoding', + None, + u"The encoding of the content. Default is UTF-8. Use 'auto-detect' if" + u' you are unsure of the encoding.', + flag_values=fv) + flags.DEFINE_string( + 'upload_filename', + '', + 'Filename to use for upload.', + flag_values=fv) + flags.DEFINE_string( + 'upload_mime_type', + '', + 'MIME type to use for the upload. Only needed if the extension on ' + '--upload_filename does not determine the correct (or any) MIME ' + 'type.', + flag_values=fv) + + def RunWithArgs(self, name): + """Import a new table. + + Args: + name: The name to be assigned to the new table. + + Flags: + delimiter: The delimiter used to separate cell values. This can only + consist of a single character. Default is ','. + encoding: The encoding of the content. Default is UTF-8. Use 'auto- + detect' if you are unsure of the encoding. + upload_filename: Filename to use for upload. + upload_mime_type: MIME type to use for the upload. Only needed if the + extension on --upload_filename does not determine the correct (or any) + MIME type. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableImportTableRequest( + name=name.decode('utf8'), + ) + if FLAGS['delimiter'].present: + request.delimiter = FLAGS.delimiter.decode('utf8') + if FLAGS['encoding'].present: + request.encoding = FLAGS.encoding.decode('utf8') + upload = None + if FLAGS.upload_filename: + upload = apitools_base.Upload.FromFile( + FLAGS.upload_filename, FLAGS.upload_mime_type, + progress_callback=apitools_base.UploadProgressPrinter, + finish_callback=apitools_base.UploadCompletePrinter) + result = client.table.ImportTable( + request, global_params=global_params, upload=upload) + print apitools_base_cli.FormatOutput(result) + + +class TableInsert(apitools_base_cli.NewCmd): + """Command wrapping table.Insert.""" + + usage = """table_insert""" + + def __init__(self, name, fv): + super(TableInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'attribution', + None, + u'Optional attribution assigned to the table.', + flag_values=fv) + flags.DEFINE_string( + 'attributionLink', + None, + u'Optional link for attribution.', + flag_values=fv) + flags.DEFINE_string( + 'baseTableIds', + None, + u'Optional base table identifier if this table is a view or merged ' + u'table.', + flag_values=fv) + flags.DEFINE_string( + 'columns', + None, + u'Columns in the table.', + flag_values=fv) + flags.DEFINE_string( + 'description', + None, + u'Optional description assigned to the table.', + flag_values=fv) + flags.DEFINE_boolean( + 'isExportable', + None, + u'Variable for whether table is exportable.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#table', + u'Type name: a template for an individual table.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Name assigned to a table.', + flag_values=fv) + flags.DEFINE_string( + 'sql', + None, + u'Optional sql that encodes the table definition for derived tables.', + flag_values=fv) + flags.DEFINE_string( + 'tableId', + None, + u'Encrypted unique alphanumeric identifier for the table.', + flag_values=fv) + + def RunWithArgs(self): + """Creates a new table. + + Flags: + attribution: Optional attribution assigned to the table. + attributionLink: Optional link for attribution. + baseTableIds: Optional base table identifier if this table is a view or + merged table. + columns: Columns in the table. + description: Optional description assigned to the table. + isExportable: Variable for whether table is exportable. + kind: Type name: a template for an individual table. + name: Name assigned to a table. + sql: Optional sql that encodes the table definition for derived tables. + tableId: Encrypted unique alphanumeric identifier for the table. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Table( + ) + if FLAGS['attribution'].present: + request.attribution = FLAGS.attribution.decode('utf8') + if FLAGS['attributionLink'].present: + request.attributionLink = FLAGS.attributionLink.decode('utf8') + if FLAGS['baseTableIds'].present: + request.baseTableIds = [x.decode('utf8') for x in FLAGS.baseTableIds] + if FLAGS['columns'].present: + request.columns = [apitools_base.JsonToMessage(messages.Column, x) for x in FLAGS.columns] + if FLAGS['description'].present: + request.description = FLAGS.description.decode('utf8') + if FLAGS['isExportable'].present: + request.isExportable = FLAGS.isExportable + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['sql'].present: + request.sql = FLAGS.sql.decode('utf8') + if FLAGS['tableId'].present: + request.tableId = FLAGS.tableId.decode('utf8') + result = client.table.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableList(apitools_base_cli.NewCmd): + """Command wrapping table.List.""" + + usage = """table_list""" + + def __init__(self, name, fv): + super(TableList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of styles to return. Optional. Default is 5.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Continuation token specifying which result page to return. ' + u'Optional.', + flag_values=fv) + + def RunWithArgs(self): + """Retrieves a list of tables a user owns. + + Flags: + maxResults: Maximum number of styles to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableListRequest( + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.table.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TablePatch(apitools_base_cli.NewCmd): + """Command wrapping table.Patch.""" + + usage = """table_patch """ + + def __init__(self, name, fv): + super(TablePatch, self).__init__(name, fv) + flags.DEFINE_boolean( + 'replaceViewDefinition', + None, + u'Should the view definition also be updated? The specified view ' + u'definition replaces the existing one. Only a view can be updated ' + u'with a new definition.', + flag_values=fv) + flags.DEFINE_string( + 'table', + None, + u'A Table resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Updates an existing table. Unless explicitly requested, only the name, + description, and attribution will be updated. This method supports patch + semantics. + + Args: + tableId: ID of the table that is being updated. + + Flags: + replaceViewDefinition: Should the view definition also be updated? The + specified view definition replaces the existing one. Only a view can + be updated with a new definition. + table: A Table resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTablePatchRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['replaceViewDefinition'].present: + request.replaceViewDefinition = FLAGS.replaceViewDefinition + if FLAGS['table'].present: + request.table = apitools_base.JsonToMessage(messages.Table, FLAGS.table) + result = client.table.Patch( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TableUpdate(apitools_base_cli.NewCmd): + """Command wrapping table.Update.""" + + usage = """table_update """ + + def __init__(self, name, fv): + super(TableUpdate, self).__init__(name, fv) + flags.DEFINE_boolean( + 'replaceViewDefinition', + None, + u'Should the view definition also be updated? The specified view ' + u'definition replaces the existing one. Only a view can be updated ' + u'with a new definition.', + flag_values=fv) + flags.DEFINE_string( + 'table', + None, + u'A Table resource to be passed as the request body.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Updates an existing table. Unless explicitly requested, only the name, + description, and attribution will be updated. + + Args: + tableId: ID of the table that is being updated. + + Flags: + replaceViewDefinition: Should the view definition also be updated? The + specified view definition replaces the existing one. Only a view can + be updated with a new definition. + table: A Table resource to be passed as the request body. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTableUpdateRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['replaceViewDefinition'].present: + request.replaceViewDefinition = FLAGS.replaceViewDefinition + if FLAGS['table'].present: + request.table = apitools_base.JsonToMessage(messages.Table, FLAGS.table) + result = client.table.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TaskDelete(apitools_base_cli.NewCmd): + """Command wrapping task.Delete.""" + + usage = """task_delete """ + + def __init__(self, name, fv): + super(TaskDelete, self).__init__(name, fv) + + def RunWithArgs(self, tableId, taskId): + """Deletes the task, unless already started. + + Args: + tableId: Table from which the task is being deleted. + taskId: A string attribute. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTaskDeleteRequest( + tableId=tableId.decode('utf8'), + taskId=taskId.decode('utf8'), + ) + result = client.task.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TaskGet(apitools_base_cli.NewCmd): + """Command wrapping task.Get.""" + + usage = """task_get """ + + def __init__(self, name, fv): + super(TaskGet, self).__init__(name, fv) + + def RunWithArgs(self, tableId, taskId): + """Retrieves a specific task by its id. + + Args: + tableId: Table to which the task belongs. + taskId: A string attribute. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTaskGetRequest( + tableId=tableId.decode('utf8'), + taskId=taskId.decode('utf8'), + ) + result = client.task.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TaskList(apitools_base_cli.NewCmd): + """Command wrapping task.List.""" + + usage = """task_list """ + + def __init__(self, name, fv): + super(TaskList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of columns to return. Optional. Default is 5.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + 'A string attribute.', + flag_values=fv) + flags.DEFINE_integer( + 'startIndex', + None, + 'A integer attribute.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Retrieves a list of tasks. + + Args: + tableId: Table whose tasks are being listed. + + Flags: + maxResults: Maximum number of columns to return. Optional. Default is 5. + pageToken: A string attribute. + startIndex: A integer attribute. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTaskListRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + if FLAGS['startIndex'].present: + request.startIndex = FLAGS.startIndex + result = client.task.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplateDelete(apitools_base_cli.NewCmd): + """Command wrapping template.Delete.""" + + usage = """template_delete """ + + def __init__(self, name, fv): + super(TemplateDelete, self).__init__(name, fv) + + def RunWithArgs(self, tableId, templateId): + """Deletes a template + + Args: + tableId: Table from which the template is being deleted + templateId: Identifier for the template which is being deleted + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTemplateDeleteRequest( + tableId=tableId.decode('utf8'), + templateId=templateId, + ) + result = client.template.Delete( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplateGet(apitools_base_cli.NewCmd): + """Command wrapping template.Get.""" + + usage = """template_get """ + + def __init__(self, name, fv): + super(TemplateGet, self).__init__(name, fv) + + def RunWithArgs(self, tableId, templateId): + """Retrieves a specific template by its id + + Args: + tableId: Table to which the template belongs + templateId: Identifier for the template that is being requested + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTemplateGetRequest( + tableId=tableId.decode('utf8'), + templateId=templateId, + ) + result = client.template.Get( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplateInsert(apitools_base_cli.NewCmd): + """Command wrapping template.Insert.""" + + usage = """template_insert """ + + def __init__(self, name, fv): + super(TemplateInsert, self).__init__(name, fv) + flags.DEFINE_string( + 'automaticColumnNames', + None, + u'List of columns from which the template is to be automatically ' + u'constructed. Only one of body or automaticColumns can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'body', + None, + u'Body of the template. It contains HTML with {column_name} to insert' + u' values from a particular column. The body is sanitized to remove ' + u'certain tags, e.g., script. Only one of body or automaticColumns ' + u'can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#template', + u'Type name: a template for the info window contents. The template ' + u'can either include an HTML body or a list of columns from which the' + u' template is computed automatically.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name assigned to a template.', + flag_values=fv) + flags.DEFINE_integer( + 'templateId', + None, + u'Identifier for the template, unique within the context of a ' + u'particular table.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Creates a new template for the table. + + Args: + tableId: Identifier for the table for which the template is defined. + + Flags: + automaticColumnNames: List of columns from which the template is to be + automatically constructed. Only one of body or automaticColumns can be + specified. + body: Body of the template. It contains HTML with {column_name} to + insert values from a particular column. The body is sanitized to + remove certain tags, e.g., script. Only one of body or + automaticColumns can be specified. + kind: Type name: a template for the info window contents. The template + can either include an HTML body or a list of columns from which the + template is computed automatically. + name: Optional name assigned to a template. + templateId: Identifier for the template, unique within the context of a + particular table. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Template( + tableId=tableId.decode('utf8'), + ) + if FLAGS['automaticColumnNames'].present: + request.automaticColumnNames = [x.decode('utf8') for x in FLAGS.automaticColumnNames] + if FLAGS['body'].present: + request.body = FLAGS.body.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + if FLAGS['templateId'].present: + request.templateId = FLAGS.templateId + result = client.template.Insert( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplateList(apitools_base_cli.NewCmd): + """Command wrapping template.List.""" + + usage = """template_list """ + + def __init__(self, name, fv): + super(TemplateList, self).__init__(name, fv) + flags.DEFINE_integer( + 'maxResults', + None, + u'Maximum number of templates to return. Optional. Default is 5.', + flag_values=fv) + flags.DEFINE_string( + 'pageToken', + None, + u'Continuation token specifying which results page to return. ' + u'Optional.', + flag_values=fv) + + def RunWithArgs(self, tableId): + """Retrieves a list of templates. + + Args: + tableId: Identifier for the table whose templates are being requested + + Flags: + maxResults: Maximum number of templates to return. Optional. Default is + 5. + pageToken: Continuation token specifying which results page to return. + Optional. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.FusiontablesTemplateListRequest( + tableId=tableId.decode('utf8'), + ) + if FLAGS['maxResults'].present: + request.maxResults = FLAGS.maxResults + if FLAGS['pageToken'].present: + request.pageToken = FLAGS.pageToken.decode('utf8') + result = client.template.List( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplatePatch(apitools_base_cli.NewCmd): + """Command wrapping template.Patch.""" + + usage = """template_patch """ + + def __init__(self, name, fv): + super(TemplatePatch, self).__init__(name, fv) + flags.DEFINE_string( + 'automaticColumnNames', + None, + u'List of columns from which the template is to be automatically ' + u'constructed. Only one of body or automaticColumns can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'body', + None, + u'Body of the template. It contains HTML with {column_name} to insert' + u' values from a particular column. The body is sanitized to remove ' + u'certain tags, e.g., script. Only one of body or automaticColumns ' + u'can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#template', + u'Type name: a template for the info window contents. The template ' + u'can either include an HTML body or a list of columns from which the' + u' template is computed automatically.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name assigned to a template.', + flag_values=fv) + + def RunWithArgs(self, tableId, templateId): + """Updates an existing template. This method supports patch semantics. + + Args: + tableId: Identifier for the table for which the template is defined. + templateId: Identifier for the template, unique within the context of a + particular table. + + Flags: + automaticColumnNames: List of columns from which the template is to be + automatically constructed. Only one of body or automaticColumns can be + specified. + body: Body of the template. It contains HTML with {column_name} to + insert values from a particular column. The body is sanitized to + remove certain tags, e.g., script. Only one of body or + automaticColumns can be specified. + kind: Type name: a template for the info window contents. The template + can either include an HTML body or a list of columns from which the + template is computed automatically. + name: Optional name assigned to a template. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Template( + tableId=tableId.decode('utf8'), + templateId=templateId, + ) + if FLAGS['automaticColumnNames'].present: + request.automaticColumnNames = [x.decode('utf8') for x in FLAGS.automaticColumnNames] + if FLAGS['body'].present: + request.body = FLAGS.body.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + result = client.template.Patch( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +class TemplateUpdate(apitools_base_cli.NewCmd): + """Command wrapping template.Update.""" + + usage = """template_update """ + + def __init__(self, name, fv): + super(TemplateUpdate, self).__init__(name, fv) + flags.DEFINE_string( + 'automaticColumnNames', + None, + u'List of columns from which the template is to be automatically ' + u'constructed. Only one of body or automaticColumns can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'body', + None, + u'Body of the template. It contains HTML with {column_name} to insert' + u' values from a particular column. The body is sanitized to remove ' + u'certain tags, e.g., script. Only one of body or automaticColumns ' + u'can be specified.', + flag_values=fv) + flags.DEFINE_string( + 'kind', + u'fusiontables#template', + u'Type name: a template for the info window contents. The template ' + u'can either include an HTML body or a list of columns from which the' + u' template is computed automatically.', + flag_values=fv) + flags.DEFINE_string( + 'name', + None, + u'Optional name assigned to a template.', + flag_values=fv) + + def RunWithArgs(self, tableId, templateId): + """Updates an existing template + + Args: + tableId: Identifier for the table for which the template is defined. + templateId: Identifier for the template, unique within the context of a + particular table. + + Flags: + automaticColumnNames: List of columns from which the template is to be + automatically constructed. Only one of body or automaticColumns can be + specified. + body: Body of the template. It contains HTML with {column_name} to + insert values from a particular column. The body is sanitized to + remove certain tags, e.g., script. Only one of body or + automaticColumns can be specified. + kind: Type name: a template for the info window contents. The template + can either include an HTML body or a list of columns from which the + template is computed automatically. + name: Optional name assigned to a template. + """ + client = GetClientFromFlags() + global_params = GetGlobalParamsFromFlags() + request = messages.Template( + tableId=tableId.decode('utf8'), + templateId=templateId, + ) + if FLAGS['automaticColumnNames'].present: + request.automaticColumnNames = [x.decode('utf8') for x in FLAGS.automaticColumnNames] + if FLAGS['body'].present: + request.body = FLAGS.body.decode('utf8') + if FLAGS['kind'].present: + request.kind = FLAGS.kind.decode('utf8') + if FLAGS['name'].present: + request.name = FLAGS.name.decode('utf8') + result = client.template.Update( + request, global_params=global_params) + print apitools_base_cli.FormatOutput(result) + + +def main(_): + appcommands.AddCmd('pyshell', PyShell) + appcommands.AddCmd('column_delete', ColumnDelete) + appcommands.AddCmd('column_get', ColumnGet) + appcommands.AddCmd('column_insert', ColumnInsert) + appcommands.AddCmd('column_list', ColumnList) + appcommands.AddCmd('column_patch', ColumnPatch) + appcommands.AddCmd('column_update', ColumnUpdate) + appcommands.AddCmd('query_sql', QuerySql) + appcommands.AddCmd('query_sqlGet', QuerySqlGet) + appcommands.AddCmd('style_delete', StyleDelete) + appcommands.AddCmd('style_get', StyleGet) + appcommands.AddCmd('style_insert', StyleInsert) + appcommands.AddCmd('style_list', StyleList) + appcommands.AddCmd('style_patch', StylePatch) + appcommands.AddCmd('style_update', StyleUpdate) + appcommands.AddCmd('table_copy', TableCopy) + appcommands.AddCmd('table_delete', TableDelete) + appcommands.AddCmd('table_get', TableGet) + appcommands.AddCmd('table_importRows', TableImportRows) + appcommands.AddCmd('table_importTable', TableImportTable) + appcommands.AddCmd('table_insert', TableInsert) + appcommands.AddCmd('table_list', TableList) + appcommands.AddCmd('table_patch', TablePatch) + appcommands.AddCmd('table_update', TableUpdate) + appcommands.AddCmd('task_delete', TaskDelete) + appcommands.AddCmd('task_get', TaskGet) + appcommands.AddCmd('task_list', TaskList) + appcommands.AddCmd('template_delete', TemplateDelete) + appcommands.AddCmd('template_get', TemplateGet) + appcommands.AddCmd('template_insert', TemplateInsert) + appcommands.AddCmd('template_list', TemplateList) + appcommands.AddCmd('template_patch', TemplatePatch) + appcommands.AddCmd('template_update', TemplateUpdate) + + apitools_base_cli.SetupLogger() + if hasattr(appcommands, 'SetDefaultCommand'): + appcommands.SetDefaultCommand('pyshell') + + +run_main = apitools_base_cli.run_main + +if __name__ == '__main__': + appcommands.Run() diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_client.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py similarity index 100% rename from apitools/base/py/testing/testclient/fusiontables_v1_client.py rename to samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py diff --git a/apitools/base/py/testing/testclient/fusiontables_v1_messages.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py similarity index 100% rename from apitools/base/py/testing/testclient/fusiontables_v1_messages.py rename to samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py diff --git a/samples/regenerate_samples.py b/samples/regenerate_samples.py index 5a65bff..c46cd70 100644 --- a/samples/regenerate_samples.py +++ b/samples/regenerate_samples.py @@ -22,6 +22,7 @@ _GEN_CLIENT_BINARY = 'gen_client' _SAMPLES = [ 'dns_sample/dns_v1.json', 'iam_sample/iam_v1.json', + 'fusiontables_sample/fusiontables_v1.json', 'storage_sample/storage_v1.json', ] -- GitLab From a6dfcfc0d59ee6de104688ae2ae585ac34c016ab Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Thu, 23 Jun 2016 16:00:29 -0400 Subject: [PATCH 248/295] Update tests after moving and regenerating fusiontables api. --- apitools/base/py/list_pager_test.py | 139 ++- apitools/base/py/testing/mock_test.py | 3 +- run_pylint.py | 1 + .../fusiontables_v1/fusiontables_v1_client.py | 1079 +++++++++++++++-- .../fusiontables_v1_messages.py | 974 +++++++++++++-- 5 files changed, 1909 insertions(+), 287 deletions(-) diff --git a/apitools/base/py/list_pager_test.py b/apitools/base/py/list_pager_test.py index 7ea5729..bffc09a 100644 --- a/apitools/base/py/list_pager_test.py +++ b/apitools/base/py/list_pager_test.py @@ -19,7 +19,12 @@ import unittest2 from apitools.base.py import list_pager from apitools.base.py.testing import mock -from apitools.base.py.testing import testclient as fusiontables +from samples.fusiontables_sample.fusiontables_v1 \ + import fusiontables_v1_client as fusiontables +from samples.fusiontables_sample.fusiontables_v1 \ + import fusiontables_v1_messages as messages +from samples.iam_sample.iam_v1 import iam_v1_client as iam_client +from samples.iam_sample.iam_v1 import iam_v1_messages as iam_messages class ListPagerTest(unittest2.TestCase): @@ -39,165 +44,177 @@ class ListPagerTest(unittest2.TestCase): def testYieldFromList(self): self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken=None, tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c0'), - fusiontables.Column(name='c1'), - fusiontables.Column(name='c2'), - fusiontables.Column(name='c3'), + messages.Column(name='c0'), + messages.Column(name='c1'), + messages.Column(name='c2'), + messages.Column(name='c3'), ], nextPageToken='x', )) self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken='x', tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c4'), - fusiontables.Column(name='c5'), - fusiontables.Column(name='c6'), - fusiontables.Column(name='c7'), + messages.Column(name='c4'), + messages.Column(name='c5'), + messages.Column(name='c6'), + messages.Column(name='c7'), ], )) client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + request = messages.FusiontablesColumnListRequest(tableId='mytable') results = list_pager.YieldFromList(client.column, request) self._AssertInstanceSequence(results, 8) def testYieldNoRecords(self): client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + request = messages.FusiontablesColumnListRequest(tableId='mytable') results = list_pager.YieldFromList(client.column, request, limit=False) self.assertEqual(0, len(list(results))) def testYieldFromListPartial(self): self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken=None, tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c0'), - fusiontables.Column(name='c1'), - fusiontables.Column(name='c2'), - fusiontables.Column(name='c3'), + messages.Column(name='c0'), + messages.Column(name='c1'), + messages.Column(name='c2'), + messages.Column(name='c3'), ], nextPageToken='x', )) self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken='x', tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c4'), - fusiontables.Column(name='c5'), - fusiontables.Column(name='c6'), - fusiontables.Column(name='c7'), + messages.Column(name='c4'), + messages.Column(name='c5'), + messages.Column(name='c6'), + messages.Column(name='c7'), ], )) client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + request = messages.FusiontablesColumnListRequest(tableId='mytable') results = list_pager.YieldFromList(client.column, request, limit=6) self._AssertInstanceSequence(results, 6) def testYieldFromListEmpty(self): self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken=None, tableId='mytable', ), - fusiontables.ColumnList()) + messages.ColumnList()) client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + request = messages.FusiontablesColumnListRequest(tableId='mytable') results = list_pager.YieldFromList(client.column, request, limit=6) self._AssertInstanceSequence(results, 0) def testYieldFromListWithPredicate(self): self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken=None, tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c0'), - fusiontables.Column(name='bad0'), - fusiontables.Column(name='c1'), - fusiontables.Column(name='bad1'), + messages.Column(name='c0'), + messages.Column(name='bad0'), + messages.Column(name='c1'), + messages.Column(name='bad1'), ], nextPageToken='x', )) self.mocked_client.column.List.Expect( - fusiontables.FusiontablesColumnListRequest( + messages.FusiontablesColumnListRequest( maxResults=100, pageToken='x', tableId='mytable', ), - fusiontables.ColumnList( + messages.ColumnList( items=[ - fusiontables.Column(name='c2'), + messages.Column(name='c2'), ], )) client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListRequest(tableId='mytable') + request = messages.FusiontablesColumnListRequest(tableId='mytable') results = list_pager.YieldFromList( client.column, request, predicate=lambda x: 'c' in x.name) self._AssertInstanceSequence(results, 3) + +class ListPagerAttributeTest(unittest2.TestCase): + + def setUp(self): + self.mocked_client = mock.Client(iam_client.IamV1) + self.mocked_client.Mock() + self.addCleanup(self.mocked_client.Unmock) + def testYieldFromListWithAttributes(self): - self.mocked_client.columnalternate.List.Expect( - fusiontables.FusiontablesColumnListAlternateRequest( + self.mocked_client.iamPolicies.GetPolicyDetails.Expect( + iam_messages.GetPolicyDetailsRequest( pageSize=100, pageToken=None, - tableId='mytable', + fullResourcePath='myresource', ), - fusiontables.ColumnListAlternate( - columns=[ - fusiontables.Column(name='c0'), - fusiontables.Column(name='c1'), + iam_messages.GetPolicyDetailsResponse( + policies=[ + iam_messages.PolicyDetail(fullResourcePath='c0'), + iam_messages.PolicyDetail(fullResourcePath='c1'), ], nextPageToken='x', )) - self.mocked_client.columnalternate.List.Expect( - fusiontables.FusiontablesColumnListAlternateRequest( + self.mocked_client.iamPolicies.GetPolicyDetails.Expect( + iam_messages.GetPolicyDetailsRequest( pageSize=100, pageToken='x', - tableId='mytable', + fullResourcePath='myresource', ), - fusiontables.ColumnListAlternate( - columns=[ - fusiontables.Column(name='c2'), + iam_messages.GetPolicyDetailsResponse( + policies=[ + iam_messages.PolicyDetail(fullResourcePath='c2'), ], )) - client = fusiontables.FusiontablesV1(get_credentials=False) - request = fusiontables.FusiontablesColumnListAlternateRequest( - tableId='mytable') + client = iam_client.IamV1(get_credentials=False) + request = iam_messages.GetPolicyDetailsRequest( + fullResourcePath='myresource') results = list_pager.YieldFromList( - client.columnalternate, request, - batch_size_attribute='pageSize', field='columns') - - self._AssertInstanceSequence(results, 3) + client.iamPolicies, request, + batch_size_attribute='pageSize', + method='GetPolicyDetails', field='policies') + + i = 0 + for i, instance in enumerate(results): + self.assertEquals('c{0}'.format(i), instance.fullResourcePath) + self.assertEquals(2, i) diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py index 88c8739..ee42ef1 100644 --- a/apitools/base/py/testing/mock_test.py +++ b/apitools/base/py/testing/mock_test.py @@ -21,7 +21,8 @@ from apitools.base.protorpclite import messages import apitools.base.py as apitools_base from apitools.base.py.testing import mock -from apitools.base.py.testing import testclient as fusiontables +from samples.fusiontables_sample.fusiontables_v1 import \ + fusiontables_v1_client as fusiontables class MockTest(unittest2.TestCase): diff --git a/run_pylint.py b/run_pylint.py index 3f0a315..2a95632 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -34,6 +34,7 @@ import sys IGNORED_DIRECTORIES = [ 'apitools/gen/testdata', 'samples/dns_sample/dns_v1', + 'samples/fusiontables_sample/fusiontables_v1', 'samples/iam_sample/iam_v1', 'samples/storage_sample/storage_v1', 'venv', diff --git a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py index 816fcad..690f148 100644 --- a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py +++ b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py @@ -1,139 +1,950 @@ -# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Modified generated client library for fusiontables version v1. - -This is a hand-customized and pruned version of the fusiontables v1 -client, designed for use in testing. - -""" - +"""Generated client library for fusiontables version v1.""" +# NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.py import base_api -from apitools.base.py.testing.testclient import fusiontables_v1_messages +from samples.fusiontables_sample.fusiontables_v1 import fusiontables_v1_messages as messages class FusiontablesV1(base_api.BaseApiClient): + """Generated client library for service fusiontables version v1.""" + + MESSAGES_MODULE = messages + BASE_URL = u'https://www.googleapis.com/fusiontables/v1/' + + _PACKAGE = u'fusiontables' + _SCOPES = [u'https://www.googleapis.com/auth/fusiontables', u'https://www.googleapis.com/auth/fusiontables.readonly'] + _VERSION = u'v1' + _CLIENT_ID = '1042881264118.apps.googleusercontent.com' + _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b' + _CLIENT_CLASS_NAME = u'FusiontablesV1' + _URL_VERSION = u'v1' + _API_KEY = None + + def __init__(self, url='', credentials=None, + get_credentials=True, http=None, model=None, + log_request=False, log_response=False, + credentials_args=None, default_global_params=None, + additional_http_headers=None): + """Create a new fusiontables handle.""" + url = url or self.BASE_URL + super(FusiontablesV1, self).__init__( + url, credentials=credentials, + get_credentials=get_credentials, http=http, model=model, + log_request=log_request, log_response=log_response, + credentials_args=credentials_args, + default_global_params=default_global_params, + additional_http_headers=additional_http_headers) + self.column = self.ColumnService(self) + self.query = self.QueryService(self) + self.style = self.StyleService(self) + self.table = self.TableService(self) + self.task = self.TaskService(self) + self.template = self.TemplateService(self) + + class ColumnService(base_api.BaseApiService): + """Service class for the column resource.""" + + _NAME = u'column' + + def __init__(self, client): + super(FusiontablesV1.ColumnService, self).__init__(client) + self._method_configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.column.delete', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field='', + request_type_name=u'FusiontablesColumnDeleteRequest', + response_type_name=u'FusiontablesColumnDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.get', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field='', + request_type_name=u'FusiontablesColumnGetRequest', + response_type_name=u'Column', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.column.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns', + request_field=u'column', + request_type_name=u'FusiontablesColumnInsertRequest', + response_type_name=u'Column', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/columns', + request_field='', + request_type_name=u'FusiontablesColumnListRequest', + response_type_name=u'ColumnList', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.column.patch', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field=u'column', + request_type_name=u'FusiontablesColumnPatchRequest', + response_type_name=u'Column', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.column.update', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field=u'column', + request_type_name=u'FusiontablesColumnUpdateRequest', + response_type_name=u'Column', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Delete(self, request, global_params=None): + """Deletes the column. + + Args: + request: (FusiontablesColumnDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (FusiontablesColumnDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Retrieves a specific column by its id. + + Args: + request: (FusiontablesColumnGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Column) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Adds a new column to the table. + + Args: + request: (FusiontablesColumnInsertRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Column) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of columns. + + Args: + request: (FusiontablesColumnListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ColumnList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates the name or type of an existing column. This method supports patch semantics. + + Args: + request: (FusiontablesColumnPatchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Column) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates the name or type of an existing column. + + Args: + request: (FusiontablesColumnUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Column) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class QueryService(base_api.BaseApiService): + """Service class for the query resource.""" + + _NAME = u'query' + + def __init__(self, client): + super(FusiontablesV1.QueryService, self).__init__(client) + self._method_configs = { + 'Sql': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.query.sql', + ordered_params=[u'sql'], + path_params=[], + query_params=[u'hdrs', u'sql', u'typed'], + relative_path=u'query', + request_field='', + request_type_name=u'FusiontablesQuerySqlRequest', + response_type_name=u'Sqlresponse', + supports_download=True, + ), + 'SqlGet': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.query.sqlGet', + ordered_params=[u'sql'], + path_params=[], + query_params=[u'hdrs', u'sql', u'typed'], + relative_path=u'query', + request_field='', + request_type_name=u'FusiontablesQuerySqlGetRequest', + response_type_name=u'Sqlresponse', + supports_download=True, + ), + } + + self._upload_configs = { + } + + def Sql(self, request, global_params=None, download=None): + """Executes an SQL SELECT/INSERT/UPDATE/DELETE/SHOW/DESCRIBE/CREATE statement. + + Args: + request: (FusiontablesQuerySqlRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Sqlresponse) The response message. + """ + config = self.GetMethodConfig('Sql') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + def SqlGet(self, request, global_params=None, download=None): + """Executes an SQL SELECT/SHOW/DESCRIBE statement. + + Args: + request: (FusiontablesQuerySqlGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + download: (Download, default: None) If present, download + data from the request via this stream. + Returns: + (Sqlresponse) The response message. + """ + config = self.GetMethodConfig('SqlGet') + return self._RunMethod( + config, request, global_params=global_params, + download=download) + + class StyleService(base_api.BaseApiService): + """Service class for the style resource.""" + + _NAME = u'style' + + def __init__(self, client): + super(FusiontablesV1.StyleService, self).__init__(client) + self._method_configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.style.delete', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'FusiontablesStyleDeleteRequest', + response_type_name=u'FusiontablesStyleDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.style.get', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'FusiontablesStyleGetRequest', + response_type_name=u'StyleSetting', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.style.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.style.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/styles', + request_field='', + request_type_name=u'FusiontablesStyleListRequest', + response_type_name=u'StyleSettingList', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.style.patch', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.style.update', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Delete(self, request, global_params=None): + """Deletes a style. + + Args: + request: (FusiontablesStyleDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (FusiontablesStyleDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Gets a specific style. + + Args: + request: (FusiontablesStyleGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StyleSetting) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Adds a new style for the table. + + Args: + request: (StyleSetting) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StyleSetting) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of styles. + + Args: + request: (FusiontablesStyleListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StyleSettingList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an existing style. This method supports patch semantics. + + Args: + request: (StyleSetting) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StyleSetting) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates an existing style. + + Args: + request: (StyleSetting) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (StyleSetting) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class TableService(base_api.BaseApiService): + """Service class for the table resource.""" + + _NAME = u'table' + + def __init__(self, client): + super(FusiontablesV1.TableService, self).__init__(client) + self._method_configs = { + 'Copy': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.copy', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'copyPresentation'], + relative_path=u'tables/{tableId}/copy', + request_field='', + request_type_name=u'FusiontablesTableCopyRequest', + response_type_name=u'Table', + supports_download=False, + ), + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.table.delete', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}', + request_field='', + request_type_name=u'FusiontablesTableDeleteRequest', + response_type_name=u'FusiontablesTableDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.table.get', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}', + request_field='', + request_type_name=u'FusiontablesTableGetRequest', + response_type_name=u'Table', + supports_download=False, + ), + 'ImportRows': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.importRows', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'delimiter', u'encoding', u'endLine', u'isStrict', u'startLine'], + relative_path=u'tables/{tableId}/import', + request_field='', + request_type_name=u'FusiontablesTableImportRowsRequest', + response_type_name=u'Import', + supports_download=False, + ), + 'ImportTable': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.importTable', + ordered_params=[u'name'], + path_params=[], + query_params=[u'delimiter', u'encoding', u'name'], + relative_path=u'tables/import', + request_field='', + request_type_name=u'FusiontablesTableImportTableRequest', + response_type_name=u'Table', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.insert', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'tables', + request_field='', + request_type_name=u'Table', + response_type_name=u'Table', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.table.list', + ordered_params=[], + path_params=[], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables', + request_field='', + request_type_name=u'FusiontablesTableListRequest', + response_type_name=u'TableList', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.table.patch', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'replaceViewDefinition'], + relative_path=u'tables/{tableId}', + request_field=u'table', + request_type_name=u'FusiontablesTablePatchRequest', + response_type_name=u'Table', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.table.update', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'replaceViewDefinition'], + relative_path=u'tables/{tableId}', + request_field=u'table', + request_type_name=u'FusiontablesTableUpdateRequest', + response_type_name=u'Table', + supports_download=False, + ), + } + + self._upload_configs = { + 'ImportRows': base_api.ApiUploadInfo( + accept=['application/octet-stream'], + max_size=262144000, + resumable_multipart=True, + resumable_path=u'/resumable/upload/fusiontables/v1/tables/{tableId}/import', + simple_multipart=True, + simple_path=u'/upload/fusiontables/v1/tables/{tableId}/import', + ), + 'ImportTable': base_api.ApiUploadInfo( + accept=['application/octet-stream'], + max_size=262144000, + resumable_multipart=True, + resumable_path=u'/resumable/upload/fusiontables/v1/tables/import', + simple_multipart=True, + simple_path=u'/upload/fusiontables/v1/tables/import', + ), + } + + def Copy(self, request, global_params=None): + """Copies a table. + + Args: + request: (FusiontablesTableCopyRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('Copy') + return self._RunMethod( + config, request, global_params=global_params) + + def Delete(self, request, global_params=None): + """Deletes a table. + + Args: + request: (FusiontablesTableDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (FusiontablesTableDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Retrieves a specific table by its id. + + Args: + request: (FusiontablesTableGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def ImportRows(self, request, global_params=None, upload=None): + """Import more rows into a table. + + Args: + request: (FusiontablesTableImportRowsRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + upload: (Upload, default: None) If present, upload + this stream with the request. + Returns: + (Import) The response message. + """ + config = self.GetMethodConfig('ImportRows') + upload_config = self.GetUploadConfig('ImportRows') + return self._RunMethod( + config, request, global_params=global_params, + upload=upload, upload_config=upload_config) + + def ImportTable(self, request, global_params=None, upload=None): + """Import a new table. + + Args: + request: (FusiontablesTableImportTableRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + upload: (Upload, default: None) If present, upload + this stream with the request. + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('ImportTable') + upload_config = self.GetUploadConfig('ImportTable') + return self._RunMethod( + config, request, global_params=global_params, + upload=upload, upload_config=upload_config) + + def Insert(self, request, global_params=None): + """Creates a new table. + + Args: + request: (Table) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of tables a user owns. + + Args: + request: (FusiontablesTableListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TableList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated. This method supports patch semantics. + + Args: + request: (FusiontablesTablePatchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated. + + Args: + request: (FusiontablesTableUpdateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Table) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) + + class TaskService(base_api.BaseApiService): + """Service class for the task resource.""" + + _NAME = u'task' + + def __init__(self, client): + super(FusiontablesV1.TaskService, self).__init__(client) + self._method_configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.task.delete', + ordered_params=[u'tableId', u'taskId'], + path_params=[u'tableId', u'taskId'], + query_params=[], + relative_path=u'tables/{tableId}/tasks/{taskId}', + request_field='', + request_type_name=u'FusiontablesTaskDeleteRequest', + response_type_name=u'FusiontablesTaskDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.task.get', + ordered_params=[u'tableId', u'taskId'], + path_params=[u'tableId', u'taskId'], + query_params=[], + relative_path=u'tables/{tableId}/tasks/{taskId}', + request_field='', + request_type_name=u'FusiontablesTaskGetRequest', + response_type_name=u'Task', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.task.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken', u'startIndex'], + relative_path=u'tables/{tableId}/tasks', + request_field='', + request_type_name=u'FusiontablesTaskListRequest', + response_type_name=u'TaskList', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Delete(self, request, global_params=None): + """Deletes the task, unless already started. + + Args: + request: (FusiontablesTaskDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (FusiontablesTaskDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Retrieves a specific task by its id. + + Args: + request: (FusiontablesTaskGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Task) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of tasks. + + Args: + request: (FusiontablesTaskListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TaskList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + class TemplateService(base_api.BaseApiService): + """Service class for the template resource.""" + + _NAME = u'template' + + def __init__(self, client): + super(FusiontablesV1.TemplateService, self).__init__(client) + self._method_configs = { + 'Delete': base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.template.delete', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'FusiontablesTemplateDeleteRequest', + response_type_name=u'FusiontablesTemplateDeleteResponse', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.template.get', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'FusiontablesTemplateGetRequest', + response_type_name=u'Template', + supports_download=False, + ), + 'Insert': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.template.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/templates', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ), + 'List': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.template.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/templates', + request_field='', + request_type_name=u'FusiontablesTemplateListRequest', + response_type_name=u'TemplateList', + supports_download=False, + ), + 'Patch': base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.template.patch', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ), + 'Update': base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.template.update', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Delete(self, request, global_params=None): + """Deletes a template. + + Args: + request: (FusiontablesTemplateDeleteRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (FusiontablesTemplateDeleteResponse) The response message. + """ + config = self.GetMethodConfig('Delete') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Retrieves a specific template by its id. + + Args: + request: (FusiontablesTemplateGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Template) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Insert(self, request, global_params=None): + """Creates a new template for the table. + + Args: + request: (Template) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Template) The response message. + """ + config = self.GetMethodConfig('Insert') + return self._RunMethod( + config, request, global_params=global_params) + + def List(self, request, global_params=None): + """Retrieves a list of templates. + + Args: + request: (FusiontablesTemplateListRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (TemplateList) The response message. + """ + config = self.GetMethodConfig('List') + return self._RunMethod( + config, request, global_params=global_params) + + def Patch(self, request, global_params=None): + """Updates an existing template. This method supports patch semantics. + + Args: + request: (Template) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Template) The response message. + """ + config = self.GetMethodConfig('Patch') + return self._RunMethod( + config, request, global_params=global_params) + + def Update(self, request, global_params=None): + """Updates an existing template. - """Generated client library for service fusiontables version v1.""" - - MESSAGES_MODULE = fusiontables_v1_messages - - _PACKAGE = u'fusiontables' - _SCOPES = [u'https://www.googleapis.com/auth/fusiontables', - u'https://www.googleapis.com/auth/fusiontables.readonly'] - _VERSION = u'v1' - _CLIENT_ID = '1042881264118.apps.googleusercontent.com' - _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' - _USER_AGENT = '' - _CLIENT_CLASS_NAME = u'FusiontablesV1' - _URL_VERSION = u'v1' - - def __init__(self, url='', credentials=None, - get_credentials=True, http=None, model=None, - log_request=False, log_response=False, - credentials_args=None, default_global_params=None, - additional_http_headers=None): - """Create a new fusiontables handle.""" - url = url or u'https://www.googleapis.com/fusiontables/v1/' - super(FusiontablesV1, self).__init__( - url, credentials=credentials, - get_credentials=get_credentials, http=http, model=model, - log_request=log_request, log_response=log_response, - credentials_args=credentials_args, - default_global_params=default_global_params, - additional_http_headers=additional_http_headers) - self.column = self.ColumnService(self) - self.columnalternate = self.ColumnAlternateService(self) - - class ColumnService(base_api.BaseApiService): - - """Service class for the column resource.""" - - _NAME = u'column' - - def __init__(self, client): - super(FusiontablesV1.ColumnService, self).__init__(client) - self._method_configs = { - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.column.list', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables/{tableId}/columns', - request_field='', - request_type_name=u'FusiontablesColumnListRequest', - response_type_name=u'ColumnList', - supports_download=False, - ), - } - - self._upload_configs = { - } - - def List(self, request, global_params=None): - """Retrieves a list of columns. - - Args: - request: (FusiontablesColumnListRequest) input message - global_params: (StandardQueryParameters, default: None) global - arguments - Returns: - (ColumnList) The response message. - """ - config = self.GetMethodConfig('List') - return self._RunMethod( - config, request, global_params=global_params) - - class ColumnAlternateService(base_api.BaseApiService): - - """Service class for the column resource.""" - - _NAME = u'columnalternate' - - def __init__(self, client): - super(FusiontablesV1.ColumnAlternateService, self).__init__(client) - self._method_configs = { - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.column.listalternate', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables/{tableId}/columns', - request_field='', - request_type_name=( - u'FusiontablesColumnListAlternateRequest'), - response_type_name=u'ColumnListAlternate', - supports_download=False, - ), - } - - self._upload_configs = { - } - - def List(self, request, global_params=None): - """Retrieves a list of columns. - - Args: - request: (FusiontablesColumnListRequest) input message - global_params: (StandardQueryParameters, default: None) global - arguments - Returns: - (ColumnList) The response message. - """ - config = self.GetMethodConfig('List') - return self._RunMethod( - config, request, global_params=global_params) + Args: + request: (Template) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Template) The response message. + """ + config = self.GetMethodConfig('Update') + return self._RunMethod( + config, request, global_params=global_params) diff --git a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py index 0111f71..6eff4b5 100644 --- a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py +++ b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py @@ -1,18 +1,3 @@ -# -# Copyright 2015 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - """Generated message classes for fusiontables version v1. API for working with Fusion Tables data. @@ -25,119 +10,926 @@ from apitools.base.protorpclite import messages as _messages package = 'fusiontables' -class Column(_messages.Message): +class Bucket(_messages.Message): + """Specifies the minimum and maximum values, the color, opacity, icon and + weight of a bucket within a StyleSetting. - """Specifies the id, name and type of a column in a table. + Fields: + color: Color of line or the interior of a polygon in #RRGGBB format. + icon: Icon name used for a point. + max: Maximum value in the selected column for a row to be styled according + to the bucket color, opacity, icon, or weight. + min: Minimum value in the selected column for a row to be styled according + to the bucket color, opacity, icon, or weight. + opacity: Opacity of the color: 0.0 (transparent) to 1.0 (opaque). + weight: Width of a line (in pixels). + """ - Messages: - BaseColumnValue: Optional identifier of the base column. If present, this - column is derived from the specified base column. + color = _messages.StringField(1) + icon = _messages.StringField(2) + max = _messages.FloatField(3) + min = _messages.FloatField(4) + opacity = _messages.FloatField(5) + weight = _messages.IntegerField(6, variant=_messages.Variant.INT32) - Fields: - baseColumn: Optional identifier of the base column. If present, this - column is derived from the specified base column. - columnId: Identifier for the column. - description: Optional column description. - graph_predicate: Optional column predicate. Used to map table to - graph data model (subject,predicate,object) See - http://www.w3.org/TR/2014/REC- - rdf11-concepts-20140225/#data-model - kind: Type name: a template for an individual column. - name: Required name of the column. - type: Required type of the column. +class Column(_messages.Message): + """Specifies the id, name and type of a column in a table. + + Messages: + BaseColumnValue: Optional identifier of the base column. If present, this + column is derived from the specified base column. + + Fields: + baseColumn: Optional identifier of the base column. If present, this + column is derived from the specified base column. + columnId: Identifier for the column. + description: Optional column description. + graph_predicate: Optional column predicate. Used to map table to graph + data model (subject,predicate,object) See http://www.w3.org/TR/2014/REC- + rdf11-concepts-20140225/#data-model + kind: Type name: a template for an individual column. + name: Required name of the column. + type: Required type of the column. + """ + + class BaseColumnValue(_messages.Message): + """Optional identifier of the base column. If present, this column is + derived from the specified base column. + + Fields: + columnId: The id of the column in the base table from which this column + is derived. + tableIndex: Offset to the entry in the list of base tables in the table + definition. """ - class BaseColumnValue(_messages.Message): + columnId = _messages.IntegerField(1, variant=_messages.Variant.INT32) + tableIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32) - """Optional identifier of the base column. If present, this column is - derived from the specified base column. + baseColumn = _messages.MessageField('BaseColumnValue', 1) + columnId = _messages.IntegerField(2, variant=_messages.Variant.INT32) + description = _messages.StringField(3) + graph_predicate = _messages.StringField(4) + kind = _messages.StringField(5, default=u'fusiontables#column') + name = _messages.StringField(6) + type = _messages.StringField(7) - Fields: - columnId: The id of the column in the base table from which - this column is derived. - tableIndex: Offset to the entry in the list of base tables - in the table definition. - """ +class ColumnList(_messages.Message): + """Represents a list of columns in a table. - columnId = _messages.IntegerField(1, variant=_messages.Variant.INT32) - tableIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32) + Fields: + items: List of all requested columns. + kind: Type name: a list of all columns. + nextPageToken: Token used to access the next page of this result. No token + is displayed if there are no more pages left. + totalItems: Total number of columns for the table. + """ - baseColumn = _messages.MessageField('BaseColumnValue', 1) - columnId = _messages.IntegerField(2, variant=_messages.Variant.INT32) - description = _messages.StringField(3) - graph_predicate = _messages.StringField(4) - kind = _messages.StringField(5, default=u'fusiontables#column') - name = _messages.StringField(6) - type = _messages.StringField(7) + items = _messages.MessageField('Column', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#columnList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) -class ColumnList(_messages.Message): +class FusiontablesColumnDeleteRequest(_messages.Message): + """A FusiontablesColumnDeleteRequest object. - """Represents a list of columns in a table. + Fields: + columnId: Name or identifier for the column being deleted. + tableId: Table from which the column is being deleted. + """ - Fields: - items: List of all requested columns. - kind: Type name: a list of all columns. - nextPageToken: Token used to access the next page of this - result. No token is displayed if there are no more pages left. - totalItems: Total number of columns for the table. + columnId = _messages.StringField(1, required=True) + tableId = _messages.StringField(2, required=True) - """ - items = _messages.MessageField('Column', 1, repeated=True) - kind = _messages.StringField(2, default=u'fusiontables#columnList') - nextPageToken = _messages.StringField(3) - totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) +class FusiontablesColumnDeleteResponse(_messages.Message): + """An empty FusiontablesColumnDelete response.""" + + +class FusiontablesColumnGetRequest(_messages.Message): + """A FusiontablesColumnGetRequest object. + + Fields: + columnId: Name or identifier for the column that is being requested. + tableId: Table to which the column belongs. + """ + + columnId = _messages.StringField(1, required=True) + tableId = _messages.StringField(2, required=True) + + +class FusiontablesColumnInsertRequest(_messages.Message): + """A FusiontablesColumnInsertRequest object. + + Fields: + column: A Column resource to be passed as the request body. + tableId: Table for which a new column is being added. + """ + + column = _messages.MessageField('Column', 1) + tableId = _messages.StringField(2, required=True) class FusiontablesColumnListRequest(_messages.Message): + """A FusiontablesColumnListRequest object. + + Fields: + maxResults: Maximum number of columns to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + tableId: Table whose columns are being listed. + """ + + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesColumnPatchRequest(_messages.Message): + """A FusiontablesColumnPatchRequest object. + + Fields: + column: A Column resource to be passed as the request body. + columnId: Name or identifier for the column that is being updated. + tableId: Table for which the column is being updated. + """ + + column = _messages.MessageField('Column', 1) + columnId = _messages.StringField(2, required=True) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesColumnUpdateRequest(_messages.Message): + """A FusiontablesColumnUpdateRequest object. + + Fields: + column: A Column resource to be passed as the request body. + columnId: Name or identifier for the column that is being updated. + tableId: Table for which the column is being updated. + """ + + column = _messages.MessageField('Column', 1) + columnId = _messages.StringField(2, required=True) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesQuerySqlGetRequest(_messages.Message): + """A FusiontablesQuerySqlGetRequest object. + + Fields: + hdrs: Should column names be included (in the first row)?. Default is + true. + sql: An SQL SELECT/SHOW/DESCRIBE statement. + typed: Should typed values be returned in the (JSON) response -- numbers + for numeric values and parsed geometries for KML values? Default is + true. + """ + + hdrs = _messages.BooleanField(1) + sql = _messages.StringField(2, required=True) + typed = _messages.BooleanField(3) + + +class FusiontablesQuerySqlRequest(_messages.Message): + """A FusiontablesQuerySqlRequest object. + + Fields: + hdrs: Should column names be included (in the first row)?. Default is + true. + sql: An SQL SELECT/SHOW/DESCRIBE/INSERT/UPDATE/DELETE/CREATE statement. + typed: Should typed values be returned in the (JSON) response -- numbers + for numeric values and parsed geometries for KML values? Default is + true. + """ + + hdrs = _messages.BooleanField(1) + sql = _messages.StringField(2, required=True) + typed = _messages.BooleanField(3) + + +class FusiontablesStyleDeleteRequest(_messages.Message): + """A FusiontablesStyleDeleteRequest object. + + Fields: + styleId: Identifier (within a table) for the style being deleted + tableId: Table from which the style is being deleted + """ + + styleId = _messages.IntegerField(1, required=True, variant=_messages.Variant.INT32) + tableId = _messages.StringField(2, required=True) + + +class FusiontablesStyleDeleteResponse(_messages.Message): + """An empty FusiontablesStyleDelete response.""" + + +class FusiontablesStyleGetRequest(_messages.Message): + """A FusiontablesStyleGetRequest object. + + Fields: + styleId: Identifier (integer) for a specific style in a table + tableId: Table to which the requested style belongs + """ + + styleId = _messages.IntegerField(1, required=True, variant=_messages.Variant.INT32) + tableId = _messages.StringField(2, required=True) + + +class FusiontablesStyleListRequest(_messages.Message): + """A FusiontablesStyleListRequest object. + + Fields: + maxResults: Maximum number of styles to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + tableId: Table whose styles are being listed + """ + + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesTableCopyRequest(_messages.Message): + """A FusiontablesTableCopyRequest object. + + Fields: + copyPresentation: Whether to also copy tabs, styles, and templates. + Default is false. + tableId: ID of the table that is being copied. + """ + + copyPresentation = _messages.BooleanField(1) + tableId = _messages.StringField(2, required=True) + + +class FusiontablesTableDeleteRequest(_messages.Message): + """A FusiontablesTableDeleteRequest object. + + Fields: + tableId: ID of the table that is being deleted. + """ + + tableId = _messages.StringField(1, required=True) + + +class FusiontablesTableDeleteResponse(_messages.Message): + """An empty FusiontablesTableDelete response.""" + + +class FusiontablesTableGetRequest(_messages.Message): + """A FusiontablesTableGetRequest object. + + Fields: + tableId: Identifier(ID) for the table being requested. + """ + + tableId = _messages.StringField(1, required=True) + + +class FusiontablesTableImportRowsRequest(_messages.Message): + """A FusiontablesTableImportRowsRequest object. + + Fields: + delimiter: The delimiter used to separate cell values. This can only + consist of a single character. Default is ','. + encoding: The encoding of the content. Default is UTF-8. Use 'auto-detect' + if you are unsure of the encoding. + endLine: The index of the last line from which to start importing, + exclusive. Thus, the number of imported lines is endLine - startLine. If + this parameter is not provided, the file will be imported until the last + line of the file. If endLine is negative, then the imported content will + exclude the last endLine lines. That is, if endline is negative, no line + will be imported whose index is greater than N + endLine where N is the + number of lines in the file, and the number of imported lines will be N + + endLine - startLine. + isStrict: Whether the CSV must have the same number of values for each + row. If false, rows with fewer values will be padded with empty values. + Default is true. + startLine: The index of the first line from which to start importing, + inclusive. Default is 0. + tableId: The table into which new rows are being imported. + """ + + delimiter = _messages.StringField(1) + encoding = _messages.StringField(2) + endLine = _messages.IntegerField(3, variant=_messages.Variant.INT32) + isStrict = _messages.BooleanField(4) + startLine = _messages.IntegerField(5, variant=_messages.Variant.INT32) + tableId = _messages.StringField(6, required=True) + + +class FusiontablesTableImportTableRequest(_messages.Message): + """A FusiontablesTableImportTableRequest object. + + Fields: + delimiter: The delimiter used to separate cell values. This can only + consist of a single character. Default is ','. + encoding: The encoding of the content. Default is UTF-8. Use 'auto-detect' + if you are unsure of the encoding. + name: The name to be assigned to the new table. + """ + + delimiter = _messages.StringField(1) + encoding = _messages.StringField(2) + name = _messages.StringField(3, required=True) + + +class FusiontablesTableListRequest(_messages.Message): + """A FusiontablesTableListRequest object. + + Fields: + maxResults: Maximum number of styles to return. Optional. Default is 5. + pageToken: Continuation token specifying which result page to return. + Optional. + """ + + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + + +class FusiontablesTablePatchRequest(_messages.Message): + """A FusiontablesTablePatchRequest object. + + Fields: + replaceViewDefinition: Should the view definition also be updated? The + specified view definition replaces the existing one. Only a view can be + updated with a new definition. + table: A Table resource to be passed as the request body. + tableId: ID of the table that is being updated. + """ - """A FusiontablesColumnListRequest object. + replaceViewDefinition = _messages.BooleanField(1) + table = _messages.MessageField('Table', 2) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesTableUpdateRequest(_messages.Message): + """A FusiontablesTableUpdateRequest object. + + Fields: + replaceViewDefinition: Should the view definition also be updated? The + specified view definition replaces the existing one. Only a view can be + updated with a new definition. + table: A Table resource to be passed as the request body. + tableId: ID of the table that is being updated. + """ + + replaceViewDefinition = _messages.BooleanField(1) + table = _messages.MessageField('Table', 2) + tableId = _messages.StringField(3, required=True) + + +class FusiontablesTaskDeleteRequest(_messages.Message): + """A FusiontablesTaskDeleteRequest object. + + Fields: + tableId: Table from which the task is being deleted. + taskId: A string attribute. + """ + + tableId = _messages.StringField(1, required=True) + taskId = _messages.StringField(2, required=True) + + +class FusiontablesTaskDeleteResponse(_messages.Message): + """An empty FusiontablesTaskDelete response.""" + + +class FusiontablesTaskGetRequest(_messages.Message): + """A FusiontablesTaskGetRequest object. + + Fields: + tableId: Table to which the task belongs. + taskId: A string attribute. + """ + + tableId = _messages.StringField(1, required=True) + taskId = _messages.StringField(2, required=True) + + +class FusiontablesTaskListRequest(_messages.Message): + """A FusiontablesTaskListRequest object. + + Fields: + maxResults: Maximum number of columns to return. Optional. Default is 5. + pageToken: A string attribute. + startIndex: A integer attribute. + tableId: Table whose tasks are being listed. + """ + + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + startIndex = _messages.IntegerField(3, variant=_messages.Variant.UINT32) + tableId = _messages.StringField(4, required=True) + + +class FusiontablesTemplateDeleteRequest(_messages.Message): + """A FusiontablesTemplateDeleteRequest object. + + Fields: + tableId: Table from which the template is being deleted + templateId: Identifier for the template which is being deleted + """ + + tableId = _messages.StringField(1, required=True) + templateId = _messages.IntegerField(2, required=True, variant=_messages.Variant.INT32) + + +class FusiontablesTemplateDeleteResponse(_messages.Message): + """An empty FusiontablesTemplateDelete response.""" + + +class FusiontablesTemplateGetRequest(_messages.Message): + """A FusiontablesTemplateGetRequest object. + + Fields: + tableId: Table to which the template belongs + templateId: Identifier for the template that is being requested + """ + + tableId = _messages.StringField(1, required=True) + templateId = _messages.IntegerField(2, required=True, variant=_messages.Variant.INT32) + + +class FusiontablesTemplateListRequest(_messages.Message): + """A FusiontablesTemplateListRequest object. + + Fields: + maxResults: Maximum number of templates to return. Optional. Default is 5. + pageToken: Continuation token specifying which results page to return. + Optional. + tableId: Identifier for the table whose templates are being requested + """ + + maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) + pageToken = _messages.StringField(2) + tableId = _messages.StringField(3, required=True) + + +class Geometry(_messages.Message): + """Represents a Geometry object. + + Fields: + geometries: The list of geometries in this geometry collection. + geometry: A extra_types.JsonValue attribute. + type: Type: A collection of geometries. + """ + + geometries = _messages.MessageField('extra_types.JsonValue', 1, repeated=True) + geometry = _messages.MessageField('extra_types.JsonValue', 2) + type = _messages.StringField(3, default=u'GeometryCollection') + + +class Import(_messages.Message): + """Represents an import request. + + Fields: + kind: Type name: a template for an import request. + numRowsReceived: The number of rows received from the import request. + """ + + kind = _messages.StringField(1, default=u'fusiontables#import') + numRowsReceived = _messages.IntegerField(2) + + +class Line(_messages.Message): + """Represents a line geometry. + + Messages: + CoordinatesValueListEntry: Single entry in a CoordinatesValue. + + Fields: + coordinates: The coordinates that define the line. + type: Type: A line geometry. + """ + + class CoordinatesValueListEntry(_messages.Message): + """Single entry in a CoordinatesValue. Fields: - maxResults: Maximum number of columns to return. Optional. Default is 5. - pageToken: Continuation token specifying which result page to return. - Optional. - tableId: Table whose columns are being listed. + entry: A number attribute. """ - maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32) - pageToken = _messages.StringField(2) - tableId = _messages.StringField(3, required=True) + entry = _messages.FloatField(1, repeated=True) + + coordinates = _messages.MessageField('CoordinatesValueListEntry', 1, repeated=True) + type = _messages.StringField(2, default=u'LineString') + + +class LineStyle(_messages.Message): + """Represents a LineStyle within a StyleSetting + + Fields: + strokeColor: Color of the line in #RRGGBB format. + strokeColorStyler: Column-value, gradient or buckets styler that is used + to determine the line color and opacity. + strokeOpacity: Opacity of the line : 0.0 (transparent) to 1.0 (opaque). + strokeWeight: Width of the line in pixels. + strokeWeightStyler: Column-value or bucket styler that is used to + determine the width of the line. + """ + + strokeColor = _messages.StringField(1) + strokeColorStyler = _messages.MessageField('StyleFunction', 2) + strokeOpacity = _messages.FloatField(3) + strokeWeight = _messages.IntegerField(4, variant=_messages.Variant.INT32) + strokeWeightStyler = _messages.MessageField('StyleFunction', 5) + + +class Point(_messages.Message): + """Represents a point object. + + Fields: + coordinates: The coordinates that define the point. + type: Point: A point geometry. + """ + + coordinates = _messages.FloatField(1, repeated=True) + type = _messages.StringField(2, default=u'Point') + + +class PointStyle(_messages.Message): + """Represents a PointStyle within a StyleSetting + + Fields: + iconName: Name of the icon. Use values defined in + http://www.google.com/fusiontables/DataSource?dsrcid=308519 + iconStyler: Column or a bucket value from which the icon name is to be + determined. + """ + + iconName = _messages.StringField(1) + iconStyler = _messages.MessageField('StyleFunction', 2) + + +class Polygon(_messages.Message): + """Represents a polygon object. + Messages: + CoordinatesValueListEntry: Single entry in a CoordinatesValue. -class FusiontablesColumnListAlternateRequest(_messages.Message): + Fields: + coordinates: The coordinates that define the polygon. + type: Type: A polygon geometry. + """ - """A FusiontablesColumnListRequest object. + class CoordinatesValueListEntry(_messages.Message): + """Single entry in a CoordinatesValue. + + Messages: + EntryValueListEntry: Single entry in a EntryValue. Fields: - pageSize: Maximum number of columns to return. Optional. Default is 5. - pageToken: Continuation token specifying which result page to return. - Optional. - tableId: Table whose columns are being listed. + entry: A EntryValueListEntry attribute. """ - pageSize = _messages.IntegerField(1, variant=_messages.Variant.UINT32) - pageToken = _messages.StringField(2) - tableId = _messages.StringField(3, required=True) + class EntryValueListEntry(_messages.Message): + """Single entry in a EntryValue. + + Fields: + entry: A number attribute. + """ + + entry = _messages.FloatField(1, repeated=True) + + entry = _messages.MessageField('EntryValueListEntry', 1, repeated=True) + + coordinates = _messages.MessageField('CoordinatesValueListEntry', 1, repeated=True) + type = _messages.StringField(2, default=u'Polygon') + + +class PolygonStyle(_messages.Message): + """Represents a PolygonStyle within a StyleSetting + + Fields: + fillColor: Color of the interior of the polygon in #RRGGBB format. + fillColorStyler: Column-value, gradient, or bucket styler that is used to + determine the interior color and opacity of the polygon. + fillOpacity: Opacity of the interior of the polygon: 0.0 (transparent) to + 1.0 (opaque). + strokeColor: Color of the polygon border in #RRGGBB format. + strokeColorStyler: Column-value, gradient or buckets styler that is used + to determine the border color and opacity. + strokeOpacity: Opacity of the polygon border: 0.0 (transparent) to 1.0 + (opaque). + strokeWeight: Width of the polyon border in pixels. + strokeWeightStyler: Column-value or bucket styler that is used to + determine the width of the polygon border. + """ + + fillColor = _messages.StringField(1) + fillColorStyler = _messages.MessageField('StyleFunction', 2) + fillOpacity = _messages.FloatField(3) + strokeColor = _messages.StringField(4) + strokeColorStyler = _messages.MessageField('StyleFunction', 5) + strokeOpacity = _messages.FloatField(6) + strokeWeight = _messages.IntegerField(7, variant=_messages.Variant.INT32) + strokeWeightStyler = _messages.MessageField('StyleFunction', 8) -class ColumnListAlternate(_messages.Message): +class Sqlresponse(_messages.Message): + """Represents a response to an sql statement. - """Represents a list of columns in a table. + Messages: + RowsValueListEntry: Single entry in a RowsValue. + + Fields: + columns: Columns in the table. + kind: Type name: a template for an individual table. + rows: The rows in the table. For each cell we print out whatever cell + value (e.g., numeric, string) exists. Thus it is important that each + cell contains only one value. + """ + + class RowsValueListEntry(_messages.Message): + """Single entry in a RowsValue. Fields: - items: List of all requested columns. - kind: Type name: a list of all columns. - nextPageToken: Token used to access the next page of this - result. No token is displayed if there are no more pages left. - totalItems: Total number of columns for the table. + entry: A extra_types.JsonValue attribute. + """ + + entry = _messages.MessageField('extra_types.JsonValue', 1, repeated=True) + + columns = _messages.StringField(1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#sqlresponse') + rows = _messages.MessageField('RowsValueListEntry', 3, repeated=True) + + +class StandardQueryParameters(_messages.Message): + """Query parameters accepted by all methods. + + Enums: + AltValueValuesEnum: Data format for the response. + + Fields: + alt: Data format for the response. + fields: Selector specifying which fields to include in a partial response. + key: API key. Your API key identifies your project and provides you with + API access, quota, and reports. Required unless you provide an OAuth 2.0 + token. + oauth_token: OAuth 2.0 token for the current user. + prettyPrint: Returns response with indentations and line breaks. + quotaUser: Available to use for quota purposes for server-side + applications. Can be any arbitrary string assigned to a user, but should + not exceed 40 characters. Overrides userIp if both are provided. + trace: A tracing token of the form "token:" to include in api + requests. + userIp: IP address of the site where the request originates. Use this if + you want to enforce per-user limits. + """ + + class AltValueValuesEnum(_messages.Enum): + """Data format for the response. + + Values: + csv: Responses with Content-Type of text/csv + json: Responses with Content-Type of application/json + """ + csv = 0 + json = 1 + + alt = _messages.EnumField('AltValueValuesEnum', 1, default=u'json') + fields = _messages.StringField(2) + key = _messages.StringField(3) + oauth_token = _messages.StringField(4) + prettyPrint = _messages.BooleanField(5, default=True) + quotaUser = _messages.StringField(6) + trace = _messages.StringField(7) + userIp = _messages.StringField(8) + + +class StyleFunction(_messages.Message): + """Represents a StyleFunction within a StyleSetting + + Messages: + GradientValue: Gradient function that interpolates a range of colors based + on column value. + + Fields: + buckets: Bucket function that assigns a style based on the range a column + value falls into. + columnName: Name of the column whose value is used in the style. + gradient: Gradient function that interpolates a range of colors based on + column value. + kind: Stylers can be one of three kinds: "fusiontables#fromColumn" if the + column value is to be used as is, i.e., the column values can have + colors in #RRGGBBAA format or integer line widths or icon names; + "fusiontables#gradient" if the styling of the row is to be based on + applying the gradient function on the column value; or + "fusiontables#buckets" if the styling is to based on the bucket into + which the the column value falls. + """ + + class GradientValue(_messages.Message): + """Gradient function that interpolates a range of colors based on column + value. + Messages: + ColorsValueListEntry: A ColorsValueListEntry object. + + Fields: + colors: Array with two or more colors. + max: Higher-end of the interpolation range: rows with this value will be + assigned to colors[n-1]. + min: Lower-end of the interpolation range: rows with this value will be + assigned to colors[0]. """ - columns = _messages.MessageField('Column', 1, repeated=True) - kind = _messages.StringField(2, default=u'fusiontables#columnList') - nextPageToken = _messages.StringField(3) - totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) + class ColorsValueListEntry(_messages.Message): + """A ColorsValueListEntry object. + + Fields: + color: Color in #RRGGBB format. + opacity: Opacity of the color: 0.0 (transparent) to 1.0 (opaque). + """ + + color = _messages.StringField(1) + opacity = _messages.FloatField(2) + + colors = _messages.MessageField('ColorsValueListEntry', 1, repeated=True) + max = _messages.FloatField(2) + min = _messages.FloatField(3) + + buckets = _messages.MessageField('Bucket', 1, repeated=True) + columnName = _messages.StringField(2) + gradient = _messages.MessageField('GradientValue', 3) + kind = _messages.StringField(4) + + +class StyleSetting(_messages.Message): + """Represents a complete StyleSettings object. The primary key is a + combination of the tableId and a styleId. + + Fields: + kind: Type name: an individual style setting. A StyleSetting contains the + style defintions for points, lines, and polygons in a table. Since a + table can have any one or all of them, a style definition can have + point, line and polygon style definitions. + markerOptions: Style definition for points in the table. + name: Optional name for the style setting. + polygonOptions: Style definition for polygons in the table. + polylineOptions: Style definition for lines in the table. + styleId: Identifier for the style setting (unique only within tables). + tableId: Identifier for the table. + """ + + kind = _messages.StringField(1, default=u'fusiontables#styleSetting') + markerOptions = _messages.MessageField('PointStyle', 2) + name = _messages.StringField(3) + polygonOptions = _messages.MessageField('PolygonStyle', 4) + polylineOptions = _messages.MessageField('LineStyle', 5) + styleId = _messages.IntegerField(6, variant=_messages.Variant.INT32) + tableId = _messages.StringField(7) + + +class StyleSettingList(_messages.Message): + """Represents a list of styles for a given table. + + Fields: + items: All requested style settings. + kind: Type name: in this case, a list of style settings. + nextPageToken: Token used to access the next page of this result. No token + is displayed if there are no more pages left. + totalItems: Total number of styles for the table. + """ + + items = _messages.MessageField('StyleSetting', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#styleSettingList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) + + +class Table(_messages.Message): + """Represents a table. Specifies the name, whether it is exportable, + description, attribution, and attribution link. + + Fields: + attribution: Optional attribution assigned to the table. + attributionLink: Optional link for attribution. + baseTableIds: Optional base table identifier if this table is a view or + merged table. + columns: Columns in the table. + description: Optional description assigned to the table. + isExportable: Variable for whether table is exportable. + kind: Type name: a template for an individual table. + name: Name assigned to a table. + sql: Optional sql that encodes the table definition for derived tables. + tableId: Encrypted unique alphanumeric identifier for the table. + """ + + attribution = _messages.StringField(1) + attributionLink = _messages.StringField(2) + baseTableIds = _messages.StringField(3, repeated=True) + columns = _messages.MessageField('Column', 4, repeated=True) + description = _messages.StringField(5) + isExportable = _messages.BooleanField(6) + kind = _messages.StringField(7, default=u'fusiontables#table') + name = _messages.StringField(8) + sql = _messages.StringField(9) + tableId = _messages.StringField(10) + + +class TableList(_messages.Message): + """Represents a list of tables. + + Fields: + items: List of all requested tables. + kind: Type name: a list of all tables. + nextPageToken: Token used to access the next page of this result. No token + is displayed if there are no more pages left. + """ + + items = _messages.MessageField('Table', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#tableList') + nextPageToken = _messages.StringField(3) + + +class Task(_messages.Message): + """Specifies the identifier, name, and type of a task in a table. + + Fields: + kind: Type of the resource. This is always "fusiontables#task". + progress: An indication of task progress. + started: false while the table is busy with some other task. true if this + background task is currently running. + taskId: Identifier for the task. + type: Type of background task. One of DELETE_ROWS Deletes one or more + rows from the table. ADD_ROWS "Adds one or more rows to a table. + Includes importing data into a new table and importing more rows into an + existing table. ADD_COLUMN Adds a new column to the table. CHANGE_TYPE + Changes the type of a column. + """ + + kind = _messages.StringField(1, default=u'fusiontables#task') + progress = _messages.StringField(2) + started = _messages.BooleanField(3) + taskId = _messages.IntegerField(4) + type = _messages.StringField(5) + + +class TaskList(_messages.Message): + """Represents a list of tasks for a table. + + Fields: + items: List of all requested tasks. + kind: Type of the resource. This is always "fusiontables#taskList". + nextPageToken: Token used to access the next page of this result. No token + is displayed if there are no more pages left. + totalItems: Total number of tasks for the table. + """ + + items = _messages.MessageField('Task', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#taskList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) + + +class Template(_messages.Message): + """Represents the contents of InfoWindow templates. + + Fields: + automaticColumnNames: List of columns from which the template is to be + automatically constructed. Only one of body or automaticColumns can be + specified. + body: Body of the template. It contains HTML with {column_name} to insert + values from a particular column. The body is sanitized to remove certain + tags, e.g., script. Only one of body or automaticColumns can be + specified. + kind: Type name: a template for the info window contents. The template can + either include an HTML body or a list of columns from which the template + is computed automatically. + name: Optional name assigned to a template. + tableId: Identifier for the table for which the template is defined. + templateId: Identifier for the template, unique within the context of a + particular table. + """ + + automaticColumnNames = _messages.StringField(1, repeated=True) + body = _messages.StringField(2) + kind = _messages.StringField(3, default=u'fusiontables#template') + name = _messages.StringField(4) + tableId = _messages.StringField(5) + templateId = _messages.IntegerField(6, variant=_messages.Variant.INT32) + + +class TemplateList(_messages.Message): + """Represents a list of templates for a given table. + + Fields: + items: List of all requested templates. + kind: Type name: a list of all templates. + nextPageToken: Token used to access the next page of this result. No token + is displayed if there are no more pages left. + totalItems: Total number of templates for the table. + """ + + items = _messages.MessageField('Template', 1, repeated=True) + kind = _messages.StringField(2, default=u'fusiontables#templateList') + nextPageToken = _messages.StringField(3) + totalItems = _messages.IntegerField(4, variant=_messages.Variant.INT32) + + -- GitLab From 86769ce19932872f441be92aab14fb20348696c2 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Fri, 15 Apr 2016 09:13:23 -0400 Subject: [PATCH 249/295] 'Make method_config class level accesible.' --- apitools/base/py/base_api.py | 18 +- apitools/base/py/testing/mock.py | 3 +- apitools/gen/service_registry.py | 30 +- samples/dns_sample/dns_v1/dns_v1_client.py | 237 ++-- .../fusiontables_v1/fusiontables_v1_client.py | 818 ++++++------ samples/iam_sample/iam_v1/iam_v1_client.py | 443 +++---- .../storage_v1/storage_v1_client.py | 1146 +++++++++-------- 7 files changed, 1373 insertions(+), 1322 deletions(-) diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py index 2bdd1ec..e3297f7 100644 --- a/apitools/base/py/base_api.py +++ b/apitools/base/py/base_api.py @@ -472,7 +472,23 @@ class BaseApiService(object): return self.__client def GetMethodConfig(self, method): - return self._method_configs[method] + """Returns service cached method config for given method.""" + method_config = self._method_configs.get(method) + if method_config: + return method_config + func = getattr(self, method, None) + if func is None: + raise KeyError(method) + method_config = getattr(func, 'method_config', None) + if method_config is None: + raise KeyError(method) + self._method_configs[method] = config = method_config() + return config + + @classmethod + def GetMethodsList(cls): + return [f.__name__ for f in six.itervalues(cls.__dict__) + if getattr(f, 'method_config', None)] def GetUploadConfig(self, method): return self._upload_configs.get(method) diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py index 91958a7..e779cfe 100644 --- a/apitools/base/py/testing/mock.py +++ b/apitools/base/py/testing/mock.py @@ -203,6 +203,7 @@ class _MockedMethod(object): self.__key = key self.__mocked_client = mocked_client self.__real_method = real_method + self.method_config = real_method.method_config def Expect(self, request, response=None, exception=None, **unused_kwargs): """Add an expectation on the mocked method. @@ -312,7 +313,7 @@ class Client(object): self.__client_class._URL_VERSION) mocked_service = _MockedService( api_name + '.' + collection_name, self, - service._method_configs.keys(), + service.GetMethodsList(), service if self.__real_client else None) mocked_constructor = _MakeMockedServiceConstructor(mocked_service) setattr(self.__client_class, name, mocked_constructor) diff --git a/apitools/gen/service_registry.py b/apitools/gen/service_registry.py index f9c0cd3..9f71592 100644 --- a/apitools/gen/service_registry.py +++ b/apitools/gen/service_registry.py @@ -104,22 +104,6 @@ class ServiceRegistry(object): with printer.Indent(): printer('super(%s.%s, self).__init__(client)', client_class_name, class_name) - printer('self._method_configs = {') - with printer.Indent(indent=' '): - for method_name, method_info in method_info_map.items(): - printer("'%s': base_api.ApiMethodInfo(", method_name) - with printer.Indent(indent=' '): - attrs = sorted( - x.name for x in method_info.all_fields()) - for attr in attrs: - if attr in ('upload_config', 'description'): - continue - value = getattr(method_info, attr) - if value is not None: - printer('%s=%r,', attr, value) - printer('),') - printer('}') - printer() printer('self._upload_configs = {') with printer.Indent(indent=' '): for method_name, method_info in method_info_map.items(): @@ -165,6 +149,20 @@ class ServiceRegistry(object): for line in arg_lines[:-1]: printer('%s,', line) printer('%s)', arg_lines[-1]) + printer() + printer('{0}.method_config = lambda: base_api.ApiMethodInfo(' + .format(method_name)) + with printer.Indent(indent=' '): + method_info = method_info_map[method_name] + attrs = sorted( + x.name for x in method_info.all_fields()) + for attr in attrs: + if attr in ('upload_config', 'description'): + continue + value = getattr(method_info, attr) + if value is not None: + printer('%s=%r,', attr, value) + printer(')') def __WriteProtoServiceDeclaration(self, printer, name, method_info_map): """Write a single service declaration to a proto file.""" diff --git a/samples/dns_sample/dns_v1/dns_v1_client.py b/samples/dns_sample/dns_v1/dns_v1_client.py index 42f5359..b15403c 100644 --- a/samples/dns_sample/dns_v1/dns_v1_client.py +++ b/samples/dns_sample/dns_v1/dns_v1_client.py @@ -46,45 +46,6 @@ class DnsV1(base_api.BaseApiClient): def __init__(self, client): super(DnsV1.ChangesService, self).__init__(client) - self._method_configs = { - 'Create': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'dns.changes.create', - ordered_params=[u'project', u'managedZone'], - path_params=[u'managedZone', u'project'], - query_params=[], - relative_path=u'projects/{project}/managedZones/{managedZone}/changes', - request_field=u'change', - request_type_name=u'DnsChangesCreateRequest', - response_type_name=u'Change', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.changes.get', - ordered_params=[u'project', u'managedZone', u'changeId'], - path_params=[u'changeId', u'managedZone', u'project'], - query_params=[], - relative_path=u'projects/{project}/managedZones/{managedZone}/changes/{changeId}', - request_field='', - request_type_name=u'DnsChangesGetRequest', - response_type_name=u'Change', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.changes.list', - ordered_params=[u'project', u'managedZone'], - path_params=[u'managedZone', u'project'], - query_params=[u'maxResults', u'pageToken', u'sortBy', u'sortOrder'], - relative_path=u'projects/{project}/managedZones/{managedZone}/changes', - request_field='', - request_type_name=u'DnsChangesListRequest', - response_type_name=u'ChangesListResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -101,6 +62,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Create.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dns.changes.create', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes', + request_field=u'change', + request_type_name=u'DnsChangesCreateRequest', + response_type_name=u'Change', + supports_download=False, + ) + def Get(self, request, global_params=None): """Fetch the representation of an existing Change. @@ -114,6 +88,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.changes.get', + ordered_params=[u'project', u'managedZone', u'changeId'], + path_params=[u'changeId', u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes/{changeId}', + request_field='', + request_type_name=u'DnsChangesGetRequest', + response_type_name=u'Change', + supports_download=False, + ) + def List(self, request, global_params=None): """Enumerate Changes to a ResourceRecordSet collection. @@ -127,6 +114,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.changes.list', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[u'maxResults', u'pageToken', u'sortBy', u'sortOrder'], + relative_path=u'projects/{project}/managedZones/{managedZone}/changes', + request_field='', + request_type_name=u'DnsChangesListRequest', + response_type_name=u'ChangesListResponse', + supports_download=False, + ) + class ManagedZonesService(base_api.BaseApiService): """Service class for the managedZones resource.""" @@ -134,57 +134,6 @@ class DnsV1(base_api.BaseApiClient): def __init__(self, client): super(DnsV1.ManagedZonesService, self).__init__(client) - self._method_configs = { - 'Create': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'dns.managedZones.create', - ordered_params=[u'project'], - path_params=[u'project'], - query_params=[], - relative_path=u'projects/{project}/managedZones', - request_field=u'managedZone', - request_type_name=u'DnsManagedZonesCreateRequest', - response_type_name=u'ManagedZone', - supports_download=False, - ), - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'dns.managedZones.delete', - ordered_params=[u'project', u'managedZone'], - path_params=[u'managedZone', u'project'], - query_params=[], - relative_path=u'projects/{project}/managedZones/{managedZone}', - request_field='', - request_type_name=u'DnsManagedZonesDeleteRequest', - response_type_name=u'DnsManagedZonesDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.managedZones.get', - ordered_params=[u'project', u'managedZone'], - path_params=[u'managedZone', u'project'], - query_params=[], - relative_path=u'projects/{project}/managedZones/{managedZone}', - request_field='', - request_type_name=u'DnsManagedZonesGetRequest', - response_type_name=u'ManagedZone', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.managedZones.list', - ordered_params=[u'project'], - path_params=[u'project'], - query_params=[u'dnsName', u'maxResults', u'pageToken'], - relative_path=u'projects/{project}/managedZones', - request_field='', - request_type_name=u'DnsManagedZonesListRequest', - response_type_name=u'ManagedZonesListResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -201,6 +150,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Create.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dns.managedZones.create', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones', + request_field=u'managedZone', + request_type_name=u'DnsManagedZonesCreateRequest', + response_type_name=u'ManagedZone', + supports_download=False, + ) + def Delete(self, request, global_params=None): """Delete a previously created ManagedZone. @@ -214,6 +176,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'dns.managedZones.delete', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}', + request_field='', + request_type_name=u'DnsManagedZonesDeleteRequest', + response_type_name=u'DnsManagedZonesDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Fetch the representation of an existing ManagedZone. @@ -227,6 +202,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.managedZones.get', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[], + relative_path=u'projects/{project}/managedZones/{managedZone}', + request_field='', + request_type_name=u'DnsManagedZonesGetRequest', + response_type_name=u'ManagedZone', + supports_download=False, + ) + def List(self, request, global_params=None): """Enumerate ManagedZones that have been created but not yet deleted. @@ -240,6 +228,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.managedZones.list', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[u'dnsName', u'maxResults', u'pageToken'], + relative_path=u'projects/{project}/managedZones', + request_field='', + request_type_name=u'DnsManagedZonesListRequest', + response_type_name=u'ManagedZonesListResponse', + supports_download=False, + ) + class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" @@ -247,21 +248,6 @@ class DnsV1(base_api.BaseApiClient): def __init__(self, client): super(DnsV1.ProjectsService, self).__init__(client) - self._method_configs = { - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.projects.get', - ordered_params=[u'project'], - path_params=[u'project'], - query_params=[], - relative_path=u'projects/{project}', - request_field='', - request_type_name=u'DnsProjectsGetRequest', - response_type_name=u'Project', - supports_download=False, - ), - } - self._upload_configs = { } @@ -278,6 +264,19 @@ class DnsV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.projects.get', + ordered_params=[u'project'], + path_params=[u'project'], + query_params=[], + relative_path=u'projects/{project}', + request_field='', + request_type_name=u'DnsProjectsGetRequest', + response_type_name=u'Project', + supports_download=False, + ) + class ResourceRecordSetsService(base_api.BaseApiService): """Service class for the resourceRecordSets resource.""" @@ -285,21 +284,6 @@ class DnsV1(base_api.BaseApiClient): def __init__(self, client): super(DnsV1.ResourceRecordSetsService, self).__init__(client) - self._method_configs = { - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'dns.resourceRecordSets.list', - ordered_params=[u'project', u'managedZone'], - path_params=[u'managedZone', u'project'], - query_params=[u'maxResults', u'name', u'pageToken', u'type'], - relative_path=u'projects/{project}/managedZones/{managedZone}/rrsets', - request_field='', - request_type_name=u'DnsResourceRecordSetsListRequest', - response_type_name=u'ResourceRecordSetsListResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -315,3 +299,16 @@ class DnsV1(base_api.BaseApiClient): config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) + + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dns.resourceRecordSets.list', + ordered_params=[u'project', u'managedZone'], + path_params=[u'managedZone', u'project'], + query_params=[u'maxResults', u'name', u'pageToken', u'type'], + relative_path=u'projects/{project}/managedZones/{managedZone}/rrsets', + request_field='', + request_type_name=u'DnsResourceRecordSetsListRequest', + response_type_name=u'ResourceRecordSetsListResponse', + supports_download=False, + ) diff --git a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py index 690f148..3376aa3 100644 --- a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py +++ b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_client.py @@ -48,81 +48,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.ColumnService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'fusiontables.column.delete', - ordered_params=[u'tableId', u'columnId'], - path_params=[u'columnId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/columns/{columnId}', - request_field='', - request_type_name=u'FusiontablesColumnDeleteRequest', - response_type_name=u'FusiontablesColumnDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.column.get', - ordered_params=[u'tableId', u'columnId'], - path_params=[u'columnId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/columns/{columnId}', - request_field='', - request_type_name=u'FusiontablesColumnGetRequest', - response_type_name=u'Column', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.column.insert', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/columns', - request_field=u'column', - request_type_name=u'FusiontablesColumnInsertRequest', - response_type_name=u'Column', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.column.list', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables/{tableId}/columns', - request_field='', - request_type_name=u'FusiontablesColumnListRequest', - response_type_name=u'ColumnList', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'fusiontables.column.patch', - ordered_params=[u'tableId', u'columnId'], - path_params=[u'columnId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/columns/{columnId}', - request_field=u'column', - request_type_name=u'FusiontablesColumnPatchRequest', - response_type_name=u'Column', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'fusiontables.column.update', - ordered_params=[u'tableId', u'columnId'], - path_params=[u'columnId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/columns/{columnId}', - request_field=u'column', - request_type_name=u'FusiontablesColumnUpdateRequest', - response_type_name=u'Column', - supports_download=False, - ), - } - self._upload_configs = { } @@ -139,6 +64,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.column.delete', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field='', + request_type_name=u'FusiontablesColumnDeleteRequest', + response_type_name=u'FusiontablesColumnDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Retrieves a specific column by its id. @@ -152,6 +90,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.get', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field='', + request_type_name=u'FusiontablesColumnGetRequest', + response_type_name=u'Column', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Adds a new column to the table. @@ -165,6 +116,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.column.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns', + request_field=u'column', + request_type_name=u'FusiontablesColumnInsertRequest', + response_type_name=u'Column', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of columns. @@ -178,6 +142,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.column.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/columns', + request_field='', + request_type_name=u'FusiontablesColumnListRequest', + response_type_name=u'ColumnList', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates the name or type of an existing column. This method supports patch semantics. @@ -191,6 +168,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.column.patch', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field=u'column', + request_type_name=u'FusiontablesColumnPatchRequest', + response_type_name=u'Column', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates the name or type of an existing column. @@ -204,6 +194,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.column.update', + ordered_params=[u'tableId', u'columnId'], + path_params=[u'columnId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/columns/{columnId}', + request_field=u'column', + request_type_name=u'FusiontablesColumnUpdateRequest', + response_type_name=u'Column', + supports_download=False, + ) + class QueryService(base_api.BaseApiService): """Service class for the query resource.""" @@ -211,33 +214,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.QueryService, self).__init__(client) - self._method_configs = { - 'Sql': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.query.sql', - ordered_params=[u'sql'], - path_params=[], - query_params=[u'hdrs', u'sql', u'typed'], - relative_path=u'query', - request_field='', - request_type_name=u'FusiontablesQuerySqlRequest', - response_type_name=u'Sqlresponse', - supports_download=True, - ), - 'SqlGet': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.query.sqlGet', - ordered_params=[u'sql'], - path_params=[], - query_params=[u'hdrs', u'sql', u'typed'], - relative_path=u'query', - request_field='', - request_type_name=u'FusiontablesQuerySqlGetRequest', - response_type_name=u'Sqlresponse', - supports_download=True, - ), - } - self._upload_configs = { } @@ -257,6 +233,19 @@ class FusiontablesV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + Sql.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.query.sql', + ordered_params=[u'sql'], + path_params=[], + query_params=[u'hdrs', u'sql', u'typed'], + relative_path=u'query', + request_field='', + request_type_name=u'FusiontablesQuerySqlRequest', + response_type_name=u'Sqlresponse', + supports_download=True, + ) + def SqlGet(self, request, global_params=None, download=None): """Executes an SQL SELECT/SHOW/DESCRIBE statement. @@ -273,6 +262,19 @@ class FusiontablesV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + SqlGet.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.query.sqlGet', + ordered_params=[u'sql'], + path_params=[], + query_params=[u'hdrs', u'sql', u'typed'], + relative_path=u'query', + request_field='', + request_type_name=u'FusiontablesQuerySqlGetRequest', + response_type_name=u'Sqlresponse', + supports_download=True, + ) + class StyleService(base_api.BaseApiService): """Service class for the style resource.""" @@ -280,81 +282,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.StyleService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'fusiontables.style.delete', - ordered_params=[u'tableId', u'styleId'], - path_params=[u'styleId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/styles/{styleId}', - request_field='', - request_type_name=u'FusiontablesStyleDeleteRequest', - response_type_name=u'FusiontablesStyleDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.style.get', - ordered_params=[u'tableId', u'styleId'], - path_params=[u'styleId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/styles/{styleId}', - request_field='', - request_type_name=u'FusiontablesStyleGetRequest', - response_type_name=u'StyleSetting', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.style.insert', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/styles', - request_field='', - request_type_name=u'StyleSetting', - response_type_name=u'StyleSetting', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.style.list', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables/{tableId}/styles', - request_field='', - request_type_name=u'FusiontablesStyleListRequest', - response_type_name=u'StyleSettingList', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'fusiontables.style.patch', - ordered_params=[u'tableId', u'styleId'], - path_params=[u'styleId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/styles/{styleId}', - request_field='', - request_type_name=u'StyleSetting', - response_type_name=u'StyleSetting', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'fusiontables.style.update', - ordered_params=[u'tableId', u'styleId'], - path_params=[u'styleId', u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/styles/{styleId}', - request_field='', - request_type_name=u'StyleSetting', - response_type_name=u'StyleSetting', - supports_download=False, - ), - } - self._upload_configs = { } @@ -371,6 +298,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.style.delete', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'FusiontablesStyleDeleteRequest', + response_type_name=u'FusiontablesStyleDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Gets a specific style. @@ -384,6 +324,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.style.get', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'FusiontablesStyleGetRequest', + response_type_name=u'StyleSetting', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Adds a new style for the table. @@ -397,6 +350,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.style.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of styles. @@ -410,6 +376,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.style.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/styles', + request_field='', + request_type_name=u'FusiontablesStyleListRequest', + response_type_name=u'StyleSettingList', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an existing style. This method supports patch semantics. @@ -423,6 +402,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.style.patch', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates an existing style. @@ -436,6 +428,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.style.update', + ordered_params=[u'tableId', u'styleId'], + path_params=[u'styleId', u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/styles/{styleId}', + request_field='', + request_type_name=u'StyleSetting', + response_type_name=u'StyleSetting', + supports_download=False, + ) + class TableService(base_api.BaseApiService): """Service class for the table resource.""" @@ -443,117 +448,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.TableService, self).__init__(client) - self._method_configs = { - 'Copy': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.table.copy', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'copyPresentation'], - relative_path=u'tables/{tableId}/copy', - request_field='', - request_type_name=u'FusiontablesTableCopyRequest', - response_type_name=u'Table', - supports_download=False, - ), - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'fusiontables.table.delete', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}', - request_field='', - request_type_name=u'FusiontablesTableDeleteRequest', - response_type_name=u'FusiontablesTableDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.table.get', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}', - request_field='', - request_type_name=u'FusiontablesTableGetRequest', - response_type_name=u'Table', - supports_download=False, - ), - 'ImportRows': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.table.importRows', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'delimiter', u'encoding', u'endLine', u'isStrict', u'startLine'], - relative_path=u'tables/{tableId}/import', - request_field='', - request_type_name=u'FusiontablesTableImportRowsRequest', - response_type_name=u'Import', - supports_download=False, - ), - 'ImportTable': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.table.importTable', - ordered_params=[u'name'], - path_params=[], - query_params=[u'delimiter', u'encoding', u'name'], - relative_path=u'tables/import', - request_field='', - request_type_name=u'FusiontablesTableImportTableRequest', - response_type_name=u'Table', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.table.insert', - ordered_params=[], - path_params=[], - query_params=[], - relative_path=u'tables', - request_field='', - request_type_name=u'Table', - response_type_name=u'Table', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.table.list', - ordered_params=[], - path_params=[], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables', - request_field='', - request_type_name=u'FusiontablesTableListRequest', - response_type_name=u'TableList', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'fusiontables.table.patch', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'replaceViewDefinition'], - relative_path=u'tables/{tableId}', - request_field=u'table', - request_type_name=u'FusiontablesTablePatchRequest', - response_type_name=u'Table', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'fusiontables.table.update', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'replaceViewDefinition'], - relative_path=u'tables/{tableId}', - request_field=u'table', - request_type_name=u'FusiontablesTableUpdateRequest', - response_type_name=u'Table', - supports_download=False, - ), - } - self._upload_configs = { 'ImportRows': base_api.ApiUploadInfo( accept=['application/octet-stream'], @@ -586,6 +480,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Copy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.copy', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'copyPresentation'], + relative_path=u'tables/{tableId}/copy', + request_field='', + request_type_name=u'FusiontablesTableCopyRequest', + response_type_name=u'Table', + supports_download=False, + ) + def Delete(self, request, global_params=None): """Deletes a table. @@ -599,6 +506,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.table.delete', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}', + request_field='', + request_type_name=u'FusiontablesTableDeleteRequest', + response_type_name=u'FusiontablesTableDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Retrieves a specific table by its id. @@ -612,6 +532,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.table.get', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}', + request_field='', + request_type_name=u'FusiontablesTableGetRequest', + response_type_name=u'Table', + supports_download=False, + ) + def ImportRows(self, request, global_params=None, upload=None): """Import more rows into a table. @@ -629,6 +562,19 @@ class FusiontablesV1(base_api.BaseApiClient): config, request, global_params=global_params, upload=upload, upload_config=upload_config) + ImportRows.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.importRows', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'delimiter', u'encoding', u'endLine', u'isStrict', u'startLine'], + relative_path=u'tables/{tableId}/import', + request_field='', + request_type_name=u'FusiontablesTableImportRowsRequest', + response_type_name=u'Import', + supports_download=False, + ) + def ImportTable(self, request, global_params=None, upload=None): """Import a new table. @@ -646,6 +592,19 @@ class FusiontablesV1(base_api.BaseApiClient): config, request, global_params=global_params, upload=upload, upload_config=upload_config) + ImportTable.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.importTable', + ordered_params=[u'name'], + path_params=[], + query_params=[u'delimiter', u'encoding', u'name'], + relative_path=u'tables/import', + request_field='', + request_type_name=u'FusiontablesTableImportTableRequest', + response_type_name=u'Table', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new table. @@ -659,6 +618,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.table.insert', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'tables', + request_field='', + request_type_name=u'Table', + response_type_name=u'Table', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of tables a user owns. @@ -672,6 +644,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.table.list', + ordered_params=[], + path_params=[], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables', + request_field='', + request_type_name=u'FusiontablesTableListRequest', + response_type_name=u'TableList', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated. This method supports patch semantics. @@ -685,6 +670,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.table.patch', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'replaceViewDefinition'], + relative_path=u'tables/{tableId}', + request_field=u'table', + request_type_name=u'FusiontablesTablePatchRequest', + response_type_name=u'Table', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates an existing table. Unless explicitly requested, only the name, description, and attribution will be updated. @@ -698,6 +696,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.table.update', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'replaceViewDefinition'], + relative_path=u'tables/{tableId}', + request_field=u'table', + request_type_name=u'FusiontablesTableUpdateRequest', + response_type_name=u'Table', + supports_download=False, + ) + class TaskService(base_api.BaseApiService): """Service class for the task resource.""" @@ -705,45 +716,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.TaskService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'fusiontables.task.delete', - ordered_params=[u'tableId', u'taskId'], - path_params=[u'tableId', u'taskId'], - query_params=[], - relative_path=u'tables/{tableId}/tasks/{taskId}', - request_field='', - request_type_name=u'FusiontablesTaskDeleteRequest', - response_type_name=u'FusiontablesTaskDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.task.get', - ordered_params=[u'tableId', u'taskId'], - path_params=[u'tableId', u'taskId'], - query_params=[], - relative_path=u'tables/{tableId}/tasks/{taskId}', - request_field='', - request_type_name=u'FusiontablesTaskGetRequest', - response_type_name=u'Task', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.task.list', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken', u'startIndex'], - relative_path=u'tables/{tableId}/tasks', - request_field='', - request_type_name=u'FusiontablesTaskListRequest', - response_type_name=u'TaskList', - supports_download=False, - ), - } - self._upload_configs = { } @@ -760,6 +732,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.task.delete', + ordered_params=[u'tableId', u'taskId'], + path_params=[u'tableId', u'taskId'], + query_params=[], + relative_path=u'tables/{tableId}/tasks/{taskId}', + request_field='', + request_type_name=u'FusiontablesTaskDeleteRequest', + response_type_name=u'FusiontablesTaskDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Retrieves a specific task by its id. @@ -773,6 +758,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.task.get', + ordered_params=[u'tableId', u'taskId'], + path_params=[u'tableId', u'taskId'], + query_params=[], + relative_path=u'tables/{tableId}/tasks/{taskId}', + request_field='', + request_type_name=u'FusiontablesTaskGetRequest', + response_type_name=u'Task', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of tasks. @@ -786,6 +784,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.task.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken', u'startIndex'], + relative_path=u'tables/{tableId}/tasks', + request_field='', + request_type_name=u'FusiontablesTaskListRequest', + response_type_name=u'TaskList', + supports_download=False, + ) + class TemplateService(base_api.BaseApiService): """Service class for the template resource.""" @@ -793,81 +804,6 @@ class FusiontablesV1(base_api.BaseApiClient): def __init__(self, client): super(FusiontablesV1.TemplateService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'fusiontables.template.delete', - ordered_params=[u'tableId', u'templateId'], - path_params=[u'tableId', u'templateId'], - query_params=[], - relative_path=u'tables/{tableId}/templates/{templateId}', - request_field='', - request_type_name=u'FusiontablesTemplateDeleteRequest', - response_type_name=u'FusiontablesTemplateDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.template.get', - ordered_params=[u'tableId', u'templateId'], - path_params=[u'tableId', u'templateId'], - query_params=[], - relative_path=u'tables/{tableId}/templates/{templateId}', - request_field='', - request_type_name=u'FusiontablesTemplateGetRequest', - response_type_name=u'Template', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'fusiontables.template.insert', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[], - relative_path=u'tables/{tableId}/templates', - request_field='', - request_type_name=u'Template', - response_type_name=u'Template', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'fusiontables.template.list', - ordered_params=[u'tableId'], - path_params=[u'tableId'], - query_params=[u'maxResults', u'pageToken'], - relative_path=u'tables/{tableId}/templates', - request_field='', - request_type_name=u'FusiontablesTemplateListRequest', - response_type_name=u'TemplateList', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'fusiontables.template.patch', - ordered_params=[u'tableId', u'templateId'], - path_params=[u'tableId', u'templateId'], - query_params=[], - relative_path=u'tables/{tableId}/templates/{templateId}', - request_field='', - request_type_name=u'Template', - response_type_name=u'Template', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'fusiontables.template.update', - ordered_params=[u'tableId', u'templateId'], - path_params=[u'tableId', u'templateId'], - query_params=[], - relative_path=u'tables/{tableId}/templates/{templateId}', - request_field='', - request_type_name=u'Template', - response_type_name=u'Template', - supports_download=False, - ), - } - self._upload_configs = { } @@ -884,6 +820,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'fusiontables.template.delete', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'FusiontablesTemplateDeleteRequest', + response_type_name=u'FusiontablesTemplateDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Retrieves a specific template by its id. @@ -897,6 +846,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.template.get', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'FusiontablesTemplateGetRequest', + response_type_name=u'Template', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new template for the table. @@ -910,6 +872,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'fusiontables.template.insert', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[], + relative_path=u'tables/{tableId}/templates', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of templates. @@ -923,6 +898,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'fusiontables.template.list', + ordered_params=[u'tableId'], + path_params=[u'tableId'], + query_params=[u'maxResults', u'pageToken'], + relative_path=u'tables/{tableId}/templates', + request_field='', + request_type_name=u'FusiontablesTemplateListRequest', + response_type_name=u'TemplateList', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an existing template. This method supports patch semantics. @@ -936,6 +924,19 @@ class FusiontablesV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'fusiontables.template.patch', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates an existing template. @@ -948,3 +949,16 @@ class FusiontablesV1(base_api.BaseApiClient): config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) + + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'fusiontables.template.update', + ordered_params=[u'tableId', u'templateId'], + path_params=[u'tableId', u'templateId'], + query_params=[], + relative_path=u'tables/{tableId}/templates/{templateId}', + request_field='', + request_type_name=u'Template', + response_type_name=u'Template', + supports_download=False, + ) diff --git a/samples/iam_sample/iam_v1/iam_v1_client.py b/samples/iam_sample/iam_v1/iam_v1_client.py index 88f153b..883c4d4 100644 --- a/samples/iam_sample/iam_v1/iam_v1_client.py +++ b/samples/iam_sample/iam_v1/iam_v1_client.py @@ -47,21 +47,6 @@ class IamV1(base_api.BaseApiClient): def __init__(self, client): super(IamV1.IamPoliciesService, self).__init__(client) - self._method_configs = { - 'GetPolicyDetails': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'iam.iamPolicies.getPolicyDetails', - ordered_params=[], - path_params=[], - query_params=[], - relative_path=u'v1/iamPolicies:getPolicyDetails', - request_field='', - request_type_name=u'GetPolicyDetailsRequest', - response_type_name=u'GetPolicyDetailsResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -79,6 +64,19 @@ that the user has access to. return self._RunMethod( config, request, global_params=global_params) + GetPolicyDetails.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.iamPolicies.getPolicyDetails', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'v1/iamPolicies:getPolicyDetails', + request_field='', + request_type_name=u'GetPolicyDetailsRequest', + response_type_name=u'GetPolicyDetailsResponse', + supports_download=False, + ) + class ProjectsServiceAccountsKeysService(base_api.BaseApiService): """Service class for the projects_serviceAccounts_keys resource.""" @@ -86,61 +84,6 @@ that the user has access to. def __init__(self, client): super(IamV1.ProjectsServiceAccountsKeysService, self).__init__(client) - self._method_configs = { - 'Create': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.keys.create', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}/keys', - request_field=u'createServiceAccountKeyRequest', - request_type_name=u'IamProjectsServiceAccountsKeysCreateRequest', - response_type_name=u'ServiceAccountKey', - supports_download=False, - ), - 'Delete': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', - http_method=u'DELETE', - method_id=u'iam.projects.serviceAccounts.keys.delete', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}', - request_field='', - request_type_name=u'IamProjectsServiceAccountsKeysDeleteRequest', - response_type_name=u'Empty', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', - http_method=u'GET', - method_id=u'iam.projects.serviceAccounts.keys.get', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[u'publicKeyType'], - relative_path=u'v1/{+name}', - request_field='', - request_type_name=u'IamProjectsServiceAccountsKeysGetRequest', - response_type_name=u'ServiceAccountKey', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', - http_method=u'GET', - method_id=u'iam.projects.serviceAccounts.keys.list', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[u'keyTypes'], - relative_path=u'v1/{+name}/keys', - request_field='', - request_type_name=u'IamProjectsServiceAccountsKeysListRequest', - response_type_name=u'ListServiceAccountKeysResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -158,6 +101,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + Create.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.keys.create', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}/keys', + request_field=u'createServiceAccountKeyRequest', + request_type_name=u'IamProjectsServiceAccountsKeysCreateRequest', + response_type_name=u'ServiceAccountKey', + supports_download=False, + ) + def Delete(self, request, global_params=None): """Deletes a ServiceAccountKey. @@ -171,6 +128,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', + http_method=u'DELETE', + method_id=u'iam.projects.serviceAccounts.keys.delete', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysDeleteRequest', + response_type_name=u'Empty', + supports_download=False, + ) + def Get(self, request, global_params=None): """Gets the ServiceAccountKey. by key id. @@ -185,6 +156,20 @@ by key id. return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys/{keysId}', + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.keys.get', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'publicKeyType'], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysGetRequest', + response_type_name=u'ServiceAccountKey', + supports_download=False, + ) + def List(self, request, global_params=None): """Lists ServiceAccountKeys. @@ -198,6 +183,20 @@ by key id. return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}/keys', + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.keys.list', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'keyTypes'], + relative_path=u'v1/{+name}/keys', + request_field='', + request_type_name=u'IamProjectsServiceAccountsKeysListRequest', + response_type_name=u'ListServiceAccountKeysResponse', + supports_download=False, + ) + class ProjectsServiceAccountsService(base_api.BaseApiService): """Service class for the projects_serviceAccounts resource.""" @@ -205,139 +204,6 @@ by key id. def __init__(self, client): super(IamV1.ProjectsServiceAccountsService, self).__init__(client) - self._method_configs = { - 'Create': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.create', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}/serviceAccounts', - request_field=u'createServiceAccountRequest', - request_type_name=u'IamProjectsServiceAccountsCreateRequest', - response_type_name=u'ServiceAccount', - supports_download=False, - ), - 'Delete': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', - http_method=u'DELETE', - method_id=u'iam.projects.serviceAccounts.delete', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}', - request_field='', - request_type_name=u'IamProjectsServiceAccountsDeleteRequest', - response_type_name=u'Empty', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', - http_method=u'GET', - method_id=u'iam.projects.serviceAccounts.get', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}', - request_field='', - request_type_name=u'IamProjectsServiceAccountsGetRequest', - response_type_name=u'ServiceAccount', - supports_download=False, - ), - 'GetIamPolicy': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:getIamPolicy', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.getIamPolicy', - ordered_params=[u'resource'], - path_params=[u'resource'], - query_params=[], - relative_path=u'v1/{+resource}:getIamPolicy', - request_field='', - request_type_name=u'IamProjectsServiceAccountsGetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts', - http_method=u'GET', - method_id=u'iam.projects.serviceAccounts.list', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[u'pageSize', u'pageToken', u'removeDeletedServiceAccounts'], - relative_path=u'v1/{+name}/serviceAccounts', - request_field='', - request_type_name=u'IamProjectsServiceAccountsListRequest', - response_type_name=u'ListServiceAccountsResponse', - supports_download=False, - ), - 'SetIamPolicy': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:setIamPolicy', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.setIamPolicy', - ordered_params=[u'resource'], - path_params=[u'resource'], - query_params=[], - relative_path=u'v1/{+resource}:setIamPolicy', - request_field=u'setIamPolicyRequest', - request_type_name=u'IamProjectsServiceAccountsSetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'SignBlob': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlob', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.signBlob', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}:signBlob', - request_field=u'signBlobRequest', - request_type_name=u'IamProjectsServiceAccountsSignBlobRequest', - response_type_name=u'SignBlobResponse', - supports_download=False, - ), - 'SignJwt': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwt', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.signJwt', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}:signJwt', - request_field=u'signJwtRequest', - request_type_name=u'IamProjectsServiceAccountsSignJwtRequest', - response_type_name=u'SignJwtResponse', - supports_download=False, - ), - 'TestIamPermissions': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:testIamPermissions', - http_method=u'POST', - method_id=u'iam.projects.serviceAccounts.testIamPermissions', - ordered_params=[u'resource'], - path_params=[u'resource'], - query_params=[], - relative_path=u'v1/{+resource}:testIamPermissions', - request_field=u'testIamPermissionsRequest', - request_type_name=u'IamProjectsServiceAccountsTestIamPermissionsRequest', - response_type_name=u'TestIamPermissionsResponse', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', - http_method=u'PUT', - method_id=u'iam.projects.serviceAccounts.update', - ordered_params=[u'name'], - path_params=[u'name'], - query_params=[], - relative_path=u'v1/{+name}', - request_field='', - request_type_name=u'ServiceAccount', - response_type_name=u'ServiceAccount', - supports_download=False, - ), - } - self._upload_configs = { } @@ -355,6 +221,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + Create.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.create', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}/serviceAccounts', + request_field=u'createServiceAccountRequest', + request_type_name=u'IamProjectsServiceAccountsCreateRequest', + response_type_name=u'ServiceAccount', + supports_download=False, + ) + def Delete(self, request, global_params=None): """Deletes a ServiceAccount. @@ -368,6 +248,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', + http_method=u'DELETE', + method_id=u'iam.projects.serviceAccounts.delete', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsDeleteRequest', + response_type_name=u'Empty', + supports_download=False, + ) + def Get(self, request, global_params=None): """Gets a ServiceAccount. @@ -381,6 +275,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.get', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'IamProjectsServiceAccountsGetRequest', + response_type_name=u'ServiceAccount', + supports_download=False, + ) + def GetIamPolicy(self, request, global_params=None): """Returns the IAM access control policy for specified IAM resource. @@ -394,6 +302,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:getIamPolicy', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.getIamPolicy', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:getIamPolicy', + request_field='', + request_type_name=u'IamProjectsServiceAccountsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def List(self, request, global_params=None): """Lists ServiceAccounts for a project. @@ -407,6 +329,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts', + http_method=u'GET', + method_id=u'iam.projects.serviceAccounts.list', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[u'pageSize', u'pageToken', u'removeDeletedServiceAccounts'], + relative_path=u'v1/{+name}/serviceAccounts', + request_field='', + request_type_name=u'IamProjectsServiceAccountsListRequest', + response_type_name=u'ListServiceAccountsResponse', + supports_download=False, + ) + def SetIamPolicy(self, request, global_params=None): """Sets the IAM access control policy for the specified IAM resource. @@ -420,6 +356,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:setIamPolicy', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.setIamPolicy', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:setIamPolicy', + request_field=u'setIamPolicyRequest', + request_type_name=u'IamProjectsServiceAccountsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def SignBlob(self, request, global_params=None): """Signs a blob using a service account's system-managed private key. @@ -433,6 +383,20 @@ and returns it. return self._RunMethod( config, request, global_params=global_params) + SignBlob.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlob', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.signBlob', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}:signBlob', + request_field=u'signBlobRequest', + request_type_name=u'IamProjectsServiceAccountsSignBlobRequest', + response_type_name=u'SignBlobResponse', + supports_download=False, + ) + def SignJwt(self, request, global_params=None): """Signs a JWT using a service account's system-managed private key. @@ -451,6 +415,20 @@ will fail. return self._RunMethod( config, request, global_params=global_params) + SignJwt.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwt', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.signJwt', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}:signJwt', + request_field=u'signJwtRequest', + request_type_name=u'IamProjectsServiceAccountsSignJwtRequest', + response_type_name=u'SignJwtResponse', + supports_download=False, + ) + def TestIamPermissions(self, request, global_params=None): """Tests the specified permissions against the IAM access control policy. for the specified IAM resource. @@ -465,6 +443,20 @@ for the specified IAM resource. return self._RunMethod( config, request, global_params=global_params) + TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:testIamPermissions', + http_method=u'POST', + method_id=u'iam.projects.serviceAccounts.testIamPermissions', + ordered_params=[u'resource'], + path_params=[u'resource'], + query_params=[], + relative_path=u'v1/{+resource}:testIamPermissions', + request_field=u'testIamPermissionsRequest', + request_type_name=u'IamProjectsServiceAccountsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates a ServiceAccount. @@ -482,6 +474,20 @@ The `etag` is mandatory. return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}', + http_method=u'PUT', + method_id=u'iam.projects.serviceAccounts.update', + ordered_params=[u'name'], + path_params=[u'name'], + query_params=[], + relative_path=u'v1/{+name}', + request_field='', + request_type_name=u'ServiceAccount', + response_type_name=u'ServiceAccount', + supports_download=False, + ) + class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" @@ -489,9 +495,6 @@ The `etag` is mandatory. def __init__(self, client): super(IamV1.ProjectsService, self).__init__(client) - self._method_configs = { - } - self._upload_configs = { } @@ -502,21 +505,6 @@ The `etag` is mandatory. def __init__(self, client): super(IamV1.RolesService, self).__init__(client) - self._method_configs = { - 'QueryGrantableRoles': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'iam.roles.queryGrantableRoles', - ordered_params=[], - path_params=[], - query_params=[], - relative_path=u'v1/roles:queryGrantableRoles', - request_field='', - request_type_name=u'QueryGrantableRolesRequest', - response_type_name=u'QueryGrantableRolesResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -532,3 +520,16 @@ The `etag` is mandatory. config = self.GetMethodConfig('QueryGrantableRoles') return self._RunMethod( config, request, global_params=global_params) + + QueryGrantableRoles.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'iam.roles.queryGrantableRoles', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'v1/roles:queryGrantableRoles', + request_field='', + request_type_name=u'QueryGrantableRolesRequest', + response_type_name=u'QueryGrantableRolesResponse', + supports_download=False, + ) diff --git a/samples/storage_sample/storage_v1/storage_v1_client.py b/samples/storage_sample/storage_v1/storage_v1_client.py index 854f4ea..74dfdc4 100644 --- a/samples/storage_sample/storage_v1/storage_v1_client.py +++ b/samples/storage_sample/storage_v1/storage_v1_client.py @@ -49,81 +49,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.BucketAccessControlsService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.bucketAccessControls.delete', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'StorageBucketAccessControlsDeleteRequest', - response_type_name=u'StorageBucketAccessControlsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.bucketAccessControls.get', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'StorageBucketAccessControlsGetRequest', - response_type_name=u'BucketAccessControl', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.bucketAccessControls.insert', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/acl', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.bucketAccessControls.list', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/acl', - request_field='', - request_type_name=u'StorageBucketAccessControlsListRequest', - response_type_name=u'BucketAccessControls', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'storage.bucketAccessControls.patch', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.bucketAccessControls.update', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/acl/{entity}', - request_field='', - request_type_name=u'BucketAccessControl', - response_type_name=u'BucketAccessControl', - supports_download=False, - ), - } - self._upload_configs = { } @@ -140,6 +65,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.bucketAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'StorageBucketAccessControlsDeleteRequest', + response_type_name=u'StorageBucketAccessControlsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Returns the ACL entry for the specified entity on the specified bucket. @@ -153,6 +91,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.bucketAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'StorageBucketAccessControlsGetRequest', + response_type_name=u'BucketAccessControl', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new ACL entry on the specified bucket. @@ -166,6 +117,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.bucketAccessControls.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves ACL entries on the specified bucket. @@ -179,6 +143,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.bucketAccessControls.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/acl', + request_field='', + request_type_name=u'StorageBucketAccessControlsListRequest', + response_type_name=u'BucketAccessControls', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an ACL entry on the specified bucket. This method supports patch semantics. @@ -192,6 +169,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.bucketAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates an ACL entry on the specified bucket. @@ -205,6 +195,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.bucketAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/acl/{entity}', + request_field='', + request_type_name=u'BucketAccessControl', + response_type_name=u'BucketAccessControl', + supports_download=False, + ) + class BucketsService(base_api.BaseApiService): """Service class for the buckets resource.""" @@ -212,117 +215,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.BucketsService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.buckets.delete', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}', - request_field='', - request_type_name=u'StorageBucketsDeleteRequest', - response_type_name=u'StorageBucketsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.buckets.get', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}', - request_field='', - request_type_name=u'StorageBucketsGetRequest', - response_type_name=u'Bucket', - supports_download=False, - ), - 'GetIamPolicy': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.buckets.getIamPolicy', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/iam', - request_field='', - request_type_name=u'StorageBucketsGetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.buckets.insert', - ordered_params=[u'project'], - path_params=[], - query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection'], - relative_path=u'b', - request_field=u'bucket', - request_type_name=u'StorageBucketsInsertRequest', - response_type_name=u'Bucket', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.buckets.list', - ordered_params=[u'project'], - path_params=[], - query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection'], - relative_path=u'b', - request_field='', - request_type_name=u'StorageBucketsListRequest', - response_type_name=u'Buckets', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'storage.buckets.patch', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsPatchRequest', - response_type_name=u'Bucket', - supports_download=False, - ), - 'SetIamPolicy': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.buckets.setIamPolicy', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/iam', - request_field=u'policy', - request_type_name=u'StorageBucketsSetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'TestIamPermissions': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.buckets.testIamPermissions', - ordered_params=[u'bucket', u'permissions'], - path_params=[u'bucket'], - query_params=[u'permissions'], - relative_path=u'b/{bucket}/iam/testPermissions', - request_field='', - request_type_name=u'StorageBucketsTestIamPermissionsRequest', - response_type_name=u'TestIamPermissionsResponse', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.buckets.update', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], - relative_path=u'b/{bucket}', - request_field=u'bucketResource', - request_type_name=u'StorageBucketsUpdateRequest', - response_type_name=u'Bucket', - supports_download=False, - ), - } - self._upload_configs = { } @@ -339,6 +231,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.buckets.delete', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}', + request_field='', + request_type_name=u'StorageBucketsDeleteRequest', + response_type_name=u'StorageBucketsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Returns metadata for the specified bucket. @@ -352,6 +257,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.get', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}', + request_field='', + request_type_name=u'StorageBucketsGetRequest', + response_type_name=u'Bucket', + supports_download=False, + ) + def GetIamPolicy(self, request, global_params=None): """Returns an IAM policy for the specified bucket. @@ -365,6 +283,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.getIamPolicy', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/iam', + request_field='', + request_type_name=u'StorageBucketsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new bucket. @@ -378,6 +309,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.buckets.insert', + ordered_params=[u'project'], + path_params=[], + query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection'], + relative_path=u'b', + request_field=u'bucket', + request_type_name=u'StorageBucketsInsertRequest', + response_type_name=u'Bucket', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of buckets for a given project. @@ -391,6 +335,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.list', + ordered_params=[u'project'], + path_params=[], + query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection'], + relative_path=u'b', + request_field='', + request_type_name=u'StorageBucketsListRequest', + response_type_name=u'Buckets', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates a bucket. This method supports patch semantics. @@ -404,6 +361,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.buckets.patch', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsPatchRequest', + response_type_name=u'Bucket', + supports_download=False, + ) + def SetIamPolicy(self, request, global_params=None): """Updates an IAM policy for the specified bucket. @@ -417,6 +387,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.buckets.setIamPolicy', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/iam', + request_field=u'policy', + request_type_name=u'StorageBucketsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def TestIamPermissions(self, request, global_params=None): """Tests a set of permissions on the given bucket to see which, if any, are held by the caller. @@ -430,6 +413,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.buckets.testIamPermissions', + ordered_params=[u'bucket', u'permissions'], + path_params=[u'bucket'], + query_params=[u'permissions'], + relative_path=u'b/{bucket}/iam/testPermissions', + request_field='', + request_type_name=u'StorageBucketsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates a bucket. @@ -443,6 +439,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.buckets.update', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection'], + relative_path=u'b/{bucket}', + request_field=u'bucketResource', + request_type_name=u'StorageBucketsUpdateRequest', + response_type_name=u'Bucket', + supports_download=False, + ) + class ChannelsService(base_api.BaseApiService): """Service class for the channels resource.""" @@ -450,21 +459,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.ChannelsService, self).__init__(client) - self._method_configs = { - 'Stop': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.channels.stop', - ordered_params=[], - path_params=[], - query_params=[], - relative_path=u'channels/stop', - request_field='', - request_type_name=u'Channel', - response_type_name=u'StorageChannelsStopResponse', - supports_download=False, - ), - } - self._upload_configs = { } @@ -481,6 +475,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Stop.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.channels.stop', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'channels/stop', + request_field='', + request_type_name=u'Channel', + response_type_name=u'StorageChannelsStopResponse', + supports_download=False, + ) + class DefaultObjectAccessControlsService(base_api.BaseApiService): """Service class for the defaultObjectAccessControls resource.""" @@ -488,81 +495,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.defaultObjectAccessControls.delete', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', - response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.get', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.defaultObjectAccessControls.insert', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.defaultObjectAccessControls.list', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/defaultObjectAcl', - request_field='', - request_type_name=u'StorageDefaultObjectAccessControlsListRequest', - response_type_name=u'ObjectAccessControls', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'storage.defaultObjectAccessControls.patch', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.defaultObjectAccessControls.update', - ordered_params=[u'bucket', u'entity'], - path_params=[u'bucket', u'entity'], - query_params=[], - relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', - request_field='', - request_type_name=u'ObjectAccessControl', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - } - self._upload_configs = { } @@ -579,6 +511,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.defaultObjectAccessControls.delete', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', + response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Returns the default object ACL entry for the specified entity on the specified bucket. @@ -592,6 +537,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.defaultObjectAccessControls.get', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new default object ACL entry on the specified bucket. @@ -605,6 +563,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.defaultObjectAccessControls.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves default object ACL entries on the specified bucket. @@ -618,6 +589,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.defaultObjectAccessControls.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/defaultObjectAcl', + request_field='', + request_type_name=u'StorageDefaultObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates a default object ACL entry on the specified bucket. This method supports patch semantics. @@ -631,6 +615,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.defaultObjectAccessControls.patch', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates a default object ACL entry on the specified bucket. @@ -644,6 +641,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.defaultObjectAccessControls.update', + ordered_params=[u'bucket', u'entity'], + path_params=[u'bucket', u'entity'], + query_params=[], + relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', + request_field='', + request_type_name=u'ObjectAccessControl', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + class NotificationsService(base_api.BaseApiService): """Service class for the notifications resource.""" @@ -651,57 +661,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.NotificationsService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.notifications.delete', - ordered_params=[u'notification'], - path_params=[u'notification'], - query_params=[], - relative_path=u'notifications/{notification}', - request_field='', - request_type_name=u'StorageNotificationsDeleteRequest', - response_type_name=u'StorageNotificationsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.notifications.get', - ordered_params=[u'notification'], - path_params=[u'notification'], - query_params=[], - relative_path=u'notifications/{notification}', - request_field='', - request_type_name=u'StorageNotificationsGetRequest', - response_type_name=u'Notification', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.notifications.insert', - ordered_params=[], - path_params=[], - query_params=[], - relative_path=u'notifications', - request_field='', - request_type_name=u'Notification', - response_type_name=u'Notification', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.notifications.list', - ordered_params=[u'bucket'], - path_params=[], - query_params=[u'bucket'], - relative_path=u'notifications', - request_field='', - request_type_name=u'StorageNotificationsListRequest', - response_type_name=u'Notifications', - supports_download=False, - ), - } - self._upload_configs = { } @@ -718,6 +677,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.notifications.delete', + ordered_params=[u'notification'], + path_params=[u'notification'], + query_params=[], + relative_path=u'notifications/{notification}', + request_field='', + request_type_name=u'StorageNotificationsDeleteRequest', + response_type_name=u'StorageNotificationsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """View a notification configuration. @@ -731,6 +703,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.notifications.get', + ordered_params=[u'notification'], + path_params=[u'notification'], + query_params=[], + relative_path=u'notifications/{notification}', + request_field='', + request_type_name=u'StorageNotificationsGetRequest', + response_type_name=u'Notification', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a notification subscription for a given bucket. @@ -744,6 +729,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.notifications.insert', + ordered_params=[], + path_params=[], + query_params=[], + relative_path=u'notifications', + request_field='', + request_type_name=u'Notification', + response_type_name=u'Notification', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves a list of notification subscriptions for a given bucket. @@ -757,6 +755,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.notifications.list', + ordered_params=[u'bucket'], + path_params=[], + query_params=[u'bucket'], + relative_path=u'notifications', + request_field='', + request_type_name=u'StorageNotificationsListRequest', + response_type_name=u'Notifications', + supports_download=False, + ) + class ObjectAccessControlsService(base_api.BaseApiService): """Service class for the objectAccessControls resource.""" @@ -764,81 +775,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.ObjectAccessControlsService, self).__init__(client) - self._method_configs = { - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.objectAccessControls.delete', - ordered_params=[u'bucket', u'object', u'entity'], - path_params=[u'bucket', u'entity', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl/{entity}', - request_field='', - request_type_name=u'StorageObjectAccessControlsDeleteRequest', - response_type_name=u'StorageObjectAccessControlsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objectAccessControls.get', - ordered_params=[u'bucket', u'object', u'entity'], - path_params=[u'bucket', u'entity', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl/{entity}', - request_field='', - request_type_name=u'StorageObjectAccessControlsGetRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objectAccessControls.insert', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl', - request_field=u'objectAccessControl', - request_type_name=u'StorageObjectAccessControlsInsertRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objectAccessControls.list', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl', - request_field='', - request_type_name=u'StorageObjectAccessControlsListRequest', - response_type_name=u'ObjectAccessControls', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'storage.objectAccessControls.patch', - ordered_params=[u'bucket', u'object', u'entity'], - path_params=[u'bucket', u'entity', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl/{entity}', - request_field=u'objectAccessControl', - request_type_name=u'StorageObjectAccessControlsPatchRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.objectAccessControls.update', - ordered_params=[u'bucket', u'object', u'entity'], - path_params=[u'bucket', u'entity', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/acl/{entity}', - request_field=u'objectAccessControl', - request_type_name=u'StorageObjectAccessControlsUpdateRequest', - response_type_name=u'ObjectAccessControl', - supports_download=False, - ), - } - self._upload_configs = { } @@ -855,6 +791,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.objectAccessControls.delete', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field='', + request_type_name=u'StorageObjectAccessControlsDeleteRequest', + response_type_name=u'StorageObjectAccessControlsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None): """Returns the ACL entry for the specified entity on the specified object. @@ -868,6 +817,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objectAccessControls.get', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field='', + request_type_name=u'StorageObjectAccessControlsGetRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def Insert(self, request, global_params=None): """Creates a new ACL entry on the specified object. @@ -881,6 +843,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objectAccessControls.insert', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsInsertRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def List(self, request, global_params=None): """Retrieves ACL entries on the specified object. @@ -894,6 +869,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objectAccessControls.list', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl', + request_field='', + request_type_name=u'StorageObjectAccessControlsListRequest', + response_type_name=u'ObjectAccessControls', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an ACL entry on the specified object. This method supports patch semantics. @@ -907,6 +895,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.objectAccessControls.patch', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsPatchRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + def Update(self, request, global_params=None): """Updates an ACL entry on the specified object. @@ -920,6 +921,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objectAccessControls.update', + ordered_params=[u'bucket', u'object', u'entity'], + path_params=[u'bucket', u'entity', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/acl/{entity}', + request_field=u'objectAccessControl', + request_type_name=u'StorageObjectAccessControlsUpdateRequest', + response_type_name=u'ObjectAccessControl', + supports_download=False, + ) + class ObjectsService(base_api.BaseApiService): """Service class for the objects resource.""" @@ -927,165 +941,6 @@ class StorageV1(base_api.BaseApiClient): def __init__(self, client): super(StorageV1.ObjectsService, self).__init__(client) - self._method_configs = { - 'Compose': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.compose', - ordered_params=[u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], - relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', - request_field=u'composeRequest', - request_type_name=u'StorageObjectsComposeRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'Copy': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.copy', - ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], - relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', - request_field=u'object', - request_type_name=u'StorageObjectsCopyRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'Delete': base_api.ApiMethodInfo( - http_method=u'DELETE', - method_id=u'storage.objects.delete', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], - relative_path=u'b/{bucket}/o/{object}', - request_field='', - request_type_name=u'StorageObjectsDeleteRequest', - response_type_name=u'StorageObjectsDeleteResponse', - supports_download=False, - ), - 'Get': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objects.get', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field='', - request_type_name=u'StorageObjectsGetRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'GetIamPolicy': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objects.getIamPolicy', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/iam', - request_field='', - request_type_name=u'StorageObjectsGetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'Insert': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.insert', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o', - request_field=u'object', - request_type_name=u'StorageObjectsInsertRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'List': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objects.list', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o', - request_field='', - request_type_name=u'StorageObjectsListRequest', - response_type_name=u'Objects', - supports_download=False, - ), - 'Patch': base_api.ApiMethodInfo( - http_method=u'PATCH', - method_id=u'storage.objects.patch', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsPatchRequest', - response_type_name=u'Object', - supports_download=False, - ), - 'Rewrite': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.rewrite', - ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], - path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], - query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'maxBytesRewrittenPerCall', u'projection', u'rewriteToken', u'sourceGeneration'], - relative_path=u'b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}', - request_field=u'object', - request_type_name=u'StorageObjectsRewriteRequest', - response_type_name=u'RewriteResponse', - supports_download=False, - ), - 'SetIamPolicy': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.objects.setIamPolicy', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation'], - relative_path=u'b/{bucket}/o/{object}/iam', - request_field=u'policy', - request_type_name=u'StorageObjectsSetIamPolicyRequest', - response_type_name=u'Policy', - supports_download=False, - ), - 'TestIamPermissions': base_api.ApiMethodInfo( - http_method=u'GET', - method_id=u'storage.objects.testIamPermissions', - ordered_params=[u'bucket', u'object', u'permissions'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'permissions'], - relative_path=u'b/{bucket}/o/{object}/iam/testPermissions', - request_field='', - request_type_name=u'StorageObjectsTestIamPermissionsRequest', - response_type_name=u'TestIamPermissionsResponse', - supports_download=False, - ), - 'Update': base_api.ApiMethodInfo( - http_method=u'PUT', - method_id=u'storage.objects.update', - ordered_params=[u'bucket', u'object'], - path_params=[u'bucket', u'object'], - query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], - relative_path=u'b/{bucket}/o/{object}', - request_field=u'objectResource', - request_type_name=u'StorageObjectsUpdateRequest', - response_type_name=u'Object', - supports_download=True, - ), - 'WatchAll': base_api.ApiMethodInfo( - http_method=u'POST', - method_id=u'storage.objects.watchAll', - ordered_params=[u'bucket'], - path_params=[u'bucket'], - query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], - relative_path=u'b/{bucket}/o/watch', - request_field=u'channel', - request_type_name=u'StorageObjectsWatchAllRequest', - response_type_name=u'Channel', - supports_download=False, - ), - } - self._upload_configs = { 'Insert': base_api.ApiUploadInfo( accept=['*/*'], @@ -1113,6 +968,19 @@ class StorageV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + Compose.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.compose', + ordered_params=[u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch'], + relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', + request_field=u'composeRequest', + request_type_name=u'StorageObjectsComposeRequest', + response_type_name=u'Object', + supports_download=True, + ) + def Copy(self, request, global_params=None, download=None): """Copies a source object to a destination object. Optionally overrides metadata. @@ -1129,6 +997,19 @@ class StorageV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + Copy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.copy', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsCopyRequest', + response_type_name=u'Object', + supports_download=True, + ) + def Delete(self, request, global_params=None): """Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. @@ -1142,6 +1023,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Delete.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'DELETE', + method_id=u'storage.objects.delete', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch'], + relative_path=u'b/{bucket}/o/{object}', + request_field='', + request_type_name=u'StorageObjectsDeleteRequest', + response_type_name=u'StorageObjectsDeleteResponse', + supports_download=False, + ) + def Get(self, request, global_params=None, download=None): """Retrieves an object or its metadata. @@ -1158,6 +1052,19 @@ class StorageV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + Get.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.get', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field='', + request_type_name=u'StorageObjectsGetRequest', + response_type_name=u'Object', + supports_download=True, + ) + def GetIamPolicy(self, request, global_params=None): """Returns an IAM policy for the specified object. @@ -1171,6 +1078,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.getIamPolicy', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/iam', + request_field='', + request_type_name=u'StorageObjectsGetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def Insert(self, request, global_params=None, upload=None, download=None): """Stores a new object and metadata. @@ -1191,6 +1111,19 @@ class StorageV1(base_api.BaseApiClient): upload=upload, upload_config=upload_config, download=download) + Insert.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.insert', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'name', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o', + request_field=u'object', + request_type_name=u'StorageObjectsInsertRequest', + response_type_name=u'Object', + supports_download=True, + ) + def List(self, request, global_params=None): """Retrieves a list of objects matching the criteria. @@ -1204,6 +1137,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + List.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.list', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o', + request_field='', + request_type_name=u'StorageObjectsListRequest', + response_type_name=u'Objects', + supports_download=False, + ) + def Patch(self, request, global_params=None): """Updates an object's metadata. This method supports patch semantics. @@ -1217,6 +1163,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Patch.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PATCH', + method_id=u'storage.objects.patch', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsPatchRequest', + response_type_name=u'Object', + supports_download=False, + ) + def Rewrite(self, request, global_params=None): """Rewrites a source object to a destination object. Optionally overrides metadata. @@ -1230,6 +1189,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + Rewrite.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.rewrite', + ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], + path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], + query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'maxBytesRewrittenPerCall', u'projection', u'rewriteToken', u'sourceGeneration'], + relative_path=u'b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}', + request_field=u'object', + request_type_name=u'StorageObjectsRewriteRequest', + response_type_name=u'RewriteResponse', + supports_download=False, + ) + def SetIamPolicy(self, request, global_params=None): """Updates an IAM policy for the specified object. @@ -1243,6 +1215,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objects.setIamPolicy', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation'], + relative_path=u'b/{bucket}/o/{object}/iam', + request_field=u'policy', + request_type_name=u'StorageObjectsSetIamPolicyRequest', + response_type_name=u'Policy', + supports_download=False, + ) + def TestIamPermissions(self, request, global_params=None): """Tests a set of permissions on the given object to see which, if any, are held by the caller. @@ -1256,6 +1241,19 @@ class StorageV1(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'storage.objects.testIamPermissions', + ordered_params=[u'bucket', u'object', u'permissions'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'permissions'], + relative_path=u'b/{bucket}/o/{object}/iam/testPermissions', + request_field='', + request_type_name=u'StorageObjectsTestIamPermissionsRequest', + response_type_name=u'TestIamPermissionsResponse', + supports_download=False, + ) + def Update(self, request, global_params=None, download=None): """Updates an object's metadata. @@ -1272,6 +1270,19 @@ class StorageV1(base_api.BaseApiClient): config, request, global_params=global_params, download=download) + Update.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'PUT', + method_id=u'storage.objects.update', + ordered_params=[u'bucket', u'object'], + path_params=[u'bucket', u'object'], + query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection'], + relative_path=u'b/{bucket}/o/{object}', + request_field=u'objectResource', + request_type_name=u'StorageObjectsUpdateRequest', + response_type_name=u'Object', + supports_download=True, + ) + def WatchAll(self, request, global_params=None): """Watch for changes on all objects in a bucket. @@ -1284,3 +1295,16 @@ class StorageV1(base_api.BaseApiClient): config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) + + WatchAll.method_config = lambda: base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'storage.objects.watchAll', + ordered_params=[u'bucket'], + path_params=[u'bucket'], + query_params=[u'delimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'versions'], + relative_path=u'b/{bucket}/o/watch', + request_field=u'channel', + request_type_name=u'StorageObjectsWatchAllRequest', + response_type_name=u'Channel', + supports_download=False, + ) -- GitLab From 781b3d1f225d287a9d61e7db00b915da5d75090c Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Thu, 30 Jun 2016 08:54:22 -0400 Subject: [PATCH 250/295] Add servicemanagement api sample. --- samples/regenerate_samples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/regenerate_samples.py b/samples/regenerate_samples.py index c46cd70..8ef07ec 100644 --- a/samples/regenerate_samples.py +++ b/samples/regenerate_samples.py @@ -23,6 +23,7 @@ _SAMPLES = [ 'dns_sample/dns_v1.json', 'iam_sample/iam_v1.json', 'fusiontables_sample/fusiontables_v1.json', + 'servicemanagement_sample/servicemanagement_v1.json', 'storage_sample/storage_v1.json', ] -- GitLab From cf66a0dc95d9ef827aa895c99caf2c4403b06bc3 Mon Sep 17 00:00:00 2001 From: "Arthur D. Cherba" Date: Thu, 30 Jun 2016 11:42:25 -0400 Subject: [PATCH 251/295] Generate extra_types import when any type is used by AdditionalProperties. --- apitools/gen/message_registry.py | 3 + run_pylint.py | 1 + samples/servicemanagement_sample/__init__.py | 13 + .../servicemanagement_sample/messages_test.py | 56 + .../servicemanagement_v1.json | 3382 ++++++++++++++++ .../servicemanagement_v1/__init__.py | 5 + .../servicemanagement_v1.py | 1520 +++++++ .../servicemanagement_v1_client.py | 854 ++++ .../servicemanagement_v1_messages.py | 3505 +++++++++++++++++ 9 files changed, 9339 insertions(+) create mode 100644 samples/servicemanagement_sample/__init__.py create mode 100644 samples/servicemanagement_sample/messages_test.py create mode 100644 samples/servicemanagement_sample/servicemanagement_v1.json create mode 100644 samples/servicemanagement_sample/servicemanagement_v1/__init__.py create mode 100644 samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1.py create mode 100644 samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py create mode 100644 samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_messages.py diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py index 461b72a..a7e9a92 100644 --- a/apitools/gen/message_registry.py +++ b/apitools/gen/message_registry.py @@ -422,6 +422,9 @@ class MessageRegistry(object): if type_name in self.PRIMITIVE_TYPE_INFO_MAP: type_info = self.PRIMITIVE_TYPE_INFO_MAP[type_name] + if type_info.type_name.startswith('extra_types.'): + self.__AddImport( + 'from %s import extra_types' % self.__base_files_package) return type_info if type_name == 'array': diff --git a/run_pylint.py b/run_pylint.py index 2a95632..c644b53 100644 --- a/run_pylint.py +++ b/run_pylint.py @@ -36,6 +36,7 @@ IGNORED_DIRECTORIES = [ 'samples/dns_sample/dns_v1', 'samples/fusiontables_sample/fusiontables_v1', 'samples/iam_sample/iam_v1', + 'samples/servicemanagement_sample/servicemanagement_v1', 'samples/storage_sample/storage_v1', 'venv', ] diff --git a/samples/servicemanagement_sample/__init__.py b/samples/servicemanagement_sample/__init__.py new file mode 100644 index 0000000..58e0d91 --- /dev/null +++ b/samples/servicemanagement_sample/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/servicemanagement_sample/messages_test.py b/samples/servicemanagement_sample/messages_test.py new file mode 100644 index 0000000..a62dbd7 --- /dev/null +++ b/samples/servicemanagement_sample/messages_test.py @@ -0,0 +1,56 @@ +# +# Copyright 2016 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test for generated servicemanagement messages module.""" + +import unittest2 + +from apitools.base.py import extra_types + +from samples.servicemanagement_sample.servicemanagement_v1 \ + import servicemanagement_v1_messages as messages # nopep8 + + +class MessagesTest(unittest2.TestCase): + + def testInstantiateMessageWithAdditionalProperties(self): + PROJECT_NAME = 'test-project' + SERVICE_NAME = 'test-service' + SERVICE_VERSION = '1.0' + + prop = messages.Operation.ResponseValue.AdditionalProperty + messages.Operation( + name='operation-12345-67890', + done=False, + response=messages.Operation.ResponseValue( + additionalProperties=[ + prop(key='producerProjectId', + value=extra_types.JsonValue( + string_value=PROJECT_NAME)), + prop(key='serviceName', + value=extra_types.JsonValue( + string_value=SERVICE_NAME)), + prop(key='serviceConfig', + value=extra_types.JsonValue( + object_value=extra_types.JsonObject( + properties=[ + extra_types.JsonObject.Property( + key='id', + value=extra_types.JsonValue( + string_value=SERVICE_VERSION) + ) + ]) + )) + ])) diff --git a/samples/servicemanagement_sample/servicemanagement_v1.json b/samples/servicemanagement_sample/servicemanagement_v1.json new file mode 100644 index 0000000..55e2518 --- /dev/null +++ b/samples/servicemanagement_sample/servicemanagement_v1.json @@ -0,0 +1,3382 @@ +{ + "kind": "discovery#restDescription", + "discoveryVersion": "v1", + "id": "servicemanagement:v1", + "name": "servicemanagement", + "version": "v1", + "revision": "0", + "title": "Google Service Management API", + "description": "The service management API for Google Cloud Platform", + "ownerDomain": "google.com", + "ownerName": "Google", + "icons": { + "x16": "http://www.google.com/images/icons/product/search-16.gif", + "x32": "http://www.google.com/images/icons/product/search-32.gif" + }, + "documentationLink": "https://cloud.google.com/service-management/", + "protocol": "rest", + "rootUrl": "https://servicemanagement.googleapis.com/", + "servicePath": "", + "baseUrl": "https://servicemanagement.googleapis.com/", + "batchPath": "batch", + "parameters": { + "access_token": { + "type": "string", + "description": "OAuth access token.", + "location": "query" + }, + "alt": { + "type": "string", + "description": "Data format for response.", + "default": "json", + "enum": [ + "json", + "media", + "proto" + ], + "enumDescriptions": [ + "Responses with Content-Type of application/json", + "Media download with context-dependent Content-Type", + "Responses with Content-Type of application/x-protobuf" + ], + "location": "query" + }, + "bearer_token": { + "type": "string", + "description": "OAuth bearer token.", + "location": "query" + }, + "callback": { + "type": "string", + "description": "JSONP", + "location": "query" + }, + "fields": { + "type": "string", + "description": "Selector specifying which fields to include in a partial response.", + "location": "query" + }, + "key": { + "type": "string", + "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.", + "location": "query" + }, + "oauth_token": { + "type": "string", + "description": "OAuth 2.0 token for the current user.", + "location": "query" + }, + "pp": { + "type": "boolean", + "description": "Pretty-print response.", + "default": "true", + "location": "query" + }, + "prettyPrint": { + "type": "boolean", + "description": "Returns response with indentations and line breaks.", + "default": "true", + "location": "query" + }, + "quotaUser": { + "type": "string", + "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters.", + "location": "query" + }, + "upload_protocol": { + "type": "string", + "description": "Upload protocol for media (e.g. \"raw\", \"multipart\").", + "location": "query" + }, + "uploadType": { + "type": "string", + "description": "Legacy upload protocol for media (e.g. \"media\", \"multipart\").", + "location": "query" + }, + "$.xgafv": { + "type": "string", + "description": "V1 error format.", + "enum": [ + "1", + "2" + ], + "enumDescriptions": [ + "v1 error format", + "v2 error format" + ], + "location": "query" + } + }, + "auth": { + "oauth2": { + "scopes": { + "https://www.googleapis.com/auth/cloud-platform": { + "description": "View and manage your data across Google Cloud Platform services" + }, + "https://www.googleapis.com/auth/service.management": { + "description": "Manage your Google API service configuration" + } + } + } + }, + "schemas": { + "ListServicesResponse": { + "id": "ListServicesResponse", + "description": "Response message for `ListServices` method.", + "type": "object", + "properties": { + "services": { + "description": "The results of the query.", + "type": "array", + "items": { + "$ref": "ManagedService" + } + }, + "nextPageToken": { + "description": "Token that can be passed to `ListServices` to resume a paginated query.", + "type": "string" + } + } + }, + "ManagedService": { + "id": "ManagedService", + "description": "The full representation of an API Service that is managed by the\n`ServiceManager` API. Includes both the service configuration, as well as\nother control plane deployment related information.", + "type": "object", + "properties": { + "serviceName": { + "description": "The name of the service. See the `ServiceManager` overview for naming\nrequirements. This name must match `google.api.Service.name` in the\n`service_config` field.", + "type": "string" + }, + "producerProjectId": { + "description": "ID of the project that produces and owns this service.", + "type": "string" + }, + "generation": { + "description": "A server-assigned monotonically increasing number that changes whenever a\nmutation is made to the `ManagedService` or any of its components via the\n`ServiceManager` API.", + "type": "string", + "format": "int64" + }, + "serviceConfig": { + "description": "The service's generated configuration.", + "$ref": "Service" + }, + "configSource": { + "description": "User-supplied source configuration for the service. This is distinct from\nthe generated configuration provided in `google.api.Service`.\nThis is NOT populated on GetService calls at the moment.\nNOTE: Any upsert operation that contains both a service_config\nand a config_source is considered invalid and will result in\nan error being returned.", + "$ref": "ConfigSource" + }, + "operations": { + "description": "Read-only view of pending operations affecting this resource, if requested.", + "type": "array", + "items": { + "$ref": "Operation" + } + }, + "projectSettings": { + "description": "Read-only view of settings for a particular consumer project, if requested.", + "$ref": "ProjectSettings" + } + } + }, + "Service": { + "id": "Service", + "description": "`Service` is the root object of the configuration schema. It\ndescribes basic information like the name of the service and the\nexposed API interfaces, and delegates other aspects to configuration\nsub-sections.\n\nExample:\n\n type: google.api.Service\n config_version: 1\n name: calendar.googleapis.com\n title: Google Calendar API\n apis:\n - name: google.calendar.Calendar\n backend:\n rules:\n - selector: \"*\"\n address: calendar.example.com", + "type": "object", + "properties": { + "configVersion": { + "description": "The version of the service configuration. The config version may\ninfluence interpretation of the configuration, for example, to\ndetermine defaults. This is documented together with applicable\noptions. The current default for the config version itself is `3`.", + "type": "integer", + "format": "uint32" + }, + "name": { + "description": "The DNS address at which this service is available,\ne.g. `calendar.googleapis.com`.", + "type": "string" + }, + "id": { + "description": "A unique ID for a specific instance of this message, typically assigned\nby the client for tracking purpose. If empty, the server may choose to\ngenerate one instead.", + "type": "string" + }, + "title": { + "description": "The product title associated with this service.", + "type": "string" + }, + "producerProjectId": { + "description": "The id of the Google developer project that owns the service.\nMembers of this project can manage the service configuration,\nmanage consumption of the service, etc.", + "type": "string" + }, + "apis": { + "description": "A list of API interfaces exported by this service. Only the `name` field\nof the google.protobuf.Api needs to be provided by the configuration\nauthor, as the remaining fields will be derived from the IDL during the\nnormalization process. It is an error to specify an API interface here\nwhich cannot be resolved against the associated IDL files.", + "type": "array", + "items": { + "$ref": "Api" + } + }, + "types": { + "description": "A list of all proto message types included in this API service.\nTypes referenced directly or indirectly by the `apis` are\nautomatically included. Messages which are not referenced but\nshall be included, such as types used by the `google.protobuf.Any` type,\nshould be listed here by name. Example:\n\n types:\n - name: google.protobuf.Int32", + "type": "array", + "items": { + "$ref": "Type" + } + }, + "enums": { + "description": "A list of all enum types included in this API service. Enums\nreferenced directly or indirectly by the `apis` are automatically\nincluded. Enums which are not referenced but shall be included\nshould be listed here by name. Example:\n\n enums:\n - name: google.someapi.v1.SomeEnum", + "type": "array", + "items": { + "$ref": "Enum" + } + }, + "documentation": { + "description": "Additional API documentation.", + "$ref": "Documentation" + }, + "visibility": { + "description": "API visibility configuration.", + "$ref": "Visibility" + }, + "backend": { + "description": "API backend configuration.", + "$ref": "Backend" + }, + "http": { + "description": "HTTP configuration.", + "$ref": "Http" + }, + "quota": { + "description": "Quota configuration.", + "$ref": "Quota" + }, + "authentication": { + "description": "Auth configuration.", + "$ref": "Authentication" + }, + "context": { + "description": "Context configuration.", + "$ref": "Context" + }, + "usage": { + "description": "Configuration controlling usage of this service.", + "$ref": "Usage" + }, + "customError": { + "description": "Custom error configuration.", + "$ref": "CustomError" + }, + "projectProperties": { + "description": "Configuration of per-consumer project properties.", + "$ref": "ProjectProperties" + }, + "control": { + "description": "Configuration for the service control plane.", + "$ref": "Control" + }, + "logs": { + "description": "Defines the logs used by this service.", + "type": "array", + "items": { + "$ref": "LogDescriptor" + } + }, + "metrics": { + "description": "Defines the metrics used by this service.", + "type": "array", + "items": { + "$ref": "MetricDescriptor" + } + }, + "monitoredResources": { + "description": "Defines the monitored resources used by this service. This is required\nby the Service.monitoring and Service.logging configurations.\n", + "type": "array", + "items": { + "$ref": "MonitoredResourceDescriptor" + } + }, + "billing": { + "description": "Billing configuration of the service.", + "$ref": "Billing" + }, + "logging": { + "description": "Logging configuration of the service.", + "$ref": "Logging" + }, + "monitoring": { + "description": "Monitoring configuration of the service.", + "$ref": "Monitoring" + }, + "systemParameters": { + "description": "Configuration for system parameters.", + "$ref": "SystemParameters" + }, + "systemTypes": { + "description": "A list of all proto message types included in this API service.\nIt serves similar purpose as [google.api.Service.types], except that\nthese types are not needed by user-defined APIs. Therefore, they will not\nshow up in the generated discovery doc. This field should only be used\nto define system APIs in ESF.", + "type": "array", + "items": { + "$ref": "Type" + } + } + } + }, + "Api": { + "id": "Api", + "description": "Api is a light-weight descriptor for a protocol buffer service.", + "type": "object", + "properties": { + "name": { + "description": "The fully qualified name of this api, including package name\nfollowed by the api's simple name.", + "type": "string" + }, + "methods": { + "description": "The methods of this api, in unspecified order.", + "type": "array", + "items": { + "$ref": "Method" + } + }, + "options": { + "description": "Any metadata attached to the API.", + "type": "array", + "items": { + "$ref": "Option" + } + }, + "version": { + "description": "A version string for this api. If specified, must have the form\n`major-version.minor-version`, as in `1.10`. If the minor version\nis omitted, it defaults to zero. If the entire version field is\nempty, the major version is derived from the package name, as\noutlined below. If the field is not empty, the version in the\npackage name will be verified to be consistent with what is\nprovided here.\n\nThe versioning schema uses [semantic\nversioning](http:\/\/semver.org) where the major version number\nindicates a breaking change and the minor version an additive,\nnon-breaking change. Both version numbers are signals to users\nwhat to expect from different versions, and should be carefully\nchosen based on the product plan.\n\nThe major version is also reflected in the package name of the\nAPI, which must end in `v`, as in\n`google.feature.v1`. For major versions 0 and 1, the suffix can\nbe omitted. Zero major versions must only be used for\nexperimental, none-GA apis.\n\n", + "type": "string" + }, + "sourceContext": { + "description": "Source context for the protocol buffer service represented by this\nmessage.", + "$ref": "SourceContext" + }, + "mixins": { + "description": "Included APIs. See Mixin.", + "type": "array", + "items": { + "$ref": "Mixin" + } + }, + "syntax": { + "description": "The source syntax of the service.", + "enumDescriptions": [ + "Syntax `proto2`.", + "Syntax `proto3`." + ], + "type": "string", + "enum": [ + "SYNTAX_PROTO2", + "SYNTAX_PROTO3" + ] + } + } + }, + "Method": { + "id": "Method", + "description": "Method represents a method of an api.", + "type": "object", + "properties": { + "name": { + "description": "The simple name of this method.", + "type": "string" + }, + "requestTypeUrl": { + "description": "A URL of the input message type.", + "type": "string" + }, + "requestStreaming": { + "description": "If true, the request is streamed.", + "type": "boolean" + }, + "responseTypeUrl": { + "description": "The URL of the output message type.", + "type": "string" + }, + "responseStreaming": { + "description": "If true, the response is streamed.", + "type": "boolean" + }, + "options": { + "description": "Any metadata attached to the method.", + "type": "array", + "items": { + "$ref": "Option" + } + }, + "syntax": { + "description": "The source syntax of this method.", + "enumDescriptions": [ + "Syntax `proto2`.", + "Syntax `proto3`." + ], + "type": "string", + "enum": [ + "SYNTAX_PROTO2", + "SYNTAX_PROTO3" + ] + } + } + }, + "Option": { + "id": "Option", + "description": "A protocol buffer option, which can be attached to a message, field,\nenumeration, etc.", + "type": "object", + "properties": { + "name": { + "description": "The option's name. For example, `\"java_package\"`.", + "type": "string" + }, + "value": { + "description": "The option's value. For example, `\"com.google.protobuf\"`.", + "type": "object", + "additionalProperties": { + "type": "any", + "description": "Properties of the object. Contains field @type with type URL." + } + } + } + }, + "SourceContext": { + "id": "SourceContext", + "description": "`SourceContext` represents information about the source of a\nprotobuf element, like the file in which it is defined.", + "type": "object", + "properties": { + "fileName": { + "description": "The path-qualified name of the .proto file that contained the associated\nprotobuf element. For example: `\"google\/protobuf\/source_context.proto\"`.", + "type": "string" + } + } + }, + "Mixin": { + "id": "Mixin", + "description": "Declares an API to be included in this API. The including API must\nredeclare all the methods from the included API, but documentation\nand options are inherited as follows:\n\n- If after comment and whitespace stripping, the documentation\n string of the redeclared method is empty, it will be inherited\n from the original method.\n\n- Each annotation belonging to the service config (http,\n visibility) which is not set in the redeclared method will be\n inherited.\n\n- If an http annotation is inherited, the path pattern will be\n modified as follows. Any version prefix will be replaced by the\n version of the including API plus the root path if specified.\n\nExample of a simple mixin:\n\n package google.acl.v1;\n service AccessControl {\n \/\/ Get the underlying ACL object.\n rpc GetAcl(GetAclRequest) returns (Acl) {\n option (google.api.http).get = \"\/v1\/{resource=**}:getAcl\";\n }\n }\n\n package google.storage.v2;\n service Storage {\n \/\/ rpc GetAcl(GetAclRequest) returns (Acl);\n\n \/\/ Get a data record.\n rpc GetData(GetDataRequest) returns (Data) {\n option (google.api.http).get = \"\/v2\/{resource=**}\";\n }\n }\n\nExample of a mixin configuration:\n\n apis:\n - name: google.storage.v2.Storage\n mixins:\n - name: google.acl.v1.AccessControl\n\nThe mixin construct implies that all methods in `AccessControl` are\nalso declared with same name and request\/response types in\n`Storage`. A documentation generator or annotation processor will\nsee the effective `Storage.GetAcl` method after inherting\ndocumentation and annotations as follows:\n\n service Storage {\n \/\/ Get the underlying ACL object.\n rpc GetAcl(GetAclRequest) returns (Acl) {\n option (google.api.http).get = \"\/v2\/{resource=**}:getAcl\";\n }\n ...\n }\n\nNote how the version in the path pattern changed from `v1` to `v2`.\n\nIf the `root` field in the mixin is specified, it should be a\nrelative path under which inherited HTTP paths are placed. Example:\n\n apis:\n - name: google.storage.v2.Storage\n mixins:\n - name: google.acl.v1.AccessControl\n root: acls\n\nThis implies the following inherited HTTP annotation:\n\n service Storage {\n \/\/ Get the underlying ACL object.\n rpc GetAcl(GetAclRequest) returns (Acl) {\n option (google.api.http).get = \"\/v2\/acls\/{resource=**}:getAcl\";\n }\n ...\n }", + "type": "object", + "properties": { + "name": { + "description": "The fully qualified name of the API which is included.", + "type": "string" + }, + "root": { + "description": "If non-empty specifies a path under which inherited HTTP paths\nare rooted.", + "type": "string" + } + } + }, + "Type": { + "id": "Type", + "description": "A protocol buffer message type.", + "type": "object", + "properties": { + "name": { + "description": "The fully qualified message name.", + "type": "string" + }, + "fields": { + "description": "The list of fields.", + "type": "array", + "items": { + "$ref": "Field" + } + }, + "oneofs": { + "description": "The list of types appearing in `oneof` definitions in this type.", + "type": "array", + "items": { + "type": "string" + } + }, + "options": { + "description": "The protocol buffer options.", + "type": "array", + "items": { + "$ref": "Option" + } + }, + "sourceContext": { + "description": "The source context.", + "$ref": "SourceContext" + }, + "syntax": { + "description": "The source syntax.", + "enumDescriptions": [ + "Syntax `proto2`.", + "Syntax `proto3`." + ], + "type": "string", + "enum": [ + "SYNTAX_PROTO2", + "SYNTAX_PROTO3" + ] + } + } + }, + "Field": { + "id": "Field", + "description": "A single field of a message type.", + "type": "object", + "properties": { + "kind": { + "description": "The field type.", + "enumDescriptions": [ + "Field type unknown.", + "Field type double.", + "Field type float.", + "Field type int64.", + "Field type uint64.", + "Field type int32.", + "Field type fixed64.", + "Field type fixed32.", + "Field type bool.", + "Field type string.", + "Field type group. Proto2 syntax only, and deprecated.", + "Field type message.", + "Field type bytes.", + "Field type uint32.", + "Field type enum.", + "Field type sfixed32.", + "Field type sfixed64.", + "Field type sint32.", + "Field type sint64." + ], + "type": "string", + "enum": [ + "TYPE_UNKNOWN", + "TYPE_DOUBLE", + "TYPE_FLOAT", + "TYPE_INT64", + "TYPE_UINT64", + "TYPE_INT32", + "TYPE_FIXED64", + "TYPE_FIXED32", + "TYPE_BOOL", + "TYPE_STRING", + "TYPE_GROUP", + "TYPE_MESSAGE", + "TYPE_BYTES", + "TYPE_UINT32", + "TYPE_ENUM", + "TYPE_SFIXED32", + "TYPE_SFIXED64", + "TYPE_SINT32", + "TYPE_SINT64" + ] + }, + "cardinality": { + "description": "The field cardinality.", + "enumDescriptions": [ + "For fields with unknown cardinality.", + "For optional fields.", + "For required fields. Proto2 syntax only.", + "For repeated fields." + ], + "type": "string", + "enum": [ + "CARDINALITY_UNKNOWN", + "CARDINALITY_OPTIONAL", + "CARDINALITY_REQUIRED", + "CARDINALITY_REPEATED" + ] + }, + "number": { + "description": "The field number.", + "type": "integer", + "format": "int32" + }, + "name": { + "description": "The field name.", + "type": "string" + }, + "typeUrl": { + "description": "The field type URL, without the scheme, for message or enumeration\ntypes. Example: `\"type.googleapis.com\/google.protobuf.Timestamp\"`.", + "type": "string" + }, + "oneofIndex": { + "description": "The index of the field type in `Type.oneofs`, for message or enumeration\ntypes. The first type has index 1; zero means the type is not in the list.", + "type": "integer", + "format": "int32" + }, + "packed": { + "description": "Whether to use alternative packed wire representation.", + "type": "boolean" + }, + "options": { + "description": "The protocol buffer options.", + "type": "array", + "items": { + "$ref": "Option" + } + }, + "jsonName": { + "description": "The field JSON name.", + "type": "string" + }, + "defaultValue": { + "description": "The string value of the default value of this field. Proto2 syntax only.", + "type": "string" + } + } + }, + "Enum": { + "id": "Enum", + "description": "Enum type definition.", + "type": "object", + "properties": { + "name": { + "description": "Enum type name.", + "type": "string" + }, + "enumvalue": { + "description": "Enum value definitions.", + "type": "array", + "items": { + "$ref": "EnumValue" + } + }, + "options": { + "description": "Protocol buffer options.", + "type": "array", + "items": { + "$ref": "Option" + } + }, + "sourceContext": { + "description": "The source context.", + "$ref": "SourceContext" + }, + "syntax": { + "description": "The source syntax.", + "enumDescriptions": [ + "Syntax `proto2`.", + "Syntax `proto3`." + ], + "type": "string", + "enum": [ + "SYNTAX_PROTO2", + "SYNTAX_PROTO3" + ] + } + } + }, + "EnumValue": { + "id": "EnumValue", + "description": "Enum value definition.", + "type": "object", + "properties": { + "name": { + "description": "Enum value name.", + "type": "string" + }, + "number": { + "description": "Enum value number.", + "type": "integer", + "format": "int32" + }, + "options": { + "description": "Protocol buffer options.", + "type": "array", + "items": { + "$ref": "Option" + } + } + } + }, + "Documentation": { + "id": "Documentation", + "description": "`Documentation` provides the information for describing a service.\n\nExample:\n
documentation:\n  summary: >\n    The Google Calendar API gives access\n    to most calendar features.\n  pages:\n  - name: Overview\n    content: (== include google\/foo\/overview.md ==)\n  - name: Tutorial\n    content: (== include google\/foo\/tutorial.md ==)\n    subpages;\n    - name: Java\n      content: (== include google\/foo\/tutorial_java.md ==)\n  rules:\n  - selector: google.calendar.Calendar.Get\n    description: >\n      ...\n  - selector: google.calendar.Calendar.Put\n    description: >\n      ...\n<\/code><\/pre>\nDocumentation is provided in markdown syntax. In addition to\nstandard markdown features, definition lists, tables and fenced\ncode blocks are supported. Section headers can be provided and are\ninterpreted relative to the section nesting of the context where\na documentation fragment is embedded.\n\nDocumentation from the IDL is merged with documentation defined\nvia the config at normalization time, where documentation provided\nby config rules overrides IDL provided.\n\nA number of constructs specific to the API platform are supported\nin documentation text.\n\nIn order to reference a proto element, the following\nnotation can be used:\n
[fully.qualified.proto.name][]<\/code><\/pre>\nTo override the display text used for the link, this can be used:\n
[display text][fully.qualified.proto.name]<\/code><\/pre>\nText can be excluded from doc using the following notation:\n
(-- internal comment --)<\/code><\/pre>\nComments can be made conditional using a visibility label. The below\ntext will be only rendered if the `BETA` label is available:\n
(--BETA: comment for BETA users --)<\/code><\/pre>\nA few directives are available in documentation. Note that\ndirectives must appear on a single line to be properly\nidentified. The `include` directive includes a markdown file from\nan external source:\n
(== include path\/to\/file ==)<\/code><\/pre>\nThe `resource_for` directive marks a message to be the resource of\na collection in REST view. If it is not specified, tools attempt\nto infer the resource from the operations in a collection:\n
(== resource_for v1.shelves.books ==)<\/code><\/pre>\nThe directive `suppress_warning` does not directly affect documentation\nand is documented together with service config validation.",
+      "type": "object",
+      "properties": {
+        "summary": {
+          "description": "A short summary of what the service does. Can only be provided by\nplain text.",
+          "type": "string"
+        },
+        "pages": {
+          "description": "The top level pages for the documentation set.",
+          "type": "array",
+          "items": {
+            "$ref": "Page"
+          }
+        },
+        "rules": {
+          "description": "Documentation rules for individual elements of the service.",
+          "type": "array",
+          "items": {
+            "$ref": "DocumentationRule"
+          }
+        },
+        "documentationRootUrl": {
+          "description": "The URL to the root of documentation.",
+          "type": "string"
+        },
+        "overview": {
+          "description": "Declares a single overview page. For example:\n
documentation:\n  summary: ...\n  overview: (== include overview.md ==)\n<\/code><\/pre>\nThis is a shortcut for the following declaration (using pages style):\n
documentation:\n  summary: ...\n  pages:\n  - name: Overview\n    content: (== include overview.md ==)\n<\/code><\/pre>\nNote: you cannot specify both `overview` field and `pages` field.",
+          "type": "string"
+        }
+      }
+    },
+    "Page": {
+      "id": "Page",
+      "description": "Represents a documentation page. A page can contain subpages to represent\nnested documentation set structure.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The name of the page. It will be used as an identity of the page to\ngenerate URI of the page, text of the link to this page in navigation,\netc. The full page name (start from the root page name to this page\nconcatenated with `.`) can be used as reference to the page in your\ndocumentation. For example:\n
pages:\n- name: Tutorial\n  content: (== include tutorial.md ==)\n  subpages:\n  - name: Java\n    content: (== include tutorial_java.md ==)\n<\/code><\/pre>\nYou can reference `Java` page using Markdown reference link syntax:\n`Java`.",
+          "type": "string"
+        },
+        "content": {
+          "description": "The Markdown content of the page. You can use (== include {path} ==)<\/code>\nto include content from a Markdown file.",
+          "type": "string"
+        },
+        "subpages": {
+          "description": "Subpages of this page. The order of subpages specified here will be\nhonored in the generated docset.",
+          "type": "array",
+          "items": {
+            "$ref": "Page"
+          }
+        }
+      }
+    },
+    "DocumentationRule": {
+      "id": "DocumentationRule",
+      "description": "A documentation rule provides information about individual API elements.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "The selector is a comma-separated list of patterns. Each pattern is a\nqualified name of the element which may end in \"*\", indicating a wildcard.\nWildcards are only allowed at the end and for a whole component of the\nqualified name, i.e. \"foo.*\" is ok, but not \"foo.b*\" or \"foo.*.bar\". To\nspecify a default for all applicable elements, the whole pattern \"*\"\nis used.",
+          "type": "string"
+        },
+        "description": {
+          "description": "Description of the selected API(s).",
+          "type": "string"
+        },
+        "deprecationDescription": {
+          "description": "Deprecation description of the selected element(s). It can be provided if an\nelement is marked as `deprecated`.",
+          "type": "string"
+        }
+      }
+    },
+    "Visibility": {
+      "id": "Visibility",
+      "description": "`Visibility` defines restrictions for the visibility of service\nelements.  Restrictions are specified using visibility labels\n(e.g., TRUSTED_TESTER) that are elsewhere linked to users and projects.\n\nUsers and projects can have access to more than one visibility label. The\neffective visibility for multiple labels is the union of each label's\nelements, plus any unrestricted elements.\n\nIf an element and its parents have no restrictions, visibility is\nunconditionally granted.\n\nExample:\n\n    visibility:\n      rules:\n      - selector: google.calendar.Calendar.EnhancedSearch\n        restriction: TRUSTED_TESTER\n      - selector: google.calendar.Calendar.Delegate\n        restriction: GOOGLE_INTERNAL\n\nHere, all methods are publicly visible except for the restricted methods\nEnhancedSearch and Delegate.",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "A list of visibility rules providing visibility configuration for\nindividual API elements.",
+          "type": "array",
+          "items": {
+            "$ref": "VisibilityRule"
+          }
+        },
+        "enforceRuntimeVisibility": {
+          "description": "Controls whether visibility rules are enforced at runtime for requests to\nall APIs and methods.\n\nIf true, requests without method visibility will receive a\nNOT_FOUND error, and any non-visible fields will be scrubbed from\nthe response messages. In service config version 0, the default is false.\nIn later config versions, it's true.\n\nNote, the `enforce_runtime_visibility` specified in a visibility rule\noverrides this setting for the APIs or methods asscoiated with the rule.",
+          "type": "boolean"
+        }
+      }
+    },
+    "VisibilityRule": {
+      "id": "VisibilityRule",
+      "description": "A visibility rule provides visibility configuration for an individual API\nelement.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects methods, messages, fields, enums, etc. to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "restriction": {
+          "description": "Lists the visibility labels for this rule. Any of the listed labels grants\nvisibility to the element.\n\nIf a rule has multiple labels, removing one of the labels but not all of\nthem can break clients.\n\nExample:\n\n    visibility:\n      rules:\n      - selector: google.calendar.Calendar.EnhancedSearch\n        restriction: GOOGLE_INTERNAL, TRUSTED_TESTER\n\nRemoving GOOGLE_INTERNAL from this restriction will break clients that\nrely on this method and only had access to it through GOOGLE_INTERNAL.",
+          "type": "string"
+        },
+        "enforceRuntimeVisibility": {
+          "description": "Controls whether visibility is enforced at runtime for requests to an API\nmethod. This setting has meaning only when the selector applies to a method\nor an API.\n\nIf true, requests without method visibility will receive a\nNOT_FOUND error, and any non-visible fields will be scrubbed from\nthe response messages. The default is determined by the value of\ngoogle.api.Visibility.enforce_runtime_visibility.",
+          "type": "boolean"
+        }
+      }
+    },
+    "Backend": {
+      "id": "Backend",
+      "description": "`Backend` defines the backend configuration for a service.",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "A list of backend rules providing configuration for individual API\nelements.",
+          "type": "array",
+          "items": {
+            "$ref": "BackendRule"
+          }
+        }
+      }
+    },
+    "BackendRule": {
+      "id": "BackendRule",
+      "description": "A backend rule provides configuration for an individual API element.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the methods to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "address": {
+          "description": "The address of the API backend.\n",
+          "type": "string"
+        },
+        "deadline": {
+          "description": "The number of seconds to wait for a response from a request.  The\ndefault depends on the deployment context.",
+          "type": "number",
+          "format": "double"
+        }
+      }
+    },
+    "Http": {
+      "id": "Http",
+      "description": "Defines the HTTP configuration for a service. It contains a list of\nHttpRule, each specifying the mapping of an RPC method\nto one or more HTTP REST API methods.",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "A list of HTTP rules for configuring the HTTP REST API methods.",
+          "type": "array",
+          "items": {
+            "$ref": "HttpRule"
+          }
+        }
+      }
+    },
+    "HttpRule": {
+      "id": "HttpRule",
+      "description": "`HttpRule` defines the mapping of an RPC method to one or more HTTP\nREST APIs.  The mapping determines what portions of the request\nmessage are populated from the path, query parameters, or body of\nthe HTTP request.  The mapping is typically specified as an\n`google.api.http` annotation, see \"google\/api\/annotations.proto\"\nfor details.\n\nThe mapping consists of a field specifying the path template and\nmethod kind.  The path template can refer to fields in the request\nmessage, as in the example below which describes a REST GET\noperation on a resource collection of messages:\n\n```proto\nservice Messaging {\n  rpc GetMessage(GetMessageRequest) returns (Message) {\n    option (google.api.http).get = \"\/v1\/messages\/{message_id}\/{sub.subfield}\";\n  }\n}\nmessage GetMessageRequest {\n  message SubMessage {\n    string subfield = 1;\n  }\n  string message_id = 1; \/\/ mapped to the URL\n  SubMessage sub = 2;    \/\/ `sub.subfield` is url-mapped\n}\nmessage Message {\n  string text = 1; \/\/ content of the resource\n}\n```\n\nThis definition enables an automatic, bidrectional mapping of HTTP\nJSON to RPC. Example:\n\nHTTP | RPC\n-----|-----\n`GET \/v1\/messages\/123456\/foo`  | `GetMessage(message_id: \"123456\" sub: SubMessage(subfield: \"foo\"))`\n\nIn general, not only fields but also field paths can be referenced\nfrom a path pattern. Fields mapped to the path pattern cannot be\nrepeated and must have a primitive (non-message) type.\n\nAny fields in the request message which are not bound by the path\npattern automatically become (optional) HTTP query\nparameters. Assume the following definition of the request message:\n\n```proto\nmessage GetMessageRequest {\n  message SubMessage {\n    string subfield = 1;\n  }\n  string message_id = 1; \/\/ mapped to the URL\n  int64 revision = 2;    \/\/ becomes a parameter\n  SubMessage sub = 3;    \/\/ `sub.subfield` becomes a parameter\n}\n```\n\nThis enables a HTTP JSON to RPC mapping as below:\n\nHTTP | RPC\n-----|-----\n`GET \/v1\/messages\/123456?revision=2&sub.subfield=foo` | `GetMessage(message_id: \"123456\" revision: 2 sub: SubMessage(subfield: \"foo\"))`\n\nNote that fields which are mapped to HTTP parameters must have a\nprimitive type or a repeated primitive type. Message types are not\nallowed. In the case of a repeated type, the parameter can be\nrepeated in the URL, as in `...?param=A¶m=B`.\n\nFor HTTP method kinds which allow a request body, the `body` field\nspecifies the mapping. Consider a REST update method on the\nmessage resource collection:\n\n```proto\nservice Messaging {\n  rpc UpdateMessage(UpdateMessageRequest) returns (Message) {\n    option (google.api.http) = {\n      put: \"\/v1\/messages\/{message_id}\"\n      body: \"message\"\n    };\n  }\n}\nmessage UpdateMessageRequest {\n  string message_id = 1; \/\/ mapped to the URL\n  Message message = 2;   \/\/ mapped to the body\n}\n```\n\nThe following HTTP JSON to RPC mapping is enabled, where the\nrepresentation of the JSON in the request body is determined by\nprotos JSON encoding:\n\nHTTP | RPC\n-----|-----\n`PUT \/v1\/messages\/123456 { \"text\": \"Hi!\" }` | `UpdateMessage(message_id: \"123456\" message { text: \"Hi!\" })`\n\nThe special name `*` can be used in the body mapping to define that\nevery field not bound by the path template should be mapped to the\nrequest body.  This enables the following alternative definition of\nthe update method:\n\n```proto\nservice Messaging {\n  rpc UpdateMessage(Message) returns (Message) {\n    option (google.api.http) = {\n      put: \"\/v1\/messages\/{message_id}\"\n      body: \"*\"\n    };\n  }\n}\nmessage Message {\n  string message_id = 1;\n  string text = 2;\n}\n```\n\nThe following HTTP JSON to RPC mapping is enabled:\n\nHTTP | RPC\n-----|-----\n`PUT \/v1\/messages\/123456 { \"text\": \"Hi!\" }` | `UpdateMessage(message_id: \"123456\" text: \"Hi!\")`\n\nNote that when using `*` in the body mapping, it is not possible to\nhave HTTP parameters, as all fields not bound by the path end in\nthe body. This makes this option more rarely used in practice of\ndefining REST APIs. The common usage of `*` is in custom methods\nwhich don't use the URL at all for transferring data.\n\nIt is possible to define multiple HTTP methods for one RPC by using\nthe `additional_bindings` option. Example:\n\n```proto\nservice Messaging {\n  rpc GetMessage(GetMessageRequest) returns (Message) {\n    option (google.api.http) = {\n      get: \"\/v1\/messages\/{message_id}\"\n      additional_bindings {\n        get: \"\/v1\/users\/{user_id}\/messages\/{message_id}\"\n      }\n    };\n  }\n}\nmessage GetMessageRequest {\n  string message_id = 1;\n  string user_id = 2;\n}\n```\n\nThis enables the following two alternative HTTP JSON to RPC\nmappings:\n\nHTTP | RPC\n-----|-----\n`GET \/v1\/messages\/123456` | `GetMessage(message_id: \"123456\")`\n`GET \/v1\/users\/me\/messages\/123456` | `GetMessage(user_id: \"me\" message_id: \"123456\")`\n\n# Rules for HTTP mapping\n\nThe rules for mapping HTTP path, query parameters, and body fields\nto the request message are as follows:\n\n1. The `body` field specifies either `*` or a field path, or is\n   omitted. If omitted, it assumes there is no HTTP body.\n2. Leaf fields (recursive expansion of nested messages in the\n   request) can be classified into three types:\n    (a) Matched in the URL template.\n    (b) Covered by body (if body is `*`, everything except (a) fields;\n        else everything under the body field)\n    (c) All other fields.\n3. URL query parameters found in the HTTP request are mapped to (c) fields.\n4. Any body sent with an HTTP request can contain only (b) fields.\n\nThe syntax of the path template is as follows:\n\n    Template = \"\/\" Segments [ Verb ] ;\n    Segments = Segment { \"\/\" Segment } ;\n    Segment  = \"*\" | \"**\" | LITERAL | Variable ;\n    Variable = \"{\" FieldPath [ \"=\" Segments ] \"}\" ;\n    FieldPath = IDENT { \".\" IDENT } ;\n    Verb     = \":\" LITERAL ;\n\nThe syntax `*` matches a single path segment. It follows the semantics of\n[RFC 6570](https:\/\/tools.ietf.org\/html\/rfc6570) Section 3.2.2 Simple String\nExpansion.\n\nThe syntax `**` matches zero or more path segments. It follows the semantics\nof [RFC 6570](https:\/\/tools.ietf.org\/html\/rfc6570) Section 3.2.3 Reserved\nExpansion.\n\nThe syntax `LITERAL` matches literal text in the URL path.\n\nThe syntax `Variable` matches the entire path as specified by its template;\nthis nested template must not contain further variables. If a variable\nmatches a single path segment, its template may be omitted, e.g. `{var}`\nis equivalent to `{var=*}`.\n\nNOTE: the field paths in variables and in the `body` must not refer to\nrepeated fields or map fields.\n\nUse CustomHttpPattern to specify any HTTP method that is not included in the\n`pattern` field, such as HEAD, or \"*\" to leave the HTTP method unspecified for\na given URL path rule. The wild-card rule is useful for services that provide\ncontent to Web (HTML) clients.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects methods to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "get": {
+          "description": "Used for listing and getting information about resources.",
+          "type": "string"
+        },
+        "put": {
+          "description": "Used for updating a resource.",
+          "type": "string"
+        },
+        "post": {
+          "description": "Used for creating a resource.",
+          "type": "string"
+        },
+        "delete": {
+          "description": "Used for deleting a resource.",
+          "type": "string"
+        },
+        "patch": {
+          "description": "Used for updating a resource.",
+          "type": "string"
+        },
+        "custom": {
+          "description": "Custom pattern is used for defining custom verbs.",
+          "$ref": "CustomHttpPattern"
+        },
+        "body": {
+          "description": "The name of the request field whose value is mapped to the HTTP body, or\n`*` for mapping all fields not captured by the path pattern to the HTTP\nbody. NOTE: the referred field must not be a repeated field.",
+          "type": "string"
+        },
+        "mediaUpload": {
+          "description": "Do not use this. For media support, add instead\n[][google.bytestream.RestByteStream] as an API to your\nconfiguration.",
+          "$ref": "MediaUpload"
+        },
+        "mediaDownload": {
+          "description": "Do not use this. For media support, add instead\n[][google.bytestream.RestByteStream] as an API to your\nconfiguration.",
+          "$ref": "MediaDownload"
+        },
+        "additionalBindings": {
+          "description": "Additional HTTP bindings for the selector. Nested bindings must\nnot contain an `additional_bindings` field themselves (that is,\nthe nesting may only be one level deep).",
+          "type": "array",
+          "items": {
+            "$ref": "HttpRule"
+          }
+        }
+      }
+    },
+    "CustomHttpPattern": {
+      "id": "CustomHttpPattern",
+      "description": "A custom pattern is used for defining custom HTTP verb.",
+      "type": "object",
+      "properties": {
+        "kind": {
+          "description": "The name of this custom HTTP verb.",
+          "type": "string"
+        },
+        "path": {
+          "description": "The path matched by this custom verb.",
+          "type": "string"
+        }
+      }
+    },
+    "MediaUpload": {
+      "id": "MediaUpload",
+      "description": "Do not use this. For media support, add instead\n[][google.bytestream.RestByteStream] as an API to your\nconfiguration.",
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "description": "Whether upload is enabled.",
+          "type": "boolean"
+        }
+      }
+    },
+    "MediaDownload": {
+      "id": "MediaDownload",
+      "description": "Do not use this. For media support, add instead\n[][google.bytestream.RestByteStream] as an API to your\nconfiguration.",
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "description": "Whether download is enabled.",
+          "type": "boolean"
+        }
+      }
+    },
+    "Quota": {
+      "id": "Quota",
+      "description": "Quota configuration helps to achieve fairness and budgeting in service\nusage.\n\n- Fairness is achieved through the use of short-term quota limits\n  that are usually defined over a time window of several seconds or\n  minutes. When such a limit is applied, for example at the user\n  level, it ensures that no single user will monopolize the service\n  or a given customer's allocated portion of it.\n- Budgeting is achieved through the use of long-term quota limits\n  that are usually defined over a time window of one or more\n  days. These limits help client application developers predict the\n  usage and help budgeting.\n\nQuota enforcement uses a simple token-based algorithm for resource sharing.\n\nThe quota configuration structure is as follows:\n\n- `QuotaLimit` defines a single enforceable limit with a specified\n  token amount that can be consumed over a specific duration and\n  applies to a particular entity, like a project or an end user. If\n  the limit applies to a user, each user making the request will\n  get the specified number of tokens to consume. When the tokens\n  run out, the requests from that user will be blocked until the\n  duration elapses and the next duration window starts.\n\n- `QuotaGroup` groups a set of quota limits.\n\n- `QuotaRule` maps a method to a set of quota groups. This allows\n  sharing of quota groups across methods as well as one method\n  consuming tokens from more than one quota group. When a group\n  contains multiple limits, requests to a method consuming tokens\n  from that group must satisfy all the limits in that group.\n\nExample:\n\n    quota:\n      groups:\n      - name: ReadGroup\n        limits:\n        - description: Daily Limit\n          name: ProjectQpd\n          default_limit: 10000\n          duration: 1d\n          limit_by: CLIENT_PROJECT\n\n        - description: Per-second Limit\n          name: UserQps\n          default_limit: 20000\n          duration: 100s\n          limit_by: USER\n\n      - name: WriteGroup\n        limits:\n        - description: Daily Limit\n          name: ProjectQpd\n          default_limit: 1000\n          max_limit: 1000\n          duration: 1d\n          limit_by: CLIENT_PROJECT\n\n        - description: Per-second Limit\n          name: UserQps\n          default_limit: 2000\n          max_limit: 4000\n          duration: 100s\n          limit_by: USER\n\n      rules:\n      - selector: \"*\"\n        groups:\n        - group: ReadGroup\n      - selector: google.calendar.Calendar.Update\n        groups:\n        - group: WriteGroup\n          cost: 2\n      - selector: google.calendar.Calendar.Delete\n        groups:\n        - group: WriteGroup\n\nHere, the configuration defines two quota groups: ReadGroup and WriteGroup,\neach defining its own daily and per-second limits. Note that One Platform\nenforces per-second limits averaged over a duration of 100 seconds. The rules\nmap ReadGroup for all methods, except for the Update and Delete methods.\nThese two methods consume from WriteGroup, with Update method consuming at\ntwice the rate as Delete method.\n\nMultiple quota groups can be specified for a method. The quota limits in all\nof those groups will be enforced. Example:\n\n    quota:\n      groups:\n      - name: WriteGroup\n        limits:\n        - description: Daily Limit\n          name: ProjectQpd\n          default_limit: 1000\n          max_limit: 1000\n          duration: 1d\n          limit_by: CLIENT_PROJECT\n\n        - description: Per-second Limit\n          name: UserQps\n          default_limit: 2000\n          max_limit: 4000\n          duration: 100s\n          limit_by: USER\n\n      - name: StorageGroup\n        limits:\n        - description: Storage Quota\n          name: StorageQuota\n          default_limit: 1000\n          duration: 0\n          limit_by: USER\n\n      rules:\n      - selector: google.calendar.Calendar.Create\n        groups:\n        - group: StorageGroup\n        - group: WriteGroup\n      - selector: google.calendar.Calendar.Delete\n        groups:\n        - group: StorageGroup\n\nIn the above example, the Create and Delete methods manage the user's\nstorage space. In addition, Create method uses WriteGroup to manage the\nrequests. In this case, requests to Create method need to satisfy all quota\nlimits defined in both quota groups.\n\nOne can disable quota for selected method(s) identified by the selector by\nsetting disable_quota to ture. For example,\n\n      rules:\n      - selector: \"*\"\n        group:\n        - group ReadGroup\n      - selector: google.calendar.Calendar.Select\n        disable_quota: true\n",
+      "type": "object",
+      "properties": {
+        "groups": {
+          "description": "List of `QuotaGroup` definitions for the service.",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaGroup"
+          }
+        },
+        "rules": {
+          "description": "List of `QuotaRule` definitions, each one mapping a selected method to one\nor more quota groups.",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaRule"
+          }
+        }
+      }
+    },
+    "QuotaGroup": {
+      "id": "QuotaGroup",
+      "description": "`QuotaGroup` defines a set of quota limits to enforce.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Name of this quota group. Must be unique within the service.\n\nQuota group name is used as part of the id for quota limits. Once the quota\ngroup has been put into use, the name of the quota group should be\nimmutable.",
+          "type": "string"
+        },
+        "description": {
+          "description": "User-visible description of this quota group.",
+          "type": "string"
+        },
+        "limits": {
+          "description": "Quota limits to be enforced when this quota group is used. A request must\nsatisfy all the limits in a group for it to be permitted.",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaLimit"
+          }
+        },
+        "billable": {
+          "description": "Indicates if the quota limits defined in this quota group apply to\nconsumers who have active billing. Quota limits defined in billable\ngroups will be applied only to consumers who have active billing. The\namount of tokens consumed from billable quota group will also be reported\nfor billing. Quota limits defined in non-billable groups will be applied\nonly to consumers who have no active billing.",
+          "type": "boolean"
+        }
+      }
+    },
+    "QuotaLimit": {
+      "id": "QuotaLimit",
+      "description": "`QuotaLimit` defines a specific limit that applies over a specified duration\nfor a limit type. There can be at most one limit for a duration and limit\ntype combination defined within a `QuotaGroup`.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Name of the quota limit.  Must be unique within the quota group.\nThis name is used to refer to the limit when overriding the limit on\na per-project basis.  If a name is not provided, it will be generated\nfrom the limit_by and duration fields.\n\nThe maximum length of the limit name is 64 characters.\n\nThe name of a limit is used as a unique identifier for this limit.\nTherefore, once a limit has been put into use, its name should be\nimmutable. You can use the display_name field to provide a user-friendly\nname for the limit. The display name can be evolved over time without\naffecting the identity of the limit.\n",
+          "type": "string"
+        },
+        "limitBy": {
+          "description": "Limit type to use for enforcing this quota limit. Each unique value gets\nthe defined number of tokens to consume from. For a quota limit that uses\nuser type, each user making requests through the same client application\nproject will get his\/her own pool of tokens to consume, whereas for a limit\nthat uses client project type, all users making requests through the same\nclient application project share a single pool of tokens.",
+          "enumDescriptions": [
+            "ID of the project owned by the client application developer making the\nrequest.",
+            "ID of the end user making the request using the client application."
+          ],
+          "type": "string",
+          "enum": [
+            "CLIENT_PROJECT",
+            "USER"
+          ]
+        },
+        "description": {
+          "description": "Optional. User-visible, extended description for this quota limit.\nShould be used only when more context is needed to understand this limit\nthan provided by the limit's display name (see: `display_name`).",
+          "type": "string"
+        },
+        "defaultLimit": {
+          "description": "Default number of tokens that can be consumed during the specified\nduration. This is the number of tokens assigned when a client\napplication developer activates the service for his\/her project.\n\nSpecifying a value of 0 will block all requests. This can be used if you\nare provisioning quota to selected consumers and blocking others.\nSimilarly, a value of -1 will indicate an unlimited quota. No other\nnegative values are allowed.",
+          "type": "string",
+          "format": "int64"
+        },
+        "maxLimit": {
+          "description": "Maximum number of tokens that can be consumed during the specified\nduration. Client application developers can override the default limit up\nto this maximum. If specified, this value cannot be set to a value less\nthan the default limit. If not specified, it is set to the default limit.\n\nTo allow clients to apply overrides with no upper bound, set this to -1,\nindicating unlimited maximum quota.",
+          "type": "string",
+          "format": "int64"
+        },
+        "freeTier": {
+          "description": "Free tier value displayed in the Developers Console for this limit.\nThe free tier is the number of tokens that will be subtracted from the\nbilled amount when billing is enabled.\nThis field can only be set on a limit with duration \"1d\", in a billable\ngroup; it is invalid on any other limit. If this field is not set, it\ndefaults to 0, indicating that there is no free tier for this service.",
+          "type": "string",
+          "format": "int64"
+        },
+        "duration": {
+          "description": "Duration of this limit in textual notation. Example: \"100s\", \"24h\", \"1d\".\nFor duration longer than a day, only multiple of days is supported. We\nsupport only \"100s\" and \"1d\" for now. Additional support will be added in\nthe future. \"0\" indicates indefinite duration.",
+          "type": "string"
+        },
+        "displayName": {
+          "description": "User-visible display name for this limit.\nOptional. If not set, the UI will provide a default display name based on\nthe quota configuration. This field can be used to override the default\ndisplay name generated from the configuration.",
+          "type": "string"
+        }
+      }
+    },
+    "QuotaRule": {
+      "id": "QuotaRule",
+      "description": "`QuotaRule` maps a method to a set of `QuotaGroup`s.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects methods to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "groups": {
+          "description": "Quota groups to be used for this method. This supports associating a cost\nwith each quota group.",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaGroupMapping"
+          }
+        },
+        "disableQuota": {
+          "description": "Indicates if quota checking should be enforced. Quota will be disabled for\nmethods without quota rules or with quota rules having this field set to\ntrue. When this field is set to true, no quota group mapping is allowed.",
+          "type": "boolean"
+        }
+      }
+    },
+    "QuotaGroupMapping": {
+      "id": "QuotaGroupMapping",
+      "description": "A quota group mapping.",
+      "type": "object",
+      "properties": {
+        "group": {
+          "description": "The `QuotaGroup.name` of the group. Requests for the mapped methods will\nconsume tokens from each of the limits defined in this group.",
+          "type": "string"
+        },
+        "cost": {
+          "description": "Number of tokens to consume for each request. This allows different cost\nto be associated with different methods that consume from the same quota\ngroup. By default, each request will cost one token.",
+          "type": "integer",
+          "format": "int32"
+        }
+      }
+    },
+    "Authentication": {
+      "id": "Authentication",
+      "description": "`Authentication` defines the authentication configuration for an API.\n\nExample for an API targeted for external use:\n\n    name: calendar.googleapis.com\n    authentication:\n      rules:\n      - selector: \"*\"\n        oauth:\n          canonical_scopes: https:\/\/www.googleapis.com\/auth\/calendar\n\n      - selector: google.calendar.Delegate\n        oauth:\n          canonical_scopes: https:\/\/www.googleapis.com\/auth\/calendar.read",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "Individual rules for authentication.",
+          "type": "array",
+          "items": {
+            "$ref": "AuthenticationRule"
+          }
+        },
+        "providers": {
+          "description": "Defines a set of authentication providers that a service supports.",
+          "type": "array",
+          "items": {
+            "$ref": "AuthProvider"
+          }
+        }
+      }
+    },
+    "AuthenticationRule": {
+      "id": "AuthenticationRule",
+      "description": "Authentication rules for the service.\n\nBy default, if a method has any authentication requirements, every request\nmust include a valid credential matching one of the requirements.\nIt's an error to include more than one kind of credential in a single\nrequest.\n\nIf a method doesn't have any auth requirements, request credentials will be\nignored.\n",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the methods to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "oauth": {
+          "description": "The requirements for OAuth credentials.",
+          "$ref": "OAuthRequirements"
+        },
+        "allowWithoutCredential": {
+          "description": "Whether to allow requests without a credential.  If quota is enabled, an\nAPI key is required for such request to pass the quota check.\n",
+          "type": "boolean"
+        },
+        "requirements": {
+          "description": "Requirements for additional authentication providers.",
+          "type": "array",
+          "items": {
+            "$ref": "AuthRequirement"
+          }
+        }
+      }
+    },
+    "OAuthRequirements": {
+      "id": "OAuthRequirements",
+      "description": "OAuth scopes are a way to define data and permissions on data. For example,\nthere are scopes defined for \"Read-only access to Google Calendar\" and\n\"Access to Cloud Platform\". Users can consent to a scope for an application,\ngiving it permission to access that data on their behalf.\n\nOAuth scope specifications should be fairly coarse grained; a user will need\nto see and understand the text description of what your scope means.\n\nIn most cases: use one or at most two OAuth scopes for an entire family of\nproducts. If your product has multiple APIs, you should probably be sharing\nthe OAuth scope across all of those APIs.\n\nWhen you need finer grained OAuth consent screens: talk with your product\nmanagement about how developers will use them in practice.\n\nPlease note that even though each of the canonical scopes is enough for a\nrequest to be accepted and passed to the backend, a request can still fail\ndue to the backend requiring additional scopes or permissions.\n",
+      "type": "object",
+      "properties": {
+        "canonicalScopes": {
+          "description": "The list of publicly documented OAuth scopes that are allowed access. An\nOAuth token containing any of these scopes will be accepted.\n\nExample:\n\n     canonical_scopes: https:\/\/www.googleapis.com\/auth\/calendar,\n                       https:\/\/www.googleapis.com\/auth\/calendar.read",
+          "type": "string"
+        }
+      }
+    },
+    "AuthRequirement": {
+      "id": "AuthRequirement",
+      "description": "User-defined authentication requirements, including support for\n[JSON Web Token (JWT)](https:\/\/tools.ietf.org\/html\/draft-ietf-oauth-json-web-token-32).",
+      "type": "object",
+      "properties": {
+        "providerId": {
+          "description": "id from authentication provider.\n\nExample:\n\n    provider_id: bookstore_auth",
+          "type": "string"
+        },
+        "audiences": {
+          "description": "The list of JWT\n[audiences](https:\/\/tools.ietf.org\/html\/draft-ietf-oauth-json-web-token-32#section-4.1.3).\nthat are allowed to access. A JWT containing any of these audiences will\nbe accepted. When this setting is absent, only JWTs with audience\n\"https:\/\/Service_name\/API_name\"\nwill be accepted. For example, if no audiences are in the setting,\nLibraryService API will only accept JWTs with the following audience\n\"https:\/\/library-example.googleapis.com\/google.example.library.v1.LibraryService\".\n\nExample:\n\n    audiences: bookstore_android.apps.googleusercontent.com,\n               bookstore_web.apps.googleusercontent.com",
+          "type": "string"
+        }
+      }
+    },
+    "AuthProvider": {
+      "id": "AuthProvider",
+      "description": "Configuration for an anthentication provider, including support for\n[JSON Web Token (JWT)](https:\/\/tools.ietf.org\/html\/draft-ietf-oauth-json-web-token-32).",
+      "type": "object",
+      "properties": {
+        "id": {
+          "description": "The unique identifier of the auth provider. It will be referred to by\n`AuthRequirement.provider_id`.\n\nExample: \"bookstore_auth\".",
+          "type": "string"
+        },
+        "issuer": {
+          "description": "Identifies the principal that issued the JWT. See\nhttps:\/\/tools.ietf.org\/html\/draft-ietf-oauth-json-web-token-32#section-4.1.1\nUsually a URL or an email address.\n\nExample: https:\/\/securetoken.google.com\nExample: 1234567-compute@developer.gserviceaccount.com",
+          "type": "string"
+        },
+        "jwksUri": {
+          "description": "URL of the provider's public key set to validate signature of the JWT. See\n[OpenID Discovery](https:\/\/openid.net\/specs\/openid-connect-discovery-1_0.html#ProviderMetadata).\nOptional if the key set document:\n - can be retrieved from\n   [OpenID Discovery](https:\/\/openid.net\/specs\/openid-connect-discovery-1_0.html\n   of the issuer.\n - can be inferred from the email domain of the issuer (e.g. a Google service account).\n\nExample: https:\/\/www.googleapis.com\/oauth2\/v1\/certs",
+          "type": "string"
+        }
+      }
+    },
+    "Context": {
+      "id": "Context",
+      "description": "`Context` defines which contexts an API requests.\n\nExample:\n\n    context:\n      rules:\n      - selector: \"*\"\n        requested:\n        - google.rpc.context.ProjectContext\n        - google.rpc.context.OriginContext\n\nThe above specifies that all methods in the API request\n`google.rpc.context.ProjectContext` and\n`google.rpc.context.OriginContext`.\n\nAvailable context types are defined in package\n`google.rpc.context`.",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "List of rules for context, applicable to methods.",
+          "type": "array",
+          "items": {
+            "$ref": "ContextRule"
+          }
+        }
+      }
+    },
+    "ContextRule": {
+      "id": "ContextRule",
+      "description": "A context rule provides information about the context for an individual API\nelement.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the methods to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "requested": {
+          "description": "A list of full type names of requested contexts.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "provided": {
+          "description": "A list of full type names of provided contexts.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "Usage": {
+      "id": "Usage",
+      "description": "Configuration controlling usage of a service.",
+      "type": "object",
+      "properties": {
+        "serviceAccess": {
+          "description": "Controls which users can see or activate the service.",
+          "enumDescriptions": [
+            "The service can only be seen\/used by users identified in the service's\naccess control policy.\n\nIf the service has not been whitelisted by your domain administrator\nfor out-of-org publishing, then this mode will be treated like\nORG_RESTRICTED.",
+            "The service can be seen\/used by anyone.\n\nIf the service has not been whitelisted by your domain administrator\nfor out-of-org publishing, then this mode will be treated like\nORG_PUBLIC.\n\nThe discovery document for the service will also be public and allow\nunregistered access.",
+            "The service can be seen\/used by users identified in the service's\naccess control policy and they are within the organization that owns the\nservice.\n\nAccess is further constrained to the group\ncontrolled by the administrator of the project\/org that owns the\nservice.",
+            "The service can be seen\/used by the group of users controlled by the\nadministrator of the project\/org that owns the service."
+          ],
+          "type": "string",
+          "enum": [
+            "RESTRICTED",
+            "PUBLIC",
+            "ORG_RESTRICTED",
+            "ORG_PUBLIC"
+          ]
+        },
+        "requirements": {
+          "description": "Requirements that must be satisfied before a consumer project can use the\nservice. Each requirement is of the form \/;\nfor example 'serviceusage.googleapis.com\/billing-enabled'.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "dependsOnServices": {
+          "description": "Services that must be activated in order for this service to be used.\nThe set of services activated as a result of these relations are all\nactivated in parallel with no guaranteed order of activation.\nEach string is a service name, e.g. `calendar.googleapis.com`.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "activationHooks": {
+          "description": "Services that must be contacted before a consumer can begin using the\nservice. Each service will be contacted in sequence, and, if any activation\ncall fails, the entire activation will fail. Each hook is of the form\n\/, where  is optional; for example:\n'robotservice.googleapis.com\/default'.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "deactivationHooks": {
+          "description": "Services that must be contacted before a consumer can deactivate a\nservice. Each service will be contacted in sequence, and, if any\ndeactivation call fails, the entire deactivation will fail. Each hook is\nof the form \/, where  is optional; for\nexample:\n'compute.googleapis.com\/'.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "rules": {
+          "description": "Individual rules for configuring usage on selected methods.",
+          "type": "array",
+          "items": {
+            "$ref": "UsageRule"
+          }
+        }
+      }
+    },
+    "UsageRule": {
+      "id": "UsageRule",
+      "description": "Usage configuration rules for the service.\n\nNOTE: Under development.\n\n\nUse this rule to configure unregistered calls for the service. Unregistered\ncalls are calls that do not contain consumer project identity.\n(Example: calls that do not contain an API key).\nBy default, API methods do not allow unregistered calls, and each method call\nmust be identified by a consumer project identity. Use this rule to\nallow\/disallow unregistered calls.\n\nExample of an API that wants to allow unregistered calls for entire service.\n\n    usage:\n      rules:\n      - selector: \"*\"\n        allow_unregistered_calls: true\n\nExample of a method that wants to allow unregistered calls.\n\n    usage:\n      rules:\n      - selector: \"google.example.library.v1.LibraryService.CreateBook\"\n        allow_unregistered_calls: true",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the methods to which this rule applies. Use '*' to indicate all\nmethods in all APIs.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "allowUnregisteredCalls": {
+          "description": "True, if the method allows unregistered calls; false otherwise.",
+          "type": "boolean"
+        }
+      }
+    },
+    "CustomError": {
+      "id": "CustomError",
+      "description": "Customize service error responses.  For example, list any service\nspecific protobuf types that can appear in error detail lists of\nerror responses.\n\nExample:\n\n    custom_error:\n      types:\n      - google.foo.v1.CustomError\n      - google.foo.v1.AnotherError\n",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "The list of custom error rules to select to which messages this should\napply.",
+          "type": "array",
+          "items": {
+            "$ref": "CustomErrorRule"
+          }
+        },
+        "types": {
+          "description": "The list of custom error detail types, e.g. 'google.foo.v1.CustomError'.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "CustomErrorRule": {
+      "id": "CustomErrorRule",
+      "description": "A custom error rule.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects messages to which this rule applies.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "isErrorType": {
+          "description": "Mark this message as possible payload in error response.  Otherwise,\nobjects of this type will be filtered when they appear in error payload.",
+          "type": "boolean"
+        }
+      }
+    },
+    "ProjectProperties": {
+      "id": "ProjectProperties",
+      "description": "A descriptor for defining project properties for a service. One service may\nhave many consumer projects, and the service may want to behave differently\ndepending on some properties on the project. For example, a project may be\nassociated with a school, or a business, or a government agency, a business\ntype property on the project may affect how a service responds to the client.\nThis descriptor defines which properties are allowed to be set on a project.\n\nExample:\n\n   project_properties:\n     properties:\n     - name: NO_WATERMARK\n       type: BOOL\n       description: Allows usage of the API without watermarks.\n     - name: EXTENDED_TILE_CACHE_PERIOD\n       type: INT64",
+      "type": "object",
+      "properties": {
+        "properties": {
+          "description": "List of per consumer project-specific properties.",
+          "type": "array",
+          "items": {
+            "$ref": "Property"
+          }
+        }
+      }
+    },
+    "Property": {
+      "id": "Property",
+      "description": "Defines project properties.\n\nAPI services can define properties that can be assigned to consumer projects\nso that backends can perform response customization without having to make\nadditional calls or maintain additional storage. For example, Maps API\ndefines properties that controls map tile cache period, or whether to embed a\nwatermark in a result.\n\nThese values can be set via API producer console. Only API providers can\ndefine and set these properties.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The name of the property (a.k.a key).",
+          "type": "string"
+        },
+        "type": {
+          "description": "The type of this property.",
+          "enumDescriptions": [
+            "The type is unspecified, and will result in an error.",
+            "The type is `int64`.",
+            "The type is `bool`.",
+            "The type is `string`.",
+            "The type is 'double'."
+          ],
+          "type": "string",
+          "enum": [
+            "UNSPECIFIED",
+            "INT64",
+            "BOOL",
+            "STRING",
+            "DOUBLE"
+          ]
+        },
+        "description": {
+          "description": "The description of the property",
+          "type": "string"
+        }
+      }
+    },
+    "Control": {
+      "id": "Control",
+      "description": "Selects and configures the service controller used by the service.  The\nservice controller handles features like abuse, quota, billing, logging,\nmonitoring, etc.\n",
+      "type": "object",
+      "properties": {
+        "environment": {
+          "description": "The service control environment to use. If empty, no control plane\nfeature (like quota and billing) will be enabled.",
+          "type": "string"
+        }
+      }
+    },
+    "LogDescriptor": {
+      "id": "LogDescriptor",
+      "description": "A description of a log type. Example in YAML format:\n\n    - name: library.googleapis.com\/activity_history\n      description: The history of borrowing and returning library items.\n      display_name: Activity\n      labels:\n      - key: \/customer_id\n        description: Identifier of a library customer",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The name of the log. It must be less than 512 characters long and can\ninclude the following characters: upper- and lower-case alphanumeric\ncharacters [A-Za-z0-9], and punctuation characters including\nslash, underscore, hyphen, period [\/_-.].",
+          "type": "string"
+        },
+        "labels": {
+          "description": "The set of labels that are available to describe a specific log entry.\nRuntime requests that contain labels not specified here are\nconsidered invalid.",
+          "type": "array",
+          "items": {
+            "$ref": "LabelDescriptor"
+          }
+        },
+        "description": {
+          "description": "A human-readable description of this log. This information appears in\nthe documentation and can contain details.",
+          "type": "string"
+        },
+        "displayName": {
+          "description": "The human-readable name for this log. This information appears on\nthe user interface and should be concise.",
+          "type": "string"
+        }
+      }
+    },
+    "LabelDescriptor": {
+      "id": "LabelDescriptor",
+      "description": "A description of a label.",
+      "type": "object",
+      "properties": {
+        "key": {
+          "description": "The label key.",
+          "type": "string"
+        },
+        "valueType": {
+          "description": "The type of data that can be assigned to the label.",
+          "enumDescriptions": [
+            "A variable-length string. This is the default.",
+            "Boolean; true or false.",
+            "A 64-bit signed integer."
+          ],
+          "type": "string",
+          "enum": [
+            "STRING",
+            "BOOL",
+            "INT64"
+          ]
+        },
+        "description": {
+          "description": "A human-readable description for the label.",
+          "type": "string"
+        }
+      }
+    },
+    "MetricDescriptor": {
+      "id": "MetricDescriptor",
+      "description": "Defines a metric type and its schema.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Resource name. The format of the name may vary between different\nimplementations. For examples:\n\n    projects\/{project_id}\/metricDescriptors\/{type=**}\n    metricDescriptors\/{type=**}",
+          "type": "string"
+        },
+        "type": {
+          "description": "The metric type including a DNS name prefix, for example\n`\"compute.googleapis.com\/instance\/cpu\/utilization\"`. Metric types\nshould use a natural hierarchical grouping such as the following:\n\n    compute.googleapis.com\/instance\/cpu\/utilization\n    compute.googleapis.com\/instance\/disk\/read_ops_count\n    compute.googleapis.com\/instance\/network\/received_bytes_count\n\nNote that if the metric type changes, the monitoring data will be\ndiscontinued, and anything depends on it will break, such as monitoring\ndashboards, alerting rules and quota limits. Therefore, once a metric has\nbeen published, its type should be immutable.",
+          "type": "string"
+        },
+        "labels": {
+          "description": "The set of labels that can be used to describe a specific instance of this\nmetric type. For example, the\n`compute.googleapis.com\/instance\/network\/received_bytes_count` metric type\nhas a label, `loadbalanced`, that specifies whether the traffic was\nreceived through a load balanced IP address.",
+          "type": "array",
+          "items": {
+            "$ref": "LabelDescriptor"
+          }
+        },
+        "metricKind": {
+          "description": "Whether the metric records instantaneous values, changes to a value, etc.",
+          "enumDescriptions": [
+            "Do not use this default value.",
+            "Instantaneous measurements of a varying quantity.",
+            "Changes over non-overlapping time intervals.",
+            "Cumulative value over time intervals that can overlap.\nThe overlapping intervals must have the same start time."
+          ],
+          "type": "string",
+          "enum": [
+            "METRIC_KIND_UNSPECIFIED",
+            "GAUGE",
+            "DELTA",
+            "CUMULATIVE"
+          ]
+        },
+        "valueType": {
+          "description": "Whether the measurement is an integer, a floating-point number, etc.",
+          "enumDescriptions": [
+            "Do not use this default value.",
+            "The value is a boolean.\nThis value type can be used only if the metric kind is `GAUGE`.",
+            "The value is a signed 64-bit integer.",
+            "The value is a double precision floating point number.",
+            "The value is a text string.\nThis value type can be used only if the metric kind is `GAUGE`.",
+            "The value is a `Distribution`.",
+            "The value is money."
+          ],
+          "type": "string",
+          "enum": [
+            "VALUE_TYPE_UNSPECIFIED",
+            "BOOL",
+            "INT64",
+            "DOUBLE",
+            "STRING",
+            "DISTRIBUTION",
+            "MONEY"
+          ]
+        },
+        "unit": {
+          "description": "The unit in which the metric value is reported. It is only applicable\nif the `value_type` is `INT64`, `DOUBLE`, or `DISTRIBUTION`. The\nsupported units are a subset of [The Unified Code for Units of\nMeasure](http:\/\/unitsofmeasure.org\/ucum.html) standard:\n\n**Basic units (UNIT)**\n\n* `bit`   bit\n* `By`    byte\n* `s`     second\n* `min`   minute\n* `h`     hour\n* `d`     day\n\n**Prefixes (PREFIX)**\n\n* `k`     kilo    (10**3)\n* `M`     mega    (10**6)\n* `G`     giga    (10**9)\n* `T`     tera    (10**12)\n* `P`     peta    (10**15)\n* `E`     exa     (10**18)\n* `Z`     zetta   (10**21)\n* `Y`     yotta   (10**24)\n* `m`     milli   (10**-3)\n* `u`     micro   (10**-6)\n* `n`     nano    (10**-9)\n* `p`     pico    (10**-12)\n* `f`     femto   (10**-15)\n* `a`     atto    (10**-18)\n* `z`     zepto   (10**-21)\n* `y`     yocto   (10**-24)\n* `Ki`    kibi    (2**10)\n* `Mi`    mebi    (2**20)\n* `Gi`    gibi    (2**30)\n* `Ti`    tebi    (2**40)\n\n**Grammar**\n\nThe grammar includes the dimensionless unit `1`, such as `1\/s`.\n\nThe grammar also includes these connectors:\n\n* `\/`    division (as an infix operator, e.g. `1\/s`).\n* `.`    multiplication (as an infix operator, e.g. `GBy.d`)\n\nThe grammar for a unit is as follows:\n\n    Expression = Component { \".\" Component } { \"\/\" Component } ;\n\n    Component = [ PREFIX ] UNIT [ Annotation ]\n              | Annotation\n              | \"1\"\n              ;\n\n    Annotation = \"{\" NAME \"}\" ;\n\nNotes:\n\n* `Annotation` is just a comment if it follows a `UNIT` and is\n   equivalent to `1` if it is used alone. For examples,\n   `{requests}\/s == 1\/s`, `By{transmitted}\/s == By\/s`.\n* `NAME` is a sequence of non-blank printable ASCII characters not\n   containing '{' or '}'.",
+          "type": "string"
+        },
+        "description": {
+          "description": "A detailed description of the metric, which can be used in documentation.",
+          "type": "string"
+        },
+        "displayName": {
+          "description": "A concise name for the metric, which can be displayed in user interfaces.\nUse sentence case without an ending period, for example \"Request count\".",
+          "type": "string"
+        }
+      }
+    },
+    "MonitoredResourceDescriptor": {
+      "id": "MonitoredResourceDescriptor",
+      "description": "An object that describes the schema of a MonitoredResource object using a\ntype name and a set of labels.  For example, the monitored resource\ndescriptor for Google Compute Engine VM instances has a type of\n`\"gce_instance\"` and specifies the use of the labels `\"instance_id\"` and\n`\"zone\"` to identify particular VM instances.\n\nDifferent APIs can support different monitored resource types. APIs generally\nprovide a `list` method that returns the monitored resource descriptors used\nby the API.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Optional. The resource name of the monitored resource descriptor:\n`\"projects\/{project_id}\/monitoredResourceDescriptors\/{type}\"` where\n{type} is the value of the `type` field in this object and\n{project_id} is a project ID that provides API-specific context for\naccessing the type.  APIs that do not use project information can use the\nresource name format `\"monitoredResourceDescriptors\/{type}\"`.",
+          "type": "string"
+        },
+        "type": {
+          "description": "Required. The monitored resource type. For example, the type\n`\"cloudsql_database\"` represents databases in Google Cloud SQL.\nThe maximum length of this value is 256 characters.",
+          "type": "string"
+        },
+        "displayName": {
+          "description": "Optional. A concise name for the monitored resource type that might be\ndisplayed in user interfaces. For example, `\"Google Cloud SQL Database\"`.",
+          "type": "string"
+        },
+        "description": {
+          "description": "Optional. A detailed description of the monitored resource type that might\nbe used in documentation.",
+          "type": "string"
+        },
+        "labels": {
+          "description": "Required. A set of labels used to describe instances of this monitored\nresource type. For example, an individual Google Cloud SQL database is\nidentified by values for the labels `\"database_id\"` and `\"zone\"`.",
+          "type": "array",
+          "items": {
+            "$ref": "LabelDescriptor"
+          }
+        }
+      }
+    },
+    "Billing": {
+      "id": "Billing",
+      "description": "Billing related configuration of the service.\n\nThe following example shows how to configure metrics for billing:\n\n    metrics:\n    - name: library.googleapis.com\/read_calls\n      metric_kind: DELTA\n      value_type: INT64\n    - name: library.googleapis.com\/write_calls\n      metric_kind: DELTA\n      value_type: INT64\n    billing:\n      metrics:\n      - library.googleapis.com\/read_calls\n      - library.googleapis.com\/write_calls\n\nThe next example shows how to enable billing status check and customize the\ncheck behavior. It makes sure billing status check is included in the `Check`\nmethod of [Service Control API](https:\/\/cloud.google.com\/service-control\/).\nIn the example, \"google.storage.Get\" method can be served when the billing\nstatus is either `current` or `delinquent`, while \"google.storage.Write\"\nmethod can only be served when the billing status is `current`:\n\n    billing:\n      rules:\n      - selector: google.storage.Get\n        allowed_statuses:\n        - current\n        - delinquent\n      - selector: google.storage.Write\n        allowed_statuses: current\n\nMostly services should only allow `current` status when serving requests.\nIn addition, services can choose to allow both `current` and `delinquent`\nstatuses when serving read-only requests to resources. If there's no\nmatching selector for operation, no billing status check will be performed.\n",
+      "type": "object",
+      "properties": {
+        "metrics": {
+          "description": "Names of the metrics to report to billing. Each name must\nbe defined in Service.metrics section.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "rules": {
+          "description": "A list of billing status rules for configuring billing status check.",
+          "type": "array",
+          "items": {
+            "$ref": "BillingStatusRule"
+          }
+        },
+        "areaUnderCurveParams": {
+          "description": "Per resource grouping for delta billing based resource configs.",
+          "type": "array",
+          "items": {
+            "$ref": "AreaUnderCurveParams"
+          }
+        }
+      }
+    },
+    "BillingStatusRule": {
+      "id": "BillingStatusRule",
+      "description": "Defines the billing status requirements for operations.\n\nWhen used with\n[Service Control API](https:\/\/cloud.google.com\/service-control\/), the\nfollowing statuses are supported:\n\n- **current**: the associated billing account is up to date and capable of\n               paying for resource usages.\n- **delinquent**: the associated billing account has a correctable problem,\n                  such as late payment.\n\nMostly services should only allow `current` status when serving requests.\nIn addition, services can choose to allow both `current` and `delinquent`\nstatuses when serving read-only requests to resources. If the list of\nallowed_statuses is empty, it means no billing requirement.\n",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the operation names to which this rule applies.\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "allowedStatuses": {
+          "description": "Allowed billing statuses. The billing status check passes if the actual\nbilling status matches any of the provided values here.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "AreaUnderCurveParams": {
+      "id": "AreaUnderCurveParams",
+      "description": "AreaUnderCurveParams groups the metrics relevant to generating\nduration based metric from base (snapshot) metric and delta\n(change) metric.  The generated metric has two dimensions:\n   resource usage metric and the duration the metric applies.\n\nEssentially the generated metric is the Area Under Curve(AUC) of\nthe \"duration - resource\" usage curve. This AUC metric is readily\nappliable to billing since \"billable resource usage\" depends on\nresource usage and duration of the resource used.\n\nA service config may contain multiple resources and corresponding\nmetrics. AreaUnderCurveParams groups the relevant ones: which\nsnapshot_metric and change_metric are used to produce which\ngenerated_metric.\n",
+      "type": "object",
+      "properties": {
+        "snapshotMetric": {
+          "description": "Total usage of a resource at a particular timestamp. This should be\na GAUGE metric.",
+          "type": "string"
+        },
+        "changeMetric": {
+          "description": "Change of resource usage at a particular timestamp. This should a\nDELTA metric.",
+          "type": "string"
+        },
+        "generatedMetric": {
+          "description": "Metric generated from snapshot_metric and change_metric. This\nis also a DELTA metric.",
+          "type": "string"
+        }
+      }
+    },
+    "Logging": {
+      "id": "Logging",
+      "description": "Logging configuration of the service.\n\nThe following example shows how to configure logs to be sent to the\nproducer and consumer projects. In the example,\nthe `library.googleapis.com\/activity_history` log is\nsent to both the producer and consumer projects, whereas\nthe `library.googleapis.com\/purchase_history` log is only sent to the\nproducer project:\n\n    monitored_resources:\n    - type: library.googleapis.com\/branch\n      labels:\n      - key: \/city\n        description: The city where the library branch is located in.\n      - key: \/name\n        description: The name of the branch.\n    logs:\n    - name: library.googleapis.com\/activity_history\n      labels:\n      - key: \/customer_id\n    - name: library.googleapis.com\/purchase_history\n    logging:\n      producer_destinations:\n      - monitored_resource: library.googleapis.com\/branch\n        logs:\n        - library.googleapis.com\/activity_history\n        - library.googleapis.com\/purchase_history\n      consumer_destinations:\n      - monitored_resource: library.googleapis.com\/branch\n        logs:\n        - library.googleapis.com\/activity_history\n",
+      "type": "object",
+      "properties": {
+        "producerDestinations": {
+          "description": "Logging configurations for sending logs to the producer project.\nThere can be multiple producer destinations, each one must have a\ndifferent monitored resource type. A log can be used in at most\none producer destination.",
+          "type": "array",
+          "items": {
+            "$ref": "LoggingDestination"
+          }
+        },
+        "consumerDestinations": {
+          "description": "Logging configurations for sending logs to the consumer project.\nThere can be multiple consumer destinations, each one must have a\ndifferent monitored resource type. A log can be used in at most\none consumer destination.",
+          "type": "array",
+          "items": {
+            "$ref": "LoggingDestination"
+          }
+        }
+      }
+    },
+    "LoggingDestination": {
+      "id": "LoggingDestination",
+      "description": "Configuration of a specific logging destination (the producer project\nor the consumer project).",
+      "type": "object",
+      "properties": {
+        "monitoredResource": {
+          "description": "The monitored resource type. The type must be defined in\nService.monitored_resources section.",
+          "type": "string"
+        },
+        "logs": {
+          "description": "Names of the logs to be sent to this destination. Each name must\nbe defined in the Service.logs section.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "Monitoring": {
+      "id": "Monitoring",
+      "description": "Monitoring configuration of the service.\n\nThe example below shows how to configure monitored resources and metrics\nfor monitoring. In the example, a monitored resource and two metrics are\ndefined. The `library.googleapis.com\/book\/returned_count` metric is sent\nto both producer and consumer projects, whereas the\n`library.googleapis.com\/book\/overdue_count` metric is only sent to the\nconsumer project.\n\n    monitored_resources:\n    - type: library.googleapis.com\/branch\n      labels:\n      - key: \/city\n        description: The city where the library branch is located in.\n      - key: \/name\n        description: The name of the branch.\n    metrics:\n    - name: library.googleapis.com\/book\/returned_count\n      metric_kind: DELTA\n      value_type: INT64\n      labels:\n      - key: \/customer_id\n    - name: library.googleapis.com\/book\/overdue_count\n      metric_kind: GAUGE\n      value_type: INT64\n      labels:\n      - key: \/customer_id\n    monitoring:\n      producer_destinations:\n      - monitored_resource: library.googleapis.com\/branch\n        metrics:\n        - library.googleapis.com\/book\/returned_count\n      consumer_destinations:\n      - monitored_resource: library.googleapis.com\/branch\n        metrics:\n        - library.googleapis.com\/book\/returned_count\n        - library.googleapis.com\/book\/overdue_count\n",
+      "type": "object",
+      "properties": {
+        "producerDestinations": {
+          "description": "Monitoring configurations for sending metrics to the producer project.\nThere can be multiple producer destinations, each one must have a\ndifferent monitored resource type. A metric can be used in at most\none producer destination.",
+          "type": "array",
+          "items": {
+            "$ref": "MonitoringDestination"
+          }
+        },
+        "consumerDestinations": {
+          "description": "Monitoring configurations for sending metrics to the consumer project.\nThere can be multiple consumer destinations, each one must have a\ndifferent monitored resource type. A metric can be used in at most\none consumer destination.",
+          "type": "array",
+          "items": {
+            "$ref": "MonitoringDestination"
+          }
+        }
+      }
+    },
+    "MonitoringDestination": {
+      "id": "MonitoringDestination",
+      "description": "Configuration of a specific monitoring destination (the producer project\nor the consumer project).",
+      "type": "object",
+      "properties": {
+        "monitoredResource": {
+          "description": "The monitored resource type. The type must be defined in\nService.monitored_resources section.",
+          "type": "string"
+        },
+        "metrics": {
+          "description": "Names of the metrics to report to this monitoring destination.\nEach name must be defined in Service.metrics section.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "SystemParameters": {
+      "id": "SystemParameters",
+      "description": "### System parameter configuration\n\nA system parameter is a special kind of parameter defined by the API\nsystem, not by an individual API. It is typically mapped to an HTTP header\nand\/or a URL query parameter. This configuration specifies which methods\nchange the names of the system parameters.",
+      "type": "object",
+      "properties": {
+        "rules": {
+          "description": "Define system parameters.\n\nThe parameters defined here will override the default parameters\nimplemented by the system. If this field is missing from the service\nconfig, default system parameters will be used. Default system parameters\nand names is implementation-dependent.\n\nExample: define api key and alt name for all methods\n\nsystem_parameters\n  rules:\n    - selector: \"*\"\n      parameters:\n        - name: api_key\n          url_query_parameter: api_key\n        - name: alt\n          http_header: Response-Content-Type\n\nExample: define 2 api key names for a specific method.\n\nsystem_parameters\n  rules:\n    - selector: \"\/ListShelves\"\n      parameters:\n        - name: api_key\n          http_header: Api-Key1\n        - name: api_key\n          http_header: Api-Key2",
+          "type": "array",
+          "items": {
+            "$ref": "SystemParameterRule"
+          }
+        }
+      }
+    },
+    "SystemParameterRule": {
+      "id": "SystemParameterRule",
+      "description": "Define a system parameter rule mapping system parameter definitions to\nmethods.",
+      "type": "object",
+      "properties": {
+        "selector": {
+          "description": "Selects the methods to which this rule applies. Use '*' to indicate all\nmethods in all APIs.\n\nRefer to selector for syntax details.",
+          "type": "string"
+        },
+        "parameters": {
+          "description": "Define parameters. Multiple names may be defined for a parameter.\nFor a given method call, only one of them should be used. If multiple\nnames are used the behavior is implementation-dependent.\nIf none of the specified names are present the behavior is\nparameter-dependent.",
+          "type": "array",
+          "items": {
+            "$ref": "SystemParameter"
+          }
+        }
+      }
+    },
+    "SystemParameter": {
+      "id": "SystemParameter",
+      "description": "Define a parameter's name and location. The parameter may be passed as either\nan HTTP header or a URL query parameter, and if both are passed the behavior\nis implementation-dependent.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "Define the name of the parameter, such as \"api_key\", \"alt\", \"callback\",\nand etc. It is case sensitive.",
+          "type": "string"
+        },
+        "httpHeader": {
+          "description": "Define the HTTP header name to use for the parameter. It is case\ninsensitive.",
+          "type": "string"
+        },
+        "urlQueryParameter": {
+          "description": "Define the URL query parameter name to use for the parameter. It is case\nsensitive.",
+          "type": "string"
+        }
+      }
+    },
+    "ConfigSource": {
+      "id": "ConfigSource",
+      "description": "Represents a user-specified configuration for a service (as opposed to the\nthe generated service config form provided by `google.api.Service`). This is\nmeant to encode service config as manipulated directly by customers,\nrather than the config form resulting from toolchain generation and\nnormalization.",
+      "type": "object",
+      "properties": {
+        "id": {
+          "description": "A unique ID for a specific instance of this message, typically assigned\nby the client for tracking purpose. If empty, the server may choose to\ngenerate one instead.",
+          "type": "string"
+        },
+        "options": {
+          "description": "Options to cover use of source config within ServiceManager and tools",
+          "$ref": "ConfigOptions"
+        },
+        "files": {
+          "description": "Set of source configuration files that are used to generate a service\nconfig (`google.api.Service`).",
+          "type": "array",
+          "items": {
+            "$ref": "ConfigFile"
+          }
+        },
+        "openApiSpec": {
+          "description": "OpenAPI specification",
+          "$ref": "OpenApiSpec"
+        },
+        "protoSpec": {
+          "description": "Protocol buffer API specification",
+          "$ref": "ProtoSpec"
+        }
+      }
+    },
+    "ConfigOptions": {
+      "id": "ConfigOptions",
+      "description": "A set of options to cover use of source config within `ServiceManager`\nand related tools.",
+      "type": "object",
+      "properties": {
+      }
+    },
+    "ConfigFile": {
+      "id": "ConfigFile",
+      "description": "Generic specification of a source configuration file",
+      "type": "object",
+      "properties": {
+        "filePath": {
+          "description": "The file name of the configuration file (full or relative path).",
+          "type": "string"
+        },
+        "contents": {
+          "description": "DEPRECATED. The contents of the configuration file. Use file_contents\nmoving forward.",
+          "type": "string"
+        },
+        "fileContents": {
+          "description": "The bytes that constitute the file.",
+          "type": "string",
+          "format": "byte"
+        },
+        "fileType": {
+          "description": "The kind of configuration file represented. This is used to determine\nthe method for generating `google.api.Service` using this file.",
+          "enumDescriptions": [
+            "Unknown file type.",
+            "YAML-specification of service.",
+            "OpenAPI specification, serialized in JSON.",
+            "OpenAPI specification, serialized in YAML.",
+            "FileDescriptorSet, generated by protoc.\n\nTo generate, use protoc with imports and source info included.\nFor an example test.proto file, the following command would put the value\nin a new file named out.pb.\n\n$protoc --include_imports --include_source_info test.proto -o out.pb"
+          ],
+          "type": "string",
+          "enum": [
+            "FILE_TYPE_UNSPECIFIED",
+            "SERVICE_CONFIG_YAML",
+            "OPEN_API_JSON",
+            "OPEN_API_YAML",
+            "FILE_DESCRIPTOR_SET_PROTO"
+          ]
+        }
+      }
+    },
+    "OpenApiSpec": {
+      "id": "OpenApiSpec",
+      "description": "A collection of OpenAPI specification files.",
+      "type": "object",
+      "properties": {
+        "openApiFiles": {
+          "description": "Individual files.",
+          "type": "array",
+          "items": {
+            "$ref": "ConfigFile"
+          }
+        }
+      }
+    },
+    "ProtoSpec": {
+      "id": "ProtoSpec",
+      "description": "A collection of protocol buffer service specification files.",
+      "type": "object",
+      "properties": {
+        "protoDescriptor": {
+          "description": "A complete descriptor of a protocol buffer specification",
+          "$ref": "ProtoDescriptor"
+        }
+      }
+    },
+    "ProtoDescriptor": {
+      "id": "ProtoDescriptor",
+      "description": "Contains a serialized protoc-generated protocol buffer message descriptor set\nalong with a URL that describes the type of the descriptor message.",
+      "type": "object",
+      "properties": {
+        "typeUrl": {
+          "description": "A URL\/resource name whose content describes the type of the\nserialized protocol buffer message.\n\nOnly 'type.googleapis.com\/google.protobuf.FileDescriptorSet' is supported.\nIf the type_url is not specificed,\n'type.googleapis.com\/google.protobuf.FileDescriptorSet' will be assumed.\n",
+          "type": "string"
+        },
+        "value": {
+          "description": "Must be a valid serialized protocol buffer descriptor set.\n\nTo generate, use protoc with imports and source info included.\nFor an example test.proto file, the following command would put the value\nin a new file named descriptor.pb.\n\n$protoc --include_imports --include_source_info test.proto -o descriptor.pb",
+          "type": "string",
+          "format": "byte"
+        }
+      }
+    },
+    "Operation": {
+      "id": "Operation",
+      "description": "This resource represents a long-running operation that is the result of a\nnetwork API call.",
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The server-assigned name, which is only unique within the same service that\noriginally returns it. If you use the default HTTP mapping, the\n`name` should have the format of `operations\/some\/unique\/name`.",
+          "type": "string"
+        },
+        "metadata": {
+          "description": "Service-specific metadata associated with the operation.  It typically\ncontains progress information and common metadata such as create time.\nSome services might not provide such metadata.  Any method that returns a\nlong-running operation should document the metadata type, if any.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "any",
+            "description": "Properties of the object. Contains field @type with type URL."
+          }
+        },
+        "done": {
+          "description": "If the value is `false`, it means the operation is still in progress.\nIf true, the operation is completed, and either `error` or `response` is\navailable.",
+          "type": "boolean"
+        },
+        "error": {
+          "description": "The error result of the operation in case of failure.",
+          "$ref": "Status"
+        },
+        "response": {
+          "description": "The normal response of the operation in case of success.  If the original\nmethod returns no data on success, such as `Delete`, the response is\n`google.protobuf.Empty`.  If the original method is standard\n`Get`\/`Create`\/`Update`, the response should be the resource.  For other\nmethods, the response should have the type `XxxResponse`, where `Xxx`\nis the original method name.  For example, if the original method name\nis `TakeSnapshot()`, the inferred response type is\n`TakeSnapshotResponse`.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "any",
+            "description": "Properties of the object. Contains field @type with type URL."
+          }
+        }
+      }
+    },
+    "Status": {
+      "id": "Status",
+      "description": "The `Status` type defines a logical error model that is suitable for different\nprogramming environments, including REST APIs and RPC APIs. It is used by\n[gRPC](https:\/\/github.com\/grpc). The error model is designed to be:\n\n- Simple to use and understand for most users\n- Flexible enough to meet unexpected needs\n\n# Overview\n\nThe `Status` message contains three pieces of data: error code, error message,\nand error details. The error code should be an enum value of\ngoogle.rpc.Code, but it may accept additional error codes if needed.  The\nerror message should be a developer-facing English message that helps\ndevelopers *understand* and *resolve* the error. If a localized user-facing\nerror message is needed, put the localized message in the error details or\nlocalize it in the client. The optional error details may contain arbitrary\ninformation about the error. There is a predefined set of error detail types\nin the package `google.rpc` which can be used for common error conditions.\n\n# Language mapping\n\nThe `Status` message is the logical representation of the error model, but it\nis not necessarily the actual wire format. When the `Status` message is\nexposed in different client libraries and different wire protocols, it can be\nmapped differently. For example, it will likely be mapped to some exceptions\nin Java, but more likely mapped to some error codes in C.\n\n# Other uses\n\nThe error model and the `Status` message can be used in a variety of\nenvironments, either with or without APIs, to provide a\nconsistent developer experience across different environments.\n\nExample uses of this error model include:\n\n- Partial errors. If a service needs to return partial errors to the client,\n    it may embed the `Status` in the normal response to indicate the partial\n    errors.\n\n- Workflow errors. A typical workflow has multiple steps. Each step may\n    have a `Status` message for error reporting purpose.\n\n- Batch operations. If a client uses batch request and batch response, the\n    `Status` message should be used directly inside batch response, one for\n    each error sub-response.\n\n- Asynchronous operations. If an API call embeds asynchronous operation\n    results in its response, the status of those operations should be\n    represented directly using the `Status` message.\n\n- Logging. If some API errors are stored in logs, the message `Status` could\n    be used directly after any stripping needed for security\/privacy reasons.",
+      "type": "object",
+      "properties": {
+        "code": {
+          "description": "The status code, which should be an enum value of google.rpc.Code.",
+          "type": "integer",
+          "format": "int32"
+        },
+        "message": {
+          "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\ngoogle.rpc.Status.details field, or localized by the client.",
+          "type": "string"
+        },
+        "details": {
+          "description": "A list of messages that carry the error details.  There will be a\ncommon set of message types for APIs to use.",
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "any",
+              "description": "Properties of the object. Contains field @type with type URL."
+            }
+          }
+        }
+      }
+    },
+    "ProjectSettings": {
+      "id": "ProjectSettings",
+      "description": "Settings that control how a consumer project uses a service.",
+      "type": "object",
+      "properties": {
+        "serviceName": {
+          "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.",
+          "type": "string"
+        },
+        "consumerProjectId": {
+          "description": "ID for the project consuming this service.",
+          "type": "string"
+        },
+        "usageSettings": {
+          "description": "Settings that control whether this service is usable by the consumer\nproject.",
+          "$ref": "UsageSettings"
+        },
+        "quotaSettings": {
+          "description": "Settings that control how much or how fast the service can be used by the\nconsumer project.",
+          "$ref": "QuotaSettings"
+        },
+        "visibilitySettings": {
+          "description": "Settings that control which features of the service are visible to the\nconsumer project.",
+          "$ref": "VisibilitySettings"
+        },
+        "properties": {
+          "description": "Service-defined per-consumer properties.\n\nA key-value mapping a string key to a google.protobuf.ListValue proto.\nValues in the list are typed as defined in the Service configuration's\nconsumer.properties field.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "array",
+            "items": {
+              "type": "any"
+            }
+          }
+        },
+        "operations": {
+          "description": "Read-only view of pending operations affecting this resource, if requested.",
+          "type": "array",
+          "items": {
+            "$ref": "Operation"
+          }
+        }
+      }
+    },
+    "UsageSettings": {
+      "id": "UsageSettings",
+      "description": "Usage settings for a consumer of a service.",
+      "type": "object",
+      "properties": {
+        "consumerEnableStatus": {
+          "description": "Consumer controlled setting to enable\/disable use of this service by the\nconsumer project. The default value of this is controlled by the service\nconfiguration.",
+          "enumDescriptions": [
+            "The service is disabled.",
+            "The service is enabled."
+          ],
+          "type": "string",
+          "enum": [
+            "DISABLED",
+            "ENABLED"
+          ]
+        }
+      }
+    },
+    "QuotaSettings": {
+      "id": "QuotaSettings",
+      "description": "Per-consumer overrides for quota settings. See google\/api\/quota.proto\nfor the corresponding service configuration which provides the default\nvalues.",
+      "type": "object",
+      "properties": {
+        "consumerOverrides": {
+          "description": "Quota overrides set by the consumer. Consumer overrides will only have\nan effect up to the max_limit specified in the service config, or the\nthe producer override, if one exists.\n\nThe key for this map is one of the following:\n\n- '\/' for quotas defined within quota groups,\nwhere GROUP_NAME is the google.api.QuotaGroup.name field and\nLIMIT_NAME is the google.api.QuotaLimit.name field from the service\nconfig.  For example: 'ReadGroup\/ProjectDaily'.\n\n- '' for quotas defined without quota groups, where LIMIT_NAME\nis the google.api.QuotaLimit.name field from the service config. For\nexample: 'borrowedCountPerOrganization'.",
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "QuotaLimitOverride"
+          }
+        },
+        "producerOverrides": {
+          "description": "Quota overrides set by the producer. Note that if a consumer override is\nalso specified, then the minimum of the two will be used. This allows\nconsumers to cap their usage voluntarily.\n\nThe key for this map is one of the following:\n\n- '\/' for quotas defined within quota groups,\nwhere GROUP_NAME is the google.api.QuotaGroup.name field and\nLIMIT_NAME is the google.api.QuotaLimit.name field from the service\nconfig.  For example: 'ReadGroup\/ProjectDaily'.\n\n- '' for quotas defined without quota groups, where LIMIT_NAME\nis the google.api.QuotaLimit.name field from the service config. For\nexample: 'borrowedCountPerOrganization'.",
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "QuotaLimitOverride"
+          }
+        },
+        "effectiveQuotas": {
+          "description": "The effective quota limits for each group, derived from the service\ndefaults together with any producer or consumer overrides.\nFor each limit, the effective value is the minimum of the producer\nand consumer overrides if either is present, or else the service default\nif neither is present.\nDEPRECATED. Use effective_quota_groups instead.",
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "QuotaLimitOverride"
+          }
+        },
+        "variableTermQuotas": {
+          "description": "Quotas that are active over a specified time period. Only writeable\nby the producer.",
+          "type": "array",
+          "items": {
+            "$ref": "VariableTermQuota"
+          }
+        },
+        "effectiveQuotaGroups": {
+          "description": "Use this field for quota limits defined under quota groups.\nCombines service quota configuration and project-specific settings, as\na map from quota group name to the effective quota information for that\ngroup.\nOutput-only.",
+          "type": "array",
+          "items": {
+            "$ref": "EffectiveQuotaGroup"
+          }
+        }
+      }
+    },
+    "QuotaLimitOverride": {
+      "id": "QuotaLimitOverride",
+      "description": "Specifies a custom quota limit that is applied for this consumer project.\nThis overrides the default value in google.api.QuotaLimit.",
+      "type": "object",
+      "properties": {
+        "limit": {
+          "description": "The new limit for this project.\nMay be -1 (unlimited), 0 (block), or any positive integer.",
+          "type": "string",
+          "format": "int64"
+        },
+        "unlimited": {
+          "description": "Indicates the override is to provide unlimited quota.  If true,\nany value set for limit will be ignored.\nDEPRECATED. Use a limit value of -1 instead.",
+          "type": "boolean"
+        }
+      }
+    },
+    "VariableTermQuota": {
+      "id": "VariableTermQuota",
+      "description": "A variable term quota is a bucket of tokens that is consumed over a\nspecified (usually long) time period. When present, it overrides any\n\"1d\" duration per-project quota specified on the group.\n\nVariable terms run from midnight to midnight, start_date to end_date\n(inclusive) in the America\/Los_Angeles time zone.",
+      "type": "object",
+      "properties": {
+        "groupName": {
+          "description": "The quota group that has the variable term quota applied to it.\nThis must be a google.api.QuotaGroup.name specified in the\nservice configuration.",
+          "type": "string"
+        },
+        "startDate": {
+          "description": "The beginning of the active period for the variable term quota.\nYYYYMMdd date format, e.g. 20140730.",
+          "type": "string"
+        },
+        "endDate": {
+          "description": "The effective end of the active period for the variable term quota\n(inclusive). This must be no more than 5 years after start_date.\nYYYYMMdd date format, e.g. 20140730.",
+          "type": "string"
+        },
+        "displayEndDate": {
+          "description": "The displayed end of the active period for the variable term quota.\nThis may be before the effective end to give the user a grace period.\nYYYYMMdd date format, e.g. 20140730.",
+          "type": "string"
+        },
+        "createTime": {
+          "description": "Time when this variable term quota was created. If multiple quotas\nare simultaneously active, then the quota with the latest create_time\nis the effective one.",
+          "type": "string",
+          "format": "google-datetime"
+        },
+        "limit": {
+          "description": "The number of tokens available during the configured term.",
+          "type": "string",
+          "format": "int64"
+        },
+        "quotaUsage": {
+          "description": "The usage data of this quota.",
+          "$ref": "QuotaUsage"
+        }
+      }
+    },
+    "QuotaUsage": {
+      "id": "QuotaUsage",
+      "description": "Specifies the used quota amount for a quota limit at a particular time.",
+      "type": "object",
+      "properties": {
+        "usage": {
+          "description": "The used quota value at the \"query_time\".",
+          "type": "string",
+          "format": "int64"
+        },
+        "startTime": {
+          "description": "The time the quota duration started.",
+          "type": "string",
+          "format": "google-datetime"
+        },
+        "endTime": {
+          "description": "The time the quota duration ended.",
+          "type": "string",
+          "format": "google-datetime"
+        },
+        "queryTime": {
+          "description": "The time the quota usage data was queried.",
+          "type": "string",
+          "format": "google-datetime"
+        }
+      }
+    },
+    "EffectiveQuotaGroup": {
+      "id": "EffectiveQuotaGroup",
+      "description": "An effective quota group contains both the metadata for a quota group\nas derived from the service config, and the effective limits in that\ngroup as calculated from producer and consumer overrides together with\nservice defaults.",
+      "type": "object",
+      "properties": {
+        "baseGroup": {
+          "description": "The service configuration for this quota group, minus the quota limits,\nwhich are replaced by the effective limits below.",
+          "$ref": "QuotaGroup"
+        },
+        "billingInteraction": {
+
+          "enumDescriptions": [
+            "The interaction between this quota group and the project billing status\nis unspecified.",
+            "This quota group is enforced only when the consumer project\nis not billable.",
+            "This quota group is enforced only when the consumer project\nis billable.",
+            "This quota group is enforced regardless of the consumer project's\nbilling status."
+          ],
+          "type": "string",
+          "enum": [
+            "BILLING_INTERACTION_UNSPECIFIED",
+            "NONBILLABLE_ONLY",
+            "BILLABLE_ONLY",
+            "ANY_BILLING_STATUS"
+          ]
+        },
+        "quotas": {
+          "description": "The usage and limit information for each limit within this quota group.",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaInfo"
+          }
+        }
+      }
+    },
+    "QuotaInfo": {
+      "id": "QuotaInfo",
+      "description": "Metadata about an individual quota, containing usage and limit information.",
+      "type": "object",
+      "properties": {
+        "limit": {
+          "description": "The effective limit for this quota.",
+          "$ref": "EffectiveQuotaLimit"
+        },
+        "currentUsage": {
+          "description": "The usage data for this quota as it applies to the current limit.",
+          "$ref": "QuotaUsage"
+        },
+        "historicalUsage": {
+          "description": "The historical usage data of this quota limit. Currently it is only\navailable for daily quota limit, that is, base_limit.duration = \"1d\".",
+          "type": "array",
+          "items": {
+            "$ref": "QuotaUsage"
+          }
+        }
+      }
+    },
+    "EffectiveQuotaLimit": {
+      "id": "EffectiveQuotaLimit",
+      "description": "An effective quota limit contains the metadata for a quota limit\nas derived from the service config, together with fields that describe\nthe effective limit value and what overrides can be applied to it.",
+      "type": "object",
+      "properties": {
+        "baseLimit": {
+          "description": "The service's configuration for this quota limit.",
+          "$ref": "QuotaLimit"
+        },
+        "key": {
+          "description": "The key used to identify this limit when applying overrides.\nThe consumer_overrides and producer_overrides maps are keyed\nby strings of the form \"QuotaGroupName\/QuotaLimitName\".",
+          "type": "string"
+        },
+        "maxConsumerOverrideAllowed": {
+          "description": "The maximum override value that a consumer may specify.",
+          "type": "string",
+          "format": "int64"
+        },
+        "effectiveLimit": {
+          "description": "The effective limit value, based on the stored producer and consumer\noverrides and the service defaults.",
+          "type": "string",
+          "format": "int64"
+        }
+      }
+    },
+    "VisibilitySettings": {
+      "id": "VisibilitySettings",
+      "description": "Settings that control which features of the service are visible to the\nconsumer project.",
+      "type": "object",
+      "properties": {
+        "visibilityLabels": {
+          "description": "The set of visibility labels that are used to determine what API surface is\nvisible to calls made by this project. The visible surface is a union of\nthe surface features associated with each label listed here, plus the\npublicly visible (unrestricted) surface.\n\nThe service producer may add or remove labels at any time. The service\nconsumer may add a label if the calling user has been granted permission\nto do so by the producer.  The service consumer may also remove any label\nat any time.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "ListServiceConfigsResponse": {
+      "id": "ListServiceConfigsResponse",
+      "description": "Response message for ListServiceConfigs method.",
+      "type": "object",
+      "properties": {
+        "serviceConfigs": {
+          "description": "The list of service config resources.",
+          "type": "array",
+          "items": {
+            "$ref": "Service"
+          }
+        },
+        "nextPageToken": {
+          "description": "The token of the next page of results.",
+          "type": "string"
+        }
+      }
+    },
+    "SubmitConfigSourceRequest": {
+      "id": "SubmitConfigSourceRequest",
+      "description": "Request message for SubmitConfigSource method.",
+      "type": "object",
+      "properties": {
+        "configSource": {
+          "description": "The source configuration for the service.",
+          "$ref": "ConfigSource"
+        },
+        "validateOnly": {
+          "description": "Optional. If set, this will result in the generation of a\n`google.api.Service` configuration based on the `ConfigSource` provided,\nbut the generated config and the sources will NOT be persisted.",
+          "type": "boolean"
+        }
+      }
+    },
+    "ConvertConfigRequest": {
+      "id": "ConvertConfigRequest",
+      "description": "Request message for `ConvertConfig` method.",
+      "type": "object",
+      "properties": {
+        "swaggerSpec": {
+          "description": "The swagger specification for an API.",
+          "$ref": "SwaggerSpec"
+        },
+        "openApiSpec": {
+          "description": "The OpenAPI specification for an API.",
+          "$ref": "OpenApiSpec"
+        },
+        "serviceName": {
+          "description": "The service name to use for constructing the normalized service\nconfiguration equivalent of the provided configuration specification.",
+          "type": "string"
+        },
+        "configSpec": {
+          "description": "Input configuration\nFor this version of API, the supported type is OpenApiSpec",
+          "type": "object",
+          "additionalProperties": {
+            "type": "any",
+            "description": "Properties of the object. Contains field @type with type URL."
+          }
+        }
+      }
+    },
+    "SwaggerSpec": {
+      "id": "SwaggerSpec",
+      "description": "A collection of swagger specification files.",
+      "type": "object",
+      "properties": {
+        "swaggerFiles": {
+          "description": "The individual files.",
+          "type": "array",
+          "items": {
+            "$ref": "File"
+          }
+        }
+      }
+    },
+    "File": {
+      "id": "File",
+      "description": "A single swagger specification file.",
+      "type": "object",
+      "properties": {
+        "path": {
+          "description": "The relative path of the swagger spec file.",
+          "type": "string"
+        },
+        "contents": {
+          "description": "The contents of the swagger spec file.",
+          "type": "string"
+        }
+      }
+    },
+    "ConvertConfigResponse": {
+      "id": "ConvertConfigResponse",
+      "description": "Response message for `ConvertConfig` method.",
+      "type": "object",
+      "properties": {
+        "serviceConfig": {
+          "description": "The service configuration. Not set if errors occured during conversion.",
+          "$ref": "Service"
+        },
+        "diagnostics": {
+          "description": "Any errors or warnings that occured during config conversion.",
+          "type": "array",
+          "items": {
+            "$ref": "Diagnostic"
+          }
+        }
+      }
+    },
+    "Diagnostic": {
+      "id": "Diagnostic",
+      "description": "A collection that represents a diagnostic message (error or warning)",
+      "type": "object",
+      "properties": {
+        "location": {
+          "description": "Location of the cause or context of the diagnostic information.",
+          "type": "string"
+        },
+        "kind": {
+          "description": "The kind of diagnostic information provided.",
+          "enumDescriptions": [
+            "Warnings and errors",
+            "Only errors"
+          ],
+          "type": "string",
+          "enum": [
+            "WARNING",
+            "ERROR"
+          ]
+        },
+        "message": {
+          "description": "The string message of the diagnostic information.",
+          "type": "string"
+        }
+      }
+    },
+    "EnableServiceRequest": {
+      "id": "EnableServiceRequest",
+      "description": "Request message for EnableService method.",
+      "type": "object",
+      "properties": {
+        "consumerId": {
+          "description": "The identity of consumer resource which service enablement will be\napplied to.\n\nThe Google Service Management implementation accepts the following\nforms: \"project:\", \"project_number:\".\n\nNote: this is made compatible with\ngoogle.api.servicecontrol.v1.Operation.consumer_id.",
+          "type": "string"
+        }
+      }
+    },
+    "DisableServiceRequest": {
+      "id": "DisableServiceRequest",
+      "description": "Request message for DisableService method.",
+      "type": "object",
+      "properties": {
+        "consumerId": {
+          "description": "The identity of consumer resource which service disablement will be\napplied to.\n\nThe Google Service Management implementation accepts the following\nforms: \"project:\", \"project_number:\".\n\nNote: this is made compatible with\ngoogle.api.servicecontrol.v1.Operation.consumer_id.",
+          "type": "string"
+        }
+      }
+    },
+    "ServiceAccessPolicy": {
+      "id": "ServiceAccessPolicy",
+      "description": "Policy describing who can access a service and any visibility labels on that\nservice.",
+      "type": "object",
+      "properties": {
+        "serviceName": {
+          "description": "The service protected by this policy.",
+          "type": "string"
+        },
+        "accessList": {
+          "description": "ACL for access to the unrestricted surface of the service.",
+          "$ref": "ServiceAccessList"
+        },
+        "visibilityLabelAccessLists": {
+          "description": "ACLs for access to restricted parts of the service.  The map key is the\nvisibility label that is being controlled.  Note that access to any label\nalso implies access to the unrestricted surface.",
+          "type": "object",
+          "additionalProperties": {
+            "$ref": "ServiceAccessList"
+          }
+        }
+      }
+    },
+    "ServiceAccessList": {
+      "id": "ServiceAccessList",
+      "description": "List of users and groups that are granted access to a service or visibility\nlabel.",
+      "type": "object",
+      "properties": {
+        "members": {
+          "description": "Members that are granted access.\n\n- \"user:{$user_email}\" - Grant access to an individual user\n- \"group:{$group_email}\" - Grant access to direct members of the group\n- \"domain:{$domain}\" - Grant access to all members of the domain. For now,\n     domain membership check will be similar to Devconsole\/TT check:\n     compare domain part of the user email to configured domain name.\n     When IAM integration is complete, this will be replaced with IAM\n     check.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "QueryUserAccessResponse": {
+      "id": "QueryUserAccessResponse",
+      "description": "Request message for QueryUserAccess method.",
+      "type": "object",
+      "properties": {
+        "canAccessService": {
+          "description": "True if the user can access the service and any unrestricted API surface.",
+          "type": "boolean"
+        },
+        "accessibleVisibilityLabels": {
+          "description": "Any visibility labels on the service that are accessible by the user.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "CustomerSettings": {
+      "id": "CustomerSettings",
+      "description": "Settings that control how a customer (identified by a billing account) uses\na service",
+      "type": "object",
+      "properties": {
+        "serviceName": {
+          "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.",
+          "type": "string"
+        },
+        "customerId": {
+          "description": "ID for the customer that consumes the service (see above).\nThe supported types of customers are:\n\n1. domain:{domain}\nA Google Apps domain name. For example, google.com.\n\n2. billingAccount:{billing_account_id}\nA Google Cloud Plafrom billing account. For Example, 123456-7890ab-cdef12.\n",
+          "type": "string"
+        },
+        "quotaSettings": {
+          "description": "Settings that control how much or how fast the service can be used by the\nconsumer projects owned by the customer collectively.",
+          "$ref": "QuotaSettings"
+        }
+      }
+    },
+    "CompositeOperationMetadata": {
+      "id": "CompositeOperationMetadata",
+      "description": "Metadata for composite operations.",
+      "type": "object",
+      "properties": {
+        "childOperations": {
+          "description": "The child operations. The details of the asynchronous\nchild operations are stored in a separate row and not in this\nmetadata. Only the operation name is stored here.",
+          "type": "array",
+          "items": {
+            "$ref": "Operation"
+          }
+        },
+        "originalRequest": {
+          "description": "Original request that triggered this operation.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "any",
+            "description": "Properties of the object. Contains field @type with type URL."
+          }
+        },
+        "responseFieldMasks": {
+          "description": "Defines which part of the response a child operation will contribute.\nEach key of the map is the name of a child operation. Each value is a\nfield mask that identifies what that child operation contributes to the\nresponse, for example, \"quota_settings\", \"visiblity_settings\", etc.",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string",
+            "format": "google-fieldmask"
+          }
+        },
+        "persisted": {
+          "description": "Indicates whether the requested state change has been persisted. Once this\nfield is set, it is guaranteed to propagate to all backends eventually, but\nit may not be visible immediately. Clients that are not concerned with\nwaiting on propagation can stop polling the operation once the persisted\nfield is set",
+          "type": "boolean"
+        }
+      }
+    },
+    "OperationMetadata": {
+      "id": "OperationMetadata",
+      "description": "The metadata associated with a long running operation resource.",
+      "type": "object",
+      "properties": {
+        "resourceNames": {
+          "description": "The full name of the resources that this operation is directly\nassociated with.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "steps": {
+          "description": "Detailed status information for each step. The order is undetermined.",
+          "type": "array",
+          "items": {
+            "$ref": "Step"
+          }
+        },
+        "progressPercentage": {
+          "description": "Percentage of completion of this operation, ranging from 0 to 100.",
+          "type": "integer",
+          "format": "int32"
+        },
+        "startTime": {
+          "description": "The start time of the operation.",
+          "type": "string",
+          "format": "google-datetime"
+        }
+      }
+    },
+    "Step": {
+      "id": "Step",
+      "description": "Represents the status of one operation step.",
+      "type": "object",
+      "properties": {
+        "description": {
+          "description": "The short description of the step.",
+          "type": "string"
+        },
+        "status": {
+          "description": "The status code.",
+          "enumDescriptions": [
+            "Unspecifed code.",
+            "The step has completed without errors.",
+            "The step has not started yet.",
+            "The step is in progress.",
+            "The step has completed with errors."
+          ],
+          "type": "string",
+          "enum": [
+            "STATUS_UNSPECIFIED",
+            "DONE",
+            "NOT_STARTED",
+            "IN_PROGRESS",
+            "FAILED"
+          ]
+        }
+      }
+    }
+  },
+  "resources": {
+    "services": {
+      "methods": {
+        "list": {
+          "id": "servicemanagement.services.list",
+          "path": "v1/services",
+          "flatPath": "v1/services",
+          "httpMethod": "GET",
+          "description": "Lists all managed services. If the `consumer_project_id` is specified,\nthe project's settings for the specified service are also returned.",
+          "parameters": {
+            "producerProjectId": {
+              "description": "Include services produced by the specified project.",
+              "location": "query",
+              "type": "string"
+            },
+            "category": {
+              "description": "Include services only in the specified category. Supported categories are\nservicemanagement.googleapis.com\/categories\/google-services or\nservicemanagement.googleapis.com\/categories\/play-games.",
+              "location": "query",
+              "type": "string"
+            },
+            "consumerProjectId": {
+              "description": "Include services consumed by the specified project.\n\nIf project_settings is expanded, then this field controls which project\nproject_settings is populated for.",
+              "location": "query",
+              "type": "string"
+            },
+            "expand": {
+              "description": "Fields to expand in any results.  By default, the following fields\nare not fully included in list results:\n- `operations`\n- `project_settings`\n- `project_settings.operations`\n- `quota_usage` (It requires `project_settings`)",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            },
+            "pageSize": {
+              "description": "Requested size of the next page of data.",
+              "location": "query",
+              "type": "integer",
+              "format": "int32"
+            },
+            "pageToken": {
+              "description": "Token identifying which result to start with; returned by a previous list\ncall.",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+          ],
+          "response": {
+            "$ref": "ListServicesResponse"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "get": {
+          "id": "servicemanagement.services.get",
+          "path": "v1/services/{serviceName}",
+          "flatPath": "v1/services/{serviceName}",
+          "httpMethod": "GET",
+          "description": "Gets a managed service. If the `consumer_project_id` is specified,\nthe project's settings for the specified service are also returned.",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "expand": {
+              "description": "Fields to expand in any results.  By default, the following fields\nare not present in the result:\n- `operations`\n- `project_settings`\n- `project_settings.operations`\n- `quota_usage` (It requires `project_settings`)\n- `historical_quota_usage` (It requires `project_settings`)",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            },
+            "consumerProjectId": {
+              "description": "If project_settings is expanded, return settings for the specified\nconsumer project.",
+              "location": "query",
+              "type": "string"
+            },
+            "view": {
+              "description": "If project_settings is expanded, request only fields for the specified\nview.",
+              "location": "query",
+              "type": "string",
+              "enum": [
+                "PROJECT_SETTINGS_VIEW_UNSPECIFIED",
+                "CONSUMER_VIEW",
+                "PRODUCER_VIEW",
+                "ALL"
+              ]
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "response": {
+            "$ref": "ManagedService"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "create": {
+          "id": "servicemanagement.services.create",
+          "path": "v1/services",
+          "flatPath": "v1/services",
+          "httpMethod": "POST",
+          "description": "Creates a new managed service.\n\nOperation",
+          "parameters": {
+          },
+          "parameterOrder": [
+          ],
+          "request": {
+            "$ref": "ManagedService"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "delete": {
+          "id": "servicemanagement.services.delete",
+          "path": "v1/services/{serviceName}",
+          "flatPath": "v1/services/{serviceName}",
+          "httpMethod": "DELETE",
+          "description": "Deletes a managed service.\n\nOperation",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "getConfig": {
+          "id": "servicemanagement.services.getConfig",
+          "path": "v1/services/{serviceName}/config",
+          "flatPath": "v1/services/{serviceName}/config",
+          "httpMethod": "GET",
+          "description": "Gets a service config (version) for a managed service. If `config_id` is\nnot specified, the latest service config will be returned.",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "configId": {
+              "description": "The id of the service config resource.\nOptional. If it is not specified, the latest version of config will be\nreturned.",
+              "location": "query",
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "response": {
+            "$ref": "Service"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "convertConfig": {
+          "id": "servicemanagement.services.convertConfig",
+          "path": "v1/services:convertConfig",
+          "flatPath": "v1/services:convertConfig",
+          "httpMethod": "POST",
+          "description": "DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide\nconfig conversion moving forward.\n\nConverts an API specification (e.g. Swagger spec) to an\nequivalent `google.api.Service`.",
+          "parameters": {
+          },
+          "parameterOrder": [
+          ],
+          "request": {
+            "$ref": "ConvertConfigRequest"
+          },
+          "response": {
+            "$ref": "ConvertConfigResponse"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "enable": {
+          "id": "servicemanagement.services.enable",
+          "path": "v1/services/{serviceName}:enable",
+          "flatPath": "v1/services/{serviceName}:enable",
+          "httpMethod": "POST",
+          "description": "Enable a managed service for a project with default setting.\nIf the managed service has dependencies, they will be enabled as well.\n\nOperation\n",
+          "parameters": {
+            "serviceName": {
+              "description": "Name of the service to enable. Specifying an unknown service name will\ncause the request to fail.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "EnableServiceRequest"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "disable": {
+          "id": "servicemanagement.services.disable",
+          "path": "v1/services/{serviceName}:disable",
+          "flatPath": "v1/services/{serviceName}:disable",
+          "httpMethod": "POST",
+          "description": "Disable a managed service for a project.\nGoogle Service Management will only disable the managed service even if\nthere are other services depend on the managed service.\n\nOperation\n",
+          "parameters": {
+            "serviceName": {
+              "description": "Name of the service to disable. Specifying an unknown service name\nwill cause the request to fail.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "DisableServiceRequest"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "update": {
+          "id": "servicemanagement.services.update",
+          "path": "v1/services/{serviceName}",
+          "flatPath": "v1/services/{serviceName}",
+          "httpMethod": "PUT",
+          "description": "Updates the configuration of a service.  If the specified service does not\nalready exist, then it is created.\n\nOperation",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "updateMask": {
+              "description": "A mask specifying which fields to update.\nUpdate mask has been deprecated on UpdateService service method. Please\nuse PatchService method instead to do partial updates.",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "ManagedService"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "patch": {
+          "id": "servicemanagement.services.patch",
+          "path": "v1/services/{serviceName}",
+          "flatPath": "v1/services/{serviceName}",
+          "httpMethod": "PATCH",
+          "description": "Updates the specified subset of the configuration. If the specified service\ndoes not exists the patch operation fails.\n\nOperation",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "updateMask": {
+              "description": "A mask specifying which fields to update.",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "ManagedService"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "updateConfig": {
+          "id": "servicemanagement.services.updateConfig",
+          "path": "v1/services/{serviceName}/config",
+          "flatPath": "v1/services/{serviceName}/config",
+          "httpMethod": "PUT",
+          "description": "Updates the specified subset of the service resource. Equivalent to\ncalling `UpdateService` with only the `service_config` field updated.\n\nOperation",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "updateMask": {
+              "description": "A mask specifying which fields to update.\nUpdate mask has been deprecated on UpdateServiceConfig service method.\nPlease use PatchServiceConfig method instead to do partial updates.",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "Service"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "patchConfig": {
+          "id": "servicemanagement.services.patchConfig",
+          "path": "v1/services/{serviceName}/config",
+          "flatPath": "v1/services/{serviceName}/config",
+          "httpMethod": "PATCH",
+          "description": "Updates the specified subset of the service resource. Equivalent to\ncalling `PatchService` with only the `service_config` field updated.\n\nOperation",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            },
+            "updateMask": {
+              "description": "A mask specifying which fields to update.",
+              "location": "query",
+              "type": "string",
+              "format": "google-fieldmask"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "Service"
+          },
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "getAccessPolicy": {
+          "id": "servicemanagement.services.getAccessPolicy",
+          "path": "v1/services/{serviceName}/accessPolicy",
+          "flatPath": "v1/services/{serviceName}/accessPolicy",
+          "httpMethod": "GET",
+          "description": "Producer method to retrieve current policy.",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  For example: `example.googleapis.com`.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "response": {
+            "$ref": "ServiceAccessPolicy"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        },
+        "updateAccessPolicy": {
+          "id": "servicemanagement.services.updateAccessPolicy",
+          "path": "v1/services/{serviceName}/accessPolicy",
+          "flatPath": "v1/services/{serviceName}/accessPolicy",
+          "httpMethod": "PUT",
+          "description": "Producer method to update the current policy.  This method will return an\nerror if the policy is too large (more than 50 entries across all lists).",
+          "parameters": {
+            "serviceName": {
+              "description": "The name of the service.  For example: `example.googleapis.com`.\nIf set, policy's service_name should be same as this one.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "serviceName"
+          ],
+          "request": {
+            "$ref": "ServiceAccessPolicy"
+          },
+          "response": {
+            "$ref": "ServiceAccessPolicy"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        }
+      }
+      ,
+      "resources": {
+        "configs": {
+          "methods": {
+            "list": {
+              "id": "servicemanagement.services.configs.list",
+              "path": "v1/services/{serviceName}/configs",
+              "flatPath": "v1/services/{serviceName}/configs",
+              "httpMethod": "GET",
+              "description": "Lists the history of the service config for a managed service,\nfrom the newest to the oldest.\n",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "pageToken": {
+                  "description": "The token of the page to retrieve.",
+                  "location": "query",
+                  "type": "string"
+                },
+                "pageSize": {
+                  "description": "The max number of items to include in the response list.",
+                  "location": "query",
+                  "type": "integer",
+                  "format": "int32"
+                }
+              },
+              "parameterOrder": [
+                "serviceName"
+              ],
+              "response": {
+                "$ref": "ListServiceConfigsResponse"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "get": {
+              "id": "servicemanagement.services.configs.get",
+              "path": "v1/services/{serviceName}/configs/{configId}",
+              "flatPath": "v1/services/{serviceName}/configs/{configId}",
+              "httpMethod": "GET",
+              "description": "Gets a service config (version) for a managed service. If `config_id` is\nnot specified, the latest service config will be returned.",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "configId": {
+                  "description": "The id of the service config resource.\nOptional. If it is not specified, the latest version of config will be\nreturned.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "configId"
+              ],
+              "response": {
+                "$ref": "Service"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "create": {
+              "id": "servicemanagement.services.configs.create",
+              "path": "v1/services/{serviceName}/configs",
+              "flatPath": "v1/services/{serviceName}/configs",
+              "httpMethod": "POST",
+              "description": "Creates a new service config (version) for a managed service. This method\nonly stores the service config, but does not apply the service config to\nany backend services.\n",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                }
+              },
+              "parameterOrder": [
+                "serviceName"
+              ],
+              "request": {
+                "$ref": "Service"
+              },
+              "response": {
+                "$ref": "Service"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "submit": {
+              "id": "servicemanagement.services.configs.submit",
+              "path": "v1/services/{serviceName}/configs:submit",
+              "flatPath": "v1/services/{serviceName}/configs:submit",
+              "httpMethod": "POST",
+              "description": "Creates a new service config (version) for a managed service based on\nuser-supplied configuration sources files (for example: OpenAPI\nSpecification). This method stores the source configurations as well as the\ngenerated service config. It does NOT apply the service config to any\nbackend services.\n\nOperation\n",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                }
+              },
+              "parameterOrder": [
+                "serviceName"
+              ],
+              "request": {
+                "$ref": "SubmitConfigSourceRequest"
+              },
+              "response": {
+                "$ref": "Operation"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            }
+          }
+        },
+        "accessPolicy": {
+          "methods": {
+            "query": {
+              "id": "servicemanagement.services.accessPolicy.query",
+              "path": "v1/services/{serviceName}/accessPolicy:query",
+              "flatPath": "v1/services/{serviceName}/accessPolicy:query",
+              "httpMethod": "POST",
+              "description": "Method to query the accessibility of a service and any associated\nvisibility labels for a specified user.\n\nMembers of the producer project may call this method and specify any user.\n\nAny user may call this method, but must specify their own email address.\nIn this case the method will return NOT_FOUND if the user has no access to\nthe service.",
+              "parameters": {
+                "serviceName": {
+                  "description": "The service to query access for.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "userEmail": {
+                  "description": "The user to query access for.",
+                  "location": "query",
+                  "type": "string"
+                }
+              },
+              "parameterOrder": [
+                "serviceName"
+              ],
+              "response": {
+                "$ref": "QueryUserAccessResponse"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            }
+          }
+        },
+        "customerSettings": {
+          "methods": {
+            "get": {
+              "id": "servicemanagement.services.customerSettings.get",
+              "path": "v1/services/{serviceName}/customerSettings/{customerId}",
+              "flatPath": "v1/services/{serviceName}/customerSettings/{customerId}",
+              "httpMethod": "GET",
+              "description": "Retrieves the settings that control the specified customer's usage of the\nservice.",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`. This field is\nrequired.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "customerId": {
+                  "description": "ID for the customer. See the comment for `CustomerSettings.customer_id`\nfield of message for its format. This field is required.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "expand": {
+                  "description": "Fields to expand in any results.",
+                  "location": "query",
+                  "type": "string",
+                  "format": "google-fieldmask"
+                },
+                "view": {
+                  "description": "Request only fields for the specified view.",
+                  "location": "query",
+                  "type": "string",
+                  "enum": [
+                    "PROJECT_SETTINGS_VIEW_UNSPECIFIED",
+                    "CONSUMER_VIEW",
+                    "PRODUCER_VIEW",
+                    "ALL"
+                  ]
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "customerId"
+              ],
+              "response": {
+                "$ref": "CustomerSettings"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "patch": {
+              "id": "servicemanagement.services.customerSettings.patch",
+              "path": "v1/services/{serviceName}/customerSettings/{customerId}",
+              "flatPath": "v1/services/{serviceName}/customerSettings/{customerId}",
+              "httpMethod": "PATCH",
+              "description": "Updates specified subset of the settings that control the specified\ncustomer's usage of the service.  Attempts to update a field not\ncontrolled by the caller will result in an access denied error.\n\nOperation",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`. This field is\nrequired.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "customerId": {
+                  "description": "ID for the customer. See the comment for `CustomerSettings.customer_id`\nfield of message for its format. This field is required.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "updateMask": {
+                  "description": "The field mask specifying which fields are to be updated.",
+                  "location": "query",
+                  "type": "string",
+                  "format": "google-fieldmask"
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "customerId"
+              ],
+              "request": {
+                "$ref": "CustomerSettings"
+              },
+              "response": {
+                "$ref": "Operation"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            }
+          }
+        },
+        "projectSettings": {
+          "methods": {
+            "get": {
+              "id": "servicemanagement.services.projectSettings.get",
+              "path": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "flatPath": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "httpMethod": "GET",
+              "description": "Retrieves the settings that control the specified consumer project's usage\nof the service.",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "consumerProjectId": {
+                  "description": "The project ID of the consumer.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "expand": {
+                  "description": "Fields to expand in any results.  By default, the following fields\nare not present in the result:\n- `operations`\n- `quota_usage`",
+                  "location": "query",
+                  "type": "string",
+                  "format": "google-fieldmask"
+                },
+                "view": {
+                  "description": "Request only the fields for the specified view.",
+                  "location": "query",
+                  "type": "string",
+                  "enum": [
+                    "PROJECT_SETTINGS_VIEW_UNSPECIFIED",
+                    "CONSUMER_VIEW",
+                    "PRODUCER_VIEW",
+                    "ALL"
+                  ]
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "consumerProjectId"
+              ],
+              "response": {
+                "$ref": "ProjectSettings"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "update": {
+              "id": "servicemanagement.services.projectSettings.update",
+              "path": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "flatPath": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "httpMethod": "PUT",
+              "description": "NOTE: Currently unsupported.  Use PatchProjectSettings instead.\n\nUpdates the settings that control the specified consumer project's usage\nof the service.  Attempts to update a field not controlled by the caller\nwill result in an access denied error.\n\nOperation",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "consumerProjectId": {
+                  "description": "The project ID of the consumer.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "consumerProjectId"
+              ],
+              "request": {
+                "$ref": "ProjectSettings"
+              },
+              "response": {
+                "$ref": "Operation"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            },
+            "patch": {
+              "id": "servicemanagement.services.projectSettings.patch",
+              "path": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "flatPath": "v1/services/{serviceName}/projectSettings/{consumerProjectId}",
+              "httpMethod": "PATCH",
+              "description": "Updates specified subset of the settings that control the specified\nconsumer project's usage of the service.  Attempts to update a field not\ncontrolled by the caller will result in an access denied error.\n\nOperation",
+              "parameters": {
+                "serviceName": {
+                  "description": "The name of the service.  See the `ServiceManager` overview for naming\nrequirements.  For example: `example.googleapis.com`.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "consumerProjectId": {
+                  "description": "The project ID of the consumer.",
+                  "location": "path",
+                  "required": true,
+                  "type": "string"
+                },
+                "updateMask": {
+                  "description": "The field mask specifying which fields are to be updated.",
+                  "location": "query",
+                  "type": "string",
+                  "format": "google-fieldmask"
+                }
+              },
+              "parameterOrder": [
+                "serviceName",
+                "consumerProjectId"
+              ],
+              "request": {
+                "$ref": "ProjectSettings"
+              },
+              "response": {
+                "$ref": "Operation"
+              },
+              "scopes": [
+                "https://www.googleapis.com/auth/cloud-platform",
+                "https://www.googleapis.com/auth/service.management"
+              ]
+            }
+          }
+        }
+      }
+    },
+    "v1": {
+      "methods": {
+        "convertConfig": {
+          "id": "servicemanagement.convertConfig",
+          "path": "v1:convertConfig",
+          "flatPath": "v1:convertConfig",
+          "httpMethod": "POST",
+          "description": "DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide\nconfig conversion moving forward.\n\nConverts an API specification (e.g. Swagger spec) to an\nequivalent `google.api.Service`.",
+          "parameters": {
+          },
+          "parameterOrder": [
+          ],
+          "request": {
+            "$ref": "ConvertConfigRequest"
+          },
+          "response": {
+            "$ref": "ConvertConfigResponse"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        }
+      }
+    },
+    "operations": {
+      "methods": {
+        "get": {
+          "id": "servicemanagement.operations.get",
+          "path": "v1/operations/{operationsId}",
+          "flatPath": "v1/operations/{operationsId}",
+          "httpMethod": "GET",
+          "description": "Gets the latest state of a long-running operation.  Clients can use this\nmethod to poll the operation result at intervals as recommended by the API\nservice.",
+          "parameters": {
+            "operationsId": {
+              "description": "Part of `name`. The name of the operation resource.",
+              "location": "path",
+              "required": true,
+              "type": "string"
+            }
+          },
+          "parameterOrder": [
+            "operationsId"
+          ],
+          "response": {
+            "$ref": "Operation"
+          },
+          "scopes": [
+            "https://www.googleapis.com/auth/cloud-platform",
+            "https://www.googleapis.com/auth/service.management"
+          ]
+        }
+      }
+    }
+  },
+  "basePath": ""
+}
diff --git a/samples/servicemanagement_sample/servicemanagement_v1/__init__.py b/samples/servicemanagement_sample/servicemanagement_v1/__init__.py
new file mode 100644
index 0000000..2816da8
--- /dev/null
+++ b/samples/servicemanagement_sample/servicemanagement_v1/__init__.py
@@ -0,0 +1,5 @@
+"""Package marker file."""
+
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)
diff --git a/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1.py b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1.py
new file mode 100644
index 0000000..d1a4ab8
--- /dev/null
+++ b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1.py
@@ -0,0 +1,1520 @@
+#!/usr/bin/env python
+"""CLI for servicemanagement, version v1."""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+import code
+import os
+import platform
+import sys
+
+from apitools.base.protorpclite import message_types
+from apitools.base.protorpclite import messages
+
+from google.apputils import appcommands
+import gflags as flags
+
+import apitools.base.py as apitools_base
+from apitools.base.py import cli as apitools_base_cli
+import servicemanagement_v1_client as client_lib
+import servicemanagement_v1_messages as messages
+
+
+def _DeclareServicemanagementFlags():
+  """Declare global flags in an idempotent way."""
+  if 'api_endpoint' in flags.FLAGS:
+    return
+  flags.DEFINE_string(
+      'api_endpoint',
+      u'https://servicemanagement.googleapis.com/',
+      'URL of the API endpoint to use.',
+      short_name='servicemanagement_url')
+  flags.DEFINE_string(
+      'history_file',
+      u'~/.servicemanagement.v1.history',
+      'File with interactive shell history.')
+  flags.DEFINE_multistring(
+      'add_header', [],
+      'Additional http headers (as key=value strings). '
+      'Can be specified multiple times.')
+  flags.DEFINE_string(
+      'service_account_json_keyfile', '',
+      'Filename for a JSON service account key downloaded'
+      ' from the Developer Console.')
+  flags.DEFINE_enum(
+      'f__xgafv',
+      u'_1',
+      [u'_1', u'_2'],
+      u'V1 error format.')
+  flags.DEFINE_string(
+      'access_token',
+      None,
+      u'OAuth access token.')
+  flags.DEFINE_enum(
+      'alt',
+      u'json',
+      [u'json', u'media', u'proto'],
+      u'Data format for response.')
+  flags.DEFINE_string(
+      'bearer_token',
+      None,
+      u'OAuth bearer token.')
+  flags.DEFINE_string(
+      'callback',
+      None,
+      u'JSONP')
+  flags.DEFINE_string(
+      'fields',
+      None,
+      u'Selector specifying which fields to include in a partial response.')
+  flags.DEFINE_string(
+      'key',
+      None,
+      u'API key. Your API key identifies your project and provides you with '
+      u'API access, quota, and reports. Required unless you provide an OAuth '
+      u'2.0 token.')
+  flags.DEFINE_string(
+      'oauth_token',
+      None,
+      u'OAuth 2.0 token for the current user.')
+  flags.DEFINE_boolean(
+      'pp',
+      'True',
+      u'Pretty-print response.')
+  flags.DEFINE_boolean(
+      'prettyPrint',
+      'True',
+      u'Returns response with indentations and line breaks.')
+  flags.DEFINE_string(
+      'quotaUser',
+      None,
+      u'Available to use for quota purposes for server-side applications. Can'
+      u' be any arbitrary string assigned to a user, but should not exceed 40'
+      u' characters.')
+  flags.DEFINE_string(
+      'trace',
+      None,
+      'A tracing token of the form "token:" to include in api '
+      'requests.')
+  flags.DEFINE_string(
+      'uploadType',
+      None,
+      u'Legacy upload protocol for media (e.g. "media", "multipart").')
+  flags.DEFINE_string(
+      'upload_protocol',
+      None,
+      u'Upload protocol for media (e.g. "raw", "multipart").')
+
+
+FLAGS = flags.FLAGS
+apitools_base_cli.DeclareBaseFlags()
+_DeclareServicemanagementFlags()
+
+
+def GetGlobalParamsFromFlags():
+  """Return a StandardQueryParameters based on flags."""
+  result = messages.StandardQueryParameters()
+  if FLAGS['f__xgafv'].present:
+    result.f__xgafv = messages.StandardQueryParameters.FXgafvValueValuesEnum(FLAGS.f__xgafv)
+  if FLAGS['access_token'].present:
+    result.access_token = FLAGS.access_token.decode('utf8')
+  if FLAGS['alt'].present:
+    result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt)
+  if FLAGS['bearer_token'].present:
+    result.bearer_token = FLAGS.bearer_token.decode('utf8')
+  if FLAGS['callback'].present:
+    result.callback = FLAGS.callback.decode('utf8')
+  if FLAGS['fields'].present:
+    result.fields = FLAGS.fields.decode('utf8')
+  if FLAGS['key'].present:
+    result.key = FLAGS.key.decode('utf8')
+  if FLAGS['oauth_token'].present:
+    result.oauth_token = FLAGS.oauth_token.decode('utf8')
+  if FLAGS['pp'].present:
+    result.pp = FLAGS.pp
+  if FLAGS['prettyPrint'].present:
+    result.prettyPrint = FLAGS.prettyPrint
+  if FLAGS['quotaUser'].present:
+    result.quotaUser = FLAGS.quotaUser.decode('utf8')
+  if FLAGS['trace'].present:
+    result.trace = FLAGS.trace.decode('utf8')
+  if FLAGS['uploadType'].present:
+    result.uploadType = FLAGS.uploadType.decode('utf8')
+  if FLAGS['upload_protocol'].present:
+    result.upload_protocol = FLAGS.upload_protocol.decode('utf8')
+  return result
+
+
+def GetClientFromFlags():
+  """Return a client object, configured from flags."""
+  log_request = FLAGS.log_request or FLAGS.log_request_response
+  log_response = FLAGS.log_response or FLAGS.log_request_response
+  api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint)
+  additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header)
+  credentials_args = {
+      'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile)
+  }
+  try:
+    client = client_lib.ServicemanagementV1(
+        api_endpoint, log_request=log_request,
+        log_response=log_response,
+        credentials_args=credentials_args,
+        additional_http_headers=additional_http_headers)
+  except apitools_base.CredentialsError as e:
+    print 'Error creating credentials: %s' % e
+    sys.exit(1)
+  return client
+
+
+class PyShell(appcommands.Cmd):
+
+  def Run(self, _):
+    """Run an interactive python shell with the client."""
+    client = GetClientFromFlags()
+    params = GetGlobalParamsFromFlags()
+    for field in params.all_fields():
+      value = params.get_assigned_value(field.name)
+      if value != field.default:
+        client.AddGlobalParam(field.name, value)
+    banner = """
+           == servicemanagement interactive console ==
+                 client: a servicemanagement client
+          apitools_base: base apitools module
+         messages: the generated messages module
+    """
+    local_vars = {
+        'apitools_base': apitools_base,
+        'client': client,
+        'client_lib': client_lib,
+        'messages': messages,
+    }
+    if platform.system() == 'Linux':
+      console = apitools_base_cli.ConsoleWithReadline(
+          local_vars, histfile=FLAGS.history_file)
+    else:
+      console = code.InteractiveConsole(local_vars)
+    try:
+      console.interact(banner)
+    except SystemExit as e:
+      return e.code
+
+
+class OperationsGet(apitools_base_cli.NewCmd):
+  """Command wrapping operations.Get."""
+
+  usage = """operations_get """
+
+  def __init__(self, name, fv):
+    super(OperationsGet, self).__init__(name, fv)
+
+  def RunWithArgs(self, operationsId):
+    """Gets the latest state of a long-running operation.  Clients can use
+    this method to poll the operation result at intervals as recommended by
+    the API service.
+
+    Args:
+      operationsId: Part of `name`. The name of the operation resource.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementOperationsGetRequest(
+        operationsId=operationsId.decode('utf8'),
+        )
+    result = client.operations.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesConvertConfig(apitools_base_cli.NewCmd):
+  """Command wrapping services.ConvertConfig."""
+
+  usage = """services_convertConfig"""
+
+  def __init__(self, name, fv):
+    super(ServicesConvertConfig, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'configSpec',
+        None,
+        u'Input configuration For this version of API, the supported type is '
+        u'OpenApiSpec',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'openApiSpec',
+        None,
+        u'The OpenAPI specification for an API.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'serviceName',
+        None,
+        u'The service name to use for constructing the normalized service '
+        u'configuration equivalent of the provided configuration '
+        u'specification.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'swaggerSpec',
+        None,
+        u'The swagger specification for an API.',
+        flag_values=fv)
+
+  def RunWithArgs(self):
+    """DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide
+    config conversion moving forward.  Converts an API specification (e.g.
+    Swagger spec) to an equivalent `google.api.Service`.
+
+    Flags:
+      configSpec: Input configuration For this version of API, the supported
+        type is OpenApiSpec
+      openApiSpec: The OpenAPI specification for an API.
+      serviceName: The service name to use for constructing the normalized
+        service configuration equivalent of the provided configuration
+        specification.
+      swaggerSpec: The swagger specification for an API.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ConvertConfigRequest(
+        )
+    if FLAGS['configSpec'].present:
+      request.configSpec = apitools_base.JsonToMessage(messages.ConvertConfigRequest.ConfigSpecValue, FLAGS.configSpec)
+    if FLAGS['openApiSpec'].present:
+      request.openApiSpec = apitools_base.JsonToMessage(messages.OpenApiSpec, FLAGS.openApiSpec)
+    if FLAGS['serviceName'].present:
+      request.serviceName = FLAGS.serviceName.decode('utf8')
+    if FLAGS['swaggerSpec'].present:
+      request.swaggerSpec = apitools_base.JsonToMessage(messages.SwaggerSpec, FLAGS.swaggerSpec)
+    result = client.services.ConvertConfig(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesCreate(apitools_base_cli.NewCmd):
+  """Command wrapping services.Create."""
+
+  usage = """services_create"""
+
+  def __init__(self, name, fv):
+    super(ServicesCreate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'configSource',
+        None,
+        u'User-supplied source configuration for the service. This is '
+        u'distinct from the generated configuration provided in '
+        u'`google.api.Service`. This is NOT populated on GetService calls at '
+        u'the moment. NOTE: Any upsert operation that contains both a '
+        u'service_config and a config_source is considered invalid and will '
+        u'result in an error being returned.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'generation',
+        None,
+        u'A server-assigned monotonically increasing number that changes '
+        u'whenever a mutation is made to the `ManagedService` or any of its '
+        u'components via the `ServiceManager` API.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'operations',
+        None,
+        u'Read-only view of pending operations affecting this resource, if '
+        u'requested.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'producerProjectId',
+        None,
+        u'ID of the project that produces and owns this service.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'projectSettings',
+        None,
+        u'Read-only view of settings for a particular consumer project, if '
+        u'requested.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'serviceConfig',
+        None,
+        u"The service's generated configuration.",
+        flag_values=fv)
+    flags.DEFINE_string(
+        'serviceName',
+        None,
+        u'The name of the service.  See the `ServiceManager` overview for '
+        u'naming requirements.  This name must match '
+        u'`google.api.Service.name` in the `service_config` field.',
+        flag_values=fv)
+
+  def RunWithArgs(self):
+    """Creates a new managed service.  Operation
+
+    Flags:
+      configSource: User-supplied source configuration for the service. This
+        is distinct from the generated configuration provided in
+        `google.api.Service`. This is NOT populated on GetService calls at the
+        moment. NOTE: Any upsert operation that contains both a service_config
+        and a config_source is considered invalid and will result in an error
+        being returned.
+      generation: A server-assigned monotonically increasing number that
+        changes whenever a mutation is made to the `ManagedService` or any of
+        its components via the `ServiceManager` API.
+      operations: Read-only view of pending operations affecting this
+        resource, if requested.
+      producerProjectId: ID of the project that produces and owns this
+        service.
+      projectSettings: Read-only view of settings for a particular consumer
+        project, if requested.
+      serviceConfig: The service's generated configuration.
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  This name must match
+        `google.api.Service.name` in the `service_config` field.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ManagedService(
+        )
+    if FLAGS['configSource'].present:
+      request.configSource = apitools_base.JsonToMessage(messages.ConfigSource, FLAGS.configSource)
+    if FLAGS['generation'].present:
+      request.generation = int(FLAGS.generation)
+    if FLAGS['operations'].present:
+      request.operations = [apitools_base.JsonToMessage(messages.Operation, x) for x in FLAGS.operations]
+    if FLAGS['producerProjectId'].present:
+      request.producerProjectId = FLAGS.producerProjectId.decode('utf8')
+    if FLAGS['projectSettings'].present:
+      request.projectSettings = apitools_base.JsonToMessage(messages.ProjectSettings, FLAGS.projectSettings)
+    if FLAGS['serviceConfig'].present:
+      request.serviceConfig = apitools_base.JsonToMessage(messages.Service, FLAGS.serviceConfig)
+    if FLAGS['serviceName'].present:
+      request.serviceName = FLAGS.serviceName.decode('utf8')
+    result = client.services.Create(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesDelete(apitools_base_cli.NewCmd):
+  """Command wrapping services.Delete."""
+
+  usage = """services_delete """
+
+  def __init__(self, name, fv):
+    super(ServicesDelete, self).__init__(name, fv)
+
+  def RunWithArgs(self, serviceName):
+    """Deletes a managed service.  Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesDeleteRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    result = client.services.Delete(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesDisable(apitools_base_cli.NewCmd):
+  """Command wrapping services.Disable."""
+
+  usage = """services_disable """
+
+  def __init__(self, name, fv):
+    super(ServicesDisable, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'disableServiceRequest',
+        None,
+        u'A DisableServiceRequest resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Disable a managed service for a project. Google Service Management will
+    only disable the managed service even if there are other services depend
+    on the managed service.  Operation
+
+    Args:
+      serviceName: Name of the service to disable. Specifying an unknown
+        service name will cause the request to fail.
+
+    Flags:
+      disableServiceRequest: A DisableServiceRequest resource to be passed as
+        the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesDisableRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['disableServiceRequest'].present:
+      request.disableServiceRequest = apitools_base.JsonToMessage(messages.DisableServiceRequest, FLAGS.disableServiceRequest)
+    result = client.services.Disable(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesEnable(apitools_base_cli.NewCmd):
+  """Command wrapping services.Enable."""
+
+  usage = """services_enable """
+
+  def __init__(self, name, fv):
+    super(ServicesEnable, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'enableServiceRequest',
+        None,
+        u'A EnableServiceRequest resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Enable a managed service for a project with default setting. If the
+    managed service has dependencies, they will be enabled as well.
+    Operation
+
+    Args:
+      serviceName: Name of the service to enable. Specifying an unknown
+        service name will cause the request to fail.
+
+    Flags:
+      enableServiceRequest: A EnableServiceRequest resource to be passed as
+        the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesEnableRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['enableServiceRequest'].present:
+      request.enableServiceRequest = apitools_base.JsonToMessage(messages.EnableServiceRequest, FLAGS.enableServiceRequest)
+    result = client.services.Enable(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesGet(apitools_base_cli.NewCmd):
+  """Command wrapping services.Get."""
+
+  usage = """services_get """
+
+  def __init__(self, name, fv):
+    super(ServicesGet, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'consumerProjectId',
+        None,
+        u'If project_settings is expanded, return settings for the specified '
+        u'consumer project.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'expand',
+        None,
+        u'Fields to expand in any results.  By default, the following fields '
+        u'are not present in the result: - `operations` - `project_settings` '
+        u'- `project_settings.operations` - `quota_usage` (It requires '
+        u'`project_settings`) - `historical_quota_usage` (It requires '
+        u'`project_settings`)',
+        flag_values=fv)
+    flags.DEFINE_enum(
+        'view',
+        u'PROJECT_SETTINGS_VIEW_UNSPECIFIED',
+        [u'PROJECT_SETTINGS_VIEW_UNSPECIFIED', u'CONSUMER_VIEW', u'PRODUCER_VIEW', u'ALL'],
+        u'If project_settings is expanded, request only fields for the '
+        u'specified view.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Gets a managed service. If the `consumer_project_id` is specified, the
+    project's settings for the specified service are also returned.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      consumerProjectId: If project_settings is expanded, return settings for
+        the specified consumer project.
+      expand: Fields to expand in any results.  By default, the following
+        fields are not present in the result: - `operations` -
+        `project_settings` - `project_settings.operations` - `quota_usage` (It
+        requires `project_settings`) - `historical_quota_usage` (It requires
+        `project_settings`)
+      view: If project_settings is expanded, request only fields for the
+        specified view.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesGetRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['consumerProjectId'].present:
+      request.consumerProjectId = FLAGS.consumerProjectId.decode('utf8')
+    if FLAGS['expand'].present:
+      request.expand = FLAGS.expand.decode('utf8')
+    if FLAGS['view'].present:
+      request.view = messages.ServicemanagementServicesGetRequest.ViewValueValuesEnum(FLAGS.view)
+    result = client.services.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesGetAccessPolicy(apitools_base_cli.NewCmd):
+  """Command wrapping services.GetAccessPolicy."""
+
+  usage = """services_getAccessPolicy """
+
+  def __init__(self, name, fv):
+    super(ServicesGetAccessPolicy, self).__init__(name, fv)
+
+  def RunWithArgs(self, serviceName):
+    """Producer method to retrieve current policy.
+
+    Args:
+      serviceName: The name of the service.  For example:
+        `example.googleapis.com`.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesGetAccessPolicyRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    result = client.services.GetAccessPolicy(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesGetConfig(apitools_base_cli.NewCmd):
+  """Command wrapping services.GetConfig."""
+
+  usage = """services_getConfig """
+
+  def __init__(self, name, fv):
+    super(ServicesGetConfig, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'configId',
+        None,
+        u'The id of the service config resource. Optional. If it is not '
+        u'specified, the latest version of config will be returned.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Gets a service config (version) for a managed service. If `config_id`
+    is not specified, the latest service config will be returned.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      configId: The id of the service config resource. Optional. If it is not
+        specified, the latest version of config will be returned.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesGetConfigRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['configId'].present:
+      request.configId = FLAGS.configId.decode('utf8')
+    result = client.services.GetConfig(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesList(apitools_base_cli.NewCmd):
+  """Command wrapping services.List."""
+
+  usage = """services_list"""
+
+  def __init__(self, name, fv):
+    super(ServicesList, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'category',
+        None,
+        u'Include services only in the specified category. Supported '
+        u'categories are servicemanagement.googleapis.com/categories/google-'
+        u'services or servicemanagement.googleapis.com/categories/play-games.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'consumerProjectId',
+        None,
+        u'Include services consumed by the specified project.  If '
+        u'project_settings is expanded, then this field controls which '
+        u'project project_settings is populated for.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'expand',
+        None,
+        u'Fields to expand in any results.  By default, the following fields '
+        u'are not fully included in list results: - `operations` - '
+        u'`project_settings` - `project_settings.operations` - `quota_usage` '
+        u'(It requires `project_settings`)',
+        flag_values=fv)
+    flags.DEFINE_integer(
+        'pageSize',
+        None,
+        u'Requested size of the next page of data.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Token identifying which result to start with; returned by a '
+        u'previous list call.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'producerProjectId',
+        None,
+        u'Include services produced by the specified project.',
+        flag_values=fv)
+
+  def RunWithArgs(self):
+    """Lists all managed services. If the `consumer_project_id` is specified,
+    the project's settings for the specified service are also returned.
+
+    Flags:
+      category: Include services only in the specified category. Supported
+        categories are servicemanagement.googleapis.com/categories/google-
+        services or servicemanagement.googleapis.com/categories/play-games.
+      consumerProjectId: Include services consumed by the specified project.
+        If project_settings is expanded, then this field controls which
+        project project_settings is populated for.
+      expand: Fields to expand in any results.  By default, the following
+        fields are not fully included in list results: - `operations` -
+        `project_settings` - `project_settings.operations` - `quota_usage` (It
+        requires `project_settings`)
+      pageSize: Requested size of the next page of data.
+      pageToken: Token identifying which result to start with; returned by a
+        previous list call.
+      producerProjectId: Include services produced by the specified project.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesListRequest(
+        )
+    if FLAGS['category'].present:
+      request.category = FLAGS.category.decode('utf8')
+    if FLAGS['consumerProjectId'].present:
+      request.consumerProjectId = FLAGS.consumerProjectId.decode('utf8')
+    if FLAGS['expand'].present:
+      request.expand = FLAGS.expand.decode('utf8')
+    if FLAGS['pageSize'].present:
+      request.pageSize = FLAGS.pageSize
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    if FLAGS['producerProjectId'].present:
+      request.producerProjectId = FLAGS.producerProjectId.decode('utf8')
+    result = client.services.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesPatch(apitools_base_cli.NewCmd):
+  """Command wrapping services.Patch."""
+
+  usage = """services_patch """
+
+  def __init__(self, name, fv):
+    super(ServicesPatch, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'managedService',
+        None,
+        u'A ManagedService resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'A mask specifying which fields to update.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Updates the specified subset of the configuration. If the specified
+    service does not exists the patch operation fails.  Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      managedService: A ManagedService resource to be passed as the request
+        body.
+      updateMask: A mask specifying which fields to update.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesPatchRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['managedService'].present:
+      request.managedService = apitools_base.JsonToMessage(messages.ManagedService, FLAGS.managedService)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services.Patch(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesPatchConfig(apitools_base_cli.NewCmd):
+  """Command wrapping services.PatchConfig."""
+
+  usage = """services_patchConfig """
+
+  def __init__(self, name, fv):
+    super(ServicesPatchConfig, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'service',
+        None,
+        u'A Service resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'A mask specifying which fields to update.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Updates the specified subset of the service resource. Equivalent to
+    calling `PatchService` with only the `service_config` field updated.
+    Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      service: A Service resource to be passed as the request body.
+      updateMask: A mask specifying which fields to update.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesPatchConfigRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['service'].present:
+      request.service = apitools_base.JsonToMessage(messages.Service, FLAGS.service)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services.PatchConfig(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesUpdate(apitools_base_cli.NewCmd):
+  """Command wrapping services.Update."""
+
+  usage = """services_update """
+
+  def __init__(self, name, fv):
+    super(ServicesUpdate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'managedService',
+        None,
+        u'A ManagedService resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'A mask specifying which fields to update. Update mask has been '
+        u'deprecated on UpdateService service method. Please use PatchService'
+        u' method instead to do partial updates.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Updates the configuration of a service.  If the specified service does
+    not already exist, then it is created.  Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      managedService: A ManagedService resource to be passed as the request
+        body.
+      updateMask: A mask specifying which fields to update. Update mask has
+        been deprecated on UpdateService service method. Please use
+        PatchService method instead to do partial updates.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesUpdateRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['managedService'].present:
+      request.managedService = apitools_base.JsonToMessage(messages.ManagedService, FLAGS.managedService)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services.Update(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesUpdateAccessPolicy(apitools_base_cli.NewCmd):
+  """Command wrapping services.UpdateAccessPolicy."""
+
+  usage = """services_updateAccessPolicy """
+
+  def __init__(self, name, fv):
+    super(ServicesUpdateAccessPolicy, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'accessList',
+        None,
+        u'ACL for access to the unrestricted surface of the service.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'visibilityLabelAccessLists',
+        None,
+        u'ACLs for access to restricted parts of the service.  The map key is'
+        u' the visibility label that is being controlled.  Note that access '
+        u'to any label also implies access to the unrestricted surface.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Producer method to update the current policy.  This method will return
+    an error if the policy is too large (more than 50 entries across all
+    lists).
+
+    Args:
+      serviceName: The service protected by this policy.
+
+    Flags:
+      accessList: ACL for access to the unrestricted surface of the service.
+      visibilityLabelAccessLists: ACLs for access to restricted parts of the
+        service.  The map key is the visibility label that is being
+        controlled.  Note that access to any label also implies access to the
+        unrestricted surface.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServiceAccessPolicy(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['accessList'].present:
+      request.accessList = apitools_base.JsonToMessage(messages.ServiceAccessList, FLAGS.accessList)
+    if FLAGS['visibilityLabelAccessLists'].present:
+      request.visibilityLabelAccessLists = apitools_base.JsonToMessage(messages.ServiceAccessPolicy.VisibilityLabelAccessListsValue, FLAGS.visibilityLabelAccessLists)
+    result = client.services.UpdateAccessPolicy(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesUpdateConfig(apitools_base_cli.NewCmd):
+  """Command wrapping services.UpdateConfig."""
+
+  usage = """services_updateConfig """
+
+  def __init__(self, name, fv):
+    super(ServicesUpdateConfig, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'service',
+        None,
+        u'A Service resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'A mask specifying which fields to update. Update mask has been '
+        u'deprecated on UpdateServiceConfig service method. Please use '
+        u'PatchServiceConfig method instead to do partial updates.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Updates the specified subset of the service resource. Equivalent to
+    calling `UpdateService` with only the `service_config` field updated.
+    Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      service: A Service resource to be passed as the request body.
+      updateMask: A mask specifying which fields to update. Update mask has
+        been deprecated on UpdateServiceConfig service method. Please use
+        PatchServiceConfig method instead to do partial updates.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesUpdateConfigRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['service'].present:
+      request.service = apitools_base.JsonToMessage(messages.Service, FLAGS.service)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services.UpdateConfig(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesAccessPolicyQuery(apitools_base_cli.NewCmd):
+  """Command wrapping services_accessPolicy.Query."""
+
+  usage = """services_accessPolicy_query """
+
+  def __init__(self, name, fv):
+    super(ServicesAccessPolicyQuery, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'userEmail',
+        None,
+        u'The user to query access for.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Method to query the accessibility of a service and any associated
+    visibility labels for a specified user.  Members of the producer project
+    may call this method and specify any user.  Any user may call this method,
+    but must specify their own email address. In this case the method will
+    return NOT_FOUND if the user has no access to the service.
+
+    Args:
+      serviceName: The service to query access for.
+
+    Flags:
+      userEmail: The user to query access for.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesAccessPolicyQueryRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['userEmail'].present:
+      request.userEmail = FLAGS.userEmail.decode('utf8')
+    result = client.services_accessPolicy.Query(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesConfigsCreate(apitools_base_cli.NewCmd):
+  """Command wrapping services_configs.Create."""
+
+  usage = """services_configs_create """
+
+  def __init__(self, name, fv):
+    super(ServicesConfigsCreate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'service',
+        None,
+        u'A Service resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Creates a new service config (version) for a managed service. This
+    method only stores the service config, but does not apply the service
+    config to any backend services.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      service: A Service resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesConfigsCreateRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['service'].present:
+      request.service = apitools_base.JsonToMessage(messages.Service, FLAGS.service)
+    result = client.services_configs.Create(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesConfigsGet(apitools_base_cli.NewCmd):
+  """Command wrapping services_configs.Get."""
+
+  usage = """services_configs_get  """
+
+  def __init__(self, name, fv):
+    super(ServicesConfigsGet, self).__init__(name, fv)
+
+  def RunWithArgs(self, serviceName, configId):
+    """Gets a service config (version) for a managed service. If `config_id`
+    is not specified, the latest service config will be returned.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+      configId: The id of the service config resource. Optional. If it is not
+        specified, the latest version of config will be returned.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesConfigsGetRequest(
+        serviceName=serviceName.decode('utf8'),
+        configId=configId.decode('utf8'),
+        )
+    result = client.services_configs.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesConfigsList(apitools_base_cli.NewCmd):
+  """Command wrapping services_configs.List."""
+
+  usage = """services_configs_list """
+
+  def __init__(self, name, fv):
+    super(ServicesConfigsList, self).__init__(name, fv)
+    flags.DEFINE_integer(
+        'pageSize',
+        None,
+        u'The max number of items to include in the response list.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'The token of the page to retrieve.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Lists the history of the service config for a managed service, from the
+    newest to the oldest.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      pageSize: The max number of items to include in the response list.
+      pageToken: The token of the page to retrieve.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesConfigsListRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['pageSize'].present:
+      request.pageSize = FLAGS.pageSize
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    result = client.services_configs.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesConfigsSubmit(apitools_base_cli.NewCmd):
+  """Command wrapping services_configs.Submit."""
+
+  usage = """services_configs_submit """
+
+  def __init__(self, name, fv):
+    super(ServicesConfigsSubmit, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'submitConfigSourceRequest',
+        None,
+        u'A SubmitConfigSourceRequest resource to be passed as the request '
+        u'body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName):
+    """Creates a new service config (version) for a managed service based on
+    user-supplied configuration sources files (for example: OpenAPI
+    Specification). This method stores the source configurations as well as
+    the generated service config. It does NOT apply the service config to any
+    backend services.  Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+
+    Flags:
+      submitConfigSourceRequest: A SubmitConfigSourceRequest resource to be
+        passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesConfigsSubmitRequest(
+        serviceName=serviceName.decode('utf8'),
+        )
+    if FLAGS['submitConfigSourceRequest'].present:
+      request.submitConfigSourceRequest = apitools_base.JsonToMessage(messages.SubmitConfigSourceRequest, FLAGS.submitConfigSourceRequest)
+    result = client.services_configs.Submit(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesCustomerSettingsGet(apitools_base_cli.NewCmd):
+  """Command wrapping services_customerSettings.Get."""
+
+  usage = """services_customerSettings_get  """
+
+  def __init__(self, name, fv):
+    super(ServicesCustomerSettingsGet, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'expand',
+        None,
+        u'Fields to expand in any results.',
+        flag_values=fv)
+    flags.DEFINE_enum(
+        'view',
+        u'PROJECT_SETTINGS_VIEW_UNSPECIFIED',
+        [u'PROJECT_SETTINGS_VIEW_UNSPECIFIED', u'CONSUMER_VIEW', u'PRODUCER_VIEW', u'ALL'],
+        u'Request only fields for the specified view.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName, customerId):
+    """Retrieves the settings that control the specified customer's usage of
+    the service.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`. This
+        field is required.
+      customerId: ID for the customer. See the comment for
+        `CustomerSettings.customer_id` field of message for its format. This
+        field is required.
+
+    Flags:
+      expand: Fields to expand in any results.
+      view: Request only fields for the specified view.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesCustomerSettingsGetRequest(
+        serviceName=serviceName.decode('utf8'),
+        customerId=customerId.decode('utf8'),
+        )
+    if FLAGS['expand'].present:
+      request.expand = FLAGS.expand.decode('utf8')
+    if FLAGS['view'].present:
+      request.view = messages.ServicemanagementServicesCustomerSettingsGetRequest.ViewValueValuesEnum(FLAGS.view)
+    result = client.services_customerSettings.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesCustomerSettingsPatch(apitools_base_cli.NewCmd):
+  """Command wrapping services_customerSettings.Patch."""
+
+  usage = """services_customerSettings_patch  """
+
+  def __init__(self, name, fv):
+    super(ServicesCustomerSettingsPatch, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'customerSettings',
+        None,
+        u'A CustomerSettings resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'The field mask specifying which fields are to be updated.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName, customerId):
+    """Updates specified subset of the settings that control the specified
+    customer's usage of the service.  Attempts to update a field not
+    controlled by the caller will result in an access denied error.
+    Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`. This
+        field is required.
+      customerId: ID for the customer. See the comment for
+        `CustomerSettings.customer_id` field of message for its format. This
+        field is required.
+
+    Flags:
+      customerSettings: A CustomerSettings resource to be passed as the
+        request body.
+      updateMask: The field mask specifying which fields are to be updated.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesCustomerSettingsPatchRequest(
+        serviceName=serviceName.decode('utf8'),
+        customerId=customerId.decode('utf8'),
+        )
+    if FLAGS['customerSettings'].present:
+      request.customerSettings = apitools_base.JsonToMessage(messages.CustomerSettings, FLAGS.customerSettings)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services_customerSettings.Patch(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesProjectSettingsGet(apitools_base_cli.NewCmd):
+  """Command wrapping services_projectSettings.Get."""
+
+  usage = """services_projectSettings_get  """
+
+  def __init__(self, name, fv):
+    super(ServicesProjectSettingsGet, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'expand',
+        None,
+        u'Fields to expand in any results.  By default, the following fields '
+        u'are not present in the result: - `operations` - `quota_usage`',
+        flag_values=fv)
+    flags.DEFINE_enum(
+        'view',
+        u'PROJECT_SETTINGS_VIEW_UNSPECIFIED',
+        [u'PROJECT_SETTINGS_VIEW_UNSPECIFIED', u'CONSUMER_VIEW', u'PRODUCER_VIEW', u'ALL'],
+        u'Request only the fields for the specified view.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName, consumerProjectId):
+    """Retrieves the settings that control the specified consumer project's
+    usage of the service.
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+      consumerProjectId: The project ID of the consumer.
+
+    Flags:
+      expand: Fields to expand in any results.  By default, the following
+        fields are not present in the result: - `operations` - `quota_usage`
+      view: Request only the fields for the specified view.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesProjectSettingsGetRequest(
+        serviceName=serviceName.decode('utf8'),
+        consumerProjectId=consumerProjectId.decode('utf8'),
+        )
+    if FLAGS['expand'].present:
+      request.expand = FLAGS.expand.decode('utf8')
+    if FLAGS['view'].present:
+      request.view = messages.ServicemanagementServicesProjectSettingsGetRequest.ViewValueValuesEnum(FLAGS.view)
+    result = client.services_projectSettings.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesProjectSettingsPatch(apitools_base_cli.NewCmd):
+  """Command wrapping services_projectSettings.Patch."""
+
+  usage = """services_projectSettings_patch  """
+
+  def __init__(self, name, fv):
+    super(ServicesProjectSettingsPatch, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'projectSettings',
+        None,
+        u'A ProjectSettings resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'updateMask',
+        None,
+        u'The field mask specifying which fields are to be updated.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName, consumerProjectId):
+    """Updates specified subset of the settings that control the specified
+    consumer project's usage of the service.  Attempts to update a field not
+    controlled by the caller will result in an access denied error.
+    Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.  For example: `example.googleapis.com`.
+      consumerProjectId: The project ID of the consumer.
+
+    Flags:
+      projectSettings: A ProjectSettings resource to be passed as the request
+        body.
+      updateMask: The field mask specifying which fields are to be updated.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ServicemanagementServicesProjectSettingsPatchRequest(
+        serviceName=serviceName.decode('utf8'),
+        consumerProjectId=consumerProjectId.decode('utf8'),
+        )
+    if FLAGS['projectSettings'].present:
+      request.projectSettings = apitools_base.JsonToMessage(messages.ProjectSettings, FLAGS.projectSettings)
+    if FLAGS['updateMask'].present:
+      request.updateMask = FLAGS.updateMask.decode('utf8')
+    result = client.services_projectSettings.Patch(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ServicesProjectSettingsUpdate(apitools_base_cli.NewCmd):
+  """Command wrapping services_projectSettings.Update."""
+
+  usage = """services_projectSettings_update  """
+
+  def __init__(self, name, fv):
+    super(ServicesProjectSettingsUpdate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'operations',
+        None,
+        u'Read-only view of pending operations affecting this resource, if '
+        u'requested.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'properties',
+        None,
+        u'Service-defined per-consumer properties.  A key-value mapping a '
+        u'string key to a google.protobuf.ListValue proto. Values in the list'
+        u" are typed as defined in the Service configuration's "
+        u'consumer.properties field.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'quotaSettings',
+        None,
+        u'Settings that control how much or how fast the service can be used '
+        u'by the consumer project.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'usageSettings',
+        None,
+        u'Settings that control whether this service is usable by the '
+        u'consumer project.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'visibilitySettings',
+        None,
+        u'Settings that control which features of the service are visible to '
+        u'the consumer project.',
+        flag_values=fv)
+
+  def RunWithArgs(self, serviceName, consumerProjectId):
+    """NOTE: Currently unsupported.  Use PatchProjectSettings instead.
+    Updates the settings that control the specified consumer project's usage
+    of the service.  Attempts to update a field not controlled by the caller
+    will result in an access denied error.  Operation
+
+    Args:
+      serviceName: The name of the service.  See the `ServiceManager` overview
+        for naming requirements.
+      consumerProjectId: ID for the project consuming this service.
+
+    Flags:
+      operations: Read-only view of pending operations affecting this
+        resource, if requested.
+      properties: Service-defined per-consumer properties.  A key-value
+        mapping a string key to a google.protobuf.ListValue proto. Values in
+        the list are typed as defined in the Service configuration's
+        consumer.properties field.
+      quotaSettings: Settings that control how much or how fast the service
+        can be used by the consumer project.
+      usageSettings: Settings that control whether this service is usable by
+        the consumer project.
+      visibilitySettings: Settings that control which features of the service
+        are visible to the consumer project.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ProjectSettings(
+        serviceName=serviceName.decode('utf8'),
+        consumerProjectId=consumerProjectId.decode('utf8'),
+        )
+    if FLAGS['operations'].present:
+      request.operations = [apitools_base.JsonToMessage(messages.Operation, x) for x in FLAGS.operations]
+    if FLAGS['properties'].present:
+      request.properties = apitools_base.JsonToMessage(messages.ProjectSettings.PropertiesValue, FLAGS.properties)
+    if FLAGS['quotaSettings'].present:
+      request.quotaSettings = apitools_base.JsonToMessage(messages.QuotaSettings, FLAGS.quotaSettings)
+    if FLAGS['usageSettings'].present:
+      request.usageSettings = apitools_base.JsonToMessage(messages.UsageSettings, FLAGS.usageSettings)
+    if FLAGS['visibilitySettings'].present:
+      request.visibilitySettings = apitools_base.JsonToMessage(messages.VisibilitySettings, FLAGS.visibilitySettings)
+    result = client.services_projectSettings.Update(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ConvertConfig(apitools_base_cli.NewCmd):
+  """Command wrapping v1.ConvertConfig."""
+
+  usage = """convertConfig"""
+
+  def __init__(self, name, fv):
+    super(ConvertConfig, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'configSpec',
+        None,
+        u'Input configuration For this version of API, the supported type is '
+        u'OpenApiSpec',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'openApiSpec',
+        None,
+        u'The OpenAPI specification for an API.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'serviceName',
+        None,
+        u'The service name to use for constructing the normalized service '
+        u'configuration equivalent of the provided configuration '
+        u'specification.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'swaggerSpec',
+        None,
+        u'The swagger specification for an API.',
+        flag_values=fv)
+
+  def RunWithArgs(self):
+    """DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide
+    config conversion moving forward.  Converts an API specification (e.g.
+    Swagger spec) to an equivalent `google.api.Service`.
+
+    Flags:
+      configSpec: Input configuration For this version of API, the supported
+        type is OpenApiSpec
+      openApiSpec: The OpenAPI specification for an API.
+      serviceName: The service name to use for constructing the normalized
+        service configuration equivalent of the provided configuration
+        specification.
+      swaggerSpec: The swagger specification for an API.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.ConvertConfigRequest(
+        )
+    if FLAGS['configSpec'].present:
+      request.configSpec = apitools_base.JsonToMessage(messages.ConvertConfigRequest.ConfigSpecValue, FLAGS.configSpec)
+    if FLAGS['openApiSpec'].present:
+      request.openApiSpec = apitools_base.JsonToMessage(messages.OpenApiSpec, FLAGS.openApiSpec)
+    if FLAGS['serviceName'].present:
+      request.serviceName = FLAGS.serviceName.decode('utf8')
+    if FLAGS['swaggerSpec'].present:
+      request.swaggerSpec = apitools_base.JsonToMessage(messages.SwaggerSpec, FLAGS.swaggerSpec)
+    result = client.v1.ConvertConfig(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+def main(_):
+  appcommands.AddCmd('pyshell', PyShell)
+  appcommands.AddCmd('operations_get', OperationsGet)
+  appcommands.AddCmd('services_convertConfig', ServicesConvertConfig)
+  appcommands.AddCmd('services_create', ServicesCreate)
+  appcommands.AddCmd('services_delete', ServicesDelete)
+  appcommands.AddCmd('services_disable', ServicesDisable)
+  appcommands.AddCmd('services_enable', ServicesEnable)
+  appcommands.AddCmd('services_get', ServicesGet)
+  appcommands.AddCmd('services_getAccessPolicy', ServicesGetAccessPolicy)
+  appcommands.AddCmd('services_getConfig', ServicesGetConfig)
+  appcommands.AddCmd('services_list', ServicesList)
+  appcommands.AddCmd('services_patch', ServicesPatch)
+  appcommands.AddCmd('services_patchConfig', ServicesPatchConfig)
+  appcommands.AddCmd('services_update', ServicesUpdate)
+  appcommands.AddCmd('services_updateAccessPolicy', ServicesUpdateAccessPolicy)
+  appcommands.AddCmd('services_updateConfig', ServicesUpdateConfig)
+  appcommands.AddCmd('services_accessPolicy_query', ServicesAccessPolicyQuery)
+  appcommands.AddCmd('services_configs_create', ServicesConfigsCreate)
+  appcommands.AddCmd('services_configs_get', ServicesConfigsGet)
+  appcommands.AddCmd('services_configs_list', ServicesConfigsList)
+  appcommands.AddCmd('services_configs_submit', ServicesConfigsSubmit)
+  appcommands.AddCmd('services_customerSettings_get', ServicesCustomerSettingsGet)
+  appcommands.AddCmd('services_customerSettings_patch', ServicesCustomerSettingsPatch)
+  appcommands.AddCmd('services_projectSettings_get', ServicesProjectSettingsGet)
+  appcommands.AddCmd('services_projectSettings_patch', ServicesProjectSettingsPatch)
+  appcommands.AddCmd('services_projectSettings_update', ServicesProjectSettingsUpdate)
+  appcommands.AddCmd('convertConfig', ConvertConfig)
+
+  apitools_base_cli.SetupLogger()
+  if hasattr(appcommands, 'SetDefaultCommand'):
+    appcommands.SetDefaultCommand('pyshell')
+
+
+run_main = apitools_base_cli.run_main
+
+if __name__ == '__main__':
+  appcommands.Run()
diff --git a/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py
new file mode 100644
index 0000000..9a1362d
--- /dev/null
+++ b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py
@@ -0,0 +1,854 @@
+"""Generated client library for servicemanagement version v1."""
+# NOTE: This file is autogenerated and should not be edited by hand.
+from apitools.base.py import base_api
+from samples.servicemanagement_sample.servicemanagement_v1 import servicemanagement_v1_messages as messages
+
+
+class ServicemanagementV1(base_api.BaseApiClient):
+  """Generated client library for service servicemanagement version v1."""
+
+  MESSAGES_MODULE = messages
+  BASE_URL = u'https://servicemanagement.googleapis.com/'
+
+  _PACKAGE = u'servicemanagement'
+  _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/service.management']
+  _VERSION = u'v1'
+  _CLIENT_ID = '1042881264118.apps.googleusercontent.com'
+  _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+  _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+  _CLIENT_CLASS_NAME = u'ServicemanagementV1'
+  _URL_VERSION = u'v1'
+  _API_KEY = None
+
+  def __init__(self, url='', credentials=None,
+               get_credentials=True, http=None, model=None,
+               log_request=False, log_response=False,
+               credentials_args=None, default_global_params=None,
+               additional_http_headers=None):
+    """Create a new servicemanagement handle."""
+    url = url or self.BASE_URL
+    super(ServicemanagementV1, self).__init__(
+        url, credentials=credentials,
+        get_credentials=get_credentials, http=http, model=model,
+        log_request=log_request, log_response=log_response,
+        credentials_args=credentials_args,
+        default_global_params=default_global_params,
+        additional_http_headers=additional_http_headers)
+    self.operations = self.OperationsService(self)
+    self.services_accessPolicy = self.ServicesAccessPolicyService(self)
+    self.services_configs = self.ServicesConfigsService(self)
+    self.services_customerSettings = self.ServicesCustomerSettingsService(self)
+    self.services_projectSettings = self.ServicesProjectSettingsService(self)
+    self.services = self.ServicesService(self)
+    self.v1 = self.V1Service(self)
+
+  class OperationsService(base_api.BaseApiService):
+    """Service class for the operations resource."""
+
+    _NAME = u'operations'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.OperationsService, self).__init__(client)
+      self._method_configs = {
+          'Get': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.operations.get',
+              ordered_params=[u'operationsId'],
+              path_params=[u'operationsId'],
+              query_params=[],
+              relative_path=u'v1/operations/{operationsId}',
+              request_field='',
+              request_type_name=u'ServicemanagementOperationsGetRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def Get(self, request, global_params=None):
+      """Gets the latest state of a long-running operation.  Clients can use this.
+method to poll the operation result at intervals as recommended by the API
+service.
+
+      Args:
+        request: (ServicemanagementOperationsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class ServicesAccessPolicyService(base_api.BaseApiService):
+    """Service class for the services_accessPolicy resource."""
+
+    _NAME = u'services_accessPolicy'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.ServicesAccessPolicyService, self).__init__(client)
+      self._method_configs = {
+          'Query': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.accessPolicy.query',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'userEmail'],
+              relative_path=u'v1/services/{serviceName}/accessPolicy:query',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesAccessPolicyQueryRequest',
+              response_type_name=u'QueryUserAccessResponse',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def Query(self, request, global_params=None):
+      """Method to query the accessibility of a service and any associated.
+visibility labels for a specified user.
+
+Members of the producer project may call this method and specify any user.
+
+Any user may call this method, but must specify their own email address.
+In this case the method will return NOT_FOUND if the user has no access to
+the service.
+
+      Args:
+        request: (ServicemanagementServicesAccessPolicyQueryRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (QueryUserAccessResponse) The response message.
+      """
+      config = self.GetMethodConfig('Query')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class ServicesConfigsService(base_api.BaseApiService):
+    """Service class for the services_configs resource."""
+
+    _NAME = u'services_configs'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.ServicesConfigsService, self).__init__(client)
+      self._method_configs = {
+          'Create': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.configs.create',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/configs',
+              request_field=u'service',
+              request_type_name=u'ServicemanagementServicesConfigsCreateRequest',
+              response_type_name=u'Service',
+              supports_download=False,
+          ),
+          'Get': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.configs.get',
+              ordered_params=[u'serviceName', u'configId'],
+              path_params=[u'configId', u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/configs/{configId}',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesConfigsGetRequest',
+              response_type_name=u'Service',
+              supports_download=False,
+          ),
+          'List': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.configs.list',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'pageSize', u'pageToken'],
+              relative_path=u'v1/services/{serviceName}/configs',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesConfigsListRequest',
+              response_type_name=u'ListServiceConfigsResponse',
+              supports_download=False,
+          ),
+          'Submit': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.configs.submit',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/configs:submit',
+              request_field=u'submitConfigSourceRequest',
+              request_type_name=u'ServicemanagementServicesConfigsSubmitRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def Create(self, request, global_params=None):
+      """Creates a new service config (version) for a managed service. This method.
+only stores the service config, but does not apply the service config to
+any backend services.
+
+      Args:
+        request: (ServicemanagementServicesConfigsCreateRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Service) The response message.
+      """
+      config = self.GetMethodConfig('Create')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Get(self, request, global_params=None):
+      """Gets a service config (version) for a managed service. If `config_id` is.
+not specified, the latest service config will be returned.
+
+      Args:
+        request: (ServicemanagementServicesConfigsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Service) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def List(self, request, global_params=None):
+      """Lists the history of the service config for a managed service,.
+from the newest to the oldest.
+
+      Args:
+        request: (ServicemanagementServicesConfigsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ListServiceConfigsResponse) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Submit(self, request, global_params=None):
+      """Creates a new service config (version) for a managed service based on.
+user-supplied configuration sources files (for example: OpenAPI
+Specification). This method stores the source configurations as well as the
+generated service config. It does NOT apply the service config to any
+backend services.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesConfigsSubmitRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Submit')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class ServicesCustomerSettingsService(base_api.BaseApiService):
+    """Service class for the services_customerSettings resource."""
+
+    _NAME = u'services_customerSettings'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.ServicesCustomerSettingsService, self).__init__(client)
+      self._method_configs = {
+          'Get': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.customerSettings.get',
+              ordered_params=[u'serviceName', u'customerId'],
+              path_params=[u'customerId', u'serviceName'],
+              query_params=[u'expand', u'view'],
+              relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesCustomerSettingsGetRequest',
+              response_type_name=u'CustomerSettings',
+              supports_download=False,
+          ),
+          'Patch': base_api.ApiMethodInfo(
+              http_method=u'PATCH',
+              method_id=u'servicemanagement.services.customerSettings.patch',
+              ordered_params=[u'serviceName', u'customerId'],
+              path_params=[u'customerId', u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
+              request_field=u'customerSettings',
+              request_type_name=u'ServicemanagementServicesCustomerSettingsPatchRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def Get(self, request, global_params=None):
+      """Retrieves the settings that control the specified customer's usage of the.
+service.
+
+      Args:
+        request: (ServicemanagementServicesCustomerSettingsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (CustomerSettings) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Patch(self, request, global_params=None):
+      """Updates specified subset of the settings that control the specified.
+customer's usage of the service.  Attempts to update a field not
+controlled by the caller will result in an access denied error.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesCustomerSettingsPatchRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Patch')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class ServicesProjectSettingsService(base_api.BaseApiService):
+    """Service class for the services_projectSettings resource."""
+
+    _NAME = u'services_projectSettings'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.ServicesProjectSettingsService, self).__init__(client)
+      self._method_configs = {
+          'Get': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.projectSettings.get',
+              ordered_params=[u'serviceName', u'consumerProjectId'],
+              path_params=[u'consumerProjectId', u'serviceName'],
+              query_params=[u'expand', u'view'],
+              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesProjectSettingsGetRequest',
+              response_type_name=u'ProjectSettings',
+              supports_download=False,
+          ),
+          'Patch': base_api.ApiMethodInfo(
+              http_method=u'PATCH',
+              method_id=u'servicemanagement.services.projectSettings.patch',
+              ordered_params=[u'serviceName', u'consumerProjectId'],
+              path_params=[u'consumerProjectId', u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+              request_field=u'projectSettings',
+              request_type_name=u'ServicemanagementServicesProjectSettingsPatchRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Update': base_api.ApiMethodInfo(
+              http_method=u'PUT',
+              method_id=u'servicemanagement.services.projectSettings.update',
+              ordered_params=[u'serviceName', u'consumerProjectId'],
+              path_params=[u'consumerProjectId', u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+              request_field='',
+              request_type_name=u'ProjectSettings',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def Get(self, request, global_params=None):
+      """Retrieves the settings that control the specified consumer project's usage.
+of the service.
+
+      Args:
+        request: (ServicemanagementServicesProjectSettingsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ProjectSettings) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Patch(self, request, global_params=None):
+      """Updates specified subset of the settings that control the specified.
+consumer project's usage of the service.  Attempts to update a field not
+controlled by the caller will result in an access denied error.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesProjectSettingsPatchRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Patch')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Update(self, request, global_params=None):
+      """NOTE: Currently unsupported.  Use PatchProjectSettings instead.
+
+Updates the settings that control the specified consumer project's usage
+of the service.  Attempts to update a field not controlled by the caller
+will result in an access denied error.
+
+Operation
+
+      Args:
+        request: (ProjectSettings) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Update')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class ServicesService(base_api.BaseApiService):
+    """Service class for the services resource."""
+
+    _NAME = u'services'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.ServicesService, self).__init__(client)
+      self._method_configs = {
+          'ConvertConfig': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.convertConfig',
+              ordered_params=[],
+              path_params=[],
+              query_params=[],
+              relative_path=u'v1/services:convertConfig',
+              request_field='',
+              request_type_name=u'ConvertConfigRequest',
+              response_type_name=u'ConvertConfigResponse',
+              supports_download=False,
+          ),
+          'Create': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.create',
+              ordered_params=[],
+              path_params=[],
+              query_params=[],
+              relative_path=u'v1/services',
+              request_field='',
+              request_type_name=u'ManagedService',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Delete': base_api.ApiMethodInfo(
+              http_method=u'DELETE',
+              method_id=u'servicemanagement.services.delete',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesDeleteRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Disable': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.disable',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}:disable',
+              request_field=u'disableServiceRequest',
+              request_type_name=u'ServicemanagementServicesDisableRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Enable': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.services.enable',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}:enable',
+              request_field=u'enableServiceRequest',
+              request_type_name=u'ServicemanagementServicesEnableRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Get': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.get',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'consumerProjectId', u'expand', u'view'],
+              relative_path=u'v1/services/{serviceName}',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesGetRequest',
+              response_type_name=u'ManagedService',
+              supports_download=False,
+          ),
+          'GetAccessPolicy': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.getAccessPolicy',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/accessPolicy',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesGetAccessPolicyRequest',
+              response_type_name=u'ServiceAccessPolicy',
+              supports_download=False,
+          ),
+          'GetConfig': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.getConfig',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'configId'],
+              relative_path=u'v1/services/{serviceName}/config',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesGetConfigRequest',
+              response_type_name=u'Service',
+              supports_download=False,
+          ),
+          'List': base_api.ApiMethodInfo(
+              http_method=u'GET',
+              method_id=u'servicemanagement.services.list',
+              ordered_params=[],
+              path_params=[],
+              query_params=[u'category', u'consumerProjectId', u'expand', u'pageSize', u'pageToken', u'producerProjectId'],
+              relative_path=u'v1/services',
+              request_field='',
+              request_type_name=u'ServicemanagementServicesListRequest',
+              response_type_name=u'ListServicesResponse',
+              supports_download=False,
+          ),
+          'Patch': base_api.ApiMethodInfo(
+              http_method=u'PATCH',
+              method_id=u'servicemanagement.services.patch',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}',
+              request_field=u'managedService',
+              request_type_name=u'ServicemanagementServicesPatchRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'PatchConfig': base_api.ApiMethodInfo(
+              http_method=u'PATCH',
+              method_id=u'servicemanagement.services.patchConfig',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}/config',
+              request_field=u'service',
+              request_type_name=u'ServicemanagementServicesPatchConfigRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'Update': base_api.ApiMethodInfo(
+              http_method=u'PUT',
+              method_id=u'servicemanagement.services.update',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}',
+              request_field=u'managedService',
+              request_type_name=u'ServicemanagementServicesUpdateRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          'UpdateAccessPolicy': base_api.ApiMethodInfo(
+              http_method=u'PUT',
+              method_id=u'servicemanagement.services.updateAccessPolicy',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[],
+              relative_path=u'v1/services/{serviceName}/accessPolicy',
+              request_field='',
+              request_type_name=u'ServiceAccessPolicy',
+              response_type_name=u'ServiceAccessPolicy',
+              supports_download=False,
+          ),
+          'UpdateConfig': base_api.ApiMethodInfo(
+              http_method=u'PUT',
+              method_id=u'servicemanagement.services.updateConfig',
+              ordered_params=[u'serviceName'],
+              path_params=[u'serviceName'],
+              query_params=[u'updateMask'],
+              relative_path=u'v1/services/{serviceName}/config',
+              request_field=u'service',
+              request_type_name=u'ServicemanagementServicesUpdateConfigRequest',
+              response_type_name=u'Operation',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def ConvertConfig(self, request, global_params=None):
+      """DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide.
+config conversion moving forward.
+
+Converts an API specification (e.g. Swagger spec) to an
+equivalent `google.api.Service`.
+
+      Args:
+        request: (ConvertConfigRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ConvertConfigResponse) The response message.
+      """
+      config = self.GetMethodConfig('ConvertConfig')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Create(self, request, global_params=None):
+      """Creates a new managed service.
+
+Operation
+
+      Args:
+        request: (ManagedService) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Create')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Delete(self, request, global_params=None):
+      """Deletes a managed service.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesDeleteRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Delete')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Disable(self, request, global_params=None):
+      """Disable a managed service for a project.
+Google Service Management will only disable the managed service even if
+there are other services depend on the managed service.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesDisableRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Disable')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Enable(self, request, global_params=None):
+      """Enable a managed service for a project with default setting.
+If the managed service has dependencies, they will be enabled as well.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesEnableRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Enable')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Get(self, request, global_params=None):
+      """Gets a managed service. If the `consumer_project_id` is specified,.
+the project's settings for the specified service are also returned.
+
+      Args:
+        request: (ServicemanagementServicesGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ManagedService) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def GetAccessPolicy(self, request, global_params=None):
+      """Producer method to retrieve current policy.
+
+      Args:
+        request: (ServicemanagementServicesGetAccessPolicyRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ServiceAccessPolicy) The response message.
+      """
+      config = self.GetMethodConfig('GetAccessPolicy')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def GetConfig(self, request, global_params=None):
+      """Gets a service config (version) for a managed service. If `config_id` is.
+not specified, the latest service config will be returned.
+
+      Args:
+        request: (ServicemanagementServicesGetConfigRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Service) The response message.
+      """
+      config = self.GetMethodConfig('GetConfig')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def List(self, request, global_params=None):
+      """Lists all managed services. If the `consumer_project_id` is specified,.
+the project's settings for the specified service are also returned.
+
+      Args:
+        request: (ServicemanagementServicesListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ListServicesResponse) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Patch(self, request, global_params=None):
+      """Updates the specified subset of the configuration. If the specified service.
+does not exists the patch operation fails.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesPatchRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Patch')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def PatchConfig(self, request, global_params=None):
+      """Updates the specified subset of the service resource. Equivalent to.
+calling `PatchService` with only the `service_config` field updated.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesPatchConfigRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('PatchConfig')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def Update(self, request, global_params=None):
+      """Updates the configuration of a service.  If the specified service does not.
+already exist, then it is created.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesUpdateRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('Update')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def UpdateAccessPolicy(self, request, global_params=None):
+      """Producer method to update the current policy.  This method will return an.
+error if the policy is too large (more than 50 entries across all lists).
+
+      Args:
+        request: (ServiceAccessPolicy) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ServiceAccessPolicy) The response message.
+      """
+      config = self.GetMethodConfig('UpdateAccessPolicy')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    def UpdateConfig(self, request, global_params=None):
+      """Updates the specified subset of the service resource. Equivalent to.
+calling `UpdateService` with only the `service_config` field updated.
+
+Operation
+
+      Args:
+        request: (ServicemanagementServicesUpdateConfigRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Operation) The response message.
+      """
+      config = self.GetMethodConfig('UpdateConfig')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+  class V1Service(base_api.BaseApiService):
+    """Service class for the v1 resource."""
+
+    _NAME = u'v1'
+
+    def __init__(self, client):
+      super(ServicemanagementV1.V1Service, self).__init__(client)
+      self._method_configs = {
+          'ConvertConfig': base_api.ApiMethodInfo(
+              http_method=u'POST',
+              method_id=u'servicemanagement.convertConfig',
+              ordered_params=[],
+              path_params=[],
+              query_params=[],
+              relative_path=u'v1:convertConfig',
+              request_field='',
+              request_type_name=u'ConvertConfigRequest',
+              response_type_name=u'ConvertConfigResponse',
+              supports_download=False,
+          ),
+          }
+
+      self._upload_configs = {
+          }
+
+    def ConvertConfig(self, request, global_params=None):
+      """DEPRECATED. `SubmitConfigSource` with `validate_only=true` will provide.
+config conversion moving forward.
+
+Converts an API specification (e.g. Swagger spec) to an
+equivalent `google.api.Service`.
+
+      Args:
+        request: (ConvertConfigRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ConvertConfigResponse) The response message.
+      """
+      config = self.GetMethodConfig('ConvertConfig')
+      return self._RunMethod(
+          config, request, global_params=global_params)
diff --git a/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_messages.py b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_messages.py
new file mode 100644
index 0000000..9291cf3
--- /dev/null
+++ b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_messages.py
@@ -0,0 +1,3505 @@
+"""Generated message classes for servicemanagement version v1.
+
+The service management API for Google Cloud Platform
+"""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import encoding
+from apitools.base.py import extra_types
+
+
+package = 'servicemanagement'
+
+
+class Api(_messages.Message):
+  """Api is a light-weight descriptor for a protocol buffer service.
+
+  Enums:
+    SyntaxValueValuesEnum: The source syntax of the service.
+
+  Fields:
+    methods: The methods of this api, in unspecified order.
+    mixins: Included APIs. See Mixin.
+    name: The fully qualified name of this api, including package name
+      followed by the api's simple name.
+    options: Any metadata attached to the API.
+    sourceContext: Source context for the protocol buffer service represented
+      by this message.
+    syntax: The source syntax of the service.
+    version: A version string for this api. If specified, must have the form
+      `major-version.minor-version`, as in `1.10`. If the minor version is
+      omitted, it defaults to zero. If the entire version field is empty, the
+      major version is derived from the package name, as outlined below. If
+      the field is not empty, the version in the package name will be verified
+      to be consistent with what is provided here.  The versioning schema uses
+      [semantic versioning](http://semver.org) where the major version number
+      indicates a breaking change and the minor version an additive, non-
+      breaking change. Both version numbers are signals to users what to
+      expect from different versions, and should be carefully chosen based on
+      the product plan.  The major version is also reflected in the package
+      name of the API, which must end in `v`, as in
+      `google.feature.v1`. For major versions 0 and 1, the suffix can be
+      omitted. Zero major versions must only be used for experimental, none-GA
+      apis.
+  """
+
+  class SyntaxValueValuesEnum(_messages.Enum):
+    """The source syntax of the service.
+
+    Values:
+      SYNTAX_PROTO2: Syntax `proto2`.
+      SYNTAX_PROTO3: Syntax `proto3`.
+    """
+    SYNTAX_PROTO2 = 0
+    SYNTAX_PROTO3 = 1
+
+  methods = _messages.MessageField('Method', 1, repeated=True)
+  mixins = _messages.MessageField('Mixin', 2, repeated=True)
+  name = _messages.StringField(3)
+  options = _messages.MessageField('Option', 4, repeated=True)
+  sourceContext = _messages.MessageField('SourceContext', 5)
+  syntax = _messages.EnumField('SyntaxValueValuesEnum', 6)
+  version = _messages.StringField(7)
+
+
+class AreaUnderCurveParams(_messages.Message):
+  """AreaUnderCurveParams groups the metrics relevant to generating duration
+  based metric from base (snapshot) metric and delta (change) metric.  The
+  generated metric has two dimensions:    resource usage metric and the
+  duration the metric applies.  Essentially the generated metric is the Area
+  Under Curve(AUC) of the "duration - resource" usage curve. This AUC metric
+  is readily appliable to billing since "billable resource usage" depends on
+  resource usage and duration of the resource used.  A service config may
+  contain multiple resources and corresponding metrics. AreaUnderCurveParams
+  groups the relevant ones: which snapshot_metric and change_metric are used
+  to produce which generated_metric.
+
+  Fields:
+    changeMetric: Change of resource usage at a particular timestamp. This
+      should a DELTA metric.
+    generatedMetric: Metric generated from snapshot_metric and change_metric.
+      This is also a DELTA metric.
+    snapshotMetric: Total usage of a resource at a particular timestamp. This
+      should be a GAUGE metric.
+  """
+
+  changeMetric = _messages.StringField(1)
+  generatedMetric = _messages.StringField(2)
+  snapshotMetric = _messages.StringField(3)
+
+
+class AuthProvider(_messages.Message):
+  """Configuration for an anthentication provider, including support for [JSON
+  Web Token (JWT)](https://tools.ietf.org/html/draft-ietf-oauth-json-web-
+  token-32).
+
+  Fields:
+    id: The unique identifier of the auth provider. It will be referred to by
+      `AuthRequirement.provider_id`.  Example: "bookstore_auth".
+    issuer: Identifies the principal that issued the JWT. See
+      https://tools.ietf.org/html/draft-ietf-oauth-json-web-
+      token-32#section-4.1.1 Usually a URL or an email address.  Example:
+      https://securetoken.google.com Example:
+      1234567-compute@developer.gserviceaccount.com
+    jwksUri: URL of the provider's public key set to validate signature of the
+      JWT. See [OpenID Discovery](https://openid.net/specs/openid-connect-
+      discovery-1_0.html#ProviderMetadata). Optional if the key set document:
+      - can be retrieved from    [OpenID Discovery](https://openid.net/specs
+      /openid-connect-discovery-1_0.html    of the issuer.  - can be inferred
+      from the email domain of the issuer (e.g. a Google service account).
+      Example: https://www.googleapis.com/oauth2/v1/certs
+  """
+
+  id = _messages.StringField(1)
+  issuer = _messages.StringField(2)
+  jwksUri = _messages.StringField(3)
+
+
+class AuthRequirement(_messages.Message):
+  """User-defined authentication requirements, including support for [JSON Web
+  Token (JWT)](https://tools.ietf.org/html/draft-ietf-oauth-json-web-
+  token-32).
+
+  Fields:
+    audiences: The list of JWT [audiences](https://tools.ietf.org/html/draft-
+      ietf-oauth-json-web-token-32#section-4.1.3). that are allowed to access.
+      A JWT containing any of these audiences will be accepted. When this
+      setting is absent, only JWTs with audience
+      "https://Service_name/API_name" will be accepted. For example, if no
+      audiences are in the setting, LibraryService API will only accept JWTs
+      with the following audience "https://library-
+      example.googleapis.com/google.example.library.v1.LibraryService".
+      Example:      audiences: bookstore_android.apps.googleusercontent.com,
+      bookstore_web.apps.googleusercontent.com
+    providerId: id from authentication provider.  Example:      provider_id:
+      bookstore_auth
+  """
+
+  audiences = _messages.StringField(1)
+  providerId = _messages.StringField(2)
+
+
+class Authentication(_messages.Message):
+  """`Authentication` defines the authentication configuration for an API.
+  Example for an API targeted for external use:      name:
+  calendar.googleapis.com     authentication:       rules:       - selector:
+  "*"         oauth:           canonical_scopes:
+  https://www.googleapis.com/auth/calendar        - selector:
+  google.calendar.Delegate         oauth:           canonical_scopes:
+  https://www.googleapis.com/auth/calendar.read
+
+  Fields:
+    providers: Defines a set of authentication providers that a service
+      supports.
+    rules: Individual rules for authentication.
+  """
+
+  providers = _messages.MessageField('AuthProvider', 1, repeated=True)
+  rules = _messages.MessageField('AuthenticationRule', 2, repeated=True)
+
+
+class AuthenticationRule(_messages.Message):
+  """Authentication rules for the service.  By default, if a method has any
+  authentication requirements, every request must include a valid credential
+  matching one of the requirements. It's an error to include more than one
+  kind of credential in a single request.  If a method doesn't have any auth
+  requirements, request credentials will be ignored.
+
+  Fields:
+    allowWithoutCredential: Whether to allow requests without a credential.
+      If quota is enabled, an API key is required for such request to pass the
+      quota check.
+    oauth: The requirements for OAuth credentials.
+    requirements: Requirements for additional authentication providers.
+    selector: Selects the methods to which this rule applies.  Refer to
+      selector for syntax details.
+  """
+
+  allowWithoutCredential = _messages.BooleanField(1)
+  oauth = _messages.MessageField('OAuthRequirements', 2)
+  requirements = _messages.MessageField('AuthRequirement', 3, repeated=True)
+  selector = _messages.StringField(4)
+
+
+class Backend(_messages.Message):
+  """`Backend` defines the backend configuration for a service.
+
+  Fields:
+    rules: A list of backend rules providing configuration for individual API
+      elements.
+  """
+
+  rules = _messages.MessageField('BackendRule', 1, repeated=True)
+
+
+class BackendRule(_messages.Message):
+  """A backend rule provides configuration for an individual API element.
+
+  Fields:
+    address: The address of the API backend.
+    deadline: The number of seconds to wait for a response from a request.
+      The default depends on the deployment context.
+    selector: Selects the methods to which this rule applies.  Refer to
+      selector for syntax details.
+  """
+
+  address = _messages.StringField(1)
+  deadline = _messages.FloatField(2)
+  selector = _messages.StringField(3)
+
+
+class Billing(_messages.Message):
+  """Billing related configuration of the service.  The following example
+  shows how to configure metrics for billing:      metrics:     - name:
+  library.googleapis.com/read_calls       metric_kind: DELTA       value_type:
+  INT64     - name: library.googleapis.com/write_calls       metric_kind:
+  DELTA       value_type: INT64     billing:       metrics:       -
+  library.googleapis.com/read_calls       - library.googleapis.com/write_calls
+  The next example shows how to enable billing status check and customize the
+  check behavior. It makes sure billing status check is included in the
+  `Check` method of [Service Control API](https://cloud.google.com/service-
+  control/). In the example, "google.storage.Get" method can be served when
+  the billing status is either `current` or `delinquent`, while
+  "google.storage.Write" method can only be served when the billing status is
+  `current`:      billing:       rules:       - selector: google.storage.Get
+  allowed_statuses:         - current         - delinquent       - selector:
+  google.storage.Write         allowed_statuses: current  Mostly services
+  should only allow `current` status when serving requests. In addition,
+  services can choose to allow both `current` and `delinquent` statuses when
+  serving read-only requests to resources. If there's no matching selector for
+  operation, no billing status check will be performed.
+
+  Fields:
+    areaUnderCurveParams: Per resource grouping for delta billing based
+      resource configs.
+    metrics: Names of the metrics to report to billing. Each name must be
+      defined in Service.metrics section.
+    rules: A list of billing status rules for configuring billing status
+      check.
+  """
+
+  areaUnderCurveParams = _messages.MessageField('AreaUnderCurveParams', 1, repeated=True)
+  metrics = _messages.StringField(2, repeated=True)
+  rules = _messages.MessageField('BillingStatusRule', 3, repeated=True)
+
+
+class BillingStatusRule(_messages.Message):
+  """Defines the billing status requirements for operations.  When used with
+  [Service Control API](https://cloud.google.com/service-control/), the
+  following statuses are supported:  - **current**: the associated billing
+  account is up to date and capable of                paying for resource
+  usages. - **delinquent**: the associated billing account has a correctable
+  problem,                   such as late payment.  Mostly services should
+  only allow `current` status when serving requests. In addition, services can
+  choose to allow both `current` and `delinquent` statuses when serving read-
+  only requests to resources. If the list of allowed_statuses is empty, it
+  means no billing requirement.
+
+  Fields:
+    allowedStatuses: Allowed billing statuses. The billing status check passes
+      if the actual billing status matches any of the provided values here.
+    selector: Selects the operation names to which this rule applies. Refer to
+      selector for syntax details.
+  """
+
+  allowedStatuses = _messages.StringField(1, repeated=True)
+  selector = _messages.StringField(2)
+
+
+class CompositeOperationMetadata(_messages.Message):
+  """Metadata for composite operations.
+
+  Messages:
+    OriginalRequestValue: Original request that triggered this operation.
+    ResponseFieldMasksValue: Defines which part of the response a child
+      operation will contribute. Each key of the map is the name of a child
+      operation. Each value is a field mask that identifies what that child
+      operation contributes to the response, for example, "quota_settings",
+      "visiblity_settings", etc.
+
+  Fields:
+    childOperations: The child operations. The details of the asynchronous
+      child operations are stored in a separate row and not in this metadata.
+      Only the operation name is stored here.
+    originalRequest: Original request that triggered this operation.
+    persisted: Indicates whether the requested state change has been
+      persisted. Once this field is set, it is guaranteed to propagate to all
+      backends eventually, but it may not be visible immediately. Clients that
+      are not concerned with waiting on propagation can stop polling the
+      operation once the persisted field is set
+    responseFieldMasks: Defines which part of the response a child operation
+      will contribute. Each key of the map is the name of a child operation.
+      Each value is a field mask that identifies what that child operation
+      contributes to the response, for example, "quota_settings",
+      "visiblity_settings", etc.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class OriginalRequestValue(_messages.Message):
+    """Original request that triggered this operation.
+
+    Messages:
+      AdditionalProperty: An additional property for a OriginalRequestValue
+        object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a OriginalRequestValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ResponseFieldMasksValue(_messages.Message):
+    """Defines which part of the response a child operation will contribute.
+    Each key of the map is the name of a child operation. Each value is a
+    field mask that identifies what that child operation contributes to the
+    response, for example, "quota_settings", "visiblity_settings", etc.
+
+    Messages:
+      AdditionalProperty: An additional property for a ResponseFieldMasksValue
+        object.
+
+    Fields:
+      additionalProperties: Additional properties of type
+        ResponseFieldMasksValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ResponseFieldMasksValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A string attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.StringField(2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  childOperations = _messages.MessageField('Operation', 1, repeated=True)
+  originalRequest = _messages.MessageField('OriginalRequestValue', 2)
+  persisted = _messages.BooleanField(3)
+  responseFieldMasks = _messages.MessageField('ResponseFieldMasksValue', 4)
+
+
+class ConfigFile(_messages.Message):
+  """Generic specification of a source configuration file
+
+  Enums:
+    FileTypeValueValuesEnum: The kind of configuration file represented. This
+      is used to determine the method for generating `google.api.Service`
+      using this file.
+
+  Fields:
+    contents: DEPRECATED. The contents of the configuration file. Use
+      file_contents moving forward.
+    fileContents: The bytes that constitute the file.
+    filePath: The file name of the configuration file (full or relative path).
+    fileType: The kind of configuration file represented. This is used to
+      determine the method for generating `google.api.Service` using this
+      file.
+  """
+
+  class FileTypeValueValuesEnum(_messages.Enum):
+    """The kind of configuration file represented. This is used to determine
+    the method for generating `google.api.Service` using this file.
+
+    Values:
+      FILE_TYPE_UNSPECIFIED: Unknown file type.
+      SERVICE_CONFIG_YAML: YAML-specification of service.
+      OPEN_API_JSON: OpenAPI specification, serialized in JSON.
+      OPEN_API_YAML: OpenAPI specification, serialized in YAML.
+      FILE_DESCRIPTOR_SET_PROTO: FileDescriptorSet, generated by protoc.  To
+        generate, use protoc with imports and source info included. For an
+        example test.proto file, the following command would put the value in
+        a new file named out.pb.  $protoc --include_imports
+        --include_source_info test.proto -o out.pb
+    """
+    FILE_TYPE_UNSPECIFIED = 0
+    SERVICE_CONFIG_YAML = 1
+    OPEN_API_JSON = 2
+    OPEN_API_YAML = 3
+    FILE_DESCRIPTOR_SET_PROTO = 4
+
+  contents = _messages.StringField(1)
+  fileContents = _messages.BytesField(2)
+  filePath = _messages.StringField(3)
+  fileType = _messages.EnumField('FileTypeValueValuesEnum', 4)
+
+
+class ConfigOptions(_messages.Message):
+  """A set of options to cover use of source config within `ServiceManager`
+  and related tools.
+  """
+
+
+
+class ConfigSource(_messages.Message):
+  """Represents a user-specified configuration for a service (as opposed to
+  the the generated service config form provided by `google.api.Service`).
+  This is meant to encode service config as manipulated directly by customers,
+  rather than the config form resulting from toolchain generation and
+  normalization.
+
+  Fields:
+    files: Set of source configuration files that are used to generate a
+      service config (`google.api.Service`).
+    id: A unique ID for a specific instance of this message, typically
+      assigned by the client for tracking purpose. If empty, the server may
+      choose to generate one instead.
+    openApiSpec: OpenAPI specification
+    options: Options to cover use of source config within ServiceManager and
+      tools
+    protoSpec: Protocol buffer API specification
+  """
+
+  files = _messages.MessageField('ConfigFile', 1, repeated=True)
+  id = _messages.StringField(2)
+  openApiSpec = _messages.MessageField('OpenApiSpec', 3)
+  options = _messages.MessageField('ConfigOptions', 4)
+  protoSpec = _messages.MessageField('ProtoSpec', 5)
+
+
+class Context(_messages.Message):
+  """`Context` defines which contexts an API requests.  Example:      context:
+  rules:       - selector: "*"         requested:         -
+  google.rpc.context.ProjectContext         - google.rpc.context.OriginContext
+  The above specifies that all methods in the API request
+  `google.rpc.context.ProjectContext` and `google.rpc.context.OriginContext`.
+  Available context types are defined in package `google.rpc.context`.
+
+  Fields:
+    rules: List of rules for context, applicable to methods.
+  """
+
+  rules = _messages.MessageField('ContextRule', 1, repeated=True)
+
+
+class ContextRule(_messages.Message):
+  """A context rule provides information about the context for an individual
+  API element.
+
+  Fields:
+    provided: A list of full type names of provided contexts.
+    requested: A list of full type names of requested contexts.
+    selector: Selects the methods to which this rule applies.  Refer to
+      selector for syntax details.
+  """
+
+  provided = _messages.StringField(1, repeated=True)
+  requested = _messages.StringField(2, repeated=True)
+  selector = _messages.StringField(3)
+
+
+class Control(_messages.Message):
+  """Selects and configures the service controller used by the service.  The
+  service controller handles features like abuse, quota, billing, logging,
+  monitoring, etc.
+
+  Fields:
+    environment: The service control environment to use. If empty, no control
+      plane feature (like quota and billing) will be enabled.
+  """
+
+  environment = _messages.StringField(1)
+
+
+class ConvertConfigRequest(_messages.Message):
+  """Request message for `ConvertConfig` method.
+
+  Messages:
+    ConfigSpecValue: Input configuration For this version of API, the
+      supported type is OpenApiSpec
+
+  Fields:
+    configSpec: Input configuration For this version of API, the supported
+      type is OpenApiSpec
+    openApiSpec: The OpenAPI specification for an API.
+    serviceName: The service name to use for constructing the normalized
+      service configuration equivalent of the provided configuration
+      specification.
+    swaggerSpec: The swagger specification for an API.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ConfigSpecValue(_messages.Message):
+    """Input configuration For this version of API, the supported type is
+    OpenApiSpec
+
+    Messages:
+      AdditionalProperty: An additional property for a ConfigSpecValue object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ConfigSpecValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  configSpec = _messages.MessageField('ConfigSpecValue', 1)
+  openApiSpec = _messages.MessageField('OpenApiSpec', 2)
+  serviceName = _messages.StringField(3)
+  swaggerSpec = _messages.MessageField('SwaggerSpec', 4)
+
+
+class ConvertConfigResponse(_messages.Message):
+  """Response message for `ConvertConfig` method.
+
+  Fields:
+    diagnostics: Any errors or warnings that occured during config conversion.
+    serviceConfig: The service configuration. Not set if errors occured during
+      conversion.
+  """
+
+  diagnostics = _messages.MessageField('Diagnostic', 1, repeated=True)
+  serviceConfig = _messages.MessageField('Service', 2)
+
+
+class CustomError(_messages.Message):
+  """Customize service error responses.  For example, list any service
+  specific protobuf types that can appear in error detail lists of error
+  responses.  Example:      custom_error:       types:       -
+  google.foo.v1.CustomError       - google.foo.v1.AnotherError
+
+  Fields:
+    rules: The list of custom error rules to select to which messages this
+      should apply.
+    types: The list of custom error detail types, e.g.
+      'google.foo.v1.CustomError'.
+  """
+
+  rules = _messages.MessageField('CustomErrorRule', 1, repeated=True)
+  types = _messages.StringField(2, repeated=True)
+
+
+class CustomErrorRule(_messages.Message):
+  """A custom error rule.
+
+  Fields:
+    isErrorType: Mark this message as possible payload in error response.
+      Otherwise, objects of this type will be filtered when they appear in
+      error payload.
+    selector: Selects messages to which this rule applies.  Refer to selector
+      for syntax details.
+  """
+
+  isErrorType = _messages.BooleanField(1)
+  selector = _messages.StringField(2)
+
+
+class CustomHttpPattern(_messages.Message):
+  """A custom pattern is used for defining custom HTTP verb.
+
+  Fields:
+    kind: The name of this custom HTTP verb.
+    path: The path matched by this custom verb.
+  """
+
+  kind = _messages.StringField(1)
+  path = _messages.StringField(2)
+
+
+class CustomerSettings(_messages.Message):
+  """Settings that control how a customer (identified by a billing account)
+  uses a service
+
+  Fields:
+    customerId: ID for the customer that consumes the service (see above). The
+      supported types of customers are:  1. domain:{domain} A Google Apps
+      domain name. For example, google.com.  2.
+      billingAccount:{billing_account_id} A Google Cloud Plafrom billing
+      account. For Example, 123456-7890ab-cdef12.
+    quotaSettings: Settings that control how much or how fast the service can
+      be used by the consumer projects owned by the customer collectively.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.
+  """
+
+  customerId = _messages.StringField(1)
+  quotaSettings = _messages.MessageField('QuotaSettings', 2)
+  serviceName = _messages.StringField(3)
+
+
+class Diagnostic(_messages.Message):
+  """A collection that represents a diagnostic message (error or warning)
+
+  Enums:
+    KindValueValuesEnum: The kind of diagnostic information provided.
+
+  Fields:
+    kind: The kind of diagnostic information provided.
+    location: Location of the cause or context of the diagnostic information.
+    message: The string message of the diagnostic information.
+  """
+
+  class KindValueValuesEnum(_messages.Enum):
+    """The kind of diagnostic information provided.
+
+    Values:
+      WARNING: Warnings and errors
+      ERROR: Only errors
+    """
+    WARNING = 0
+    ERROR = 1
+
+  kind = _messages.EnumField('KindValueValuesEnum', 1)
+  location = _messages.StringField(2)
+  message = _messages.StringField(3)
+
+
+class DisableServiceRequest(_messages.Message):
+  """Request message for DisableService method.
+
+  Fields:
+    consumerId: The identity of consumer resource which service disablement
+      will be applied to.  The Google Service Management implementation
+      accepts the following forms: "project:",
+      "project_number:".  Note: this is made compatible with
+      google.api.servicecontrol.v1.Operation.consumer_id.
+  """
+
+  consumerId = _messages.StringField(1)
+
+
+class Documentation(_messages.Message):
+  """`Documentation` provides the information for describing a service.
+  Example: 
documentation:   summary: >     The Google Calendar API
+  gives access     to most calendar features.   pages:   - name: Overview
+  content: (== include google/foo/overview.md ==)   - name: Tutorial
+  content: (== include google/foo/tutorial.md ==)     subpages;     -
+  name: Java       content: (== include google/foo/tutorial_java.md
+  ==)   rules:   - selector: google.calendar.Calendar.Get     description:
+  >       ...   - selector: google.calendar.Calendar.Put     description: >
+  ... 
 Documentation is provided in markdown syntax. In addition
+  to standard markdown features, definition lists, tables and fenced code
+  blocks are supported. Section headers can be provided and are interpreted
+  relative to the section nesting of the context where a documentation
+  fragment is embedded.  Documentation from the IDL is merged with
+  documentation defined via the config at normalization time, where
+  documentation provided by config rules overrides IDL provided.  A number of
+  constructs specific to the API platform are supported in documentation text.
+  In order to reference a proto element, the following notation can be used:
+  
[fully.qualified.proto.name][]
 To override
+  the display text used for the link, this can be used:
+  
[display text][fully.qualified.proto.name]

+  Text can be excluded from doc using the following notation:
+  
(-- internal comment --)
 Comments can be
+  made conditional using a visibility label. The below text will be only
+  rendered if the `BETA` label is available: 
(--BETA: comment
+  for BETA users --)
 A few directives are available in
+  documentation. Note that directives must appear on a single line to be
+  properly identified. The `include` directive includes a markdown file from
+  an external source: 
(== include path/to/file
+  ==)
 The `resource_for` directive marks a message to be the
+  resource of a collection in REST view. If it is not specified, tools attempt
+  to infer the resource from the operations in a collection:
+  
(== resource_for v1.shelves.books ==)
 The
+  directive `suppress_warning` does not directly affect documentation and is
+  documented together with service config validation.
+
+  Fields:
+    documentationRootUrl: The URL to the root of documentation.
+    overview: Declares a single overview page. For example:
+      
documentation:   summary: ...   overview: (== include
+      overview.md ==) 
 This is a shortcut for the following
+      declaration (using pages style): 
documentation:   summary:
+      ...   pages:   - name: Overview     content: (== include overview.md
+      ==) 
 Note: you cannot specify both `overview` field and
+      `pages` field.
+    pages: The top level pages for the documentation set.
+    rules: Documentation rules for individual elements of the service.
+    summary: A short summary of what the service does. Can only be provided by
+      plain text.
+  """
+
+  documentationRootUrl = _messages.StringField(1)
+  overview = _messages.StringField(2)
+  pages = _messages.MessageField('Page', 3, repeated=True)
+  rules = _messages.MessageField('DocumentationRule', 4, repeated=True)
+  summary = _messages.StringField(5)
+
+
+class DocumentationRule(_messages.Message):
+  """A documentation rule provides information about individual API elements.
+
+  Fields:
+    deprecationDescription: Deprecation description of the selected
+      element(s). It can be provided if an element is marked as `deprecated`.
+    description: Description of the selected API(s).
+    selector: The selector is a comma-separated list of patterns. Each pattern
+      is a qualified name of the element which may end in "*", indicating a
+      wildcard. Wildcards are only allowed at the end and for a whole
+      component of the qualified name, i.e. "foo.*" is ok, but not "foo.b*" or
+      "foo.*.bar". To specify a default for all applicable elements, the whole
+      pattern "*" is used.
+  """
+
+  deprecationDescription = _messages.StringField(1)
+  description = _messages.StringField(2)
+  selector = _messages.StringField(3)
+
+
+class EffectiveQuotaGroup(_messages.Message):
+  """An effective quota group contains both the metadata for a quota group as
+  derived from the service config, and the effective limits in that group as
+  calculated from producer and consumer overrides together with service
+  defaults.
+
+  Enums:
+    BillingInteractionValueValuesEnum:
+
+  Fields:
+    baseGroup: The service configuration for this quota group, minus the quota
+      limits, which are replaced by the effective limits below.
+    billingInteraction: A BillingInteractionValueValuesEnum attribute.
+    quotas: The usage and limit information for each limit within this quota
+      group.
+  """
+
+  class BillingInteractionValueValuesEnum(_messages.Enum):
+    """BillingInteractionValueValuesEnum enum type.
+
+    Values:
+      BILLING_INTERACTION_UNSPECIFIED: The interaction between this quota
+        group and the project billing status is unspecified.
+      NONBILLABLE_ONLY: This quota group is enforced only when the consumer
+        project is not billable.
+      BILLABLE_ONLY: This quota group is enforced only when the consumer
+        project is billable.
+      ANY_BILLING_STATUS: This quota group is enforced regardless of the
+        consumer project's billing status.
+    """
+    BILLING_INTERACTION_UNSPECIFIED = 0
+    NONBILLABLE_ONLY = 1
+    BILLABLE_ONLY = 2
+    ANY_BILLING_STATUS = 3
+
+  baseGroup = _messages.MessageField('QuotaGroup', 1)
+  billingInteraction = _messages.EnumField('BillingInteractionValueValuesEnum', 2)
+  quotas = _messages.MessageField('QuotaInfo', 3, repeated=True)
+
+
+class EffectiveQuotaLimit(_messages.Message):
+  """An effective quota limit contains the metadata for a quota limit as
+  derived from the service config, together with fields that describe the
+  effective limit value and what overrides can be applied to it.
+
+  Fields:
+    baseLimit: The service's configuration for this quota limit.
+    effectiveLimit: The effective limit value, based on the stored producer
+      and consumer overrides and the service defaults.
+    key: The key used to identify this limit when applying overrides. The
+      consumer_overrides and producer_overrides maps are keyed by strings of
+      the form "QuotaGroupName/QuotaLimitName".
+    maxConsumerOverrideAllowed: The maximum override value that a consumer may
+      specify.
+  """
+
+  baseLimit = _messages.MessageField('QuotaLimit', 1)
+  effectiveLimit = _messages.IntegerField(2)
+  key = _messages.StringField(3)
+  maxConsumerOverrideAllowed = _messages.IntegerField(4)
+
+
+class EnableServiceRequest(_messages.Message):
+  """Request message for EnableService method.
+
+  Fields:
+    consumerId: The identity of consumer resource which service enablement
+      will be applied to.  The Google Service Management implementation
+      accepts the following forms: "project:",
+      "project_number:".  Note: this is made compatible with
+      google.api.servicecontrol.v1.Operation.consumer_id.
+  """
+
+  consumerId = _messages.StringField(1)
+
+
+class Enum(_messages.Message):
+  """Enum type definition.
+
+  Enums:
+    SyntaxValueValuesEnum: The source syntax.
+
+  Fields:
+    enumvalue: Enum value definitions.
+    name: Enum type name.
+    options: Protocol buffer options.
+    sourceContext: The source context.
+    syntax: The source syntax.
+  """
+
+  class SyntaxValueValuesEnum(_messages.Enum):
+    """The source syntax.
+
+    Values:
+      SYNTAX_PROTO2: Syntax `proto2`.
+      SYNTAX_PROTO3: Syntax `proto3`.
+    """
+    SYNTAX_PROTO2 = 0
+    SYNTAX_PROTO3 = 1
+
+  enumvalue = _messages.MessageField('EnumValue', 1, repeated=True)
+  name = _messages.StringField(2)
+  options = _messages.MessageField('Option', 3, repeated=True)
+  sourceContext = _messages.MessageField('SourceContext', 4)
+  syntax = _messages.EnumField('SyntaxValueValuesEnum', 5)
+
+
+class EnumValue(_messages.Message):
+  """Enum value definition.
+
+  Fields:
+    name: Enum value name.
+    number: Enum value number.
+    options: Protocol buffer options.
+  """
+
+  name = _messages.StringField(1)
+  number = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+  options = _messages.MessageField('Option', 3, repeated=True)
+
+
+class Field(_messages.Message):
+  """A single field of a message type.
+
+  Enums:
+    CardinalityValueValuesEnum: The field cardinality.
+    KindValueValuesEnum: The field type.
+
+  Fields:
+    cardinality: The field cardinality.
+    defaultValue: The string value of the default value of this field. Proto2
+      syntax only.
+    jsonName: The field JSON name.
+    kind: The field type.
+    name: The field name.
+    number: The field number.
+    oneofIndex: The index of the field type in `Type.oneofs`, for message or
+      enumeration types. The first type has index 1; zero means the type is
+      not in the list.
+    options: The protocol buffer options.
+    packed: Whether to use alternative packed wire representation.
+    typeUrl: The field type URL, without the scheme, for message or
+      enumeration types. Example:
+      `"type.googleapis.com/google.protobuf.Timestamp"`.
+  """
+
+  class CardinalityValueValuesEnum(_messages.Enum):
+    """The field cardinality.
+
+    Values:
+      CARDINALITY_UNKNOWN: For fields with unknown cardinality.
+      CARDINALITY_OPTIONAL: For optional fields.
+      CARDINALITY_REQUIRED: For required fields. Proto2 syntax only.
+      CARDINALITY_REPEATED: For repeated fields.
+    """
+    CARDINALITY_UNKNOWN = 0
+    CARDINALITY_OPTIONAL = 1
+    CARDINALITY_REQUIRED = 2
+    CARDINALITY_REPEATED = 3
+
+  class KindValueValuesEnum(_messages.Enum):
+    """The field type.
+
+    Values:
+      TYPE_UNKNOWN: Field type unknown.
+      TYPE_DOUBLE: Field type double.
+      TYPE_FLOAT: Field type float.
+      TYPE_INT64: Field type int64.
+      TYPE_UINT64: Field type uint64.
+      TYPE_INT32: Field type int32.
+      TYPE_FIXED64: Field type fixed64.
+      TYPE_FIXED32: Field type fixed32.
+      TYPE_BOOL: Field type bool.
+      TYPE_STRING: Field type string.
+      TYPE_GROUP: Field type group. Proto2 syntax only, and deprecated.
+      TYPE_MESSAGE: Field type message.
+      TYPE_BYTES: Field type bytes.
+      TYPE_UINT32: Field type uint32.
+      TYPE_ENUM: Field type enum.
+      TYPE_SFIXED32: Field type sfixed32.
+      TYPE_SFIXED64: Field type sfixed64.
+      TYPE_SINT32: Field type sint32.
+      TYPE_SINT64: Field type sint64.
+    """
+    TYPE_UNKNOWN = 0
+    TYPE_DOUBLE = 1
+    TYPE_FLOAT = 2
+    TYPE_INT64 = 3
+    TYPE_UINT64 = 4
+    TYPE_INT32 = 5
+    TYPE_FIXED64 = 6
+    TYPE_FIXED32 = 7
+    TYPE_BOOL = 8
+    TYPE_STRING = 9
+    TYPE_GROUP = 10
+    TYPE_MESSAGE = 11
+    TYPE_BYTES = 12
+    TYPE_UINT32 = 13
+    TYPE_ENUM = 14
+    TYPE_SFIXED32 = 15
+    TYPE_SFIXED64 = 16
+    TYPE_SINT32 = 17
+    TYPE_SINT64 = 18
+
+  cardinality = _messages.EnumField('CardinalityValueValuesEnum', 1)
+  defaultValue = _messages.StringField(2)
+  jsonName = _messages.StringField(3)
+  kind = _messages.EnumField('KindValueValuesEnum', 4)
+  name = _messages.StringField(5)
+  number = _messages.IntegerField(6, variant=_messages.Variant.INT32)
+  oneofIndex = _messages.IntegerField(7, variant=_messages.Variant.INT32)
+  options = _messages.MessageField('Option', 8, repeated=True)
+  packed = _messages.BooleanField(9)
+  typeUrl = _messages.StringField(10)
+
+
+class File(_messages.Message):
+  """A single swagger specification file.
+
+  Fields:
+    contents: The contents of the swagger spec file.
+    path: The relative path of the swagger spec file.
+  """
+
+  contents = _messages.StringField(1)
+  path = _messages.StringField(2)
+
+
+class Http(_messages.Message):
+  """Defines the HTTP configuration for a service. It contains a list of
+  HttpRule, each specifying the mapping of an RPC method to one or more HTTP
+  REST API methods.
+
+  Fields:
+    rules: A list of HTTP rules for configuring the HTTP REST API methods.
+  """
+
+  rules = _messages.MessageField('HttpRule', 1, repeated=True)
+
+
+class HttpRule(_messages.Message):
+  """`HttpRule` defines the mapping of an RPC method to one or more HTTP REST
+  APIs.  The mapping determines what portions of the request message are
+  populated from the path, query parameters, or body of the HTTP request.  The
+  mapping is typically specified as an `google.api.http` annotation, see
+  "google/api/annotations.proto" for details.  The mapping consists of a field
+  specifying the path template and method kind.  The path template can refer
+  to fields in the request message, as in the example below which describes a
+  REST GET operation on a resource collection of messages:  ```proto service
+  Messaging {   rpc GetMessage(GetMessageRequest) returns (Message) {
+  option (google.api.http).get = "/v1/messages/{message_id}/{sub.subfield}";
+  } } message GetMessageRequest {   message SubMessage {     string subfield =
+  1;   }   string message_id = 1; // mapped to the URL   SubMessage sub = 2;
+  // `sub.subfield` is url-mapped } message Message {   string text = 1; //
+  content of the resource } ```  This definition enables an automatic,
+  bidrectional mapping of HTTP JSON to RPC. Example:  HTTP | RPC -----|-----
+  `GET /v1/messages/123456/foo`  | `GetMessage(message_id: "123456" sub:
+  SubMessage(subfield: "foo"))`  In general, not only fields but also field
+  paths can be referenced from a path pattern. Fields mapped to the path
+  pattern cannot be repeated and must have a primitive (non-message) type.
+  Any fields in the request message which are not bound by the path pattern
+  automatically become (optional) HTTP query parameters. Assume the following
+  definition of the request message:  ```proto message GetMessageRequest {
+  message SubMessage {     string subfield = 1;   }   string message_id = 1;
+  // mapped to the URL   int64 revision = 2;    // becomes a parameter
+  SubMessage sub = 3;    // `sub.subfield` becomes a parameter } ```  This
+  enables a HTTP JSON to RPC mapping as below:  HTTP | RPC -----|----- `GET
+  /v1/messages/123456?revision=2&sub.subfield=foo` | `GetMessage(message_id:
+  "123456" revision: 2 sub: SubMessage(subfield: "foo"))`  Note that fields
+  which are mapped to HTTP parameters must have a primitive type or a repeated
+  primitive type. Message types are not allowed. In the case of a repeated
+  type, the parameter can be repeated in the URL, as in `...?param=A¶m=B`.
+  For HTTP method kinds which allow a request body, the `body` field specifies
+  the mapping. Consider a REST update method on the message resource
+  collection:  ```proto service Messaging {   rpc
+  UpdateMessage(UpdateMessageRequest) returns (Message) {     option
+  (google.api.http) = {       put: "/v1/messages/{message_id}"       body:
+  "message"     };   } } message UpdateMessageRequest {   string message_id =
+  1; // mapped to the URL   Message message = 2;   // mapped to the body } ```
+  The following HTTP JSON to RPC mapping is enabled, where the representation
+  of the JSON in the request body is determined by protos JSON encoding:  HTTP
+  | RPC -----|----- `PUT /v1/messages/123456 { "text": "Hi!" }` |
+  `UpdateMessage(message_id: "123456" message { text: "Hi!" })`  The special
+  name `*` can be used in the body mapping to define that every field not
+  bound by the path template should be mapped to the request body.  This
+  enables the following alternative definition of the update method:  ```proto
+  service Messaging {   rpc UpdateMessage(Message) returns (Message) {
+  option (google.api.http) = {       put: "/v1/messages/{message_id}"
+  body: "*"     };   } } message Message {   string message_id = 1;   string
+  text = 2; } ```  The following HTTP JSON to RPC mapping is enabled:  HTTP |
+  RPC -----|----- `PUT /v1/messages/123456 { "text": "Hi!" }` |
+  `UpdateMessage(message_id: "123456" text: "Hi!")`  Note that when using `*`
+  in the body mapping, it is not possible to have HTTP parameters, as all
+  fields not bound by the path end in the body. This makes this option more
+  rarely used in practice of defining REST APIs. The common usage of `*` is in
+  custom methods which don't use the URL at all for transferring data.  It is
+  possible to define multiple HTTP methods for one RPC by using the
+  `additional_bindings` option. Example:  ```proto service Messaging {   rpc
+  GetMessage(GetMessageRequest) returns (Message) {     option
+  (google.api.http) = {       get: "/v1/messages/{message_id}"
+  additional_bindings {         get:
+  "/v1/users/{user_id}/messages/{message_id}"       }     };   } } message
+  GetMessageRequest {   string message_id = 1;   string user_id = 2; } ```
+  This enables the following two alternative HTTP JSON to RPC mappings:  HTTP
+  | RPC -----|----- `GET /v1/messages/123456` | `GetMessage(message_id:
+  "123456")` `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me"
+  message_id: "123456")`  # Rules for HTTP mapping  The rules for mapping HTTP
+  path, query parameters, and body fields to the request message are as
+  follows:  1. The `body` field specifies either `*` or a field path, or is
+  omitted. If omitted, it assumes there is no HTTP body. 2. Leaf fields
+  (recursive expansion of nested messages in the    request) can be classified
+  into three types:     (a) Matched in the URL template.     (b) Covered by
+  body (if body is `*`, everything except (a) fields;         else everything
+  under the body field)     (c) All other fields. 3. URL query parameters
+  found in the HTTP request are mapped to (c) fields. 4. Any body sent with an
+  HTTP request can contain only (b) fields.  The syntax of the path template
+  is as follows:      Template = "/" Segments [ Verb ] ;     Segments =
+  Segment { "/" Segment } ;     Segment  = "*" | "**" | LITERAL | Variable ;
+  Variable = "{" FieldPath [ "=" Segments ] "}" ;     FieldPath = IDENT { "."
+  IDENT } ;     Verb     = ":" LITERAL ;  The syntax `*` matches a single path
+  segment. It follows the semantics of [RFC
+  6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String
+  Expansion.  The syntax `**` matches zero or more path segments. It follows
+  the semantics of [RFC 6570](https://tools.ietf.org/html/rfc6570) Section
+  3.2.3 Reserved Expansion.  The syntax `LITERAL` matches literal text in the
+  URL path.  The syntax `Variable` matches the entire path as specified by its
+  template; this nested template must not contain further variables. If a
+  variable matches a single path segment, its template may be omitted, e.g.
+  `{var}` is equivalent to `{var=*}`.  NOTE: the field paths in variables and
+  in the `body` must not refer to repeated fields or map fields.  Use
+  CustomHttpPattern to specify any HTTP method that is not included in the
+  `pattern` field, such as HEAD, or "*" to leave the HTTP method unspecified
+  for a given URL path rule. The wild-card rule is useful for services that
+  provide content to Web (HTML) clients.
+
+  Fields:
+    additionalBindings: Additional HTTP bindings for the selector. Nested
+      bindings must not contain an `additional_bindings` field themselves
+      (that is, the nesting may only be one level deep).
+    body: The name of the request field whose value is mapped to the HTTP
+      body, or `*` for mapping all fields not captured by the path pattern to
+      the HTTP body. NOTE: the referred field must not be a repeated field.
+    custom: Custom pattern is used for defining custom verbs.
+    delete: Used for deleting a resource.
+    get: Used for listing and getting information about resources.
+    mediaDownload: Do not use this. For media support, add instead
+      [][google.bytestream.RestByteStream] as an API to your configuration.
+    mediaUpload: Do not use this. For media support, add instead
+      [][google.bytestream.RestByteStream] as an API to your configuration.
+    patch: Used for updating a resource.
+    post: Used for creating a resource.
+    put: Used for updating a resource.
+    selector: Selects methods to which this rule applies.  Refer to selector
+      for syntax details.
+  """
+
+  additionalBindings = _messages.MessageField('HttpRule', 1, repeated=True)
+  body = _messages.StringField(2)
+  custom = _messages.MessageField('CustomHttpPattern', 3)
+  delete = _messages.StringField(4)
+  get = _messages.StringField(5)
+  mediaDownload = _messages.MessageField('MediaDownload', 6)
+  mediaUpload = _messages.MessageField('MediaUpload', 7)
+  patch = _messages.StringField(8)
+  post = _messages.StringField(9)
+  put = _messages.StringField(10)
+  selector = _messages.StringField(11)
+
+
+class LabelDescriptor(_messages.Message):
+  """A description of a label.
+
+  Enums:
+    ValueTypeValueValuesEnum: The type of data that can be assigned to the
+      label.
+
+  Fields:
+    description: A human-readable description for the label.
+    key: The label key.
+    valueType: The type of data that can be assigned to the label.
+  """
+
+  class ValueTypeValueValuesEnum(_messages.Enum):
+    """The type of data that can be assigned to the label.
+
+    Values:
+      STRING: A variable-length string. This is the default.
+      BOOL: Boolean; true or false.
+      INT64: A 64-bit signed integer.
+    """
+    STRING = 0
+    BOOL = 1
+    INT64 = 2
+
+  description = _messages.StringField(1)
+  key = _messages.StringField(2)
+  valueType = _messages.EnumField('ValueTypeValueValuesEnum', 3)
+
+
+class ListServiceConfigsResponse(_messages.Message):
+  """Response message for ListServiceConfigs method.
+
+  Fields:
+    nextPageToken: The token of the next page of results.
+    serviceConfigs: The list of service config resources.
+  """
+
+  nextPageToken = _messages.StringField(1)
+  serviceConfigs = _messages.MessageField('Service', 2, repeated=True)
+
+
+class ListServicesResponse(_messages.Message):
+  """Response message for `ListServices` method.
+
+  Fields:
+    nextPageToken: Token that can be passed to `ListServices` to resume a
+      paginated query.
+    services: The results of the query.
+  """
+
+  nextPageToken = _messages.StringField(1)
+  services = _messages.MessageField('ManagedService', 2, repeated=True)
+
+
+class LogDescriptor(_messages.Message):
+  """A description of a log type. Example in YAML format:      - name:
+  library.googleapis.com/activity_history       description: The history of
+  borrowing and returning library items.       display_name: Activity
+  labels:       - key: /customer_id         description: Identifier of a
+  library customer
+
+  Fields:
+    description: A human-readable description of this log. This information
+      appears in the documentation and can contain details.
+    displayName: The human-readable name for this log. This information
+      appears on the user interface and should be concise.
+    labels: The set of labels that are available to describe a specific log
+      entry. Runtime requests that contain labels not specified here are
+      considered invalid.
+    name: The name of the log. It must be less than 512 characters long and
+      can include the following characters: upper- and lower-case alphanumeric
+      characters [A-Za-z0-9], and punctuation characters including slash,
+      underscore, hyphen, period [/_-.].
+  """
+
+  description = _messages.StringField(1)
+  displayName = _messages.StringField(2)
+  labels = _messages.MessageField('LabelDescriptor', 3, repeated=True)
+  name = _messages.StringField(4)
+
+
+class Logging(_messages.Message):
+  """Logging configuration of the service.  The following example shows how to
+  configure logs to be sent to the producer and consumer projects. In the
+  example, the `library.googleapis.com/activity_history` log is sent to both
+  the producer and consumer projects, whereas the
+  `library.googleapis.com/purchase_history` log is only sent to the producer
+  project:      monitored_resources:     - type: library.googleapis.com/branch
+  labels:       - key: /city         description: The city where the library
+  branch is located in.       - key: /name         description: The name of
+  the branch.     logs:     - name: library.googleapis.com/activity_history
+  labels:       - key: /customer_id     - name:
+  library.googleapis.com/purchase_history     logging:
+  producer_destinations:       - monitored_resource:
+  library.googleapis.com/branch         logs:         -
+  library.googleapis.com/activity_history         -
+  library.googleapis.com/purchase_history       consumer_destinations:       -
+  monitored_resource: library.googleapis.com/branch         logs:         -
+  library.googleapis.com/activity_history
+
+  Fields:
+    consumerDestinations: Logging configurations for sending logs to the
+      consumer project. There can be multiple consumer destinations, each one
+      must have a different monitored resource type. A log can be used in at
+      most one consumer destination.
+    producerDestinations: Logging configurations for sending logs to the
+      producer project. There can be multiple producer destinations, each one
+      must have a different monitored resource type. A log can be used in at
+      most one producer destination.
+  """
+
+  consumerDestinations = _messages.MessageField('LoggingDestination', 1, repeated=True)
+  producerDestinations = _messages.MessageField('LoggingDestination', 2, repeated=True)
+
+
+class LoggingDestination(_messages.Message):
+  """Configuration of a specific logging destination (the producer project or
+  the consumer project).
+
+  Fields:
+    logs: Names of the logs to be sent to this destination. Each name must be
+      defined in the Service.logs section.
+    monitoredResource: The monitored resource type. The type must be defined
+      in Service.monitored_resources section.
+  """
+
+  logs = _messages.StringField(1, repeated=True)
+  monitoredResource = _messages.StringField(2)
+
+
+class ManagedService(_messages.Message):
+  """The full representation of an API Service that is managed by the
+  `ServiceManager` API.  Includes both the service configuration, as well as
+  other control plane deployment related information.
+
+  Fields:
+    configSource: User-supplied source configuration for the service. This is
+      distinct from the generated configuration provided in
+      `google.api.Service`. This is NOT populated on GetService calls at the
+      moment. NOTE: Any upsert operation that contains both a service_config
+      and a config_source is considered invalid and will result in an error
+      being returned.
+    generation: A server-assigned monotonically increasing number that changes
+      whenever a mutation is made to the `ManagedService` or any of its
+      components via the `ServiceManager` API.
+    operations: Read-only view of pending operations affecting this resource,
+      if requested.
+    producerProjectId: ID of the project that produces and owns this service.
+    projectSettings: Read-only view of settings for a particular consumer
+      project, if requested.
+    serviceConfig: The service's generated configuration.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  This name must match `google.api.Service.name`
+      in the `service_config` field.
+  """
+
+  configSource = _messages.MessageField('ConfigSource', 1)
+  generation = _messages.IntegerField(2)
+  operations = _messages.MessageField('Operation', 3, repeated=True)
+  producerProjectId = _messages.StringField(4)
+  projectSettings = _messages.MessageField('ProjectSettings', 5)
+  serviceConfig = _messages.MessageField('Service', 6)
+  serviceName = _messages.StringField(7)
+
+
+class MediaDownload(_messages.Message):
+  """Do not use this. For media support, add instead
+  [][google.bytestream.RestByteStream] as an API to your configuration.
+
+  Fields:
+    enabled: Whether download is enabled.
+  """
+
+  enabled = _messages.BooleanField(1)
+
+
+class MediaUpload(_messages.Message):
+  """Do not use this. For media support, add instead
+  [][google.bytestream.RestByteStream] as an API to your configuration.
+
+  Fields:
+    enabled: Whether upload is enabled.
+  """
+
+  enabled = _messages.BooleanField(1)
+
+
+class Method(_messages.Message):
+  """Method represents a method of an api.
+
+  Enums:
+    SyntaxValueValuesEnum: The source syntax of this method.
+
+  Fields:
+    name: The simple name of this method.
+    options: Any metadata attached to the method.
+    requestStreaming: If true, the request is streamed.
+    requestTypeUrl: A URL of the input message type.
+    responseStreaming: If true, the response is streamed.
+    responseTypeUrl: The URL of the output message type.
+    syntax: The source syntax of this method.
+  """
+
+  class SyntaxValueValuesEnum(_messages.Enum):
+    """The source syntax of this method.
+
+    Values:
+      SYNTAX_PROTO2: Syntax `proto2`.
+      SYNTAX_PROTO3: Syntax `proto3`.
+    """
+    SYNTAX_PROTO2 = 0
+    SYNTAX_PROTO3 = 1
+
+  name = _messages.StringField(1)
+  options = _messages.MessageField('Option', 2, repeated=True)
+  requestStreaming = _messages.BooleanField(3)
+  requestTypeUrl = _messages.StringField(4)
+  responseStreaming = _messages.BooleanField(5)
+  responseTypeUrl = _messages.StringField(6)
+  syntax = _messages.EnumField('SyntaxValueValuesEnum', 7)
+
+
+class MetricDescriptor(_messages.Message):
+  """Defines a metric type and its schema.
+
+  Enums:
+    MetricKindValueValuesEnum: Whether the metric records instantaneous
+      values, changes to a value, etc.
+    ValueTypeValueValuesEnum: Whether the measurement is an integer, a
+      floating-point number, etc.
+
+  Fields:
+    description: A detailed description of the metric, which can be used in
+      documentation.
+    displayName: A concise name for the metric, which can be displayed in user
+      interfaces. Use sentence case without an ending period, for example
+      "Request count".
+    labels: The set of labels that can be used to describe a specific instance
+      of this metric type. For example, the
+      `compute.googleapis.com/instance/network/received_bytes_count` metric
+      type has a label, `loadbalanced`, that specifies whether the traffic was
+      received through a load balanced IP address.
+    metricKind: Whether the metric records instantaneous values, changes to a
+      value, etc.
+    name: Resource name. The format of the name may vary between different
+      implementations. For examples:
+      projects/{project_id}/metricDescriptors/{type=**}
+      metricDescriptors/{type=**}
+    type: The metric type including a DNS name prefix, for example
+      `"compute.googleapis.com/instance/cpu/utilization"`. Metric types should
+      use a natural hierarchical grouping such as the following:
+      compute.googleapis.com/instance/cpu/utilization
+      compute.googleapis.com/instance/disk/read_ops_count
+      compute.googleapis.com/instance/network/received_bytes_count  Note that
+      if the metric type changes, the monitoring data will be discontinued,
+      and anything depends on it will break, such as monitoring dashboards,
+      alerting rules and quota limits. Therefore, once a metric has been
+      published, its type should be immutable.
+    unit: The unit in which the metric value is reported. It is only
+      applicable if the `value_type` is `INT64`, `DOUBLE`, or `DISTRIBUTION`.
+      The supported units are a subset of [The Unified Code for Units of
+      Measure](http://unitsofmeasure.org/ucum.html) standard:  **Basic units
+      (UNIT)**  * `bit`   bit * `By`    byte * `s`     second * `min`   minute
+      * `h`     hour * `d`     day  **Prefixes (PREFIX)**  * `k`     kilo
+      (10**3) * `M`     mega    (10**6) * `G`     giga    (10**9) * `T`
+      tera    (10**12) * `P`     peta    (10**15) * `E`     exa     (10**18) *
+      `Z`     zetta   (10**21) * `Y`     yotta   (10**24) * `m`     milli
+      (10**-3) * `u`     micro   (10**-6) * `n`     nano    (10**-9) * `p`
+      pico    (10**-12) * `f`     femto   (10**-15) * `a`     atto
+      (10**-18) * `z`     zepto   (10**-21) * `y`     yocto   (10**-24) * `Ki`
+      kibi    (2**10) * `Mi`    mebi    (2**20) * `Gi`    gibi    (2**30) *
+      `Ti`    tebi    (2**40)  **Grammar**  The grammar includes the
+      dimensionless unit `1`, such as `1/s`.  The grammar also includes these
+      connectors:  * `/`    division (as an infix operator, e.g. `1/s`). * `.`
+      multiplication (as an infix operator, e.g. `GBy.d`)  The grammar for a
+      unit is as follows:      Expression = Component { "." Component } { "/"
+      Component } ;      Component = [ PREFIX ] UNIT [ Annotation ]
+      | Annotation               | "1"               ;      Annotation = "{"
+      NAME "}" ;  Notes:  * `Annotation` is just a comment if it follows a
+      `UNIT` and is    equivalent to `1` if it is used alone. For examples,
+      `{requests}/s == 1/s`, `By{transmitted}/s == By/s`. * `NAME` is a
+      sequence of non-blank printable ASCII characters not    containing '{'
+      or '}'.
+    valueType: Whether the measurement is an integer, a floating-point number,
+      etc.
+  """
+
+  class MetricKindValueValuesEnum(_messages.Enum):
+    """Whether the metric records instantaneous values, changes to a value,
+    etc.
+
+    Values:
+      METRIC_KIND_UNSPECIFIED: Do not use this default value.
+      GAUGE: Instantaneous measurements of a varying quantity.
+      DELTA: Changes over non-overlapping time intervals.
+      CUMULATIVE: Cumulative value over time intervals that can overlap. The
+        overlapping intervals must have the same start time.
+    """
+    METRIC_KIND_UNSPECIFIED = 0
+    GAUGE = 1
+    DELTA = 2
+    CUMULATIVE = 3
+
+  class ValueTypeValueValuesEnum(_messages.Enum):
+    """Whether the measurement is an integer, a floating-point number, etc.
+
+    Values:
+      VALUE_TYPE_UNSPECIFIED: Do not use this default value.
+      BOOL: The value is a boolean. This value type can be used only if the
+        metric kind is `GAUGE`.
+      INT64: The value is a signed 64-bit integer.
+      DOUBLE: The value is a double precision floating point number.
+      STRING: The value is a text string. This value type can be used only if
+        the metric kind is `GAUGE`.
+      DISTRIBUTION: The value is a `Distribution`.
+      MONEY: The value is money.
+    """
+    VALUE_TYPE_UNSPECIFIED = 0
+    BOOL = 1
+    INT64 = 2
+    DOUBLE = 3
+    STRING = 4
+    DISTRIBUTION = 5
+    MONEY = 6
+
+  description = _messages.StringField(1)
+  displayName = _messages.StringField(2)
+  labels = _messages.MessageField('LabelDescriptor', 3, repeated=True)
+  metricKind = _messages.EnumField('MetricKindValueValuesEnum', 4)
+  name = _messages.StringField(5)
+  type = _messages.StringField(6)
+  unit = _messages.StringField(7)
+  valueType = _messages.EnumField('ValueTypeValueValuesEnum', 8)
+
+
+class Mixin(_messages.Message):
+  """Declares an API to be included in this API. The including API must
+  redeclare all the methods from the included API, but documentation and
+  options are inherited as follows:  - If after comment and whitespace
+  stripping, the documentation   string of the redeclared method is empty, it
+  will be inherited   from the original method.  - Each annotation belonging
+  to the service config (http,   visibility) which is not set in the
+  redeclared method will be   inherited.  - If an http annotation is
+  inherited, the path pattern will be   modified as follows. Any version
+  prefix will be replaced by the   version of the including API plus the root
+  path if specified.  Example of a simple mixin:      package google.acl.v1;
+  service AccessControl {       // Get the underlying ACL object.       rpc
+  GetAcl(GetAclRequest) returns (Acl) {         option (google.api.http).get =
+  "/v1/{resource=**}:getAcl";       }     }      package google.storage.v2;
+  service Storage {       //       rpc GetAcl(GetAclRequest) returns (Acl);
+  // Get a data record.       rpc GetData(GetDataRequest) returns (Data) {
+  option (google.api.http).get = "/v2/{resource=**}";       }     }  Example
+  of a mixin configuration:      apis:     - name: google.storage.v2.Storage
+  mixins:       - name: google.acl.v1.AccessControl  The mixin construct
+  implies that all methods in `AccessControl` are also declared with same name
+  and request/response types in `Storage`. A documentation generator or
+  annotation processor will see the effective `Storage.GetAcl` method after
+  inherting documentation and annotations as follows:      service Storage {
+  // Get the underlying ACL object.       rpc GetAcl(GetAclRequest) returns
+  (Acl) {         option (google.api.http).get = "/v2/{resource=**}:getAcl";
+  }       ...     }  Note how the version in the path pattern changed from
+  `v1` to `v2`.  If the `root` field in the mixin is specified, it should be a
+  relative path under which inherited HTTP paths are placed. Example:
+  apis:     - name: google.storage.v2.Storage       mixins:       - name:
+  google.acl.v1.AccessControl         root: acls  This implies the following
+  inherited HTTP annotation:      service Storage {       // Get the
+  underlying ACL object.       rpc GetAcl(GetAclRequest) returns (Acl) {
+  option (google.api.http).get = "/v2/acls/{resource=**}:getAcl";       }
+  ...     }
+
+  Fields:
+    name: The fully qualified name of the API which is included.
+    root: If non-empty specifies a path under which inherited HTTP paths are
+      rooted.
+  """
+
+  name = _messages.StringField(1)
+  root = _messages.StringField(2)
+
+
+class MonitoredResourceDescriptor(_messages.Message):
+  """An object that describes the schema of a MonitoredResource object using a
+  type name and a set of labels.  For example, the monitored resource
+  descriptor for Google Compute Engine VM instances has a type of
+  `"gce_instance"` and specifies the use of the labels `"instance_id"` and
+  `"zone"` to identify particular VM instances.  Different APIs can support
+  different monitored resource types. APIs generally provide a `list` method
+  that returns the monitored resource descriptors used by the API.
+
+  Fields:
+    description: Optional. A detailed description of the monitored resource
+      type that might be used in documentation.
+    displayName: Optional. A concise name for the monitored resource type that
+      might be displayed in user interfaces. For example, `"Google Cloud SQL
+      Database"`.
+    labels: Required. A set of labels used to describe instances of this
+      monitored resource type. For example, an individual Google Cloud SQL
+      database is identified by values for the labels `"database_id"` and
+      `"zone"`.
+    name: Optional. The resource name of the monitored resource descriptor:
+      `"projects/{project_id}/monitoredResourceDescriptors/{type}"` where
+      {type} is the value of the `type` field in this object and {project_id}
+      is a project ID that provides API-specific context for accessing the
+      type.  APIs that do not use project information can use the resource
+      name format `"monitoredResourceDescriptors/{type}"`.
+    type: Required. The monitored resource type. For example, the type
+      `"cloudsql_database"` represents databases in Google Cloud SQL. The
+      maximum length of this value is 256 characters.
+  """
+
+  description = _messages.StringField(1)
+  displayName = _messages.StringField(2)
+  labels = _messages.MessageField('LabelDescriptor', 3, repeated=True)
+  name = _messages.StringField(4)
+  type = _messages.StringField(5)
+
+
+class Monitoring(_messages.Message):
+  """Monitoring configuration of the service.  The example below shows how to
+  configure monitored resources and metrics for monitoring. In the example, a
+  monitored resource and two metrics are defined. The
+  `library.googleapis.com/book/returned_count` metric is sent to both producer
+  and consumer projects, whereas the
+  `library.googleapis.com/book/overdue_count` metric is only sent to the
+  consumer project.      monitored_resources:     - type:
+  library.googleapis.com/branch       labels:       - key: /city
+  description: The city where the library branch is located in.       - key:
+  /name         description: The name of the branch.     metrics:     - name:
+  library.googleapis.com/book/returned_count       metric_kind: DELTA
+  value_type: INT64       labels:       - key: /customer_id     - name:
+  library.googleapis.com/book/overdue_count       metric_kind: GAUGE
+  value_type: INT64       labels:       - key: /customer_id     monitoring:
+  producer_destinations:       - monitored_resource:
+  library.googleapis.com/branch         metrics:         -
+  library.googleapis.com/book/returned_count       consumer_destinations:
+  - monitored_resource: library.googleapis.com/branch         metrics:
+  - library.googleapis.com/book/returned_count         -
+  library.googleapis.com/book/overdue_count
+
+  Fields:
+    consumerDestinations: Monitoring configurations for sending metrics to the
+      consumer project. There can be multiple consumer destinations, each one
+      must have a different monitored resource type. A metric can be used in
+      at most one consumer destination.
+    producerDestinations: Monitoring configurations for sending metrics to the
+      producer project. There can be multiple producer destinations, each one
+      must have a different monitored resource type. A metric can be used in
+      at most one producer destination.
+  """
+
+  consumerDestinations = _messages.MessageField('MonitoringDestination', 1, repeated=True)
+  producerDestinations = _messages.MessageField('MonitoringDestination', 2, repeated=True)
+
+
+class MonitoringDestination(_messages.Message):
+  """Configuration of a specific monitoring destination (the producer project
+  or the consumer project).
+
+  Fields:
+    metrics: Names of the metrics to report to this monitoring destination.
+      Each name must be defined in Service.metrics section.
+    monitoredResource: The monitored resource type. The type must be defined
+      in Service.monitored_resources section.
+  """
+
+  metrics = _messages.StringField(1, repeated=True)
+  monitoredResource = _messages.StringField(2)
+
+
+class OAuthRequirements(_messages.Message):
+  """OAuth scopes are a way to define data and permissions on data. For
+  example, there are scopes defined for "Read-only access to Google Calendar"
+  and "Access to Cloud Platform". Users can consent to a scope for an
+  application, giving it permission to access that data on their behalf.
+  OAuth scope specifications should be fairly coarse grained; a user will need
+  to see and understand the text description of what your scope means.  In
+  most cases: use one or at most two OAuth scopes for an entire family of
+  products. If your product has multiple APIs, you should probably be sharing
+  the OAuth scope across all of those APIs.  When you need finer grained OAuth
+  consent screens: talk with your product management about how developers will
+  use them in practice.  Please note that even though each of the canonical
+  scopes is enough for a request to be accepted and passed to the backend, a
+  request can still fail due to the backend requiring additional scopes or
+  permissions.
+
+  Fields:
+    canonicalScopes: The list of publicly documented OAuth scopes that are
+      allowed access. An OAuth token containing any of these scopes will be
+      accepted.  Example:       canonical_scopes:
+      https://www.googleapis.com/auth/calendar,
+      https://www.googleapis.com/auth/calendar.read
+  """
+
+  canonicalScopes = _messages.StringField(1)
+
+
+class OpenApiSpec(_messages.Message):
+  """A collection of OpenAPI specification files.
+
+  Fields:
+    openApiFiles: Individual files.
+  """
+
+  openApiFiles = _messages.MessageField('ConfigFile', 1, repeated=True)
+
+
+class Operation(_messages.Message):
+  """This resource represents a long-running operation that is the result of a
+  network API call.
+
+  Messages:
+    MetadataValue: Service-specific metadata associated with the operation.
+      It typically contains progress information and common metadata such as
+      create time. Some services might not provide such metadata.  Any method
+      that returns a long-running operation should document the metadata type,
+      if any.
+    ResponseValue: The normal response of the operation in case of success.
+      If the original method returns no data on success, such as `Delete`, the
+      response is `google.protobuf.Empty`.  If the original method is standard
+      `Get`/`Create`/`Update`, the response should be the resource.  For other
+      methods, the response should have the type `XxxResponse`, where `Xxx` is
+      the original method name.  For example, if the original method name is
+      `TakeSnapshot()`, the inferred response type is `TakeSnapshotResponse`.
+
+  Fields:
+    done: If the value is `false`, it means the operation is still in
+      progress. If true, the operation is completed, and either `error` or
+      `response` is available.
+    error: The error result of the operation in case of failure.
+    metadata: Service-specific metadata associated with the operation.  It
+      typically contains progress information and common metadata such as
+      create time. Some services might not provide such metadata.  Any method
+      that returns a long-running operation should document the metadata type,
+      if any.
+    name: The server-assigned name, which is only unique within the same
+      service that originally returns it. If you use the default HTTP mapping,
+      the `name` should have the format of `operations/some/unique/name`.
+    response: The normal response of the operation in case of success.  If the
+      original method returns no data on success, such as `Delete`, the
+      response is `google.protobuf.Empty`.  If the original method is standard
+      `Get`/`Create`/`Update`, the response should be the resource.  For other
+      methods, the response should have the type `XxxResponse`, where `Xxx` is
+      the original method name.  For example, if the original method name is
+      `TakeSnapshot()`, the inferred response type is `TakeSnapshotResponse`.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class MetadataValue(_messages.Message):
+    """Service-specific metadata associated with the operation.  It typically
+    contains progress information and common metadata such as create time.
+    Some services might not provide such metadata.  Any method that returns a
+    long-running operation should document the metadata type, if any.
+
+    Messages:
+      AdditionalProperty: An additional property for a MetadataValue object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a MetadataValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ResponseValue(_messages.Message):
+    """The normal response of the operation in case of success.  If the
+    original method returns no data on success, such as `Delete`, the response
+    is `google.protobuf.Empty`.  If the original method is standard
+    `Get`/`Create`/`Update`, the response should be the resource.  For other
+    methods, the response should have the type `XxxResponse`, where `Xxx` is
+    the original method name.  For example, if the original method name is
+    `TakeSnapshot()`, the inferred response type is `TakeSnapshotResponse`.
+
+    Messages:
+      AdditionalProperty: An additional property for a ResponseValue object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ResponseValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  done = _messages.BooleanField(1)
+  error = _messages.MessageField('Status', 2)
+  metadata = _messages.MessageField('MetadataValue', 3)
+  name = _messages.StringField(4)
+  response = _messages.MessageField('ResponseValue', 5)
+
+
+class OperationMetadata(_messages.Message):
+  """The metadata associated with a long running operation resource.
+
+  Fields:
+    progressPercentage: Percentage of completion of this operation, ranging
+      from 0 to 100.
+    resourceNames: The full name of the resources that this operation is
+      directly associated with.
+    startTime: The start time of the operation.
+    steps: Detailed status information for each step. The order is
+      undetermined.
+  """
+
+  progressPercentage = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+  resourceNames = _messages.StringField(2, repeated=True)
+  startTime = _messages.StringField(3)
+  steps = _messages.MessageField('Step', 4, repeated=True)
+
+
+class Option(_messages.Message):
+  """A protocol buffer option, which can be attached to a message, field,
+  enumeration, etc.
+
+  Messages:
+    ValueValue: The option's value. For example, `"com.google.protobuf"`.
+
+  Fields:
+    name: The option's name. For example, `"java_package"`.
+    value: The option's value. For example, `"com.google.protobuf"`.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ValueValue(_messages.Message):
+    """The option's value. For example, `"com.google.protobuf"`.
+
+    Messages:
+      AdditionalProperty: An additional property for a ValueValue object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ValueValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  name = _messages.StringField(1)
+  value = _messages.MessageField('ValueValue', 2)
+
+
+class Page(_messages.Message):
+  """Represents a documentation page. A page can contain subpages to represent
+  nested documentation set structure.
+
+  Fields:
+    content: The Markdown content of the page. You can use (==
+      include {path} ==) to include content from a Markdown file.
+    name: The name of the page. It will be used as an identity of the page to
+      generate URI of the page, text of the link to this page in navigation,
+      etc. The full page name (start from the root page name to this page
+      concatenated with `.`) can be used as reference to the page in your
+      documentation. For example: 
pages: - name: Tutorial
+      content: (== include tutorial.md ==)   subpages:   - name: Java
+      content: (== include tutorial_java.md ==) 
 You can
+      reference `Java` page using Markdown reference link syntax: `Java`.
+    subpages: Subpages of this page. The order of subpages specified here will
+      be honored in the generated docset.
+  """
+
+  content = _messages.StringField(1)
+  name = _messages.StringField(2)
+  subpages = _messages.MessageField('Page', 3, repeated=True)
+
+
+class ProjectProperties(_messages.Message):
+  """A descriptor for defining project properties for a service. One service
+  may have many consumer projects, and the service may want to behave
+  differently depending on some properties on the project. For example, a
+  project may be associated with a school, or a business, or a government
+  agency, a business type property on the project may affect how a service
+  responds to the client. This descriptor defines which properties are allowed
+  to be set on a project.  Example:     project_properties:      properties:
+  - name: NO_WATERMARK        type: BOOL        description: Allows usage of
+  the API without watermarks.      - name: EXTENDED_TILE_CACHE_PERIOD
+  type: INT64
+
+  Fields:
+    properties: List of per consumer project-specific properties.
+  """
+
+  properties = _messages.MessageField('Property', 1, repeated=True)
+
+
+class ProjectSettings(_messages.Message):
+  """Settings that control how a consumer project uses a service.
+
+  Messages:
+    PropertiesValue: Service-defined per-consumer properties.  A key-value
+      mapping a string key to a google.protobuf.ListValue proto. Values in the
+      list are typed as defined in the Service configuration's
+      consumer.properties field.
+
+  Fields:
+    consumerProjectId: ID for the project consuming this service.
+    operations: Read-only view of pending operations affecting this resource,
+      if requested.
+    properties: Service-defined per-consumer properties.  A key-value mapping
+      a string key to a google.protobuf.ListValue proto. Values in the list
+      are typed as defined in the Service configuration's consumer.properties
+      field.
+    quotaSettings: Settings that control how much or how fast the service can
+      be used by the consumer project.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.
+    usageSettings: Settings that control whether this service is usable by the
+      consumer project.
+    visibilitySettings: Settings that control which features of the service
+      are visible to the consumer project.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class PropertiesValue(_messages.Message):
+    """Service-defined per-consumer properties.  A key-value mapping a string
+    key to a google.protobuf.ListValue proto. Values in the list are typed as
+    defined in the Service configuration's consumer.properties field.
+
+    Messages:
+      AdditionalProperty: An additional property for a PropertiesValue object.
+
+    Fields:
+      additionalProperties: Additional properties of type PropertiesValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a PropertiesValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2, repeated=True)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  consumerProjectId = _messages.StringField(1)
+  operations = _messages.MessageField('Operation', 2, repeated=True)
+  properties = _messages.MessageField('PropertiesValue', 3)
+  quotaSettings = _messages.MessageField('QuotaSettings', 4)
+  serviceName = _messages.StringField(5)
+  usageSettings = _messages.MessageField('UsageSettings', 6)
+  visibilitySettings = _messages.MessageField('VisibilitySettings', 7)
+
+
+class Property(_messages.Message):
+  """Defines project properties.  API services can define properties that can
+  be assigned to consumer projects so that backends can perform response
+  customization without having to make additional calls or maintain additional
+  storage. For example, Maps API defines properties that controls map tile
+  cache period, or whether to embed a watermark in a result.  These values can
+  be set via API producer console. Only API providers can define and set these
+  properties.
+
+  Enums:
+    TypeValueValuesEnum: The type of this property.
+
+  Fields:
+    description: The description of the property
+    name: The name of the property (a.k.a key).
+    type: The type of this property.
+  """
+
+  class TypeValueValuesEnum(_messages.Enum):
+    """The type of this property.
+
+    Values:
+      UNSPECIFIED: The type is unspecified, and will result in an error.
+      INT64: The type is `int64`.
+      BOOL: The type is `bool`.
+      STRING: The type is `string`.
+      DOUBLE: The type is 'double'.
+    """
+    UNSPECIFIED = 0
+    INT64 = 1
+    BOOL = 2
+    STRING = 3
+    DOUBLE = 4
+
+  description = _messages.StringField(1)
+  name = _messages.StringField(2)
+  type = _messages.EnumField('TypeValueValuesEnum', 3)
+
+
+class ProtoDescriptor(_messages.Message):
+  """Contains a serialized protoc-generated protocol buffer message descriptor
+  set along with a URL that describes the type of the descriptor message.
+
+  Fields:
+    typeUrl: A URL/resource name whose content describes the type of the
+      serialized protocol buffer message.  Only
+      'type.googleapis.com/google.protobuf.FileDescriptorSet' is supported. If
+      the type_url is not specificed,
+      'type.googleapis.com/google.protobuf.FileDescriptorSet' will be assumed.
+    value: Must be a valid serialized protocol buffer descriptor set.  To
+      generate, use protoc with imports and source info included. For an
+      example test.proto file, the following command would put the value in a
+      new file named descriptor.pb.  $protoc --include_imports
+      --include_source_info test.proto -o descriptor.pb
+  """
+
+  typeUrl = _messages.StringField(1)
+  value = _messages.BytesField(2)
+
+
+class ProtoSpec(_messages.Message):
+  """A collection of protocol buffer service specification files.
+
+  Fields:
+    protoDescriptor: A complete descriptor of a protocol buffer specification
+  """
+
+  protoDescriptor = _messages.MessageField('ProtoDescriptor', 1)
+
+
+class QueryUserAccessResponse(_messages.Message):
+  """Request message for QueryUserAccess method.
+
+  Fields:
+    accessibleVisibilityLabels: Any visibility labels on the service that are
+      accessible by the user.
+    canAccessService: True if the user can access the service and any
+      unrestricted API surface.
+  """
+
+  accessibleVisibilityLabels = _messages.StringField(1, repeated=True)
+  canAccessService = _messages.BooleanField(2)
+
+
+class Quota(_messages.Message):
+  """Quota configuration helps to achieve fairness and budgeting in service
+  usage.  - Fairness is achieved through the use of short-term quota limits
+  that are usually defined over a time window of several seconds or   minutes.
+  When such a limit is applied, for example at the user   level, it ensures
+  that no single user will monopolize the service   or a given customer's
+  allocated portion of it. - Budgeting is achieved through the use of long-
+  term quota limits   that are usually defined over a time window of one or
+  more   days. These limits help client application developers predict the
+  usage and help budgeting.  Quota enforcement uses a simple token-based
+  algorithm for resource sharing.  The quota configuration structure is as
+  follows:  - `QuotaLimit` defines a single enforceable limit with a specified
+  token amount that can be consumed over a specific duration and   applies to
+  a particular entity, like a project or an end user. If   the limit applies
+  to a user, each user making the request will   get the specified number of
+  tokens to consume. When the tokens   run out, the requests from that user
+  will be blocked until the   duration elapses and the next duration window
+  starts.  - `QuotaGroup` groups a set of quota limits.  - `QuotaRule` maps a
+  method to a set of quota groups. This allows   sharing of quota groups
+  across methods as well as one method   consuming tokens from more than one
+  quota group. When a group   contains multiple limits, requests to a method
+  consuming tokens   from that group must satisfy all the limits in that
+  group.  Example:      quota:       groups:       - name: ReadGroup
+  limits:         - description: Daily Limit           name: ProjectQpd
+  default_limit: 10000           duration: 1d           limit_by:
+  CLIENT_PROJECT          - description: Per-second Limit           name:
+  UserQps           default_limit: 20000           duration: 100s
+  limit_by: USER        - name: WriteGroup         limits:         -
+  description: Daily Limit           name: ProjectQpd           default_limit:
+  1000           max_limit: 1000           duration: 1d           limit_by:
+  CLIENT_PROJECT          - description: Per-second Limit           name:
+  UserQps           default_limit: 2000           max_limit: 4000
+  duration: 100s           limit_by: USER        rules:       - selector: "*"
+  groups:         - group: ReadGroup       - selector:
+  google.calendar.Calendar.Update         groups:         - group: WriteGroup
+  cost: 2       - selector: google.calendar.Calendar.Delete         groups:
+  - group: WriteGroup  Here, the configuration defines two quota groups:
+  ReadGroup and WriteGroup, each defining its own daily and per-second limits.
+  Note that One Platform enforces per-second limits averaged over a duration
+  of 100 seconds. The rules map ReadGroup for all methods, except for the
+  Update and Delete methods. These two methods consume from WriteGroup, with
+  Update method consuming at twice the rate as Delete method.  Multiple quota
+  groups can be specified for a method. The quota limits in all of those
+  groups will be enforced. Example:      quota:       groups:       - name:
+  WriteGroup         limits:         - description: Daily Limit
+  name: ProjectQpd           default_limit: 1000           max_limit: 1000
+  duration: 1d           limit_by: CLIENT_PROJECT          - description: Per-
+  second Limit           name: UserQps           default_limit: 2000
+  max_limit: 4000           duration: 100s           limit_by: USER        -
+  name: StorageGroup         limits:         - description: Storage Quota
+  name: StorageQuota           default_limit: 1000           duration: 0
+  limit_by: USER        rules:       - selector:
+  google.calendar.Calendar.Create         groups:         - group:
+  StorageGroup         - group: WriteGroup       - selector:
+  google.calendar.Calendar.Delete         groups:         - group:
+  StorageGroup  In the above example, the Create and Delete methods manage the
+  user's storage space. In addition, Create method uses WriteGroup to manage
+  the requests. In this case, requests to Create method need to satisfy all
+  quota limits defined in both quota groups.  One can disable quota for
+  selected method(s) identified by the selector by setting disable_quota to
+  ture. For example,        rules:       - selector: "*"         group:
+  - group ReadGroup       - selector: google.calendar.Calendar.Select
+  disable_quota: true
+
+  Fields:
+    groups: List of `QuotaGroup` definitions for the service.
+    rules: List of `QuotaRule` definitions, each one mapping a selected method
+      to one or more quota groups.
+  """
+
+  groups = _messages.MessageField('QuotaGroup', 1, repeated=True)
+  rules = _messages.MessageField('QuotaRule', 2, repeated=True)
+
+
+class QuotaGroup(_messages.Message):
+  """`QuotaGroup` defines a set of quota limits to enforce.
+
+  Fields:
+    billable: Indicates if the quota limits defined in this quota group apply
+      to consumers who have active billing. Quota limits defined in billable
+      groups will be applied only to consumers who have active billing. The
+      amount of tokens consumed from billable quota group will also be
+      reported for billing. Quota limits defined in non-billable groups will
+      be applied only to consumers who have no active billing.
+    description: User-visible description of this quota group.
+    limits: Quota limits to be enforced when this quota group is used. A
+      request must satisfy all the limits in a group for it to be permitted.
+    name: Name of this quota group. Must be unique within the service.  Quota
+      group name is used as part of the id for quota limits. Once the quota
+      group has been put into use, the name of the quota group should be
+      immutable.
+  """
+
+  billable = _messages.BooleanField(1)
+  description = _messages.StringField(2)
+  limits = _messages.MessageField('QuotaLimit', 3, repeated=True)
+  name = _messages.StringField(4)
+
+
+class QuotaGroupMapping(_messages.Message):
+  """A quota group mapping.
+
+  Fields:
+    cost: Number of tokens to consume for each request. This allows different
+      cost to be associated with different methods that consume from the same
+      quota group. By default, each request will cost one token.
+    group: The `QuotaGroup.name` of the group. Requests for the mapped methods
+      will consume tokens from each of the limits defined in this group.
+  """
+
+  cost = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+  group = _messages.StringField(2)
+
+
+class QuotaInfo(_messages.Message):
+  """Metadata about an individual quota, containing usage and limit
+  information.
+
+  Fields:
+    currentUsage: The usage data for this quota as it applies to the current
+      limit.
+    historicalUsage: The historical usage data of this quota limit. Currently
+      it is only available for daily quota limit, that is, base_limit.duration
+      = "1d".
+    limit: The effective limit for this quota.
+  """
+
+  currentUsage = _messages.MessageField('QuotaUsage', 1)
+  historicalUsage = _messages.MessageField('QuotaUsage', 2, repeated=True)
+  limit = _messages.MessageField('EffectiveQuotaLimit', 3)
+
+
+class QuotaLimit(_messages.Message):
+  """`QuotaLimit` defines a specific limit that applies over a specified
+  duration for a limit type. There can be at most one limit for a duration and
+  limit type combination defined within a `QuotaGroup`.
+
+  Enums:
+    LimitByValueValuesEnum: Limit type to use for enforcing this quota limit.
+      Each unique value gets the defined number of tokens to consume from. For
+      a quota limit that uses user type, each user making requests through the
+      same client application project will get his/her own pool of tokens to
+      consume, whereas for a limit that uses client project type, all users
+      making requests through the same client application project share a
+      single pool of tokens.
+
+  Fields:
+    defaultLimit: Default number of tokens that can be consumed during the
+      specified duration. This is the number of tokens assigned when a client
+      application developer activates the service for his/her project.
+      Specifying a value of 0 will block all requests. This can be used if you
+      are provisioning quota to selected consumers and blocking others.
+      Similarly, a value of -1 will indicate an unlimited quota. No other
+      negative values are allowed.
+    description: Optional. User-visible, extended description for this quota
+      limit. Should be used only when more context is needed to understand
+      this limit than provided by the limit's display name (see:
+      `display_name`).
+    displayName: User-visible display name for this limit. Optional. If not
+      set, the UI will provide a default display name based on the quota
+      configuration. This field can be used to override the default display
+      name generated from the configuration.
+    duration: Duration of this limit in textual notation. Example: "100s",
+      "24h", "1d". For duration longer than a day, only multiple of days is
+      supported. We support only "100s" and "1d" for now. Additional support
+      will be added in the future. "0" indicates indefinite duration.
+    freeTier: Free tier value displayed in the Developers Console for this
+      limit. The free tier is the number of tokens that will be subtracted
+      from the billed amount when billing is enabled. This field can only be
+      set on a limit with duration "1d", in a billable group; it is invalid on
+      any other limit. If this field is not set, it defaults to 0, indicating
+      that there is no free tier for this service.
+    limitBy: Limit type to use for enforcing this quota limit. Each unique
+      value gets the defined number of tokens to consume from. For a quota
+      limit that uses user type, each user making requests through the same
+      client application project will get his/her own pool of tokens to
+      consume, whereas for a limit that uses client project type, all users
+      making requests through the same client application project share a
+      single pool of tokens.
+    maxLimit: Maximum number of tokens that can be consumed during the
+      specified duration. Client application developers can override the
+      default limit up to this maximum. If specified, this value cannot be set
+      to a value less than the default limit. If not specified, it is set to
+      the default limit.  To allow clients to apply overrides with no upper
+      bound, set this to -1, indicating unlimited maximum quota.
+    name: Name of the quota limit.  Must be unique within the quota group.
+      This name is used to refer to the limit when overriding the limit on a
+      per-project basis.  If a name is not provided, it will be generated from
+      the limit_by and duration fields.  The maximum length of the limit name
+      is 64 characters.  The name of a limit is used as a unique identifier
+      for this limit. Therefore, once a limit has been put into use, its name
+      should be immutable. You can use the display_name field to provide a
+      user-friendly name for the limit. The display name can be evolved over
+      time without affecting the identity of the limit.
+  """
+
+  class LimitByValueValuesEnum(_messages.Enum):
+    """Limit type to use for enforcing this quota limit. Each unique value
+    gets the defined number of tokens to consume from. For a quota limit that
+    uses user type, each user making requests through the same client
+    application project will get his/her own pool of tokens to consume,
+    whereas for a limit that uses client project type, all users making
+    requests through the same client application project share a single pool
+    of tokens.
+
+    Values:
+      CLIENT_PROJECT: ID of the project owned by the client application
+        developer making the request.
+      USER: ID of the end user making the request using the client
+        application.
+    """
+    CLIENT_PROJECT = 0
+    USER = 1
+
+  defaultLimit = _messages.IntegerField(1)
+  description = _messages.StringField(2)
+  displayName = _messages.StringField(3)
+  duration = _messages.StringField(4)
+  freeTier = _messages.IntegerField(5)
+  limitBy = _messages.EnumField('LimitByValueValuesEnum', 6)
+  maxLimit = _messages.IntegerField(7)
+  name = _messages.StringField(8)
+
+
+class QuotaLimitOverride(_messages.Message):
+  """Specifies a custom quota limit that is applied for this consumer project.
+  This overrides the default value in google.api.QuotaLimit.
+
+  Fields:
+    limit: The new limit for this project. May be -1 (unlimited), 0 (block),
+      or any positive integer.
+    unlimited: Indicates the override is to provide unlimited quota.  If true,
+      any value set for limit will be ignored. DEPRECATED. Use a limit value
+      of -1 instead.
+  """
+
+  limit = _messages.IntegerField(1)
+  unlimited = _messages.BooleanField(2)
+
+
+class QuotaRule(_messages.Message):
+  """`QuotaRule` maps a method to a set of `QuotaGroup`s.
+
+  Fields:
+    disableQuota: Indicates if quota checking should be enforced. Quota will
+      be disabled for methods without quota rules or with quota rules having
+      this field set to true. When this field is set to true, no quota group
+      mapping is allowed.
+    groups: Quota groups to be used for this method. This supports associating
+      a cost with each quota group.
+    selector: Selects methods to which this rule applies.  Refer to selector
+      for syntax details.
+  """
+
+  disableQuota = _messages.BooleanField(1)
+  groups = _messages.MessageField('QuotaGroupMapping', 2, repeated=True)
+  selector = _messages.StringField(3)
+
+
+class QuotaSettings(_messages.Message):
+  """Per-consumer overrides for quota settings. See google/api/quota.proto for
+  the corresponding service configuration which provides the default values.
+
+  Messages:
+    ConsumerOverridesValue: Quota overrides set by the consumer. Consumer
+      overrides will only have an effect up to the max_limit specified in the
+      service config, or the the producer override, if one exists.  The key
+      for this map is one of the following:  - '/' for
+      quotas defined within quota groups, where GROUP_NAME is the
+      google.api.QuotaGroup.name field and LIMIT_NAME is the
+      google.api.QuotaLimit.name field from the service config.  For example:
+      'ReadGroup/ProjectDaily'.  - '' for quotas defined without
+      quota groups, where LIMIT_NAME is the google.api.QuotaLimit.name field
+      from the service config. For example: 'borrowedCountPerOrganization'.
+    EffectiveQuotasValue: The effective quota limits for each group, derived
+      from the service defaults together with any producer or consumer
+      overrides. For each limit, the effective value is the minimum of the
+      producer and consumer overrides if either is present, or else the
+      service default if neither is present. DEPRECATED. Use
+      effective_quota_groups instead.
+    ProducerOverridesValue: Quota overrides set by the producer. Note that if
+      a consumer override is also specified, then the minimum of the two will
+      be used. This allows consumers to cap their usage voluntarily.  The key
+      for this map is one of the following:  - '/' for
+      quotas defined within quota groups, where GROUP_NAME is the
+      google.api.QuotaGroup.name field and LIMIT_NAME is the
+      google.api.QuotaLimit.name field from the service config.  For example:
+      'ReadGroup/ProjectDaily'.  - '' for quotas defined without
+      quota groups, where LIMIT_NAME is the google.api.QuotaLimit.name field
+      from the service config. For example: 'borrowedCountPerOrganization'.
+
+  Fields:
+    consumerOverrides: Quota overrides set by the consumer. Consumer overrides
+      will only have an effect up to the max_limit specified in the service
+      config, or the the producer override, if one exists.  The key for this
+      map is one of the following:  - '/' for quotas
+      defined within quota groups, where GROUP_NAME is the
+      google.api.QuotaGroup.name field and LIMIT_NAME is the
+      google.api.QuotaLimit.name field from the service config.  For example:
+      'ReadGroup/ProjectDaily'.  - '' for quotas defined without
+      quota groups, where LIMIT_NAME is the google.api.QuotaLimit.name field
+      from the service config. For example: 'borrowedCountPerOrganization'.
+    effectiveQuotaGroups: Use this field for quota limits defined under quota
+      groups. Combines service quota configuration and project-specific
+      settings, as a map from quota group name to the effective quota
+      information for that group. Output-only.
+    effectiveQuotas: The effective quota limits for each group, derived from
+      the service defaults together with any producer or consumer overrides.
+      For each limit, the effective value is the minimum of the producer and
+      consumer overrides if either is present, or else the service default if
+      neither is present. DEPRECATED. Use effective_quota_groups instead.
+    producerOverrides: Quota overrides set by the producer. Note that if a
+      consumer override is also specified, then the minimum of the two will be
+      used. This allows consumers to cap their usage voluntarily.  The key for
+      this map is one of the following:  - '/' for
+      quotas defined within quota groups, where GROUP_NAME is the
+      google.api.QuotaGroup.name field and LIMIT_NAME is the
+      google.api.QuotaLimit.name field from the service config.  For example:
+      'ReadGroup/ProjectDaily'.  - '' for quotas defined without
+      quota groups, where LIMIT_NAME is the google.api.QuotaLimit.name field
+      from the service config. For example: 'borrowedCountPerOrganization'.
+    variableTermQuotas: Quotas that are active over a specified time period.
+      Only writeable by the producer.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ConsumerOverridesValue(_messages.Message):
+    """Quota overrides set by the consumer. Consumer overrides will only have
+    an effect up to the max_limit specified in the service config, or the the
+    producer override, if one exists.  The key for this map is one of the
+    following:  - '/' for quotas defined within quota
+    groups, where GROUP_NAME is the google.api.QuotaGroup.name field and
+    LIMIT_NAME is the google.api.QuotaLimit.name field from the service
+    config.  For example: 'ReadGroup/ProjectDaily'.  - '' for
+    quotas defined without quota groups, where LIMIT_NAME is the
+    google.api.QuotaLimit.name field from the service config. For example:
+    'borrowedCountPerOrganization'.
+
+    Messages:
+      AdditionalProperty: An additional property for a ConsumerOverridesValue
+        object.
+
+    Fields:
+      additionalProperties: Additional properties of type
+        ConsumerOverridesValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ConsumerOverridesValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A QuotaLimitOverride attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('QuotaLimitOverride', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class EffectiveQuotasValue(_messages.Message):
+    """The effective quota limits for each group, derived from the service
+    defaults together with any producer or consumer overrides. For each limit,
+    the effective value is the minimum of the producer and consumer overrides
+    if either is present, or else the service default if neither is present.
+    DEPRECATED. Use effective_quota_groups instead.
+
+    Messages:
+      AdditionalProperty: An additional property for a EffectiveQuotasValue
+        object.
+
+    Fields:
+      additionalProperties: Additional properties of type EffectiveQuotasValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a EffectiveQuotasValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A QuotaLimitOverride attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('QuotaLimitOverride', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ProducerOverridesValue(_messages.Message):
+    """Quota overrides set by the producer. Note that if a consumer override
+    is also specified, then the minimum of the two will be used. This allows
+    consumers to cap their usage voluntarily.  The key for this map is one of
+    the following:  - '/' for quotas defined within
+    quota groups, where GROUP_NAME is the google.api.QuotaGroup.name field and
+    LIMIT_NAME is the google.api.QuotaLimit.name field from the service
+    config.  For example: 'ReadGroup/ProjectDaily'.  - '' for
+    quotas defined without quota groups, where LIMIT_NAME is the
+    google.api.QuotaLimit.name field from the service config. For example:
+    'borrowedCountPerOrganization'.
+
+    Messages:
+      AdditionalProperty: An additional property for a ProducerOverridesValue
+        object.
+
+    Fields:
+      additionalProperties: Additional properties of type
+        ProducerOverridesValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ProducerOverridesValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A QuotaLimitOverride attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('QuotaLimitOverride', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  consumerOverrides = _messages.MessageField('ConsumerOverridesValue', 1)
+  effectiveQuotaGroups = _messages.MessageField('EffectiveQuotaGroup', 2, repeated=True)
+  effectiveQuotas = _messages.MessageField('EffectiveQuotasValue', 3)
+  producerOverrides = _messages.MessageField('ProducerOverridesValue', 4)
+  variableTermQuotas = _messages.MessageField('VariableTermQuota', 5, repeated=True)
+
+
+class QuotaUsage(_messages.Message):
+  """Specifies the used quota amount for a quota limit at a particular time.
+
+  Fields:
+    endTime: The time the quota duration ended.
+    queryTime: The time the quota usage data was queried.
+    startTime: The time the quota duration started.
+    usage: The used quota value at the "query_time".
+  """
+
+  endTime = _messages.StringField(1)
+  queryTime = _messages.StringField(2)
+  startTime = _messages.StringField(3)
+  usage = _messages.IntegerField(4)
+
+
+class Service(_messages.Message):
+  """`Service` is the root object of the configuration schema. It describes
+  basic information like the name of the service and the exposed API
+  interfaces, and delegates other aspects to configuration sub-sections.
+  Example:      type: google.api.Service     config_version: 1     name:
+  calendar.googleapis.com     title: Google Calendar API     apis:     - name:
+  google.calendar.Calendar     backend:       rules:       - selector: "*"
+  address: calendar.example.com
+
+  Fields:
+    apis: A list of API interfaces exported by this service. Only the `name`
+      field of the google.protobuf.Api needs to be provided by the
+      configuration author, as the remaining fields will be derived from the
+      IDL during the normalization process. It is an error to specify an API
+      interface here which cannot be resolved against the associated IDL
+      files.
+    authentication: Auth configuration.
+    backend: API backend configuration.
+    billing: Billing configuration of the service.
+    configVersion: The version of the service configuration. The config
+      version may influence interpretation of the configuration, for example,
+      to determine defaults. This is documented together with applicable
+      options. The current default for the config version itself is `3`.
+    context: Context configuration.
+    control: Configuration for the service control plane.
+    customError: Custom error configuration.
+    documentation: Additional API documentation.
+    enums: A list of all enum types included in this API service.  Enums
+      referenced directly or indirectly by the `apis` are automatically
+      included.  Enums which are not referenced but shall be included should
+      be listed here by name. Example:      enums:     - name:
+      google.someapi.v1.SomeEnum
+    http: HTTP configuration.
+    id: A unique ID for a specific instance of this message, typically
+      assigned by the client for tracking purpose. If empty, the server may
+      choose to generate one instead.
+    logging: Logging configuration of the service.
+    logs: Defines the logs used by this service.
+    metrics: Defines the metrics used by this service.
+    monitoredResources: Defines the monitored resources used by this service.
+      This is required by the Service.monitoring and Service.logging
+      configurations.
+    monitoring: Monitoring configuration of the service.
+    name: The DNS address at which this service is available, e.g.
+      `calendar.googleapis.com`.
+    producerProjectId: The id of the Google developer project that owns the
+      service. Members of this project can manage the service configuration,
+      manage consumption of the service, etc.
+    projectProperties: Configuration of per-consumer project properties.
+    quota: Quota configuration.
+    systemParameters: Configuration for system parameters.
+    systemTypes: A list of all proto message types included in this API
+      service. It serves similar purpose as [google.api.Service.types], except
+      that these types are not needed by user-defined APIs. Therefore, they
+      will not show up in the generated discovery doc. This field should only
+      be used to define system APIs in ESF.
+    title: The product title associated with this service.
+    types: A list of all proto message types included in this API service.
+      Types referenced directly or indirectly by the `apis` are automatically
+      included.  Messages which are not referenced but shall be included, such
+      as types used by the `google.protobuf.Any` type, should be listed here
+      by name. Example:      types:     - name: google.protobuf.Int32
+    usage: Configuration controlling usage of this service.
+    visibility: API visibility configuration.
+  """
+
+  apis = _messages.MessageField('Api', 1, repeated=True)
+  authentication = _messages.MessageField('Authentication', 2)
+  backend = _messages.MessageField('Backend', 3)
+  billing = _messages.MessageField('Billing', 4)
+  configVersion = _messages.IntegerField(5, variant=_messages.Variant.UINT32)
+  context = _messages.MessageField('Context', 6)
+  control = _messages.MessageField('Control', 7)
+  customError = _messages.MessageField('CustomError', 8)
+  documentation = _messages.MessageField('Documentation', 9)
+  enums = _messages.MessageField('Enum', 10, repeated=True)
+  http = _messages.MessageField('Http', 11)
+  id = _messages.StringField(12)
+  logging = _messages.MessageField('Logging', 13)
+  logs = _messages.MessageField('LogDescriptor', 14, repeated=True)
+  metrics = _messages.MessageField('MetricDescriptor', 15, repeated=True)
+  monitoredResources = _messages.MessageField('MonitoredResourceDescriptor', 16, repeated=True)
+  monitoring = _messages.MessageField('Monitoring', 17)
+  name = _messages.StringField(18)
+  producerProjectId = _messages.StringField(19)
+  projectProperties = _messages.MessageField('ProjectProperties', 20)
+  quota = _messages.MessageField('Quota', 21)
+  systemParameters = _messages.MessageField('SystemParameters', 22)
+  systemTypes = _messages.MessageField('Type', 23, repeated=True)
+  title = _messages.StringField(24)
+  types = _messages.MessageField('Type', 25, repeated=True)
+  usage = _messages.MessageField('Usage', 26)
+  visibility = _messages.MessageField('Visibility', 27)
+
+
+class ServiceAccessList(_messages.Message):
+  """List of users and groups that are granted access to a service or
+  visibility label.
+
+  Fields:
+    members: Members that are granted access.  - "user:{$user_email}" - Grant
+      access to an individual user - "group:{$group_email}" - Grant access to
+      direct members of the group - "domain:{$domain}" - Grant access to all
+      members of the domain. For now,      domain membership check will be
+      similar to Devconsole/TT check:      compare domain part of the user
+      email to configured domain name.      When IAM integration is complete,
+      this will be replaced with IAM      check.
+  """
+
+  members = _messages.StringField(1, repeated=True)
+
+
+class ServiceAccessPolicy(_messages.Message):
+  """Policy describing who can access a service and any visibility labels on
+  that service.
+
+  Messages:
+    VisibilityLabelAccessListsValue: ACLs for access to restricted parts of
+      the service.  The map key is the visibility label that is being
+      controlled.  Note that access to any label also implies access to the
+      unrestricted surface.
+
+  Fields:
+    accessList: ACL for access to the unrestricted surface of the service.
+    serviceName: The service protected by this policy.
+    visibilityLabelAccessLists: ACLs for access to restricted parts of the
+      service.  The map key is the visibility label that is being controlled.
+      Note that access to any label also implies access to the unrestricted
+      surface.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class VisibilityLabelAccessListsValue(_messages.Message):
+    """ACLs for access to restricted parts of the service.  The map key is the
+    visibility label that is being controlled.  Note that access to any label
+    also implies access to the unrestricted surface.
+
+    Messages:
+      AdditionalProperty: An additional property for a
+        VisibilityLabelAccessListsValue object.
+
+    Fields:
+      additionalProperties: Additional properties of type
+        VisibilityLabelAccessListsValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a VisibilityLabelAccessListsValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A ServiceAccessList attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('ServiceAccessList', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  accessList = _messages.MessageField('ServiceAccessList', 1)
+  serviceName = _messages.StringField(2)
+  visibilityLabelAccessLists = _messages.MessageField('VisibilityLabelAccessListsValue', 3)
+
+
+class ServicemanagementOperationsGetRequest(_messages.Message):
+  """A ServicemanagementOperationsGetRequest object.
+
+  Fields:
+    operationsId: Part of `name`. The name of the operation resource.
+  """
+
+  operationsId = _messages.StringField(1, required=True)
+
+
+class ServicemanagementServicesAccessPolicyQueryRequest(_messages.Message):
+  """A ServicemanagementServicesAccessPolicyQueryRequest object.
+
+  Fields:
+    serviceName: The service to query access for.
+    userEmail: The user to query access for.
+  """
+
+  serviceName = _messages.StringField(1, required=True)
+  userEmail = _messages.StringField(2)
+
+
+class ServicemanagementServicesConfigsCreateRequest(_messages.Message):
+  """A ServicemanagementServicesConfigsCreateRequest object.
+
+  Fields:
+    service: A Service resource to be passed as the request body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+  """
+
+  service = _messages.MessageField('Service', 1)
+  serviceName = _messages.StringField(2, required=True)
+
+
+class ServicemanagementServicesConfigsGetRequest(_messages.Message):
+  """A ServicemanagementServicesConfigsGetRequest object.
+
+  Fields:
+    configId: The id of the service config resource. Optional. If it is not
+      specified, the latest version of config will be returned.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+  """
+
+  configId = _messages.StringField(1, required=True)
+  serviceName = _messages.StringField(2, required=True)
+
+
+class ServicemanagementServicesConfigsListRequest(_messages.Message):
+  """A ServicemanagementServicesConfigsListRequest object.
+
+  Fields:
+    pageSize: The max number of items to include in the response list.
+    pageToken: The token of the page to retrieve.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+  """
+
+  pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+  pageToken = _messages.StringField(2)
+  serviceName = _messages.StringField(3, required=True)
+
+
+class ServicemanagementServicesConfigsSubmitRequest(_messages.Message):
+  """A ServicemanagementServicesConfigsSubmitRequest object.
+
+  Fields:
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    submitConfigSourceRequest: A SubmitConfigSourceRequest resource to be
+      passed as the request body.
+  """
+
+  serviceName = _messages.StringField(1, required=True)
+  submitConfigSourceRequest = _messages.MessageField('SubmitConfigSourceRequest', 2)
+
+
+class ServicemanagementServicesCustomerSettingsGetRequest(_messages.Message):
+  """A ServicemanagementServicesCustomerSettingsGetRequest object.
+
+  Enums:
+    ViewValueValuesEnum: Request only fields for the specified view.
+
+  Fields:
+    customerId: ID for the customer. See the comment for
+      `CustomerSettings.customer_id` field of message for its format. This
+      field is required.
+    expand: Fields to expand in any results.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`. This
+      field is required.
+    view: Request only fields for the specified view.
+  """
+
+  class ViewValueValuesEnum(_messages.Enum):
+    """Request only fields for the specified view.
+
+    Values:
+      PROJECT_SETTINGS_VIEW_UNSPECIFIED: 
+      CONSUMER_VIEW: 
+      PRODUCER_VIEW: 
+      ALL: 
+    """
+    PROJECT_SETTINGS_VIEW_UNSPECIFIED = 0
+    CONSUMER_VIEW = 1
+    PRODUCER_VIEW = 2
+    ALL = 3
+
+  customerId = _messages.StringField(1, required=True)
+  expand = _messages.StringField(2)
+  serviceName = _messages.StringField(3, required=True)
+  view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class ServicemanagementServicesCustomerSettingsPatchRequest(_messages.Message):
+  """A ServicemanagementServicesCustomerSettingsPatchRequest object.
+
+  Fields:
+    customerId: ID for the customer. See the comment for
+      `CustomerSettings.customer_id` field of message for its format. This
+      field is required.
+    customerSettings: A CustomerSettings resource to be passed as the request
+      body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`. This
+      field is required.
+    updateMask: The field mask specifying which fields are to be updated.
+  """
+
+  customerId = _messages.StringField(1, required=True)
+  customerSettings = _messages.MessageField('CustomerSettings', 2)
+  serviceName = _messages.StringField(3, required=True)
+  updateMask = _messages.StringField(4)
+
+
+class ServicemanagementServicesDeleteRequest(_messages.Message):
+  """A ServicemanagementServicesDeleteRequest object.
+
+  Fields:
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+  """
+
+  serviceName = _messages.StringField(1, required=True)
+
+
+class ServicemanagementServicesDisableRequest(_messages.Message):
+  """A ServicemanagementServicesDisableRequest object.
+
+  Fields:
+    disableServiceRequest: A DisableServiceRequest resource to be passed as
+      the request body.
+    serviceName: Name of the service to disable. Specifying an unknown service
+      name will cause the request to fail.
+  """
+
+  disableServiceRequest = _messages.MessageField('DisableServiceRequest', 1)
+  serviceName = _messages.StringField(2, required=True)
+
+
+class ServicemanagementServicesEnableRequest(_messages.Message):
+  """A ServicemanagementServicesEnableRequest object.
+
+  Fields:
+    enableServiceRequest: A EnableServiceRequest resource to be passed as the
+      request body.
+    serviceName: Name of the service to enable. Specifying an unknown service
+      name will cause the request to fail.
+  """
+
+  enableServiceRequest = _messages.MessageField('EnableServiceRequest', 1)
+  serviceName = _messages.StringField(2, required=True)
+
+
+class ServicemanagementServicesGetAccessPolicyRequest(_messages.Message):
+  """A ServicemanagementServicesGetAccessPolicyRequest object.
+
+  Fields:
+    serviceName: The name of the service.  For example:
+      `example.googleapis.com`.
+  """
+
+  serviceName = _messages.StringField(1, required=True)
+
+
+class ServicemanagementServicesGetConfigRequest(_messages.Message):
+  """A ServicemanagementServicesGetConfigRequest object.
+
+  Fields:
+    configId: The id of the service config resource. Optional. If it is not
+      specified, the latest version of config will be returned.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+  """
+
+  configId = _messages.StringField(1)
+  serviceName = _messages.StringField(2, required=True)
+
+
+class ServicemanagementServicesGetRequest(_messages.Message):
+  """A ServicemanagementServicesGetRequest object.
+
+  Enums:
+    ViewValueValuesEnum: If project_settings is expanded, request only fields
+      for the specified view.
+
+  Fields:
+    consumerProjectId: If project_settings is expanded, return settings for
+      the specified consumer project.
+    expand: Fields to expand in any results.  By default, the following fields
+      are not present in the result: - `operations` - `project_settings` -
+      `project_settings.operations` - `quota_usage` (It requires
+      `project_settings`) - `historical_quota_usage` (It requires
+      `project_settings`)
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    view: If project_settings is expanded, request only fields for the
+      specified view.
+  """
+
+  class ViewValueValuesEnum(_messages.Enum):
+    """If project_settings is expanded, request only fields for the specified
+    view.
+
+    Values:
+      PROJECT_SETTINGS_VIEW_UNSPECIFIED: 
+      CONSUMER_VIEW: 
+      PRODUCER_VIEW: 
+      ALL: 
+    """
+    PROJECT_SETTINGS_VIEW_UNSPECIFIED = 0
+    CONSUMER_VIEW = 1
+    PRODUCER_VIEW = 2
+    ALL = 3
+
+  consumerProjectId = _messages.StringField(1)
+  expand = _messages.StringField(2)
+  serviceName = _messages.StringField(3, required=True)
+  view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class ServicemanagementServicesListRequest(_messages.Message):
+  """A ServicemanagementServicesListRequest object.
+
+  Fields:
+    category: Include services only in the specified category. Supported
+      categories are servicemanagement.googleapis.com/categories/google-
+      services or servicemanagement.googleapis.com/categories/play-games.
+    consumerProjectId: Include services consumed by the specified project.  If
+      project_settings is expanded, then this field controls which project
+      project_settings is populated for.
+    expand: Fields to expand in any results.  By default, the following fields
+      are not fully included in list results: - `operations` -
+      `project_settings` - `project_settings.operations` - `quota_usage` (It
+      requires `project_settings`)
+    pageSize: Requested size of the next page of data.
+    pageToken: Token identifying which result to start with; returned by a
+      previous list call.
+    producerProjectId: Include services produced by the specified project.
+  """
+
+  category = _messages.StringField(1)
+  consumerProjectId = _messages.StringField(2)
+  expand = _messages.StringField(3)
+  pageSize = _messages.IntegerField(4, variant=_messages.Variant.INT32)
+  pageToken = _messages.StringField(5)
+  producerProjectId = _messages.StringField(6)
+
+
+class ServicemanagementServicesPatchConfigRequest(_messages.Message):
+  """A ServicemanagementServicesPatchConfigRequest object.
+
+  Fields:
+    service: A Service resource to be passed as the request body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    updateMask: A mask specifying which fields to update.
+  """
+
+  service = _messages.MessageField('Service', 1)
+  serviceName = _messages.StringField(2, required=True)
+  updateMask = _messages.StringField(3)
+
+
+class ServicemanagementServicesPatchRequest(_messages.Message):
+  """A ServicemanagementServicesPatchRequest object.
+
+  Fields:
+    managedService: A ManagedService resource to be passed as the request
+      body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    updateMask: A mask specifying which fields to update.
+  """
+
+  managedService = _messages.MessageField('ManagedService', 1)
+  serviceName = _messages.StringField(2, required=True)
+  updateMask = _messages.StringField(3)
+
+
+class ServicemanagementServicesProjectSettingsGetRequest(_messages.Message):
+  """A ServicemanagementServicesProjectSettingsGetRequest object.
+
+  Enums:
+    ViewValueValuesEnum: Request only the fields for the specified view.
+
+  Fields:
+    consumerProjectId: The project ID of the consumer.
+    expand: Fields to expand in any results.  By default, the following fields
+      are not present in the result: - `operations` - `quota_usage`
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    view: Request only the fields for the specified view.
+  """
+
+  class ViewValueValuesEnum(_messages.Enum):
+    """Request only the fields for the specified view.
+
+    Values:
+      PROJECT_SETTINGS_VIEW_UNSPECIFIED: 
+      CONSUMER_VIEW: 
+      PRODUCER_VIEW: 
+      ALL: 
+    """
+    PROJECT_SETTINGS_VIEW_UNSPECIFIED = 0
+    CONSUMER_VIEW = 1
+    PRODUCER_VIEW = 2
+    ALL = 3
+
+  consumerProjectId = _messages.StringField(1, required=True)
+  expand = _messages.StringField(2)
+  serviceName = _messages.StringField(3, required=True)
+  view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class ServicemanagementServicesProjectSettingsPatchRequest(_messages.Message):
+  """A ServicemanagementServicesProjectSettingsPatchRequest object.
+
+  Fields:
+    consumerProjectId: The project ID of the consumer.
+    projectSettings: A ProjectSettings resource to be passed as the request
+      body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    updateMask: The field mask specifying which fields are to be updated.
+  """
+
+  consumerProjectId = _messages.StringField(1, required=True)
+  projectSettings = _messages.MessageField('ProjectSettings', 2)
+  serviceName = _messages.StringField(3, required=True)
+  updateMask = _messages.StringField(4)
+
+
+class ServicemanagementServicesUpdateConfigRequest(_messages.Message):
+  """A ServicemanagementServicesUpdateConfigRequest object.
+
+  Fields:
+    service: A Service resource to be passed as the request body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    updateMask: A mask specifying which fields to update. Update mask has been
+      deprecated on UpdateServiceConfig service method. Please use
+      PatchServiceConfig method instead to do partial updates.
+  """
+
+  service = _messages.MessageField('Service', 1)
+  serviceName = _messages.StringField(2, required=True)
+  updateMask = _messages.StringField(3)
+
+
+class ServicemanagementServicesUpdateRequest(_messages.Message):
+  """A ServicemanagementServicesUpdateRequest object.
+
+  Fields:
+    managedService: A ManagedService resource to be passed as the request
+      body.
+    serviceName: The name of the service.  See the `ServiceManager` overview
+      for naming requirements.  For example: `example.googleapis.com`.
+    updateMask: A mask specifying which fields to update. Update mask has been
+      deprecated on UpdateService service method. Please use PatchService
+      method instead to do partial updates.
+  """
+
+  managedService = _messages.MessageField('ManagedService', 1)
+  serviceName = _messages.StringField(2, required=True)
+  updateMask = _messages.StringField(3)
+
+
+class SourceContext(_messages.Message):
+  """`SourceContext` represents information about the source of a protobuf
+  element, like the file in which it is defined.
+
+  Fields:
+    fileName: The path-qualified name of the .proto file that contained the
+      associated protobuf element.  For example:
+      `"google/protobuf/source_context.proto"`.
+  """
+
+  fileName = _messages.StringField(1)
+
+
+class StandardQueryParameters(_messages.Message):
+  """Query parameters accepted by all methods.
+
+  Enums:
+    FXgafvValueValuesEnum: V1 error format.
+    AltValueValuesEnum: Data format for response.
+
+  Fields:
+    f__xgafv: V1 error format.
+    access_token: OAuth access token.
+    alt: Data format for response.
+    bearer_token: OAuth bearer token.
+    callback: JSONP
+    fields: Selector specifying which fields to include in a partial response.
+    key: API key. Your API key identifies your project and provides you with
+      API access, quota, and reports. Required unless you provide an OAuth 2.0
+      token.
+    oauth_token: OAuth 2.0 token for the current user.
+    pp: Pretty-print response.
+    prettyPrint: Returns response with indentations and line breaks.
+    quotaUser: Available to use for quota purposes for server-side
+      applications. Can be any arbitrary string assigned to a user, but should
+      not exceed 40 characters.
+    trace: A tracing token of the form "token:" to include in api
+      requests.
+    uploadType: Legacy upload protocol for media (e.g. "media", "multipart").
+    upload_protocol: Upload protocol for media (e.g. "raw", "multipart").
+  """
+
+  class AltValueValuesEnum(_messages.Enum):
+    """Data format for response.
+
+    Values:
+      json: Responses with Content-Type of application/json
+      media: Media download with context-dependent Content-Type
+      proto: Responses with Content-Type of application/x-protobuf
+    """
+    json = 0
+    media = 1
+    proto = 2
+
+  class FXgafvValueValuesEnum(_messages.Enum):
+    """V1 error format.
+
+    Values:
+      _1: v1 error format
+      _2: v2 error format
+    """
+    _1 = 0
+    _2 = 1
+
+  f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1)
+  access_token = _messages.StringField(2)
+  alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json')
+  bearer_token = _messages.StringField(4)
+  callback = _messages.StringField(5)
+  fields = _messages.StringField(6)
+  key = _messages.StringField(7)
+  oauth_token = _messages.StringField(8)
+  pp = _messages.BooleanField(9, default=True)
+  prettyPrint = _messages.BooleanField(10, default=True)
+  quotaUser = _messages.StringField(11)
+  trace = _messages.StringField(12)
+  uploadType = _messages.StringField(13)
+  upload_protocol = _messages.StringField(14)
+
+
+class Status(_messages.Message):
+  """The `Status` type defines a logical error model that is suitable for
+  different programming environments, including REST APIs and RPC APIs. It is
+  used by [gRPC](https://github.com/grpc). The error model is designed to be:
+  - Simple to use and understand for most users - Flexible enough to meet
+  unexpected needs  # Overview  The `Status` message contains three pieces of
+  data: error code, error message, and error details. The error code should be
+  an enum value of google.rpc.Code, but it may accept additional error codes
+  if needed.  The error message should be a developer-facing English message
+  that helps developers *understand* and *resolve* the error. If a localized
+  user-facing error message is needed, put the localized message in the error
+  details or localize it in the client. The optional error details may contain
+  arbitrary information about the error. There is a predefined set of error
+  detail types in the package `google.rpc` which can be used for common error
+  conditions.  # Language mapping  The `Status` message is the logical
+  representation of the error model, but it is not necessarily the actual wire
+  format. When the `Status` message is exposed in different client libraries
+  and different wire protocols, it can be mapped differently. For example, it
+  will likely be mapped to some exceptions in Java, but more likely mapped to
+  some error codes in C.  # Other uses  The error model and the `Status`
+  message can be used in a variety of environments, either with or without
+  APIs, to provide a consistent developer experience across different
+  environments.  Example uses of this error model include:  - Partial errors.
+  If a service needs to return partial errors to the client,     it may embed
+  the `Status` in the normal response to indicate the partial     errors.  -
+  Workflow errors. A typical workflow has multiple steps. Each step may
+  have a `Status` message for error reporting purpose.  - Batch operations. If
+  a client uses batch request and batch response, the     `Status` message
+  should be used directly inside batch response, one for     each error sub-
+  response.  - Asynchronous operations. If an API call embeds asynchronous
+  operation     results in its response, the status of those operations should
+  be     represented directly using the `Status` message.  - Logging. If some
+  API errors are stored in logs, the message `Status` could     be used
+  directly after any stripping needed for security/privacy reasons.
+
+  Messages:
+    DetailsValueListEntry: A DetailsValueListEntry object.
+
+  Fields:
+    code: The status code, which should be an enum value of google.rpc.Code.
+    details: A list of messages that carry the error details.  There will be a
+      common set of message types for APIs to use.
+    message: A developer-facing error message, which should be in English. Any
+      user-facing error message should be localized and sent in the
+      google.rpc.Status.details field, or localized by the client.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class DetailsValueListEntry(_messages.Message):
+    """A DetailsValueListEntry object.
+
+    Messages:
+      AdditionalProperty: An additional property for a DetailsValueListEntry
+        object.
+
+    Fields:
+      additionalProperties: Properties of the object. Contains field @type
+        with type URL.
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a DetailsValueListEntry object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A extra_types.JsonValue attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('extra_types.JsonValue', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  code = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+  details = _messages.MessageField('DetailsValueListEntry', 2, repeated=True)
+  message = _messages.StringField(3)
+
+
+class Step(_messages.Message):
+  """Represents the status of one operation step.
+
+  Enums:
+    StatusValueValuesEnum: The status code.
+
+  Fields:
+    description: The short description of the step.
+    status: The status code.
+  """
+
+  class StatusValueValuesEnum(_messages.Enum):
+    """The status code.
+
+    Values:
+      STATUS_UNSPECIFIED: Unspecifed code.
+      DONE: The step has completed without errors.
+      NOT_STARTED: The step has not started yet.
+      IN_PROGRESS: The step is in progress.
+      FAILED: The step has completed with errors.
+    """
+    STATUS_UNSPECIFIED = 0
+    DONE = 1
+    NOT_STARTED = 2
+    IN_PROGRESS = 3
+    FAILED = 4
+
+  description = _messages.StringField(1)
+  status = _messages.EnumField('StatusValueValuesEnum', 2)
+
+
+class SubmitConfigSourceRequest(_messages.Message):
+  """Request message for SubmitConfigSource method.
+
+  Fields:
+    configSource: The source configuration for the service.
+    validateOnly: Optional. If set, this will result in the generation of a
+      `google.api.Service` configuration based on the `ConfigSource` provided,
+      but the generated config and the sources will NOT be persisted.
+  """
+
+  configSource = _messages.MessageField('ConfigSource', 1)
+  validateOnly = _messages.BooleanField(2)
+
+
+class SwaggerSpec(_messages.Message):
+  """A collection of swagger specification files.
+
+  Fields:
+    swaggerFiles: The individual files.
+  """
+
+  swaggerFiles = _messages.MessageField('File', 1, repeated=True)
+
+
+class SystemParameter(_messages.Message):
+  """Define a parameter's name and location. The parameter may be passed as
+  either an HTTP header or a URL query parameter, and if both are passed the
+  behavior is implementation-dependent.
+
+  Fields:
+    httpHeader: Define the HTTP header name to use for the parameter. It is
+      case insensitive.
+    name: Define the name of the parameter, such as "api_key", "alt",
+      "callback", and etc. It is case sensitive.
+    urlQueryParameter: Define the URL query parameter name to use for the
+      parameter. It is case sensitive.
+  """
+
+  httpHeader = _messages.StringField(1)
+  name = _messages.StringField(2)
+  urlQueryParameter = _messages.StringField(3)
+
+
+class SystemParameterRule(_messages.Message):
+  """Define a system parameter rule mapping system parameter definitions to
+  methods.
+
+  Fields:
+    parameters: Define parameters. Multiple names may be defined for a
+      parameter. For a given method call, only one of them should be used. If
+      multiple names are used the behavior is implementation-dependent. If
+      none of the specified names are present the behavior is parameter-
+      dependent.
+    selector: Selects the methods to which this rule applies. Use '*' to
+      indicate all methods in all APIs.  Refer to selector for syntax details.
+  """
+
+  parameters = _messages.MessageField('SystemParameter', 1, repeated=True)
+  selector = _messages.StringField(2)
+
+
+class SystemParameters(_messages.Message):
+  """### System parameter configuration  A system parameter is a special kind
+  of parameter defined by the API system, not by an individual API. It is
+  typically mapped to an HTTP header and/or a URL query parameter. This
+  configuration specifies which methods change the names of the system
+  parameters.
+
+  Fields:
+    rules: Define system parameters.  The parameters defined here will
+      override the default parameters implemented by the system. If this field
+      is missing from the service config, default system parameters will be
+      used. Default system parameters and names is implementation-dependent.
+      Example: define api key and alt name for all methods  system_parameters
+      rules:     - selector: "*"       parameters:         - name: api_key
+      url_query_parameter: api_key         - name: alt           http_header:
+      Response-Content-Type  Example: define 2 api key names for a specific
+      method.  system_parameters   rules:     - selector: "/ListShelves"
+      parameters:         - name: api_key           http_header: Api-Key1
+      - name: api_key           http_header: Api-Key2
+  """
+
+  rules = _messages.MessageField('SystemParameterRule', 1, repeated=True)
+
+
+class Type(_messages.Message):
+  """A protocol buffer message type.
+
+  Enums:
+    SyntaxValueValuesEnum: The source syntax.
+
+  Fields:
+    fields: The list of fields.
+    name: The fully qualified message name.
+    oneofs: The list of types appearing in `oneof` definitions in this type.
+    options: The protocol buffer options.
+    sourceContext: The source context.
+    syntax: The source syntax.
+  """
+
+  class SyntaxValueValuesEnum(_messages.Enum):
+    """The source syntax.
+
+    Values:
+      SYNTAX_PROTO2: Syntax `proto2`.
+      SYNTAX_PROTO3: Syntax `proto3`.
+    """
+    SYNTAX_PROTO2 = 0
+    SYNTAX_PROTO3 = 1
+
+  fields = _messages.MessageField('Field', 1, repeated=True)
+  name = _messages.StringField(2)
+  oneofs = _messages.StringField(3, repeated=True)
+  options = _messages.MessageField('Option', 4, repeated=True)
+  sourceContext = _messages.MessageField('SourceContext', 5)
+  syntax = _messages.EnumField('SyntaxValueValuesEnum', 6)
+
+
+class Usage(_messages.Message):
+  """Configuration controlling usage of a service.
+
+  Enums:
+    ServiceAccessValueValuesEnum: Controls which users can see or activate the
+      service.
+
+  Fields:
+    activationHooks: Services that must be contacted before a consumer can
+      begin using the service. Each service will be contacted in sequence,
+      and, if any activation call fails, the entire activation will fail. Each
+      hook is of the form /, where  is
+      optional; for example: 'robotservice.googleapis.com/default'.
+    deactivationHooks: Services that must be contacted before a consumer can
+      deactivate a service. Each service will be contacted in sequence, and,
+      if any deactivation call fails, the entire deactivation will fail. Each
+      hook is of the form /, where  is
+      optional; for example: 'compute.googleapis.com/'.
+    dependsOnServices: Services that must be activated in order for this
+      service to be used. The set of services activated as a result of these
+      relations are all activated in parallel with no guaranteed order of
+      activation. Each string is a service name, e.g.
+      `calendar.googleapis.com`.
+    requirements: Requirements that must be satisfied before a consumer
+      project can use the service. Each requirement is of the form
+      /; for example
+      'serviceusage.googleapis.com/billing-enabled'.
+    rules: Individual rules for configuring usage on selected methods.
+    serviceAccess: Controls which users can see or activate the service.
+  """
+
+  class ServiceAccessValueValuesEnum(_messages.Enum):
+    """Controls which users can see or activate the service.
+
+    Values:
+      RESTRICTED: The service can only be seen/used by users identified in the
+        service's access control policy.  If the service has not been
+        whitelisted by your domain administrator for out-of-org publishing,
+        then this mode will be treated like ORG_RESTRICTED.
+      PUBLIC: The service can be seen/used by anyone.  If the service has not
+        been whitelisted by your domain administrator for out-of-org
+        publishing, then this mode will be treated like ORG_PUBLIC.  The
+        discovery document for the service will also be public and allow
+        unregistered access.
+      ORG_RESTRICTED: The service can be seen/used by users identified in the
+        service's access control policy and they are within the organization
+        that owns the service.  Access is further constrained to the group
+        controlled by the administrator of the project/org that owns the
+        service.
+      ORG_PUBLIC: The service can be seen/used by the group of users
+        controlled by the administrator of the project/org that owns the
+        service.
+    """
+    RESTRICTED = 0
+    PUBLIC = 1
+    ORG_RESTRICTED = 2
+    ORG_PUBLIC = 3
+
+  activationHooks = _messages.StringField(1, repeated=True)
+  deactivationHooks = _messages.StringField(2, repeated=True)
+  dependsOnServices = _messages.StringField(3, repeated=True)
+  requirements = _messages.StringField(4, repeated=True)
+  rules = _messages.MessageField('UsageRule', 5, repeated=True)
+  serviceAccess = _messages.EnumField('ServiceAccessValueValuesEnum', 6)
+
+
+class UsageRule(_messages.Message):
+  """Usage configuration rules for the service.  NOTE: Under development.
+  Use this rule to configure unregistered calls for the service. Unregistered
+  calls are calls that do not contain consumer project identity. (Example:
+  calls that do not contain an API key). By default, API methods do not allow
+  unregistered calls, and each method call must be identified by a consumer
+  project identity. Use this rule to allow/disallow unregistered calls.
+  Example of an API that wants to allow unregistered calls for entire service.
+  usage:       rules:       - selector: "*"         allow_unregistered_calls:
+  true  Example of a method that wants to allow unregistered calls.
+  usage:       rules:       - selector:
+  "google.example.library.v1.LibraryService.CreateBook"
+  allow_unregistered_calls: true
+
+  Fields:
+    allowUnregisteredCalls: True, if the method allows unregistered calls;
+      false otherwise.
+    selector: Selects the methods to which this rule applies. Use '*' to
+      indicate all methods in all APIs.  Refer to selector for syntax details.
+  """
+
+  allowUnregisteredCalls = _messages.BooleanField(1)
+  selector = _messages.StringField(2)
+
+
+class UsageSettings(_messages.Message):
+  """Usage settings for a consumer of a service.
+
+  Enums:
+    ConsumerEnableStatusValueValuesEnum: Consumer controlled setting to
+      enable/disable use of this service by the consumer project. The default
+      value of this is controlled by the service configuration.
+
+  Fields:
+    consumerEnableStatus: Consumer controlled setting to enable/disable use of
+      this service by the consumer project. The default value of this is
+      controlled by the service configuration.
+  """
+
+  class ConsumerEnableStatusValueValuesEnum(_messages.Enum):
+    """Consumer controlled setting to enable/disable use of this service by
+    the consumer project. The default value of this is controlled by the
+    service configuration.
+
+    Values:
+      DISABLED: The service is disabled.
+      ENABLED: The service is enabled.
+    """
+    DISABLED = 0
+    ENABLED = 1
+
+  consumerEnableStatus = _messages.EnumField('ConsumerEnableStatusValueValuesEnum', 1)
+
+
+class VariableTermQuota(_messages.Message):
+  """A variable term quota is a bucket of tokens that is consumed over a
+  specified (usually long) time period. When present, it overrides any "1d"
+  duration per-project quota specified on the group.  Variable terms run from
+  midnight to midnight, start_date to end_date (inclusive) in the
+  America/Los_Angeles time zone.
+
+  Fields:
+    createTime: Time when this variable term quota was created. If multiple
+      quotas are simultaneously active, then the quota with the latest
+      create_time is the effective one.
+    displayEndDate: The displayed end of the active period for the variable
+      term quota. This may be before the effective end to give the user a
+      grace period. YYYYMMdd date format, e.g. 20140730.
+    endDate: The effective end of the active period for the variable term
+      quota (inclusive). This must be no more than 5 years after start_date.
+      YYYYMMdd date format, e.g. 20140730.
+    groupName: The quota group that has the variable term quota applied to it.
+      This must be a google.api.QuotaGroup.name specified in the service
+      configuration.
+    limit: The number of tokens available during the configured term.
+    quotaUsage: The usage data of this quota.
+    startDate: The beginning of the active period for the variable term quota.
+      YYYYMMdd date format, e.g. 20140730.
+  """
+
+  createTime = _messages.StringField(1)
+  displayEndDate = _messages.StringField(2)
+  endDate = _messages.StringField(3)
+  groupName = _messages.StringField(4)
+  limit = _messages.IntegerField(5)
+  quotaUsage = _messages.MessageField('QuotaUsage', 6)
+  startDate = _messages.StringField(7)
+
+
+class Visibility(_messages.Message):
+  """`Visibility` defines restrictions for the visibility of service elements.
+  Restrictions are specified using visibility labels (e.g., TRUSTED_TESTER)
+  that are elsewhere linked to users and projects.  Users and projects can
+  have access to more than one visibility label. The effective visibility for
+  multiple labels is the union of each label's elements, plus any unrestricted
+  elements.  If an element and its parents have no restrictions, visibility is
+  unconditionally granted.  Example:      visibility:       rules:       -
+  selector: google.calendar.Calendar.EnhancedSearch         restriction:
+  TRUSTED_TESTER       - selector: google.calendar.Calendar.Delegate
+  restriction: GOOGLE_INTERNAL  Here, all methods are publicly visible except
+  for the restricted methods EnhancedSearch and Delegate.
+
+  Fields:
+    enforceRuntimeVisibility: Controls whether visibility rules are enforced
+      at runtime for requests to all APIs and methods.  If true, requests
+      without method visibility will receive a NOT_FOUND error, and any non-
+      visible fields will be scrubbed from the response messages. In service
+      config version 0, the default is false. In later config versions, it's
+      true.  Note, the `enforce_runtime_visibility` specified in a visibility
+      rule overrides this setting for the APIs or methods asscoiated with the
+      rule.
+    rules: A list of visibility rules providing visibility configuration for
+      individual API elements.
+  """
+
+  enforceRuntimeVisibility = _messages.BooleanField(1)
+  rules = _messages.MessageField('VisibilityRule', 2, repeated=True)
+
+
+class VisibilityRule(_messages.Message):
+  """A visibility rule provides visibility configuration for an individual API
+  element.
+
+  Fields:
+    enforceRuntimeVisibility: Controls whether visibility is enforced at
+      runtime for requests to an API method. This setting has meaning only
+      when the selector applies to a method or an API.  If true, requests
+      without method visibility will receive a NOT_FOUND error, and any non-
+      visible fields will be scrubbed from the response messages. The default
+      is determined by the value of
+      google.api.Visibility.enforce_runtime_visibility.
+    restriction: Lists the visibility labels for this rule. Any of the listed
+      labels grants visibility to the element.  If a rule has multiple labels,
+      removing one of the labels but not all of them can break clients.
+      Example:      visibility:       rules:       - selector:
+      google.calendar.Calendar.EnhancedSearch         restriction:
+      GOOGLE_INTERNAL, TRUSTED_TESTER  Removing GOOGLE_INTERNAL from this
+      restriction will break clients that rely on this method and only had
+      access to it through GOOGLE_INTERNAL.
+    selector: Selects methods, messages, fields, enums, etc. to which this
+      rule applies.  Refer to selector for syntax details.
+  """
+
+  enforceRuntimeVisibility = _messages.BooleanField(1)
+  restriction = _messages.StringField(2)
+  selector = _messages.StringField(3)
+
+
+class VisibilitySettings(_messages.Message):
+  """Settings that control which features of the service are visible to the
+  consumer project.
+
+  Fields:
+    visibilityLabels: The set of visibility labels that are used to determine
+      what API surface is visible to calls made by this project. The visible
+      surface is a union of the surface features associated with each label
+      listed here, plus the publicly visible (unrestricted) surface.  The
+      service producer may add or remove labels at any time. The service
+      consumer may add a label if the calling user has been granted permission
+      to do so by the producer.  The service consumer may also remove any
+      label at any time.
+  """
+
+  visibilityLabels = _messages.StringField(1, repeated=True)
+
+
+encoding.AddCustomJsonFieldMapping(
+    StandardQueryParameters, 'f__xgafv', '$.xgafv',
+    package=u'servicemanagement')
+encoding.AddCustomJsonEnumMapping(
+    StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1',
+    package=u'servicemanagement')
+encoding.AddCustomJsonEnumMapping(
+    StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2',
+    package=u'servicemanagement')
-- 
GitLab


From cce7bd1896ba611186b0a3ce6b579da0687f158e Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Mon, 18 Jul 2016 14:28:27 -0400
Subject: [PATCH 252/295] Fix doc string lint errors.

---
 apitools/base/protorpclite/messages.py | 1 +
 apitools/base/protorpclite/util.py     | 1 +
 apitools/base/py/batch.py              | 1 +
 apitools/base/py/credentials_lib.py    | 4 ++++
 apitools/gen/command_registry.py       | 7 +++++++
 apitools/scripts/oauth2l.py            | 2 +-
 default.pylintrc                       | 3 ---
 7 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py
index 4a8b03f..ade2168 100644
--- a/apitools/base/protorpclite/messages.py
+++ b/apitools/base/protorpclite/messages.py
@@ -1166,6 +1166,7 @@ class _FieldMeta(type):
 
 # TODO(rafek): Prevent additional field subclasses.
 class Field(six.with_metaclass(_FieldMeta, object)):
+    """Definition for message field."""
 
     __initialized = False  # pylint:disable=invalid-name
     __variant_to_type = {}  # pylint:disable=invalid-name
diff --git a/apitools/base/protorpclite/util.py b/apitools/base/protorpclite/util.py
index 4df0458..f6147e9 100644
--- a/apitools/base/protorpclite/util.py
+++ b/apitools/base/protorpclite/util.py
@@ -130,6 +130,7 @@ def positional(max_positional_args):
         has no arguments with default values.
     """
     def positional_decorator(wrapped):
+        """Creates a function wraper to enforce number of arguments."""
         @functools.wraps(wrapped)
         def positional_wrapper(*args, **kwargs):
             if len(args) > max_positional_args:
diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py
index 7efe6dd..61cfb25 100644
--- a/apitools/base/py/batch.py
+++ b/apitools/base/py/batch.py
@@ -56,6 +56,7 @@ class RequestResponseAndHandler(collections.namedtuple(
 
 
 class BatchApiRequest(object):
+    """Batches multiple api requests into a single request."""
 
     class ApiCall(object):
 
diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index a3819bc..efffe8c 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -335,6 +335,7 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
                     cache_file.unlock_and_close()
 
     def _ScopesFromMetadataServer(self, scopes):
+        """Returns instance scopes based on GCE metadata server."""
         if not util.DetectGce():
             raise exceptions.ResourceUnavailableError(
                 'GCE credentials requested outside a GCE instance')
@@ -497,6 +498,7 @@ class GaeAssertionCredentials(oauth2client.client.AssertionCredentials):
 
 
 def _GetRunFlowFlags(args=None):
+    """Retrieves command line flags based on gflags module."""
     # There's one rare situation where gsutil will not have argparse
     # available, but doesn't need anything depending on argparse anyway,
     # since they're bringing their own credentials. So we just allow this
@@ -591,6 +593,7 @@ def _GetUserinfoUrl(credentials):
 def _GetServiceAccountCredentials(
         client_info, service_account_name=None, service_account_keyfile=None,
         service_account_json_keyfile=None, **unused_kwds):
+    """Returns ServiceAccountCredentials from give file."""
     if ((service_account_name and not service_account_keyfile) or
             (service_account_keyfile and not service_account_name)):
         raise exceptions.CredentialsError(
@@ -623,6 +626,7 @@ def _GetGceServiceAccount(client_info, **unused_kwds):
 def _GetApplicationDefaultCredentials(
         client_info, skip_application_default_credentials=False,
         **unused_kwds):
+    """Returns ADC with right scopes."""
     scopes = client_info['scope'].split()
     if skip_application_default_credentials:
         return None
diff --git a/apitools/gen/command_registry.py b/apitools/gen/command_registry.py
index ef0c667..486934f 100644
--- a/apitools/gen/command_registry.py
+++ b/apitools/gen/command_registry.py
@@ -226,6 +226,7 @@ class CommandRegistry(object):
         return command_name
 
     def __GetConversion(self, extended_field, extended_message):
+        """Returns a template for field type."""
         field = extended_field.field_descriptor
 
         type_name = ''
@@ -262,6 +263,7 @@ class CommandRegistry(object):
         return field.label == descriptor.FieldDescriptor.Label.REPEATED
 
     def __FlagInfoFromField(self, extended_field, extended_message, fv=''):
+        """Creates FlagInfo object for given field."""
         field = extended_field.field_descriptor
         flag_info = FlagInfo()
         flag_info.name = str(field.name)
@@ -290,6 +292,7 @@ class CommandRegistry(object):
         return flag_info
 
     def __PrintFlagDeclarations(self, printer):
+        """Writes out command line flag declarations."""
         package = self.__client_info.package
         function_name = '_Declare%sFlags' % (package[0].upper() + package[1:])
         printer()
@@ -331,6 +334,7 @@ class CommandRegistry(object):
         printer('%s()', function_name)
 
     def __PrintGetGlobalParams(self, printer):
+        """Writes out GetGlobalParamsFromFlags function."""
         printer('def GetGlobalParamsFromFlags():')
         with printer.Indent():
             printer('"""Return a StandardQueryParameters based on flags."""')
@@ -348,6 +352,7 @@ class CommandRegistry(object):
         printer()
 
     def __PrintGetClient(self, printer):
+        """Writes out GetClientFromFlags function."""
         printer('def GetClientFromFlags():')
         with printer.Indent():
             printer('"""Return a client object, configured from flags."""')
@@ -393,6 +398,7 @@ class CommandRegistry(object):
             printer('"""')
 
     def __PrintFlag(self, printer, flag_info):
+        """Writes out given flag definition."""
         printer('flags.DEFINE_%s(', flag_info.type)
         with printer.Indent(indent='    '):
             printer('%r,', flag_info.name)
@@ -414,6 +420,7 @@ class CommandRegistry(object):
             printer('flags.MarkFlagAsRequired(%r)', flag_info.name)
 
     def __PrintPyShell(self, printer):
+        """Writes out PyShell class."""
         printer('class PyShell(appcommands.Cmd):')
         printer()
         with printer.Indent():
diff --git a/apitools/scripts/oauth2l.py b/apitools/scripts/oauth2l.py
index faaa859..cddba0a 100644
--- a/apitools/scripts/oauth2l.py
+++ b/apitools/scripts/oauth2l.py
@@ -235,7 +235,7 @@ def _Validate(args):
 
 
 def _GetParser():
-
+    """Returns argparse argument parser."""
     shared_flags = argparse.ArgumentParser(add_help=False)
     shared_flags.add_argument(
         '--client_secrets',
diff --git a/default.pylintrc b/default.pylintrc
index 142fee0..7b9c3c4 100644
--- a/default.pylintrc
+++ b/default.pylintrc
@@ -238,9 +238,6 @@ bad-names=
 # 
 bad-functions=input,apply,reduce
 
-# TEMPORARY
-no-docstring-rgx=.*
-
 
 [TYPECHECK]
 
-- 
GitLab


From cb528817acbd62762f737cdf93d5425e8dc5e92c Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 26 Jul 2016 08:59:11 -0400
Subject: [PATCH 253/295] Update testing.mock to closer match real api client.

---
 apitools/base/py/testing/mock.py      | 58 ++++++++++++++-------------
 apitools/base/py/testing/mock_test.py | 33 +++++++++++++++
 2 files changed, 63 insertions(+), 28 deletions(-)

diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py
index e779cfe..777f1af 100644
--- a/apitools/base/py/testing/mock.py
+++ b/apitools/base/py/testing/mock.py
@@ -180,21 +180,6 @@ class _ExpectedRequestResponse(object):
         return self.__response
 
 
-class _MockedService(base_api.BaseApiService):
-
-    def __init__(self, key, mocked_client, methods, real_service):
-        super(_MockedService, self).__init__(mocked_client)
-        self.__dict__.update(real_service.__dict__)
-        for method in methods:
-            real_method = None
-            if real_service:
-                real_method = getattr(real_service, method)
-            setattr(self, method,
-                    _MockedMethod(key + '.' + method,
-                                  mocked_client,
-                                  real_method))
-
-
 class _MockedMethod(object):
 
     """A mocked API service method."""
@@ -256,10 +241,22 @@ class _MockedMethod(object):
         return response
 
 
-def _MakeMockedServiceConstructor(mocked_service):
-    def Constructor(unused_self, unused_client):
-        return mocked_service
-    return Constructor
+def _MakeMockedService(api_name, collection_name,
+                       mock_client, service, real_service):
+    class MockedService(base_api.BaseApiService):
+        def __init__(self, real_client):
+            super(MockedService, self).__init__(real_client)
+
+    for method in service.GetMethodsList():
+        real_method = None
+        if real_service:
+            real_method = getattr(real_service, method)
+        setattr(MockedService,
+                method,
+                _MockedMethod(api_name + '.' + collection_name + '.' + method,
+                              mock_client,
+                              real_method))
+    return MockedService
 
 
 class Client(object):
@@ -283,6 +280,7 @@ class Client(object):
         if not real_client:
             real_client = client_class(get_credentials=False)
 
+        self.__orig_class = self.__class__
         self.__client_class = client_class
         self.__real_service_classes = {}
         self.__real_client = real_client
@@ -297,6 +295,11 @@ class Client(object):
         """Stub out the client class with mocked services."""
         client = self.__real_client or self.__client_class(
             get_credentials=False)
+
+        class Patched(self.__class__, self.__client_class):
+            pass
+        self.__class__ = Patched
+
         for name in dir(self.__client_class):
             service_class = getattr(self.__client_class, name)
             if not isinstance(service_class, type):
@@ -304,21 +307,19 @@ class Client(object):
             if not issubclass(service_class, base_api.BaseApiService):
                 continue
             self.__real_service_classes[name] = service_class
-            service = service_class(client)
             # pylint: disable=protected-access
-            # Some liberty is allowed with mocking.
             collection_name = service_class._NAME
             # pylint: enable=protected-access
             api_name = '%s_%s' % (self.__client_class._PACKAGE,
                                   self.__client_class._URL_VERSION)
-            mocked_service = _MockedService(
-                api_name + '.' + collection_name, self,
-                service.GetMethodsList(),
-                service if self.__real_client else None)
-            mocked_constructor = _MakeMockedServiceConstructor(mocked_service)
-            setattr(self.__client_class, name, mocked_constructor)
+            mocked_service_class = _MakeMockedService(
+                api_name, collection_name, self,
+                service_class,
+                service_class(client) if self.__real_client else None)
+
+            setattr(self.__client_class, name, mocked_service_class)
 
-            setattr(self, collection_name, mocked_service)
+            setattr(self, collection_name, mocked_service_class(self))
 
         self.__real_include_fields = self.__client_class.IncludeFields
         self.__client_class.IncludeFields = self.IncludeFields
@@ -332,6 +333,7 @@ class Client(object):
         return True
 
     def Unmock(self):
+        self.__class__ = self.__orig_class
         for name, service_class in self.__real_service_classes.items():
             setattr(self.__client_class, name, service_class)
             delattr(self, service_class._NAME)
diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py
index ee42ef1..85af559 100644
--- a/apitools/base/py/testing/mock_test.py
+++ b/apitools/base/py/testing/mock_test.py
@@ -16,6 +16,7 @@
 """Tests for apitools.base.py.testing.mock."""
 
 import unittest2
+import six
 
 from apitools.base.protorpclite import messages
 
@@ -23,6 +24,16 @@ import apitools.base.py as apitools_base
 from apitools.base.py.testing import mock
 from samples.fusiontables_sample.fusiontables_v1 import \
     fusiontables_v1_client as fusiontables
+from samples.fusiontables_sample.fusiontables_v1 import \
+    fusiontables_v1_messages as fusiontables_messages
+
+
+def _GetApiServices(api_client_class):
+    return dict(
+        (name, potential_service)
+        for name, potential_service in six.iteritems(api_client_class.__dict__)
+        if (isinstance(potential_service, type) and
+            issubclass(potential_service, apitools_base.BaseApiService)))
 
 
 class MockTest(unittest2.TestCase):
@@ -87,12 +98,34 @@ class MockTest(unittest2.TestCase):
 
     def testClientUnmock(self):
         mock_client = mock.Client(fusiontables.FusiontablesV1)
+        self.assertFalse(isinstance(mock_client, fusiontables.FusiontablesV1))
         attributes = set(mock_client.__dict__.keys())
         mock_client = mock_client.Mock()
+        self.assertTrue(isinstance(mock_client, fusiontables.FusiontablesV1))
         self.assertTrue(set(mock_client.__dict__.keys()) - attributes)
         mock_client.Unmock()
+        self.assertFalse(isinstance(mock_client, fusiontables.FusiontablesV1))
         self.assertEqual(attributes, set(mock_client.__dict__.keys()))
 
+    def testMockHasMessagesModule(self):
+        with mock.Client(fusiontables.FusiontablesV1) as mock_client:
+            self.assertEquals(fusiontables_messages,
+                              mock_client.MESSAGES_MODULE)
+
+    def testMockPreservesServiceMethods(self):
+        services = _GetApiServices(fusiontables.FusiontablesV1)
+        with mock.Client(fusiontables.FusiontablesV1):
+            mocked_services = _GetApiServices(fusiontables.FusiontablesV1)
+            self.assertEquals(services.keys(), mocked_services.keys())
+            for name, service in six.iteritems(services):
+                mocked_service = mocked_services[name]
+                methods = service.GetMethodsList()
+                for method in methods:
+                    mocked_method = getattr(mocked_service, method)
+                    mocked_method_config = mocked_method.method_config()
+                    method_config = getattr(service, method).method_config()
+                    self.assertEquals(method_config, mocked_method_config)
+
 
 class _NestedMessage(messages.Message):
     nested = messages.StringField(1)
-- 
GitLab


From e3003ded245205d2ebfcd44a003d45475826f87e Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 26 Jul 2016 15:22:29 -0400
Subject: [PATCH 254/295] Include all samples to be updated on changes to
 generation.

---
 .../fusiontables_v1_messages.py               |   1 +
 .../servicemanagement_v1_client.py            | 671 +++++++++---------
 samples/uptodate_check_test.py                |   6 +
 3 files changed, 345 insertions(+), 333 deletions(-)

diff --git a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py
index 6eff4b5..15f878e 100644
--- a/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py
+++ b/samples/fusiontables_sample/fusiontables_v1/fusiontables_v1_messages.py
@@ -5,6 +5,7 @@ API for working with Fusion Tables data.
 # NOTE: This file is autogenerated and should not be edited by hand.
 
 from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import extra_types
 
 
 package = 'fusiontables'
diff --git a/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py
index 9a1362d..26291bc 100644
--- a/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py
+++ b/samples/servicemanagement_sample/servicemanagement_v1/servicemanagement_v1_client.py
@@ -49,21 +49,6 @@ class ServicemanagementV1(base_api.BaseApiClient):
 
     def __init__(self, client):
       super(ServicemanagementV1.OperationsService, self).__init__(client)
-      self._method_configs = {
-          'Get': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.operations.get',
-              ordered_params=[u'operationsId'],
-              path_params=[u'operationsId'],
-              query_params=[],
-              relative_path=u'v1/operations/{operationsId}',
-              request_field='',
-              request_type_name=u'ServicemanagementOperationsGetRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -82,6 +67,19 @@ service.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.operations.get',
+        ordered_params=[u'operationsId'],
+        path_params=[u'operationsId'],
+        query_params=[],
+        relative_path=u'v1/operations/{operationsId}',
+        request_field='',
+        request_type_name=u'ServicemanagementOperationsGetRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
   class ServicesAccessPolicyService(base_api.BaseApiService):
     """Service class for the services_accessPolicy resource."""
 
@@ -89,21 +87,6 @@ service.
 
     def __init__(self, client):
       super(ServicemanagementV1.ServicesAccessPolicyService, self).__init__(client)
-      self._method_configs = {
-          'Query': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.accessPolicy.query',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'userEmail'],
-              relative_path=u'v1/services/{serviceName}/accessPolicy:query',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesAccessPolicyQueryRequest',
-              response_type_name=u'QueryUserAccessResponse',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -127,6 +110,19 @@ the service.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Query.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.accessPolicy.query',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'userEmail'],
+        relative_path=u'v1/services/{serviceName}/accessPolicy:query',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesAccessPolicyQueryRequest',
+        response_type_name=u'QueryUserAccessResponse',
+        supports_download=False,
+    )
+
   class ServicesConfigsService(base_api.BaseApiService):
     """Service class for the services_configs resource."""
 
@@ -134,57 +130,6 @@ the service.
 
     def __init__(self, client):
       super(ServicemanagementV1.ServicesConfigsService, self).__init__(client)
-      self._method_configs = {
-          'Create': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.configs.create',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/configs',
-              request_field=u'service',
-              request_type_name=u'ServicemanagementServicesConfigsCreateRequest',
-              response_type_name=u'Service',
-              supports_download=False,
-          ),
-          'Get': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.configs.get',
-              ordered_params=[u'serviceName', u'configId'],
-              path_params=[u'configId', u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/configs/{configId}',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesConfigsGetRequest',
-              response_type_name=u'Service',
-              supports_download=False,
-          ),
-          'List': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.configs.list',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'pageSize', u'pageToken'],
-              relative_path=u'v1/services/{serviceName}/configs',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesConfigsListRequest',
-              response_type_name=u'ListServiceConfigsResponse',
-              supports_download=False,
-          ),
-          'Submit': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.configs.submit',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/configs:submit',
-              request_field=u'submitConfigSourceRequest',
-              request_type_name=u'ServicemanagementServicesConfigsSubmitRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -203,6 +148,19 @@ any backend services.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Create.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.configs.create',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/configs',
+        request_field=u'service',
+        request_type_name=u'ServicemanagementServicesConfigsCreateRequest',
+        response_type_name=u'Service',
+        supports_download=False,
+    )
+
     def Get(self, request, global_params=None):
       """Gets a service config (version) for a managed service. If `config_id` is.
 not specified, the latest service config will be returned.
@@ -217,6 +175,19 @@ not specified, the latest service config will be returned.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.configs.get',
+        ordered_params=[u'serviceName', u'configId'],
+        path_params=[u'configId', u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/configs/{configId}',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesConfigsGetRequest',
+        response_type_name=u'Service',
+        supports_download=False,
+    )
+
     def List(self, request, global_params=None):
       """Lists the history of the service config for a managed service,.
 from the newest to the oldest.
@@ -231,6 +202,19 @@ from the newest to the oldest.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.configs.list',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'pageSize', u'pageToken'],
+        relative_path=u'v1/services/{serviceName}/configs',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesConfigsListRequest',
+        response_type_name=u'ListServiceConfigsResponse',
+        supports_download=False,
+    )
+
     def Submit(self, request, global_params=None):
       """Creates a new service config (version) for a managed service based on.
 user-supplied configuration sources files (for example: OpenAPI
@@ -250,6 +234,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Submit.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.configs.submit',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/configs:submit',
+        request_field=u'submitConfigSourceRequest',
+        request_type_name=u'ServicemanagementServicesConfigsSubmitRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
   class ServicesCustomerSettingsService(base_api.BaseApiService):
     """Service class for the services_customerSettings resource."""
 
@@ -257,33 +254,6 @@ Operation
 
     def __init__(self, client):
       super(ServicemanagementV1.ServicesCustomerSettingsService, self).__init__(client)
-      self._method_configs = {
-          'Get': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.customerSettings.get',
-              ordered_params=[u'serviceName', u'customerId'],
-              path_params=[u'customerId', u'serviceName'],
-              query_params=[u'expand', u'view'],
-              relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesCustomerSettingsGetRequest',
-              response_type_name=u'CustomerSettings',
-              supports_download=False,
-          ),
-          'Patch': base_api.ApiMethodInfo(
-              http_method=u'PATCH',
-              method_id=u'servicemanagement.services.customerSettings.patch',
-              ordered_params=[u'serviceName', u'customerId'],
-              path_params=[u'customerId', u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
-              request_field=u'customerSettings',
-              request_type_name=u'ServicemanagementServicesCustomerSettingsPatchRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -301,6 +271,19 @@ service.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.customerSettings.get',
+        ordered_params=[u'serviceName', u'customerId'],
+        path_params=[u'customerId', u'serviceName'],
+        query_params=[u'expand', u'view'],
+        relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesCustomerSettingsGetRequest',
+        response_type_name=u'CustomerSettings',
+        supports_download=False,
+    )
+
     def Patch(self, request, global_params=None):
       """Updates specified subset of the settings that control the specified.
 customer's usage of the service.  Attempts to update a field not
@@ -318,6 +301,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Patch.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'servicemanagement.services.customerSettings.patch',
+        ordered_params=[u'serviceName', u'customerId'],
+        path_params=[u'customerId', u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}/customerSettings/{customerId}',
+        request_field=u'customerSettings',
+        request_type_name=u'ServicemanagementServicesCustomerSettingsPatchRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
   class ServicesProjectSettingsService(base_api.BaseApiService):
     """Service class for the services_projectSettings resource."""
 
@@ -325,45 +321,6 @@ Operation
 
     def __init__(self, client):
       super(ServicemanagementV1.ServicesProjectSettingsService, self).__init__(client)
-      self._method_configs = {
-          'Get': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.projectSettings.get',
-              ordered_params=[u'serviceName', u'consumerProjectId'],
-              path_params=[u'consumerProjectId', u'serviceName'],
-              query_params=[u'expand', u'view'],
-              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesProjectSettingsGetRequest',
-              response_type_name=u'ProjectSettings',
-              supports_download=False,
-          ),
-          'Patch': base_api.ApiMethodInfo(
-              http_method=u'PATCH',
-              method_id=u'servicemanagement.services.projectSettings.patch',
-              ordered_params=[u'serviceName', u'consumerProjectId'],
-              path_params=[u'consumerProjectId', u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
-              request_field=u'projectSettings',
-              request_type_name=u'ServicemanagementServicesProjectSettingsPatchRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Update': base_api.ApiMethodInfo(
-              http_method=u'PUT',
-              method_id=u'servicemanagement.services.projectSettings.update',
-              ordered_params=[u'serviceName', u'consumerProjectId'],
-              path_params=[u'consumerProjectId', u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
-              request_field='',
-              request_type_name=u'ProjectSettings',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -381,6 +338,19 @@ of the service.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.projectSettings.get',
+        ordered_params=[u'serviceName', u'consumerProjectId'],
+        path_params=[u'consumerProjectId', u'serviceName'],
+        query_params=[u'expand', u'view'],
+        relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesProjectSettingsGetRequest',
+        response_type_name=u'ProjectSettings',
+        supports_download=False,
+    )
+
     def Patch(self, request, global_params=None):
       """Updates specified subset of the settings that control the specified.
 consumer project's usage of the service.  Attempts to update a field not
@@ -398,6 +368,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Patch.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'servicemanagement.services.projectSettings.patch',
+        ordered_params=[u'serviceName', u'consumerProjectId'],
+        path_params=[u'consumerProjectId', u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+        request_field=u'projectSettings',
+        request_type_name=u'ServicemanagementServicesProjectSettingsPatchRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Update(self, request, global_params=None):
       """NOTE: Currently unsupported.  Use PatchProjectSettings instead.
 
@@ -417,6 +400,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Update.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'servicemanagement.services.projectSettings.update',
+        ordered_params=[u'serviceName', u'consumerProjectId'],
+        path_params=[u'consumerProjectId', u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/projectSettings/{consumerProjectId}',
+        request_field='',
+        request_type_name=u'ProjectSettings',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
   class ServicesService(base_api.BaseApiService):
     """Service class for the services resource."""
 
@@ -424,177 +420,6 @@ Operation
 
     def __init__(self, client):
       super(ServicemanagementV1.ServicesService, self).__init__(client)
-      self._method_configs = {
-          'ConvertConfig': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.convertConfig',
-              ordered_params=[],
-              path_params=[],
-              query_params=[],
-              relative_path=u'v1/services:convertConfig',
-              request_field='',
-              request_type_name=u'ConvertConfigRequest',
-              response_type_name=u'ConvertConfigResponse',
-              supports_download=False,
-          ),
-          'Create': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.create',
-              ordered_params=[],
-              path_params=[],
-              query_params=[],
-              relative_path=u'v1/services',
-              request_field='',
-              request_type_name=u'ManagedService',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Delete': base_api.ApiMethodInfo(
-              http_method=u'DELETE',
-              method_id=u'servicemanagement.services.delete',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesDeleteRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Disable': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.disable',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}:disable',
-              request_field=u'disableServiceRequest',
-              request_type_name=u'ServicemanagementServicesDisableRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Enable': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.services.enable',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}:enable',
-              request_field=u'enableServiceRequest',
-              request_type_name=u'ServicemanagementServicesEnableRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Get': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.get',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'consumerProjectId', u'expand', u'view'],
-              relative_path=u'v1/services/{serviceName}',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesGetRequest',
-              response_type_name=u'ManagedService',
-              supports_download=False,
-          ),
-          'GetAccessPolicy': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.getAccessPolicy',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/accessPolicy',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesGetAccessPolicyRequest',
-              response_type_name=u'ServiceAccessPolicy',
-              supports_download=False,
-          ),
-          'GetConfig': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.getConfig',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'configId'],
-              relative_path=u'v1/services/{serviceName}/config',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesGetConfigRequest',
-              response_type_name=u'Service',
-              supports_download=False,
-          ),
-          'List': base_api.ApiMethodInfo(
-              http_method=u'GET',
-              method_id=u'servicemanagement.services.list',
-              ordered_params=[],
-              path_params=[],
-              query_params=[u'category', u'consumerProjectId', u'expand', u'pageSize', u'pageToken', u'producerProjectId'],
-              relative_path=u'v1/services',
-              request_field='',
-              request_type_name=u'ServicemanagementServicesListRequest',
-              response_type_name=u'ListServicesResponse',
-              supports_download=False,
-          ),
-          'Patch': base_api.ApiMethodInfo(
-              http_method=u'PATCH',
-              method_id=u'servicemanagement.services.patch',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}',
-              request_field=u'managedService',
-              request_type_name=u'ServicemanagementServicesPatchRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'PatchConfig': base_api.ApiMethodInfo(
-              http_method=u'PATCH',
-              method_id=u'servicemanagement.services.patchConfig',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}/config',
-              request_field=u'service',
-              request_type_name=u'ServicemanagementServicesPatchConfigRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'Update': base_api.ApiMethodInfo(
-              http_method=u'PUT',
-              method_id=u'servicemanagement.services.update',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}',
-              request_field=u'managedService',
-              request_type_name=u'ServicemanagementServicesUpdateRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          'UpdateAccessPolicy': base_api.ApiMethodInfo(
-              http_method=u'PUT',
-              method_id=u'servicemanagement.services.updateAccessPolicy',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[],
-              relative_path=u'v1/services/{serviceName}/accessPolicy',
-              request_field='',
-              request_type_name=u'ServiceAccessPolicy',
-              response_type_name=u'ServiceAccessPolicy',
-              supports_download=False,
-          ),
-          'UpdateConfig': base_api.ApiMethodInfo(
-              http_method=u'PUT',
-              method_id=u'servicemanagement.services.updateConfig',
-              ordered_params=[u'serviceName'],
-              path_params=[u'serviceName'],
-              query_params=[u'updateMask'],
-              relative_path=u'v1/services/{serviceName}/config',
-              request_field=u'service',
-              request_type_name=u'ServicemanagementServicesUpdateConfigRequest',
-              response_type_name=u'Operation',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -615,6 +440,19 @@ equivalent `google.api.Service`.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    ConvertConfig.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.convertConfig',
+        ordered_params=[],
+        path_params=[],
+        query_params=[],
+        relative_path=u'v1/services:convertConfig',
+        request_field='',
+        request_type_name=u'ConvertConfigRequest',
+        response_type_name=u'ConvertConfigResponse',
+        supports_download=False,
+    )
+
     def Create(self, request, global_params=None):
       """Creates a new managed service.
 
@@ -630,6 +468,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Create.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.create',
+        ordered_params=[],
+        path_params=[],
+        query_params=[],
+        relative_path=u'v1/services',
+        request_field='',
+        request_type_name=u'ManagedService',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Delete(self, request, global_params=None):
       """Deletes a managed service.
 
@@ -645,6 +496,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Delete.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'DELETE',
+        method_id=u'servicemanagement.services.delete',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesDeleteRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Disable(self, request, global_params=None):
       """Disable a managed service for a project.
 Google Service Management will only disable the managed service even if
@@ -662,6 +526,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Disable.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.disable',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}:disable',
+        request_field=u'disableServiceRequest',
+        request_type_name=u'ServicemanagementServicesDisableRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Enable(self, request, global_params=None):
       """Enable a managed service for a project with default setting.
 If the managed service has dependencies, they will be enabled as well.
@@ -678,6 +555,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Enable.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.services.enable',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}:enable',
+        request_field=u'enableServiceRequest',
+        request_type_name=u'ServicemanagementServicesEnableRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Get(self, request, global_params=None):
       """Gets a managed service. If the `consumer_project_id` is specified,.
 the project's settings for the specified service are also returned.
@@ -692,6 +582,19 @@ the project's settings for the specified service are also returned.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.get',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'consumerProjectId', u'expand', u'view'],
+        relative_path=u'v1/services/{serviceName}',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesGetRequest',
+        response_type_name=u'ManagedService',
+        supports_download=False,
+    )
+
     def GetAccessPolicy(self, request, global_params=None):
       """Producer method to retrieve current policy.
 
@@ -705,6 +608,19 @@ the project's settings for the specified service are also returned.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    GetAccessPolicy.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.getAccessPolicy',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/accessPolicy',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesGetAccessPolicyRequest',
+        response_type_name=u'ServiceAccessPolicy',
+        supports_download=False,
+    )
+
     def GetConfig(self, request, global_params=None):
       """Gets a service config (version) for a managed service. If `config_id` is.
 not specified, the latest service config will be returned.
@@ -719,6 +635,19 @@ not specified, the latest service config will be returned.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    GetConfig.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.getConfig',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'configId'],
+        relative_path=u'v1/services/{serviceName}/config',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesGetConfigRequest',
+        response_type_name=u'Service',
+        supports_download=False,
+    )
+
     def List(self, request, global_params=None):
       """Lists all managed services. If the `consumer_project_id` is specified,.
 the project's settings for the specified service are also returned.
@@ -733,6 +662,19 @@ the project's settings for the specified service are also returned.
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'servicemanagement.services.list',
+        ordered_params=[],
+        path_params=[],
+        query_params=[u'category', u'consumerProjectId', u'expand', u'pageSize', u'pageToken', u'producerProjectId'],
+        relative_path=u'v1/services',
+        request_field='',
+        request_type_name=u'ServicemanagementServicesListRequest',
+        response_type_name=u'ListServicesResponse',
+        supports_download=False,
+    )
+
     def Patch(self, request, global_params=None):
       """Updates the specified subset of the configuration. If the specified service.
 does not exists the patch operation fails.
@@ -749,6 +691,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Patch.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'servicemanagement.services.patch',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}',
+        request_field=u'managedService',
+        request_type_name=u'ServicemanagementServicesPatchRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def PatchConfig(self, request, global_params=None):
       """Updates the specified subset of the service resource. Equivalent to.
 calling `PatchService` with only the `service_config` field updated.
@@ -765,6 +720,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    PatchConfig.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'servicemanagement.services.patchConfig',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}/config',
+        request_field=u'service',
+        request_type_name=u'ServicemanagementServicesPatchConfigRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def Update(self, request, global_params=None):
       """Updates the configuration of a service.  If the specified service does not.
 already exist, then it is created.
@@ -781,6 +749,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    Update.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'servicemanagement.services.update',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}',
+        request_field=u'managedService',
+        request_type_name=u'ServicemanagementServicesUpdateRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
     def UpdateAccessPolicy(self, request, global_params=None):
       """Producer method to update the current policy.  This method will return an.
 error if the policy is too large (more than 50 entries across all lists).
@@ -795,6 +776,19 @@ error if the policy is too large (more than 50 entries across all lists).
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    UpdateAccessPolicy.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'servicemanagement.services.updateAccessPolicy',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[],
+        relative_path=u'v1/services/{serviceName}/accessPolicy',
+        request_field='',
+        request_type_name=u'ServiceAccessPolicy',
+        response_type_name=u'ServiceAccessPolicy',
+        supports_download=False,
+    )
+
     def UpdateConfig(self, request, global_params=None):
       """Updates the specified subset of the service resource. Equivalent to.
 calling `UpdateService` with only the `service_config` field updated.
@@ -811,6 +805,19 @@ Operation
       return self._RunMethod(
           config, request, global_params=global_params)
 
+    UpdateConfig.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'servicemanagement.services.updateConfig',
+        ordered_params=[u'serviceName'],
+        path_params=[u'serviceName'],
+        query_params=[u'updateMask'],
+        relative_path=u'v1/services/{serviceName}/config',
+        request_field=u'service',
+        request_type_name=u'ServicemanagementServicesUpdateConfigRequest',
+        response_type_name=u'Operation',
+        supports_download=False,
+    )
+
   class V1Service(base_api.BaseApiService):
     """Service class for the v1 resource."""
 
@@ -818,21 +825,6 @@ Operation
 
     def __init__(self, client):
       super(ServicemanagementV1.V1Service, self).__init__(client)
-      self._method_configs = {
-          'ConvertConfig': base_api.ApiMethodInfo(
-              http_method=u'POST',
-              method_id=u'servicemanagement.convertConfig',
-              ordered_params=[],
-              path_params=[],
-              query_params=[],
-              relative_path=u'v1:convertConfig',
-              request_field='',
-              request_type_name=u'ConvertConfigRequest',
-              response_type_name=u'ConvertConfigResponse',
-              supports_download=False,
-          ),
-          }
-
       self._upload_configs = {
           }
 
@@ -852,3 +844,16 @@ equivalent `google.api.Service`.
       config = self.GetMethodConfig('ConvertConfig')
       return self._RunMethod(
           config, request, global_params=global_params)
+
+    ConvertConfig.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'servicemanagement.convertConfig',
+        ordered_params=[],
+        path_params=[],
+        query_params=[],
+        relative_path=u'v1:convertConfig',
+        request_field='',
+        request_type_name=u'ConvertConfigRequest',
+        response_type_name=u'ConvertConfigResponse',
+        supports_download=False,
+    )
diff --git a/samples/uptodate_check_test.py b/samples/uptodate_check_test.py
index d584b71..73fcf3b 100644
--- a/samples/uptodate_check_test.py
+++ b/samples/uptodate_check_test.py
@@ -70,8 +70,14 @@ class ClientGenCliTest(unittest2.TestCase):
     def testGenClient_DnsDoc(self):
         self._CheckGeneratedFiles('dns', 'v1')
 
+    def testGenClient_FusiontablesDoc(self):
+        self._CheckGeneratedFiles('fusiontables', 'v1')
+
     def testGenClient_IamDoc(self):
         self._CheckGeneratedFiles('iam', 'v1')
 
+    def testGenClient_ServicemanagementDoc(self):
+        self._CheckGeneratedFiles('servicemanagement', 'v1')
+
     def testGenClient_StorageDoc(self):
         self._CheckGeneratedFiles('storage', 'v1')
-- 
GitLab


From ad75088ee9c8eed4d995fd0d52b043e9ff6c47ec Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Mon, 8 Aug 2016 10:26:12 -0400
Subject: [PATCH 255/295] Expose client mock url and http properties. (#125)

---
 apitools/base/py/testing/mock.py      |  6 ++++++
 apitools/base/py/testing/mock_test.py | 27 +++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py
index 777f1af..b5fe016 100644
--- a/apitools/base/py/testing/mock.py
+++ b/apitools/base/py/testing/mock.py
@@ -324,6 +324,10 @@ class Client(object):
         self.__real_include_fields = self.__client_class.IncludeFields
         self.__client_class.IncludeFields = self.IncludeFields
 
+        # pylint: disable=attribute-defined-outside-init
+        self._url = client._url
+        self._http = client._http
+
         return self
 
     def __exit__(self, exc_type, value, traceback):
@@ -338,6 +342,8 @@ class Client(object):
             setattr(self.__client_class, name, service_class)
             delattr(self, service_class._NAME)
         self.__real_service_classes = {}
+        del self._url
+        del self._http
 
         if self._request_responses:
             raise ExpectedRequestsException(
diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py
index 85af559..fb5b9e3 100644
--- a/apitools/base/py/testing/mock_test.py
+++ b/apitools/base/py/testing/mock_test.py
@@ -15,6 +15,7 @@
 
 """Tests for apitools.base.py.testing.mock."""
 
+import httplib2
 import unittest2
 import six
 
@@ -112,6 +113,32 @@ class MockTest(unittest2.TestCase):
             self.assertEquals(fusiontables_messages,
                               mock_client.MESSAGES_MODULE)
 
+    def testMockHasUrlProperty(self):
+        with mock.Client(fusiontables.FusiontablesV1) as mock_client:
+            self.assertEquals(fusiontables.FusiontablesV1.BASE_URL,
+                              mock_client.url)
+        self.assertFalse(hasattr(mock_client, 'url'))
+
+    def testMockHasOverrideUrlProperty(self):
+        real_client = fusiontables.FusiontablesV1(url='http://localhost:8080',
+                                                  get_credentials=False)
+        with mock.Client(fusiontables.FusiontablesV1,
+                         real_client) as mock_client:
+            self.assertEquals('http://localhost:8080/', mock_client.url)
+
+    def testMockHasHttpProperty(self):
+        with mock.Client(fusiontables.FusiontablesV1) as mock_client:
+            self.assertIsInstance(mock_client.http, httplib2.Http)
+        self.assertFalse(hasattr(mock_client, 'http'))
+
+    def testMockHasOverrideHttpProperty(self):
+        real_client = fusiontables.FusiontablesV1(url='http://localhost:8080',
+                                                  http='SomeHttpObject',
+                                                  get_credentials=False)
+        with mock.Client(fusiontables.FusiontablesV1,
+                         real_client) as mock_client:
+            self.assertEquals('SomeHttpObject', mock_client.http)
+
     def testMockPreservesServiceMethods(self):
         services = _GetApiServices(fusiontables.FusiontablesV1)
         with mock.Client(fusiontables.FusiontablesV1):
-- 
GitLab


From 9d0c388922395becf38bc0977ee8b89f7fd3f52a Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 9 Aug 2016 12:52:29 -0400
Subject: [PATCH 256/295] Provide more information when raising HttpError.

Include method_config and original request when raising HttpError. This will allow to produce more general error repoting logic.
---
 apitools/base/py/base_api.py      | 12 +++++++-----
 apitools/base/py/base_api_test.py | 30 ++++++++++++++++++++++++++++--
 apitools/base/py/exceptions.py    |  5 ++++-
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py
index e3297f7..56aae3d 100644
--- a/apitools/base/py/base_api.py
+++ b/apitools/base/py/base_api.py
@@ -590,11 +590,13 @@ class BaseApiService(object):
             url_builder.query_params = {}
         http_request.url = url_builder.url
 
-    def __ProcessHttpResponse(self, method_config, http_response):
+    def __ProcessHttpResponse(self, method_config, http_response, request):
         """Process the given http response."""
         if http_response.status_code not in (http_client.OK,
                                              http_client.NO_CONTENT):
-            raise exceptions.HttpError.FromResponse(http_response)
+            raise exceptions.HttpError(
+                http_response.info, http_response.content,
+                http_response.request_url, method_config, request)
         if http_response.status_code == http_client.NO_CONTENT:
             # TODO(craigcitro): Find out why _replace doesn't seem to work
             # here.
@@ -718,10 +720,10 @@ class BaseApiService(object):
             http_response = http_wrapper.MakeRequest(
                 http, http_request, **opts)
 
-        return self.ProcessHttpResponse(method_config, http_response)
+        return self.ProcessHttpResponse(method_config, http_response, request)
 
-    def ProcessHttpResponse(self, method_config, http_response):
+    def ProcessHttpResponse(self, method_config, http_response, request=None):
         """Convert an HTTP response to the expected message type."""
         return self.__client.ProcessResponse(
             method_config,
-            self.__ProcessHttpResponse(method_config, http_response))
+            self.__ProcessHttpResponse(method_config, http_response, request))
diff --git a/apitools/base/py/base_api_test.py b/apitools/base/py/base_api_test.py
index c41f9d5..7b23fa6 100644
--- a/apitools/base/py/base_api_test.py
+++ b/apitools/base/py/base_api_test.py
@@ -19,6 +19,7 @@ import sys
 import contextlib
 
 import six
+from six.moves import http_client
 from six.moves import urllib_parse
 import unittest2
 
@@ -26,6 +27,7 @@ from apitools.base.protorpclite import message_types
 from apitools.base.protorpclite import messages
 from apitools.base.py import base_api
 from apitools.base.py import encoding
+from apitools.base.py import exceptions
 from apitools.base.py import http_wrapper
 
 
@@ -154,7 +156,6 @@ class BaseApiTest(unittest2.TestCase):
             return http_wrapper.Response(
                 info={'status': '200'}, content='{"field": "abc"}',
                 request_url='http://www.google.com')
-        http_wrapper.MakeRequest = fakeMakeRequest
         method_config = base_api.ApiMethodInfo(
             request_type_name='SimpleMessage',
             response_type_name='SimpleMessage')
@@ -174,7 +175,6 @@ class BaseApiTest(unittest2.TestCase):
             return http_wrapper.Response(
                 info={'status': '200'}, content='{"field": "abc"}',
                 request_url='http://www.google.com')
-        http_wrapper.MakeRequest = fakeMakeRequest
         method_config = base_api.ApiMethodInfo(
             request_type_name='SimpleMessage',
             response_type_name='SimpleMessage')
@@ -185,6 +185,28 @@ class BaseApiTest(unittest2.TestCase):
         with mock(base_api.http_wrapper, 'MakeRequest', fakeMakeRequest):
             service._RunMethod(method_config, request)
 
+    def testHttpError(self):
+        def fakeMakeRequest(*unused_args, **unused_kwargs):
+            return http_wrapper.Response(
+                info={'status': http_client.BAD_REQUEST},
+                content='{"field": "abc"}',
+                request_url='http://www.google.com')
+        method_config = base_api.ApiMethodInfo(
+            request_type_name='SimpleMessage',
+            response_type_name='SimpleMessage')
+        client = self.__GetFakeClient()
+        service = FakeService(client=client)
+        request = SimpleMessage()
+        with mock(base_api.http_wrapper, 'MakeRequest', fakeMakeRequest):
+            with self.assertRaises(exceptions.HttpError) as error_context:
+                service._RunMethod(method_config, request)
+        http_error = error_context.exception
+        self.assertEquals(400, http_error.status_code)
+        self.assertEquals('http://www.google.com', http_error.url)
+        self.assertEquals('{"field": "abc"}', http_error.content)
+        self.assertEquals(method_config, http_error.method_config)
+        self.assertEquals(request, http_error.request)
+
     def testQueryEncoding(self):
         method_config = base_api.ApiMethodInfo(
             request_type_name='MessageWithTime', query_params=['timestamp'])
@@ -288,3 +310,7 @@ class BaseApiTest(unittest2.TestCase):
         http_request = service.PrepareHttpRequest(method_config, request)
         self.assertEqual('http://www.example.com/path:withJustColon',
                          http_request.url)
+
+
+if __name__ == '__main__':
+    unittest2.main()
diff --git a/apitools/base/py/exceptions.py b/apitools/base/py/exceptions.py
index a3789b9..e63b893 100644
--- a/apitools/base/py/exceptions.py
+++ b/apitools/base/py/exceptions.py
@@ -51,11 +51,14 @@ class HttpError(CommunicationError):
 
     """Error making a request. Soon to be HttpError."""
 
-    def __init__(self, response, content, url):
+    def __init__(self, response, content, url,
+                 method_config=None, request=None):
         super(HttpError, self).__init__()
         self.response = response
         self.content = content
         self.url = url
+        self.method_config = method_config
+        self.request = request
 
     def __str__(self):
         content = self.content
-- 
GitLab


From 0555347edf1b0936fcba0da0a1618236d11ecc39 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Thu, 29 Sep 2016 14:59:55 -0400
Subject: [PATCH 257/295] Fix #127 parsing property int values from strings.

---
 apitools/base/py/encoding.py      |  5 ++---
 apitools/base/py/encoding_test.py | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py
index 698304b..7eec985 100644
--- a/apitools/base/py/encoding.py
+++ b/apitools/base/py/encoding.py
@@ -422,10 +422,9 @@ def _DecodeUnrecognizedFields(message, pair_type):
         value_type = pair_type.field_by_name('value')
         if isinstance(value_type, messages.MessageField):
             decoded_value = DictToMessage(value, pair_type.value.message_type)
-        elif isinstance(value_type, messages.EnumField):
-            decoded_value = pair_type.value.type(value)
         else:
-            decoded_value = value
+            decoded_value = protojson.ProtoJson().decode_field(
+                pair_type.value, value)
         new_pair = pair_type(key=str(unknown_field), value=decoded_value)
         new_values.append(new_pair)
     return new_values
diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py
index ad51965..d306aae 100644
--- a/apitools/base/py/encoding_test.py
+++ b/apitools/base/py/encoding_test.py
@@ -53,6 +53,17 @@ class AdditionalPropertiesMessage(messages.Message):
         'AdditionalProperty', 1, repeated=True)
 
 
+@encoding.MapUnrecognizedFields('additional_properties')
+class AdditionalIntPropertiesMessage(messages.Message):
+
+    class AdditionalProperty(messages.Message):
+        key = messages.StringField(1)
+        value = messages.IntegerField(2)
+
+    additional_properties = messages.MessageField(
+        'AdditionalProperty', 1, repeated=True)
+
+
 @encoding.MapUnrecognizedFields('additional_properties')
 class UnrecognizedEnumMessage(messages.Message):
 
@@ -229,6 +240,16 @@ class EncodingTest(unittest2.TestCase):
         msg = encoding.JsonToMessage(HasNestedMessage, json_msg)
         self.assertEqual(1, len(msg.nested.additional_properties))
 
+    def testNumericPropertyValue(self):
+        json_msg = '{"key_one": "123"}'
+        msg = encoding.JsonToMessage(AdditionalIntPropertiesMessage, json_msg)
+        self.assertEqual(
+            AdditionalIntPropertiesMessage(
+                additional_properties=[
+                    AdditionalIntPropertiesMessage.AdditionalProperty(
+                        key='key_one', value=123)]),
+            msg)
+
     def testAdditionalMessageProperties(self):
         json_msg = '{"input": {"index": 0, "name": "output"}}'
         result = encoding.JsonToMessage(
-- 
GitLab


From c8066057c20a2511f0bbf754b43254e4c4ef9790 Mon Sep 17 00:00:00 2001
From: Benjamin Turner 
Date: Thu, 29 Sep 2016 15:11:13 -0400
Subject: [PATCH 258/295] Update YieldFromList to support APIs without a
 batch_size_attribute

---
 apitools/base/py/list_pager.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/apitools/base/py/list_pager.py b/apitools/base/py/list_pager.py
index 817606d..13b1cba 100644
--- a/apitools/base/py/list_pager.py
+++ b/apitools/base/py/list_pager.py
@@ -52,14 +52,15 @@ def YieldFromList(
           response message holding the page token for the next page.
       batch_size_attribute: str, The name of the attribute in a
           response message holding the maximum number of results to be
-          returned.
+          returned. None if caller-specified batch size is unsupported.
 
     Yields:
       protorpc.message.Message, The resources listed by the service.
 
     """
     request = encoding.CopyProtoMessage(request)
-    setattr(request, batch_size_attribute, batch_size)
+    if batch_size_attribute:
+        setattr(request, batch_size_attribute, batch_size)
     setattr(request, current_token_attribute, None)
     while limit is None or limit:
         response = getattr(service, method)(request,
-- 
GitLab


From e59816f1936504e4cc7e1fad6d7381afdfe078d3 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Fri, 30 Sep 2016 08:27:33 -0400
Subject: [PATCH 259/295] Force gflags dependent library version to 3.0.6.

Python 2.6 is no longer supported by gflags.
---
 setup.py | 2 +-
 tox.ini  | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index a014347..2175b4c 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@ REQUIRED_PACKAGES = [
 
 CLI_PACKAGES = [
     'google-apputils>=0.4.0',
-    'python-gflags>=2.0',
+    'python-gflags==3.0.6',  # Starting version 3.0.7 py26 is not supported.
 ]
 
 TESTING_PACKAGES = [
diff --git a/tox.ini b/tox.ini
index dbdeb78..e2d1f23 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,7 +2,9 @@
 envlist = py26,py27,pypy,py34,py35,lint,cover,py27oldoauth2client
 
 [testenv]
-deps = nose
+deps =
+    nose
+    python-gflags==3.0.6
 commands =
     pip install google-apitools[testing]
     nosetests []
@@ -53,7 +55,7 @@ commands =
     nosetests --with-xunit --with-xcoverage --cover-package=apitools --nocapture --cover-erase --cover-tests --cover-branches []
 deps =
     google-apputils
-    python-gflags
+    python-gflags==3.0.6
     mock
     nose
     unittest2
-- 
GitLab


From deac102f72664479ba0f1379d415cd98433e93ab Mon Sep 17 00:00:00 2001
From: Benjamin Turner 
Date: Fri, 30 Sep 2016 09:18:57 -0400
Subject: [PATCH 260/295] Update list_pager_test.py

---
 apitools/base/py/list_pager_test.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/apitools/base/py/list_pager_test.py b/apitools/base/py/list_pager_test.py
index bffc09a..21f33c1 100644
--- a/apitools/base/py/list_pager_test.py
+++ b/apitools/base/py/list_pager_test.py
@@ -218,3 +218,29 @@ class ListPagerAttributeTest(unittest2.TestCase):
         for i, instance in enumerate(results):
             self.assertEquals('c{0}'.format(i), instance.fullResourcePath)
         self.assertEquals(2, i)
+    
+    def testYieldFromListWithNoBatchSizeAttribute(self):
+        self.mocked_client.iamPolicies.GetPolicyDetails.Expect(
+            iam_messages.GetPolicyDetailsRequest(
+                pageToken=None,
+                fullResourcePath='myresource',
+            ),
+            iam_messages.GetPolicyDetailsResponse(
+                policies=[
+                    iam_messages.PolicyDetail(fullResourcePath='c0'),
+                    iam_messages.PolicyDetail(fullResourcePath='c1'),
+                ],
+            ))
+        
+        client = iam_client.IamV1(get_credentials=False)
+        request = iam_messages.GetPolicyDetailsRequest(
+            fullResourcePath='myresource')
+        results = list_pager.YieldFromList(
+            client.iamPolicies, request,
+            batch_size_attribute=None,
+            method='GetPolicyDetails', field='policies')
+
+        i = 0
+        for i, instance in enumerate(results):
+            self.assertEquals('c{0}'.format(i), instance.fullResourcePath)
+        self.assertEquals(1, i)
-- 
GitLab


From 61695dc10c855ffd28643d95006812e10385c3b6 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 4 Oct 2016 14:10:17 -0400
Subject: [PATCH 261/295] Do not raise exception in client mock if one already
 active.

---
 apitools/base/py/list_pager_test.py   |  4 ++--
 apitools/base/py/testing/mock.py      | 21 ++++++++++++---------
 apitools/base/py/testing/mock_test.py | 10 ++++++++++
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/apitools/base/py/list_pager_test.py b/apitools/base/py/list_pager_test.py
index 21f33c1..3aafede 100644
--- a/apitools/base/py/list_pager_test.py
+++ b/apitools/base/py/list_pager_test.py
@@ -218,7 +218,7 @@ class ListPagerAttributeTest(unittest2.TestCase):
         for i, instance in enumerate(results):
             self.assertEquals('c{0}'.format(i), instance.fullResourcePath)
         self.assertEquals(2, i)
-    
+
     def testYieldFromListWithNoBatchSizeAttribute(self):
         self.mocked_client.iamPolicies.GetPolicyDetails.Expect(
             iam_messages.GetPolicyDetailsRequest(
@@ -231,7 +231,7 @@ class ListPagerAttributeTest(unittest2.TestCase):
                     iam_messages.PolicyDetail(fullResourcePath='c1'),
                 ],
             ))
-        
+
         client = iam_client.IamV1(get_credentials=False)
         request = iam_messages.GetPolicyDetailsRequest(
             fullResourcePath='myresource')
diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py
index b5fe016..6642dc8 100644
--- a/apitools/base/py/testing/mock.py
+++ b/apitools/base/py/testing/mock.py
@@ -23,6 +23,7 @@ as it's all done within the context of a mock.
 """
 
 import difflib
+import sys
 
 import six
 
@@ -331,12 +332,13 @@ class Client(object):
         return self
 
     def __exit__(self, exc_type, value, traceback):
-        self.Unmock()
-        if value:
+        is_active_exception = value is not None
+        self.Unmock(suppress=is_active_exception)
+        if is_active_exception:
             six.reraise(exc_type, value, traceback)
         return True
 
-    def Unmock(self):
+    def Unmock(self, suppress=False):
         self.__class__ = self.__orig_class
         for name, service_class in self.__real_service_classes.items():
             setattr(self.__client_class, name, service_class)
@@ -345,15 +347,16 @@ class Client(object):
         del self._url
         del self._http
 
-        if self._request_responses:
-            raise ExpectedRequestsException(
-                [(rq_rs.key, rq_rs.request) for rq_rs
-                 in self._request_responses])
-        self._request_responses = []
-
         self.__client_class.IncludeFields = self.__real_include_fields
         self.__real_include_fields = None
 
+        requests = [(rq_rs.key, rq_rs.request)
+                    for rq_rs in self._request_responses]
+        self._request_responses = []
+
+        if requests and not suppress and sys.exc_info()[1] is None:
+            raise ExpectedRequestsException(requests)
+
     def IncludeFields(self, include_fields):
         if self.__real_client:
             return self.__real_include_fields(self.__real_client,
diff --git a/apitools/base/py/testing/mock_test.py b/apitools/base/py/testing/mock_test.py
index fb5b9e3..d295f21 100644
--- a/apitools/base/py/testing/mock_test.py
+++ b/apitools/base/py/testing/mock_test.py
@@ -37,6 +37,10 @@ def _GetApiServices(api_client_class):
             issubclass(potential_service, apitools_base.BaseApiService)))
 
 
+class CustomException(Exception):
+    pass
+
+
 class MockTest(unittest2.TestCase):
 
     def testMockFusionBasic(self):
@@ -56,6 +60,12 @@ class MockTest(unittest2.TestCase):
             with self.assertRaises(apitools_base.HttpError):
                 client.column.List(1)
 
+    def testMockIfAnotherException(self):
+        with self.assertRaises(CustomException):
+            with mock.Client(fusiontables.FusiontablesV1) as client_class:
+                client_class.column.List.Expect(request=1, response=2)
+                raise CustomException('Something when wrong')
+
     def testMockFusionOrder(self):
         with mock.Client(fusiontables.FusiontablesV1) as client_class:
             client_class.column.List.Expect(request=1, response=2)
-- 
GitLab


From 8da5d74ddc4944307ec676729619216e09c4961f Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Wed, 5 Oct 2016 08:57:06 -0400
Subject: [PATCH 262/295] Add bigquery sample.

This is one of the most popular apis. Will also migrate some existing
tests against bigquery to this repository.
---
 run_pylint.py                                 |    1 +
 samples/bigquery_sample/bigquery_v2.json      | 2636 +++++++++++++++++
 .../bigquery_sample/bigquery_v2/__init__.py   |    5 +
 .../bigquery_v2/bigquery_v2.py                | 1096 +++++++
 .../bigquery_v2/bigquery_v2_client.py         |  649 ++++
 .../bigquery_v2/bigquery_v2_messages.py       | 2050 +++++++++++++
 samples/regenerate_samples.py                 |    1 +
 samples/uptodate_check_test.py                |    3 +
 8 files changed, 6441 insertions(+)
 create mode 100644 samples/bigquery_sample/bigquery_v2.json
 create mode 100644 samples/bigquery_sample/bigquery_v2/__init__.py
 create mode 100644 samples/bigquery_sample/bigquery_v2/bigquery_v2.py
 create mode 100644 samples/bigquery_sample/bigquery_v2/bigquery_v2_client.py
 create mode 100644 samples/bigquery_sample/bigquery_v2/bigquery_v2_messages.py

diff --git a/run_pylint.py b/run_pylint.py
index c644b53..c53943f 100644
--- a/run_pylint.py
+++ b/run_pylint.py
@@ -33,6 +33,7 @@ import sys
 
 IGNORED_DIRECTORIES = [
     'apitools/gen/testdata',
+    'samples/bigquery_sample/bigquery_v2',
     'samples/dns_sample/dns_v1',
     'samples/fusiontables_sample/fusiontables_v1',
     'samples/iam_sample/iam_v1',
diff --git a/samples/bigquery_sample/bigquery_v2.json b/samples/bigquery_sample/bigquery_v2.json
new file mode 100644
index 0000000..8f4d760
--- /dev/null
+++ b/samples/bigquery_sample/bigquery_v2.json
@@ -0,0 +1,2636 @@
+{
+ "kind": "discovery#restDescription",
+ "discoveryVersion": "v1",
+ "id": "bigquery:v2",
+ "name": "bigquery",
+ "version": "v2",
+ "revision": "20160819",
+ "title": "BigQuery API",
+ "description": "A data platform for customers to create, manage, share and query data.",
+ "ownerDomain": "google.com",
+ "ownerName": "Google",
+ "icons": {
+  "x16": "https://www.google.com/images/icons/product/search-16.gif",
+  "x32": "https://www.google.com/images/icons/product/search-32.gif"
+ },
+ "documentationLink": "https://cloud.google.com/bigquery/",
+ "protocol": "rest",
+ "baseUrl": "https://www.googleapis.com/bigquery/v2/",
+ "basePath": "/bigquery/v2/",
+ "rootUrl": "https://www.googleapis.com/",
+ "servicePath": "bigquery/v2/",
+ "batchPath": "batch",
+ "parameters": {
+  "alt": {
+   "type": "string",
+   "description": "Data format for the response.",
+   "default": "json",
+   "enum": [
+    "json"
+   ],
+   "enumDescriptions": [
+    "Responses with Content-Type of application/json"
+   ],
+   "location": "query"
+  },
+  "fields": {
+   "type": "string",
+   "description": "Selector specifying which fields to include in a partial response.",
+   "location": "query"
+  },
+  "key": {
+   "type": "string",
+   "description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
+   "location": "query"
+  },
+  "oauth_token": {
+   "type": "string",
+   "description": "OAuth 2.0 token for the current user.",
+   "location": "query"
+  },
+  "prettyPrint": {
+   "type": "boolean",
+   "description": "Returns response with indentations and line breaks.",
+   "default": "true",
+   "location": "query"
+  },
+  "quotaUser": {
+   "type": "string",
+   "description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.",
+   "location": "query"
+  },
+  "userIp": {
+   "type": "string",
+   "description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.",
+   "location": "query"
+  }
+ },
+ "auth": {
+  "oauth2": {
+   "scopes": {
+    "https://www.googleapis.com/auth/bigquery": {
+     "description": "View and manage your data in Google BigQuery"
+    },
+    "https://www.googleapis.com/auth/bigquery.insertdata": {
+     "description": "Insert data into Google BigQuery"
+    },
+    "https://www.googleapis.com/auth/cloud-platform": {
+     "description": "View and manage your data across Google Cloud Platform services"
+    },
+    "https://www.googleapis.com/auth/cloud-platform.read-only": {
+     "description": "View your data across Google Cloud Platform services"
+    },
+    "https://www.googleapis.com/auth/devstorage.full_control": {
+     "description": "Manage your data and permissions in Google Cloud Storage"
+    },
+    "https://www.googleapis.com/auth/devstorage.read_only": {
+     "description": "View your data in Google Cloud Storage"
+    },
+    "https://www.googleapis.com/auth/devstorage.read_write": {
+     "description": "Manage your data in Google Cloud Storage"
+    }
+   }
+  }
+ },
+ "schemas": {
+  "BigtableColumn": {
+   "id": "BigtableColumn",
+   "type": "object",
+   "properties": {
+    "encoding": {
+     "type": "string",
+     "description": "[Optional] The encoding of the values when the type is not STRING. Acceptable encoding values are: TEXT - indicates values are alphanumeric text strings. BINARY - indicates values are encoded using HBase Bytes.toBytes family of functions. 'encoding' can also be set at the column family level. However, the setting at this level takes precedence if 'encoding' is set at both levels."
+    },
+    "fieldName": {
+     "type": "string",
+     "description": "[Optional] If the qualifier is not a valid BigQuery field identifier i.e. does not match [a-zA-Z][a-zA-Z0-9_]*, a valid identifier must be provided as the column field name and is used as field name in queries."
+    },
+    "onlyReadLatest": {
+     "type": "boolean",
+     "description": "[Optional] If this is set, only the latest version of value in this column are exposed. 'onlyReadLatest' can also be set at the column family level. However, the setting at this level takes precedence if 'onlyReadLatest' is set at both levels."
+    },
+    "qualifierEncoded": {
+     "type": "string",
+     "description": "[Required] Qualifier of the column. Columns in the parent column family that has this exact qualifier are exposed as . field. If the qualifier is valid UTF-8 string, it can be specified in the qualifier_string field. Otherwise, a base-64 encoded value must be set to qualifier_encoded. The column field name is the same as the column qualifier. However, if the qualifier is not a valid BigQuery field identifier i.e. does not match [a-zA-Z][a-zA-Z0-9_]*, a valid identifier must be provided as field_name.",
+     "format": "byte"
+    },
+    "qualifierString": {
+     "type": "string"
+    },
+    "type": {
+     "type": "string",
+     "description": "[Optional] The type to convert the value in cells of this column. The values are expected to be encoded using HBase Bytes.toBytes function when using the BINARY encoding value. Following BigQuery types are allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN Default type is BYTES. 'type' can also be set at the column family level. However, the setting at this level takes precedence if 'type' is set at both levels."
+    }
+   }
+  },
+  "BigtableColumnFamily": {
+   "id": "BigtableColumnFamily",
+   "type": "object",
+   "properties": {
+    "columns": {
+     "type": "array",
+     "description": "[Optional] Lists of columns that should be exposed as individual fields as opposed to a list of (column name, value) pairs. All columns whose qualifier matches a qualifier in this list can be accessed as .. Other columns can be accessed as a list through .Column field.",
+     "items": {
+      "$ref": "BigtableColumn"
+     }
+    },
+    "encoding": {
+     "type": "string",
+     "description": "[Optional] The encoding of the values when the type is not STRING. Acceptable encoding values are: TEXT - indicates values are alphanumeric text strings. BINARY - indicates values are encoded using HBase Bytes.toBytes family of functions. This can be overridden for a specific column by listing that column in 'columns' and specifying an encoding for it."
+    },
+    "familyId": {
+     "type": "string",
+     "description": "Identifier of the column family."
+    },
+    "onlyReadLatest": {
+     "type": "boolean",
+     "description": "[Optional] If this is set only the latest version of value are exposed for all columns in this column family. This can be overridden for a specific column by listing that column in 'columns' and specifying a different setting for that column."
+    },
+    "type": {
+     "type": "string",
+     "description": "[Optional] The type to convert the value in cells of this column family. The values are expected to be encoded using HBase Bytes.toBytes function when using the BINARY encoding value. Following BigQuery types are allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN Default type is BYTES. This can be overridden for a specific column by listing that column in 'columns' and specifying a type for it."
+    }
+   }
+  },
+  "BigtableOptions": {
+   "id": "BigtableOptions",
+   "type": "object",
+   "properties": {
+    "columnFamilies": {
+     "type": "array",
+     "description": "[Optional] List of column families to expose in the table schema along with their types. This list restricts the column families that can be referenced in queries and specifies their value types. You can use this list to do type conversions - see the 'type' field for more details. If you leave this list empty, all column families are present in the table schema and their values are read as BYTES. During a query only the column families referenced in that query are read from Bigtable.",
+     "items": {
+      "$ref": "BigtableColumnFamily"
+     }
+    },
+    "ignoreUnspecifiedColumnFamilies": {
+     "type": "boolean",
+     "description": "[Optional] If field is true, then the column families that are not specified in columnFamilies list are not exposed in the table schema. Otherwise, they are read with BYTES type values. The default value is false."
+    },
+    "readRowkeyAsString": {
+     "type": "boolean",
+     "description": "[Optional] If field is true, then the rowkey column families will be read and converted to string. Otherwise they are read with BYTES type values and users need to manually cast them with CAST if necessary. The default value is false."
+    }
+   }
+  },
+  "CsvOptions": {
+   "id": "CsvOptions",
+   "type": "object",
+   "properties": {
+    "allowJaggedRows": {
+     "type": "boolean",
+     "description": "[Optional] Indicates if BigQuery should accept rows that are missing trailing optional columns. If true, BigQuery treats missing trailing columns as null values. If false, records with missing trailing columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false."
+    },
+    "allowQuotedNewlines": {
+     "type": "boolean",
+     "description": "[Optional] Indicates if BigQuery should allow quoted data sections that contain newline characters in a CSV file. The default value is false."
+    },
+    "encoding": {
+     "type": "string",
+     "description": "[Optional] The character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split using the values of the quote and fieldDelimiter properties."
+    },
+    "fieldDelimiter": {
+     "type": "string",
+     "description": "[Optional] The separator for fields in a CSV file. BigQuery converts the string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data in its raw, binary state. BigQuery also supports the escape sequence \"\\t\" to specify a tab separator. The default value is a comma (',')."
+    },
+    "quote": {
+     "type": "string",
+     "description": "[Optional] The value that is used to quote data sections in a CSV file. BigQuery converts the string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data in its raw, binary state. The default value is a double-quote ('\"'). If your data does not contain quoted sections, set the property value to an empty string. If your data contains quoted newline characters, you must also set the allowQuotedNewlines property to true.",
+     "default": "\"",
+     "pattern": ".?"
+    },
+    "skipLeadingRows": {
+     "type": "string",
+     "description": "[Optional] The number of rows at the top of a CSV file that BigQuery will skip when reading the data. The default value is 0. This property is useful if you have header rows in the file that should be skipped.",
+     "format": "int64"
+    }
+   }
+  },
+  "Dataset": {
+   "id": "Dataset",
+   "type": "object",
+   "properties": {
+    "access": {
+     "type": "array",
+     "description": "[Optional] An array of objects that define dataset access for one or more entities. You can set this property when inserting or updating a dataset in order to control who is allowed to access the data. If unspecified at dataset creation time, BigQuery adds default dataset access for the following entities: access.specialGroup: projectReaders; access.role: READER; access.specialGroup: projectWriters; access.role: WRITER; access.specialGroup: projectOwners; access.role: OWNER; access.userByEmail: [dataset creator email]; access.role: OWNER;",
+     "items": {
+      "type": "object",
+      "properties": {
+       "domain": {
+        "type": "string",
+        "description": "[Pick one] A domain to grant access to. Any users signed in with the domain specified will be granted the specified access. Example: \"example.com\"."
+       },
+       "groupByEmail": {
+        "type": "string",
+        "description": "[Pick one] An email address of a Google Group to grant access to."
+       },
+       "role": {
+        "type": "string",
+        "description": "[Required] Describes the rights granted to the user specified by the other member of the access object. The following string values are supported: READER, WRITER, OWNER."
+       },
+       "specialGroup": {
+        "type": "string",
+        "description": "[Pick one] A special group to grant access to. Possible values include: projectOwners: Owners of the enclosing project. projectReaders: Readers of the enclosing project. projectWriters: Writers of the enclosing project. allAuthenticatedUsers: All authenticated BigQuery users."
+       },
+       "userByEmail": {
+        "type": "string",
+        "description": "[Pick one] An email address of a user to grant access to. For example: fred@example.com."
+       },
+       "view": {
+        "$ref": "TableReference",
+        "description": "[Pick one] A view from a different dataset to grant access to. Queries executed against that view will have read access to tables in this dataset. The role field is not required when this field is set. If that view is updated by any user, access to the view needs to be granted again via an update operation."
+       }
+      }
+     }
+    },
+    "creationTime": {
+     "type": "string",
+     "description": "[Output-only] The time when this dataset was created, in milliseconds since the epoch.",
+     "format": "int64"
+    },
+    "datasetReference": {
+     "$ref": "DatasetReference",
+     "description": "[Required] A reference that identifies the dataset."
+    },
+    "defaultTableExpirationMs": {
+     "type": "string",
+     "description": "[Optional] The default lifetime of all tables in the dataset, in milliseconds. The minimum value is 3600000 milliseconds (one hour). Once this property is set, all newly-created tables in the dataset will have an expirationTime property set to the creation time plus the value in this property, and changing the value will only affect new tables, not existing ones. When the expirationTime for a given table is reached, that table will be deleted automatically. If a table's expirationTime is modified or removed before the table expires, or if you provide an explicit expirationTime when creating a table, that value takes precedence over the default expiration time indicated by this property.",
+     "format": "int64"
+    },
+    "description": {
+     "type": "string",
+     "description": "[Optional] A user-friendly description of the dataset."
+    },
+    "etag": {
+     "type": "string",
+     "description": "[Output-only] A hash of the resource."
+    },
+    "friendlyName": {
+     "type": "string",
+     "description": "[Optional] A descriptive name for the dataset."
+    },
+    "id": {
+     "type": "string",
+     "description": "[Output-only] The fully-qualified unique name of the dataset in the format projectId:datasetId. The dataset name without the project name is given in the datasetId field. When creating a new dataset, leave this field blank, and instead specify the datasetId field."
+    },
+    "kind": {
+     "type": "string",
+     "description": "[Output-only] The resource type.",
+     "default": "bigquery#dataset"
+    },
+    "labels": {
+     "type": "object",
+     "description": "[Experimental] The labels associated with this dataset. You can use these to organize and group your datasets. You can set this property when inserting or updating a dataset. Label keys and values can be no longer than 63 characters, can only contain letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and must be unique within a dataset. Both keys and values are additionally constrained to be \u003c= 128 bytes in size.",
+     "additionalProperties": {
+      "type": "string"
+     }
+    },
+    "lastModifiedTime": {
+     "type": "string",
+     "description": "[Output-only] The date when this dataset or any of its tables was last modified, in milliseconds since the epoch.",
+     "format": "int64"
+    },
+    "location": {
+     "type": "string",
+     "description": "[Experimental] The geographic location where the dataset should reside. Possible values include EU and US. The default value is US."
+    },
+    "selfLink": {
+     "type": "string",
+     "description": "[Output-only] A URL that can be used to access the resource again. You can use this URL in Get or Update requests to the resource."
+    }
+   }
+  },
+  "DatasetList": {
+   "id": "DatasetList",
+   "type": "object",
+   "properties": {
+    "datasets": {
+     "type": "array",
+     "description": "An array of the dataset resources in the project. Each resource contains basic information. For full information about a particular dataset resource, use the Datasets: get method. This property is omitted when there are no datasets in the project.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "datasetReference": {
+        "$ref": "DatasetReference",
+        "description": "The dataset reference. Use this property to access specific parts of the dataset's ID, such as project ID or dataset ID."
+       },
+       "friendlyName": {
+        "type": "string",
+        "description": "A descriptive name for the dataset, if one exists."
+       },
+       "id": {
+        "type": "string",
+        "description": "The fully-qualified, unique, opaque ID of the dataset."
+       },
+       "kind": {
+        "type": "string",
+        "description": "The resource type. This property always returns the value \"bigquery#dataset\".",
+        "default": "bigquery#dataset"
+       },
+       "labels": {
+        "type": "object",
+        "description": "[Experimental] The labels associated with this dataset. You can use these to organize and group your datasets.",
+        "additionalProperties": {
+         "type": "string"
+        }
+       }
+      }
+     }
+    },
+    "etag": {
+     "type": "string",
+     "description": "A hash value of the results page. You can use this property to determine if the page has changed since the last request."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The list type. This property always returns the value \"bigquery#datasetList\".",
+     "default": "bigquery#datasetList"
+    },
+    "nextPageToken": {
+     "type": "string",
+     "description": "A token that can be used to request the next results page. This property is omitted on the final results page."
+    }
+   }
+  },
+  "DatasetReference": {
+   "id": "DatasetReference",
+   "type": "object",
+   "properties": {
+    "datasetId": {
+     "type": "string",
+     "description": "[Required] A unique ID for this dataset, without the project name. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). The maximum length is 1,024 characters.",
+     "annotations": {
+      "required": [
+       "bigquery.datasets.update"
+      ]
+     }
+    },
+    "projectId": {
+     "type": "string",
+     "description": "[Optional] The ID of the project containing this dataset.",
+     "annotations": {
+      "required": [
+       "bigquery.datasets.update"
+      ]
+     }
+    }
+   }
+  },
+  "ErrorProto": {
+   "id": "ErrorProto",
+   "type": "object",
+   "properties": {
+    "debugInfo": {
+     "type": "string",
+     "description": "Debugging information. This property is internal to Google and should not be used."
+    },
+    "location": {
+     "type": "string",
+     "description": "Specifies where the error occurred, if present."
+    },
+    "message": {
+     "type": "string",
+     "description": "A human-readable description of the error."
+    },
+    "reason": {
+     "type": "string",
+     "description": "A short error code that summarizes the error."
+    }
+   }
+  },
+  "ExplainQueryStage": {
+   "id": "ExplainQueryStage",
+   "type": "object",
+   "properties": {
+    "computeRatioAvg": {
+     "type": "number",
+     "description": "Relative amount of time the average shard spent on CPU-bound tasks.",
+     "format": "double"
+    },
+    "computeRatioMax": {
+     "type": "number",
+     "description": "Relative amount of time the slowest shard spent on CPU-bound tasks.",
+     "format": "double"
+    },
+    "id": {
+     "type": "string",
+     "description": "Unique ID for stage within plan.",
+     "format": "int64"
+    },
+    "name": {
+     "type": "string",
+     "description": "Human-readable name for stage."
+    },
+    "readRatioAvg": {
+     "type": "number",
+     "description": "Relative amount of time the average shard spent reading input.",
+     "format": "double"
+    },
+    "readRatioMax": {
+     "type": "number",
+     "description": "Relative amount of time the slowest shard spent reading input.",
+     "format": "double"
+    },
+    "recordsRead": {
+     "type": "string",
+     "description": "Number of records read into the stage.",
+     "format": "int64"
+    },
+    "recordsWritten": {
+     "type": "string",
+     "description": "Number of records written by the stage.",
+     "format": "int64"
+    },
+    "steps": {
+     "type": "array",
+     "description": "List of operations within the stage in dependency order (approximately chronological).",
+     "items": {
+      "$ref": "ExplainQueryStep"
+     }
+    },
+    "waitRatioAvg": {
+     "type": "number",
+     "description": "Relative amount of time the average shard spent waiting to be scheduled.",
+     "format": "double"
+    },
+    "waitRatioMax": {
+     "type": "number",
+     "description": "Relative amount of time the slowest shard spent waiting to be scheduled.",
+     "format": "double"
+    },
+    "writeRatioAvg": {
+     "type": "number",
+     "description": "Relative amount of time the average shard spent on writing output.",
+     "format": "double"
+    },
+    "writeRatioMax": {
+     "type": "number",
+     "description": "Relative amount of time the slowest shard spent on writing output.",
+     "format": "double"
+    }
+   }
+  },
+  "ExplainQueryStep": {
+   "id": "ExplainQueryStep",
+   "type": "object",
+   "properties": {
+    "kind": {
+     "type": "string",
+     "description": "Machine-readable operation type."
+    },
+    "substeps": {
+     "type": "array",
+     "description": "Human-readable stage descriptions.",
+     "items": {
+      "type": "string"
+     }
+    }
+   }
+  },
+  "ExternalDataConfiguration": {
+   "id": "ExternalDataConfiguration",
+   "type": "object",
+   "properties": {
+    "autodetect": {
+     "type": "boolean",
+     "description": "[Experimental] Try to detect schema and format options automatically. Any option specified explicitly will be honored."
+    },
+    "bigtableOptions": {
+     "$ref": "BigtableOptions",
+     "description": "[Optional] Additional options if sourceFormat is set to BIGTABLE."
+    },
+    "compression": {
+     "type": "string",
+     "description": "[Optional] The compression type of the data source. Possible values include GZIP and NONE. The default value is NONE. This setting is ignored for Google Cloud Bigtable, Google Cloud Datastore backups and Avro formats."
+    },
+    "csvOptions": {
+     "$ref": "CsvOptions",
+     "description": "Additional properties to set if sourceFormat is set to CSV."
+    },
+    "googleSheetsOptions": {
+     "$ref": "GoogleSheetsOptions",
+     "description": "[Optional] Additional options if sourceFormat is set to GOOGLE_SHEETS."
+    },
+    "ignoreUnknownValues": {
+     "type": "boolean",
+     "description": "[Optional] Indicates if BigQuery should allow extra values that are not represented in the table schema. If true, the extra values are ignored. If false, records with extra columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false. The sourceFormat property determines what BigQuery treats as an extra value: CSV: Trailing columns JSON: Named values that don't match any column names Google Cloud Bigtable: This setting is ignored. Google Cloud Datastore backups: This setting is ignored. Avro: This setting is ignored."
+    },
+    "maxBadRecords": {
+     "type": "integer",
+     "description": "[Optional] The maximum number of bad records that BigQuery can ignore when reading data. If the number of bad records exceeds this value, an invalid error is returned in the job result. The default value is 0, which requires that all records are valid. This setting is ignored for Google Cloud Bigtable, Google Cloud Datastore backups and Avro formats.",
+     "format": "int32"
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "[Optional] The schema for the data. Schema is required for CSV and JSON formats. Schema is disallowed for Google Cloud Bigtable, Cloud Datastore backups, and Avro formats."
+    },
+    "sourceFormat": {
+     "type": "string",
+     "description": "[Required] The data format. For CSV files, specify \"CSV\". For Google sheets, specify \"GOOGLE_SHEETS\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro files, specify \"AVRO\". For Google Cloud Datastore backups, specify \"DATASTORE_BACKUP\". [Experimental] For Google Cloud Bigtable, specify \"BIGTABLE\". Please note that reading from Google Cloud Bigtable is experimental and has to be enabled for your project. Please contact Google Cloud Support to enable this for your project."
+    },
+    "sourceUris": {
+     "type": "array",
+     "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud. For Google Cloud Storage URIs: Each URI can contain one '*' wildcard character and it must come after the 'bucket' name. Size limits related to load jobs apply to external data sources. For Google Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and valid HTTPS URL for a Google Cloud Bigtable table. For Google Cloud Datastore backups, exactly one URI can be specified, and it must end with '.backup_info'. Also, the '*' wildcard character is not allowed.",
+     "items": {
+      "type": "string"
+     }
+    }
+   }
+  },
+  "GetQueryResultsResponse": {
+   "id": "GetQueryResultsResponse",
+   "type": "object",
+   "properties": {
+    "cacheHit": {
+     "type": "boolean",
+     "description": "Whether the query result was fetched from the query cache."
+    },
+    "errors": {
+     "type": "array",
+     "description": "[Output-only] All errors and warnings encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.",
+     "items": {
+      "$ref": "ErrorProto"
+     }
+    },
+    "etag": {
+     "type": "string",
+     "description": "A hash of this response."
+    },
+    "jobComplete": {
+     "type": "boolean",
+     "description": "Whether the query has completed or not. If rows or totalRows are present, this will always be true. If this is false, totalRows will not be available."
+    },
+    "jobReference": {
+     "$ref": "JobReference",
+     "description": "Reference to the BigQuery Job that was created to run the query. This field will be present even if the original request timed out, in which case GetQueryResults can be used to read the results once the query has completed. Since this API only returns the first page of results, subsequent pages can be fetched via the same mechanism (GetQueryResults)."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#getQueryResultsResponse"
+    },
+    "numDmlAffectedRows": {
+     "type": "string",
+     "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.",
+     "format": "int64"
+    },
+    "pageToken": {
+     "type": "string",
+     "description": "A token used for paging results."
+    },
+    "rows": {
+     "type": "array",
+     "description": "An object with as many results as can be contained within the maximum permitted reply size. To get any additional rows, you can call GetQueryResults and specify the jobReference returned above. Present only when the query completes successfully.",
+     "items": {
+      "$ref": "TableRow"
+     }
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "The schema of the results. Present only when the query completes successfully."
+    },
+    "totalBytesProcessed": {
+     "type": "string",
+     "description": "The total number of bytes processed for this query.",
+     "format": "int64"
+    },
+    "totalRows": {
+     "type": "string",
+     "description": "The total number of rows in the complete query result set, which can be more than the number of rows in this single page of results. Present only when the query completes successfully.",
+     "format": "uint64"
+    }
+   }
+  },
+  "GoogleSheetsOptions": {
+   "id": "GoogleSheetsOptions",
+   "type": "object",
+   "properties": {
+    "skipLeadingRows": {
+     "type": "string",
+     "description": "[Optional] The number of rows at the top of a sheet that BigQuery will skip when reading the data. The default value is 0. This property is useful if you have header rows that should be skipped. When autodetect is on, behavior is the following: * skipLeadingRows unspecified - Autodetect tries to detect headers in the first row. If they are not detected, the row is read as data. Otherwise data is read starting from the second row. * skipLeadingRows is 0 - Instructs autodetect that there are no headers and data should be read starting from the first row. * skipLeadingRows = N \u003e 0 - Autodetect skips N-1 rows and tries to detect headers in row N. If headers are not detected, row N is just skipped. Otherwise row N is used to extract column names for the detected schema.",
+     "format": "int64"
+    }
+   }
+  },
+  "Job": {
+   "id": "Job",
+   "type": "object",
+   "properties": {
+    "configuration": {
+     "$ref": "JobConfiguration",
+     "description": "[Required] Describes the job configuration."
+    },
+    "etag": {
+     "type": "string",
+     "description": "[Output-only] A hash of this resource."
+    },
+    "id": {
+     "type": "string",
+     "description": "[Output-only] Opaque ID field of the job"
+    },
+    "jobReference": {
+     "$ref": "JobReference",
+     "description": "[Optional] Reference describing the unique-per-user name of the job."
+    },
+    "kind": {
+     "type": "string",
+     "description": "[Output-only] The type of the resource.",
+     "default": "bigquery#job"
+    },
+    "selfLink": {
+     "type": "string",
+     "description": "[Output-only] A URL that can be used to access this resource again."
+    },
+    "statistics": {
+     "$ref": "JobStatistics",
+     "description": "[Output-only] Information about the job, including starting time and ending time of the job."
+    },
+    "status": {
+     "$ref": "JobStatus",
+     "description": "[Output-only] The status of this job. Examine this value when polling an asynchronous job to see if the job is complete."
+    },
+    "user_email": {
+     "type": "string",
+     "description": "[Output-only] Email address of the user who ran the job."
+    }
+   }
+  },
+  "JobCancelResponse": {
+   "id": "JobCancelResponse",
+   "type": "object",
+   "properties": {
+    "job": {
+     "$ref": "Job",
+     "description": "The final state of the job."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#jobCancelResponse"
+    }
+   }
+  },
+  "JobConfiguration": {
+   "id": "JobConfiguration",
+   "type": "object",
+   "properties": {
+    "copy": {
+     "$ref": "JobConfigurationTableCopy",
+     "description": "[Pick one] Copies a table."
+    },
+    "dryRun": {
+     "type": "boolean",
+     "description": "[Optional] If set, don't actually run this job. A valid query will return a mostly empty response with some processing statistics, while an invalid query will return the same error it would if it wasn't a dry run. Behavior of non-query jobs is undefined."
+    },
+    "extract": {
+     "$ref": "JobConfigurationExtract",
+     "description": "[Pick one] Configures an extract job."
+    },
+    "load": {
+     "$ref": "JobConfigurationLoad",
+     "description": "[Pick one] Configures a load job."
+    },
+    "query": {
+     "$ref": "JobConfigurationQuery",
+     "description": "[Pick one] Configures a query job."
+    }
+   }
+  },
+  "JobConfigurationExtract": {
+   "id": "JobConfigurationExtract",
+   "type": "object",
+   "properties": {
+    "compression": {
+     "type": "string",
+     "description": "[Optional] The compression type to use for exported files. Possible values include GZIP and NONE. The default value is NONE."
+    },
+    "destinationFormat": {
+     "type": "string",
+     "description": "[Optional] The exported file format. Possible values include CSV, NEWLINE_DELIMITED_JSON and AVRO. The default value is CSV. Tables with nested or repeated fields cannot be exported as CSV."
+    },
+    "destinationUri": {
+     "type": "string",
+     "description": "[Pick one] DEPRECATED: Use destinationUris instead, passing only one URI as necessary. The fully-qualified Google Cloud Storage URI where the extracted table should be written."
+    },
+    "destinationUris": {
+     "type": "array",
+     "description": "[Pick one] A list of fully-qualified Google Cloud Storage URIs where the extracted table should be written.",
+     "items": {
+      "type": "string"
+     }
+    },
+    "fieldDelimiter": {
+     "type": "string",
+     "description": "[Optional] Delimiter to use between fields in the exported data. Default is ','"
+    },
+    "printHeader": {
+     "type": "boolean",
+     "description": "[Optional] Whether to print out a header row in the results. Default is true.",
+     "default": "true"
+    },
+    "sourceTable": {
+     "$ref": "TableReference",
+     "description": "[Required] A reference to the table being exported."
+    }
+   }
+  },
+  "JobConfigurationLoad": {
+   "id": "JobConfigurationLoad",
+   "type": "object",
+   "properties": {
+    "allowJaggedRows": {
+     "type": "boolean",
+     "description": "[Optional] Accept rows that are missing trailing optional columns. The missing values are treated as nulls. If false, records with missing trailing columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false. Only applicable to CSV, ignored for other formats."
+    },
+    "allowQuotedNewlines": {
+     "type": "boolean",
+     "description": "Indicates if BigQuery should allow quoted data sections that contain newline characters in a CSV file. The default value is false."
+    },
+    "autodetect": {
+     "type": "boolean",
+     "description": "[Experimental] Indicates if we should automatically infer the options and schema for CSV and JSON sources."
+    },
+    "createDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies whether the job is allowed to create new tables. The following values are supported: CREATE_IF_NEEDED: If the table does not exist, BigQuery creates the table. CREATE_NEVER: The table must already exist. If it does not, a 'notFound' error is returned in the job result. The default value is CREATE_IF_NEEDED. Creation, truncation and append actions occur as one atomic update upon job completion."
+    },
+    "destinationTable": {
+     "$ref": "TableReference",
+     "description": "[Required] The destination table to load the data into."
+    },
+    "encoding": {
+     "type": "string",
+     "description": "[Optional] The character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split using the values of the quote and fieldDelimiter properties."
+    },
+    "fieldDelimiter": {
+     "type": "string",
+     "description": "[Optional] The separator for fields in a CSV file. The separator can be any ISO-8859-1 single-byte character. To use a character in the range 128-255, you must encode the character as UTF8. BigQuery converts the string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data in its raw, binary state. BigQuery also supports the escape sequence \"\\t\" to specify a tab separator. The default value is a comma (',')."
+    },
+    "ignoreUnknownValues": {
+     "type": "boolean",
+     "description": "[Optional] Indicates if BigQuery should allow extra values that are not represented in the table schema. If true, the extra values are ignored. If false, records with extra columns are treated as bad records, and if there are too many bad records, an invalid error is returned in the job result. The default value is false. The sourceFormat property determines what BigQuery treats as an extra value: CSV: Trailing columns JSON: Named values that don't match any column names"
+    },
+    "maxBadRecords": {
+     "type": "integer",
+     "description": "[Optional] The maximum number of bad records that BigQuery can ignore when running the job. If the number of bad records exceeds this value, an invalid error is returned in the job result. The default value is 0, which requires that all records are valid.",
+     "format": "int32"
+    },
+    "projectionFields": {
+     "type": "array",
+     "description": "[Experimental] If sourceFormat is set to \"DATASTORE_BACKUP\", indicates which entity properties to load into BigQuery from a Cloud Datastore backup. Property names are case sensitive and must be top-level properties. If no properties are specified, BigQuery loads all properties. If any named property isn't found in the Cloud Datastore backup, an invalid error is returned in the job result.",
+     "items": {
+      "type": "string"
+     }
+    },
+    "quote": {
+     "type": "string",
+     "description": "[Optional] The value that is used to quote data sections in a CSV file. BigQuery converts the string to ISO-8859-1 encoding, and then uses the first byte of the encoded string to split the data in its raw, binary state. The default value is a double-quote ('\"'). If your data does not contain quoted sections, set the property value to an empty string. If your data contains quoted newline characters, you must also set the allowQuotedNewlines property to true.",
+     "default": "\"",
+     "pattern": ".?"
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "[Optional] The schema for the destination table. The schema can be omitted if the destination table already exists, or if you're loading data from Google Cloud Datastore."
+    },
+    "schemaInline": {
+     "type": "string",
+     "description": "[Deprecated] The inline schema. For CSV schemas, specify as \"Field1:Type1[,Field2:Type2]*\". For example, \"foo:STRING, bar:INTEGER, baz:FLOAT\"."
+    },
+    "schemaInlineFormat": {
+     "type": "string",
+     "description": "[Deprecated] The format of the schemaInline property."
+    },
+    "schemaUpdateOptions": {
+     "type": "array",
+     "description": "[Experimental] Allows the schema of the desitination table to be updated as a side effect of the load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.",
+     "items": {
+      "type": "string"
+     }
+    },
+    "skipLeadingRows": {
+     "type": "integer",
+     "description": "[Optional] The number of rows at the top of a CSV file that BigQuery will skip when loading the data. The default value is 0. This property is useful if you have header rows in the file that should be skipped.",
+     "format": "int32"
+    },
+    "sourceFormat": {
+     "type": "string",
+     "description": "[Optional] The format of the data files. For CSV files, specify \"CSV\". For datastore backups, specify \"DATASTORE_BACKUP\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro, specify \"AVRO\". The default value is CSV."
+    },
+    "sourceUris": {
+     "type": "array",
+     "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can contain one '*' wildcard character and it must come after the 'bucket' name.",
+     "items": {
+      "type": "string"
+     }
+    },
+    "writeDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_APPEND. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion."
+    }
+   }
+  },
+  "JobConfigurationQuery": {
+   "id": "JobConfigurationQuery",
+   "type": "object",
+   "properties": {
+    "allowLargeResults": {
+     "type": "boolean",
+     "description": "If true, allows the query to produce arbitrarily large result tables at a slight cost in performance. Requires destinationTable to be set."
+    },
+    "createDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies whether the job is allowed to create new tables. The following values are supported: CREATE_IF_NEEDED: If the table does not exist, BigQuery creates the table. CREATE_NEVER: The table must already exist. If it does not, a 'notFound' error is returned in the job result. The default value is CREATE_IF_NEEDED. Creation, truncation and append actions occur as one atomic update upon job completion."
+    },
+    "defaultDataset": {
+     "$ref": "DatasetReference",
+     "description": "[Optional] Specifies the default dataset to use for unqualified table names in the query."
+    },
+    "destinationTable": {
+     "$ref": "TableReference",
+     "description": "[Optional] Describes the table where the query results should be stored. If not present, a new table will be created to store the results."
+    },
+    "flattenResults": {
+     "type": "boolean",
+     "description": "[Optional] Flattens all nested and repeated fields in the query results. The default value is true. allowLargeResults must be true if this is set to false.",
+     "default": "true"
+    },
+    "maximumBillingTier": {
+     "type": "integer",
+     "description": "[Optional] Limits the billing tier for this job. Queries that have resource usage beyond this tier will fail (without incurring a charge). If unspecified, this will be set to your project default.",
+     "default": "1",
+     "format": "int32"
+    },
+    "maximumBytesBilled": {
+     "type": "string",
+     "description": "[Optional] Limits the bytes billed for this job. Queries that will have bytes billed beyond this limit will fail (without incurring a charge). If unspecified, this will be set to your project default.",
+     "format": "int64"
+    },
+    "preserveNulls": {
+     "type": "boolean",
+     "description": "[Deprecated] This property is deprecated."
+    },
+    "priority": {
+     "type": "string",
+     "description": "[Optional] Specifies a priority for the query. Possible values include INTERACTIVE and BATCH. The default value is INTERACTIVE."
+    },
+    "query": {
+     "type": "string",
+     "description": "[Required] BigQuery SQL query to execute."
+    },
+    "schemaUpdateOptions": {
+     "type": "array",
+     "description": "[Experimental] Allows the schema of the desitination table to be updated as a side effect of the query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.",
+     "items": {
+      "type": "string"
+     }
+    },
+    "tableDefinitions": {
+     "type": "object",
+     "description": "[Optional] If querying an external data source outside of BigQuery, describes the data format, location and other properties of the data source. By defining these properties, the data source can then be queried as if it were a standard BigQuery table.",
+     "additionalProperties": {
+      "$ref": "ExternalDataConfiguration"
+     }
+    },
+    "useLegacySql": {
+     "type": "boolean",
+     "description": "[Experimental] Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the values of allowLargeResults and flattenResults are ignored; query will be run as if allowLargeResults is true and flattenResults is false."
+    },
+    "useQueryCache": {
+     "type": "boolean",
+     "description": "[Optional] Whether to look for the result in the query cache. The query cache is a best-effort cache that will be flushed whenever tables in the query are modified. Moreover, the query cache is only available when a query does not have a destination table specified. The default value is true.",
+     "default": "true"
+    },
+    "userDefinedFunctionResources": {
+     "type": "array",
+     "description": "[Experimental] Describes user-defined function resources used in the query.",
+     "items": {
+      "$ref": "UserDefinedFunctionResource"
+     }
+    },
+    "writeDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion."
+    }
+   }
+  },
+  "JobConfigurationTableCopy": {
+   "id": "JobConfigurationTableCopy",
+   "type": "object",
+   "properties": {
+    "createDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies whether the job is allowed to create new tables. The following values are supported: CREATE_IF_NEEDED: If the table does not exist, BigQuery creates the table. CREATE_NEVER: The table must already exist. If it does not, a 'notFound' error is returned in the job result. The default value is CREATE_IF_NEEDED. Creation, truncation and append actions occur as one atomic update upon job completion."
+    },
+    "destinationTable": {
+     "$ref": "TableReference",
+     "description": "[Required] The destination table"
+    },
+    "sourceTable": {
+     "$ref": "TableReference",
+     "description": "[Pick one] Source table to copy."
+    },
+    "sourceTables": {
+     "type": "array",
+     "description": "[Pick one] Source tables to copy.",
+     "items": {
+      "$ref": "TableReference"
+     }
+    },
+    "writeDisposition": {
+     "type": "string",
+     "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion."
+    }
+   }
+  },
+  "JobList": {
+   "id": "JobList",
+   "type": "object",
+   "properties": {
+    "etag": {
+     "type": "string",
+     "description": "A hash of this page of results."
+    },
+    "jobs": {
+     "type": "array",
+     "description": "List of jobs that were requested.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "configuration": {
+        "$ref": "JobConfiguration",
+        "description": "[Full-projection-only] Specifies the job configuration."
+       },
+       "errorResult": {
+        "$ref": "ErrorProto",
+        "description": "A result object that will be present only if the job has failed."
+       },
+       "id": {
+        "type": "string",
+        "description": "Unique opaque ID of the job."
+       },
+       "jobReference": {
+        "$ref": "JobReference",
+        "description": "Job reference uniquely identifying the job."
+       },
+       "kind": {
+        "type": "string",
+        "description": "The resource type.",
+        "default": "bigquery#job"
+       },
+       "state": {
+        "type": "string",
+        "description": "Running state of the job. When the state is DONE, errorResult can be checked to determine whether the job succeeded or failed."
+       },
+       "statistics": {
+        "$ref": "JobStatistics",
+        "description": "[Output-only] Information about the job, including starting time and ending time of the job."
+       },
+       "status": {
+        "$ref": "JobStatus",
+        "description": "[Full-projection-only] Describes the state of the job."
+       },
+       "user_email": {
+        "type": "string",
+        "description": "[Full-projection-only] Email address of the user who ran the job."
+       }
+      }
+     }
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#jobList"
+    },
+    "nextPageToken": {
+     "type": "string",
+     "description": "A token to request the next page of results."
+    }
+   }
+  },
+  "JobReference": {
+   "id": "JobReference",
+   "type": "object",
+   "properties": {
+    "jobId": {
+     "type": "string",
+     "description": "[Required] The ID of the job. The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or dashes (-). The maximum length is 1,024 characters.",
+     "annotations": {
+      "required": [
+       "bigquery.jobs.getQueryResults"
+      ]
+     }
+    },
+    "projectId": {
+     "type": "string",
+     "description": "[Required] The ID of the project containing this job.",
+     "annotations": {
+      "required": [
+       "bigquery.jobs.getQueryResults"
+      ]
+     }
+    }
+   }
+  },
+  "JobStatistics": {
+   "id": "JobStatistics",
+   "type": "object",
+   "properties": {
+    "creationTime": {
+     "type": "string",
+     "description": "[Output-only] Creation time of this job, in milliseconds since the epoch. This field will be present on all jobs.",
+     "format": "int64"
+    },
+    "endTime": {
+     "type": "string",
+     "description": "[Output-only] End time of this job, in milliseconds since the epoch. This field will be present whenever a job is in the DONE state.",
+     "format": "int64"
+    },
+    "extract": {
+     "$ref": "JobStatistics4",
+     "description": "[Output-only] Statistics for an extract job."
+    },
+    "load": {
+     "$ref": "JobStatistics3",
+     "description": "[Output-only] Statistics for a load job."
+    },
+    "query": {
+     "$ref": "JobStatistics2",
+     "description": "[Output-only] Statistics for a query job."
+    },
+    "startTime": {
+     "type": "string",
+     "description": "[Output-only] Start time of this job, in milliseconds since the epoch. This field will be present when the job transitions from the PENDING state to either RUNNING or DONE.",
+     "format": "int64"
+    },
+    "totalBytesProcessed": {
+     "type": "string",
+     "description": "[Output-only] [Deprecated] Use the bytes processed in the query statistics instead.",
+     "format": "int64"
+    }
+   }
+  },
+  "JobStatistics2": {
+   "id": "JobStatistics2",
+   "type": "object",
+   "properties": {
+    "billingTier": {
+     "type": "integer",
+     "description": "[Output-only] Billing tier for the job.",
+     "format": "int32"
+    },
+    "cacheHit": {
+     "type": "boolean",
+     "description": "[Output-only] Whether the query result was fetched from the query cache."
+    },
+    "numDmlAffectedRows": {
+     "type": "string",
+     "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.",
+     "format": "int64"
+    },
+    "queryPlan": {
+     "type": "array",
+     "description": "[Output-only, Experimental] Describes execution plan for the query.",
+     "items": {
+      "$ref": "ExplainQueryStage"
+     }
+    },
+    "referencedTables": {
+     "type": "array",
+     "description": "[Output-only, Experimental] Referenced tables for the job. Queries that reference more than 50 tables will not have a complete list.",
+     "items": {
+      "$ref": "TableReference"
+     }
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "[Output-only, Experimental] The schema of the results. Present only for successful dry run of non-legacy SQL queries."
+    },
+    "totalBytesBilled": {
+     "type": "string",
+     "description": "[Output-only] Total bytes billed for the job.",
+     "format": "int64"
+    },
+    "totalBytesProcessed": {
+     "type": "string",
+     "description": "[Output-only] Total bytes processed for the job.",
+     "format": "int64"
+    }
+   }
+  },
+  "JobStatistics3": {
+   "id": "JobStatistics3",
+   "type": "object",
+   "properties": {
+    "inputFileBytes": {
+     "type": "string",
+     "description": "[Output-only] Number of bytes of source data in a load job.",
+     "format": "int64"
+    },
+    "inputFiles": {
+     "type": "string",
+     "description": "[Output-only] Number of source files in a load job.",
+     "format": "int64"
+    },
+    "outputBytes": {
+     "type": "string",
+     "description": "[Output-only] Size of the loaded data in bytes. Note that while a load job is in the running state, this value may change.",
+     "format": "int64"
+    },
+    "outputRows": {
+     "type": "string",
+     "description": "[Output-only] Number of rows imported in a load job. Note that while an import job is in the running state, this value may change.",
+     "format": "int64"
+    }
+   }
+  },
+  "JobStatistics4": {
+   "id": "JobStatistics4",
+   "type": "object",
+   "properties": {
+    "destinationUriFileCounts": {
+     "type": "array",
+     "description": "[Output-only] Number of files per destination URI or URI pattern specified in the extract configuration. These values will be in the same order as the URIs specified in the 'destinationUris' field.",
+     "items": {
+      "type": "string",
+      "format": "int64"
+     }
+    }
+   }
+  },
+  "JobStatus": {
+   "id": "JobStatus",
+   "type": "object",
+   "properties": {
+    "errorResult": {
+     "$ref": "ErrorProto",
+     "description": "[Output-only] Final error result of the job. If present, indicates that the job has completed and was unsuccessful."
+    },
+    "errors": {
+     "type": "array",
+     "description": "[Output-only] All errors encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.",
+     "items": {
+      "$ref": "ErrorProto"
+     }
+    },
+    "state": {
+     "type": "string",
+     "description": "[Output-only] Running state of the job."
+    }
+   }
+  },
+  "JsonObject": {
+   "id": "JsonObject",
+   "type": "object",
+   "description": "Represents a single JSON object.",
+   "additionalProperties": {
+    "$ref": "JsonValue"
+   }
+  },
+  "JsonValue": {
+   "id": "JsonValue",
+   "type": "any"
+  },
+  "ProjectList": {
+   "id": "ProjectList",
+   "type": "object",
+   "properties": {
+    "etag": {
+     "type": "string",
+     "description": "A hash of the page of results"
+    },
+    "kind": {
+     "type": "string",
+     "description": "The type of list.",
+     "default": "bigquery#projectList"
+    },
+    "nextPageToken": {
+     "type": "string",
+     "description": "A token to request the next page of results."
+    },
+    "projects": {
+     "type": "array",
+     "description": "Projects to which you have at least READ access.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "friendlyName": {
+        "type": "string",
+        "description": "A descriptive name for this project."
+       },
+       "id": {
+        "type": "string",
+        "description": "An opaque ID of this project."
+       },
+       "kind": {
+        "type": "string",
+        "description": "The resource type.",
+        "default": "bigquery#project"
+       },
+       "numericId": {
+        "type": "string",
+        "description": "The numeric ID of this project.",
+        "format": "uint64"
+       },
+       "projectReference": {
+        "$ref": "ProjectReference",
+        "description": "A unique reference to this project."
+       }
+      }
+     }
+    },
+    "totalItems": {
+     "type": "integer",
+     "description": "The total number of projects in the list.",
+     "format": "int32"
+    }
+   }
+  },
+  "ProjectReference": {
+   "id": "ProjectReference",
+   "type": "object",
+   "properties": {
+    "projectId": {
+     "type": "string",
+     "description": "[Required] ID of the project. Can be either the numeric ID or the assigned ID of the project."
+    }
+   }
+  },
+  "QueryRequest": {
+   "id": "QueryRequest",
+   "type": "object",
+   "properties": {
+    "defaultDataset": {
+     "$ref": "DatasetReference",
+     "description": "[Optional] Specifies the default datasetId and projectId to assume for any unqualified table names in the query. If not set, all table names in the query string must be qualified in the format 'datasetId.tableId'."
+    },
+    "dryRun": {
+     "type": "boolean",
+     "description": "[Optional] If set to true, BigQuery doesn't run the job. Instead, if the query is valid, BigQuery returns statistics about the job such as how many bytes would be processed. If the query is invalid, an error returns. The default value is false."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the request.",
+     "default": "bigquery#queryRequest"
+    },
+    "maxResults": {
+     "type": "integer",
+     "description": "[Optional] The maximum number of rows of data to return per page of results. Setting this flag to a small value such as 1000 and then paging through results might improve reliability when the query result set is large. In addition to this limit, responses are also limited to 10 MB. By default, there is no maximum row count, and only the byte limit applies.",
+     "format": "uint32"
+    },
+    "preserveNulls": {
+     "type": "boolean",
+     "description": "[Deprecated] This property is deprecated."
+    },
+    "query": {
+     "type": "string",
+     "description": "[Required] A query string, following the BigQuery query syntax, of the query to execute. Example: \"SELECT count(f1) FROM [myProjectId:myDatasetId.myTableId]\".",
+     "annotations": {
+      "required": [
+       "bigquery.jobs.query"
+      ]
+     }
+    },
+    "timeoutMs": {
+     "type": "integer",
+     "description": "[Optional] How long to wait for the query to complete, in milliseconds, before the request times out and returns. Note that this is only a timeout for the request, not the query. If the query takes longer to run than the timeout value, the call returns without any results and with the 'jobComplete' flag set to false. You can call GetQueryResults() to wait for the query to complete and read the results. The default value is 10000 milliseconds (10 seconds).",
+     "format": "uint32"
+    },
+    "useLegacySql": {
+     "type": "boolean",
+     "description": "[Experimental] Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the values of allowLargeResults and flattenResults are ignored; query will be run as if allowLargeResults is true and flattenResults is false."
+    },
+    "useQueryCache": {
+     "type": "boolean",
+     "description": "[Optional] Whether to look for the result in the query cache. The query cache is a best-effort cache that will be flushed whenever tables in the query are modified. The default value is true.",
+     "default": "true"
+    }
+   }
+  },
+  "QueryResponse": {
+   "id": "QueryResponse",
+   "type": "object",
+   "properties": {
+    "cacheHit": {
+     "type": "boolean",
+     "description": "Whether the query result was fetched from the query cache."
+    },
+    "errors": {
+     "type": "array",
+     "description": "[Output-only] All errors and warnings encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.",
+     "items": {
+      "$ref": "ErrorProto"
+     }
+    },
+    "jobComplete": {
+     "type": "boolean",
+     "description": "Whether the query has completed or not. If rows or totalRows are present, this will always be true. If this is false, totalRows will not be available."
+    },
+    "jobReference": {
+     "$ref": "JobReference",
+     "description": "Reference to the Job that was created to run the query. This field will be present even if the original request timed out, in which case GetQueryResults can be used to read the results once the query has completed. Since this API only returns the first page of results, subsequent pages can be fetched via the same mechanism (GetQueryResults)."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type.",
+     "default": "bigquery#queryResponse"
+    },
+    "numDmlAffectedRows": {
+     "type": "string",
+     "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.",
+     "format": "int64"
+    },
+    "pageToken": {
+     "type": "string",
+     "description": "A token used for paging results."
+    },
+    "rows": {
+     "type": "array",
+     "description": "An object with as many results as can be contained within the maximum permitted reply size. To get any additional rows, you can call GetQueryResults and specify the jobReference returned above.",
+     "items": {
+      "$ref": "TableRow"
+     }
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "The schema of the results. Present only when the query completes successfully."
+    },
+    "totalBytesProcessed": {
+     "type": "string",
+     "description": "The total number of bytes processed for this query. If this query was a dry run, this is the number of bytes that would be processed if the query were run.",
+     "format": "int64"
+    },
+    "totalRows": {
+     "type": "string",
+     "description": "The total number of rows in the complete query result set, which can be more than the number of rows in this single page of results.",
+     "format": "uint64"
+    }
+   }
+  },
+  "Streamingbuffer": {
+   "id": "Streamingbuffer",
+   "type": "object",
+   "properties": {
+    "estimatedBytes": {
+     "type": "string",
+     "description": "[Output-only] A lower-bound estimate of the number of bytes currently in the streaming buffer.",
+     "format": "uint64"
+    },
+    "estimatedRows": {
+     "type": "string",
+     "description": "[Output-only] A lower-bound estimate of the number of rows currently in the streaming buffer.",
+     "format": "uint64"
+    },
+    "oldestEntryTime": {
+     "type": "string",
+     "description": "[Output-only] Contains the timestamp of the oldest entry in the streaming buffer, in milliseconds since the epoch, if the streaming buffer is available.",
+     "format": "uint64"
+    }
+   }
+  },
+  "Table": {
+   "id": "Table",
+   "type": "object",
+   "properties": {
+    "creationTime": {
+     "type": "string",
+     "description": "[Output-only] The time when this table was created, in milliseconds since the epoch.",
+     "format": "int64"
+    },
+    "description": {
+     "type": "string",
+     "description": "[Optional] A user-friendly description of this table."
+    },
+    "etag": {
+     "type": "string",
+     "description": "[Output-only] A hash of this resource."
+    },
+    "expirationTime": {
+     "type": "string",
+     "description": "[Optional] The time when this table expires, in milliseconds since the epoch. If not present, the table will persist indefinitely. Expired tables will be deleted and their storage reclaimed.",
+     "format": "int64"
+    },
+    "externalDataConfiguration": {
+     "$ref": "ExternalDataConfiguration",
+     "description": "[Optional] Describes the data format, location, and other properties of a table stored outside of BigQuery. By defining these properties, the data source can then be queried as if it were a standard BigQuery table."
+    },
+    "friendlyName": {
+     "type": "string",
+     "description": "[Optional] A descriptive name for this table."
+    },
+    "id": {
+     "type": "string",
+     "description": "[Output-only] An opaque ID uniquely identifying the table."
+    },
+    "kind": {
+     "type": "string",
+     "description": "[Output-only] The type of the resource.",
+     "default": "bigquery#table"
+    },
+    "lastModifiedTime": {
+     "type": "string",
+     "description": "[Output-only] The time when this table was last modified, in milliseconds since the epoch.",
+     "format": "uint64"
+    },
+    "location": {
+     "type": "string",
+     "description": "[Output-only] The geographic location where the table resides. This value is inherited from the dataset."
+    },
+    "numBytes": {
+     "type": "string",
+     "description": "[Output-only] The size of this table in bytes, excluding any data in the streaming buffer.",
+     "format": "int64"
+    },
+    "numLongTermBytes": {
+     "type": "string",
+     "description": "[Output-only] The number of bytes in the table that are considered \"long-term storage\".",
+     "format": "int64"
+    },
+    "numRows": {
+     "type": "string",
+     "description": "[Output-only] The number of rows of data in this table, excluding any data in the streaming buffer.",
+     "format": "uint64"
+    },
+    "schema": {
+     "$ref": "TableSchema",
+     "description": "[Optional] Describes the schema of this table."
+    },
+    "selfLink": {
+     "type": "string",
+     "description": "[Output-only] A URL that can be used to access this resource again."
+    },
+    "streamingBuffer": {
+     "$ref": "Streamingbuffer",
+     "description": "[Output-only] Contains information regarding this table's streaming buffer, if one is present. This field will be absent if the table is not being streamed to or if there is no data in the streaming buffer."
+    },
+    "tableReference": {
+     "$ref": "TableReference",
+     "description": "[Required] Reference describing the ID of this table."
+    },
+    "timePartitioning": {
+     "$ref": "TimePartitioning",
+     "description": "[Experimental] If specified, configures time-based partitioning for this table."
+    },
+    "type": {
+     "type": "string",
+     "description": "[Output-only] Describes the table type. The following values are supported: TABLE: A normal BigQuery table. VIEW: A virtual table defined by a SQL query. EXTERNAL: A table that references data stored in an external storage system, such as Google Cloud Storage. The default value is TABLE."
+    },
+    "view": {
+     "$ref": "ViewDefinition",
+     "description": "[Optional] The view definition."
+    }
+   }
+  },
+  "TableCell": {
+   "id": "TableCell",
+   "type": "object",
+   "properties": {
+    "v": {
+     "type": "any"
+    }
+   }
+  },
+  "TableDataInsertAllRequest": {
+   "id": "TableDataInsertAllRequest",
+   "type": "object",
+   "properties": {
+    "ignoreUnknownValues": {
+     "type": "boolean",
+     "description": "[Optional] Accept rows that contain values that do not match the schema. The unknown values are ignored. Default is false, which treats unknown values as errors."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#tableDataInsertAllRequest"
+    },
+    "rows": {
+     "type": "array",
+     "description": "The rows to insert.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "insertId": {
+        "type": "string",
+        "description": "[Optional] A unique ID for each row. BigQuery uses this property to detect duplicate insertion requests on a best-effort basis."
+       },
+       "json": {
+        "$ref": "JsonObject",
+        "description": "[Required] A JSON object that contains a row of data. The object's properties and values must match the destination table's schema."
+       }
+      }
+     }
+    },
+    "skipInvalidRows": {
+     "type": "boolean",
+     "description": "[Optional] Insert all valid rows of a request, even if invalid rows exist. The default value is false, which causes the entire request to fail if any invalid rows exist."
+    },
+    "templateSuffix": {
+     "type": "string",
+     "description": "[Experimental] If specified, treats the destination table as a base template, and inserts the rows into an instance table named \"{destination}{templateSuffix}\". BigQuery will manage creation of the instance table, using the schema of the base template table. See https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables for considerations when working with templates tables."
+    }
+   }
+  },
+  "TableDataInsertAllResponse": {
+   "id": "TableDataInsertAllResponse",
+   "type": "object",
+   "properties": {
+    "insertErrors": {
+     "type": "array",
+     "description": "An array of errors for rows that were not inserted.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "errors": {
+        "type": "array",
+        "description": "Error information for the row indicated by the index property.",
+        "items": {
+         "$ref": "ErrorProto"
+        }
+       },
+       "index": {
+        "type": "integer",
+        "description": "The index of the row that error applies to.",
+        "format": "uint32"
+       }
+      }
+     }
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#tableDataInsertAllResponse"
+    }
+   }
+  },
+  "TableDataList": {
+   "id": "TableDataList",
+   "type": "object",
+   "properties": {
+    "etag": {
+     "type": "string",
+     "description": "A hash of this page of results."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The resource type of the response.",
+     "default": "bigquery#tableDataList"
+    },
+    "pageToken": {
+     "type": "string",
+     "description": "A token used for paging results. Providing this token instead of the startIndex parameter can help you retrieve stable results when an underlying table is changing."
+    },
+    "rows": {
+     "type": "array",
+     "description": "Rows of results.",
+     "items": {
+      "$ref": "TableRow"
+     }
+    },
+    "totalRows": {
+     "type": "string",
+     "description": "The total number of rows in the complete table.",
+     "format": "int64"
+    }
+   }
+  },
+  "TableFieldSchema": {
+   "id": "TableFieldSchema",
+   "type": "object",
+   "properties": {
+    "description": {
+     "type": "string",
+     "description": "[Optional] The field description. The maximum length is 16K characters."
+    },
+    "fields": {
+     "type": "array",
+     "description": "[Optional] Describes the nested schema fields if the type property is set to RECORD.",
+     "items": {
+      "$ref": "TableFieldSchema"
+     }
+    },
+    "mode": {
+     "type": "string",
+     "description": "[Optional] The field mode. Possible values include NULLABLE, REQUIRED and REPEATED. The default value is NULLABLE."
+    },
+    "name": {
+     "type": "string",
+     "description": "[Required] The field name. The name must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_), and must start with a letter or underscore. The maximum length is 128 characters."
+    },
+    "type": {
+     "type": "string",
+     "description": "[Required] The field data type. Possible values include STRING, BYTES, INTEGER, FLOAT, BOOLEAN, TIMESTAMP or RECORD (where RECORD indicates that the field contains a nested schema)."
+    }
+   }
+  },
+  "TableList": {
+   "id": "TableList",
+   "type": "object",
+   "properties": {
+    "etag": {
+     "type": "string",
+     "description": "A hash of this page of results."
+    },
+    "kind": {
+     "type": "string",
+     "description": "The type of list.",
+     "default": "bigquery#tableList"
+    },
+    "nextPageToken": {
+     "type": "string",
+     "description": "A token to request the next page of results."
+    },
+    "tables": {
+     "type": "array",
+     "description": "Tables in the requested dataset.",
+     "items": {
+      "type": "object",
+      "properties": {
+       "friendlyName": {
+        "type": "string",
+        "description": "The user-friendly name for this table."
+       },
+       "id": {
+        "type": "string",
+        "description": "An opaque ID of the table"
+       },
+       "kind": {
+        "type": "string",
+        "description": "The resource type.",
+        "default": "bigquery#table"
+       },
+       "tableReference": {
+        "$ref": "TableReference",
+        "description": "A reference uniquely identifying the table."
+       },
+       "type": {
+        "type": "string",
+        "description": "The type of table. Possible values are: TABLE, VIEW."
+       }
+      }
+     }
+    },
+    "totalItems": {
+     "type": "integer",
+     "description": "The total number of tables in the dataset.",
+     "format": "int32"
+    }
+   }
+  },
+  "TableReference": {
+   "id": "TableReference",
+   "type": "object",
+   "properties": {
+    "datasetId": {
+     "type": "string",
+     "description": "[Required] The ID of the dataset containing this table.",
+     "annotations": {
+      "required": [
+       "bigquery.tables.update"
+      ]
+     }
+    },
+    "projectId": {
+     "type": "string",
+     "description": "[Required] The ID of the project containing this table.",
+     "annotations": {
+      "required": [
+       "bigquery.tables.update"
+      ]
+     }
+    },
+    "tableId": {
+     "type": "string",
+     "description": "[Required] The ID of the table. The ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). The maximum length is 1,024 characters.",
+     "annotations": {
+      "required": [
+       "bigquery.tables.update"
+      ]
+     }
+    }
+   }
+  },
+  "TableRow": {
+   "id": "TableRow",
+   "type": "object",
+   "properties": {
+    "f": {
+     "type": "array",
+     "description": "Represents a single row in the result set, consisting of one or more fields.",
+     "items": {
+      "$ref": "TableCell"
+     }
+    }
+   }
+  },
+  "TableSchema": {
+   "id": "TableSchema",
+   "type": "object",
+   "properties": {
+    "fields": {
+     "type": "array",
+     "description": "Describes the fields in a table.",
+     "items": {
+      "$ref": "TableFieldSchema"
+     }
+    }
+   }
+  },
+  "TimePartitioning": {
+   "id": "TimePartitioning",
+   "type": "object",
+   "properties": {
+    "expirationMs": {
+     "type": "string",
+     "description": "[Optional] Number of milliseconds for which to keep the storage for a partition.",
+     "format": "int64"
+    },
+    "type": {
+     "type": "string",
+     "description": "[Required] The only type supported is DAY, which will generate one partition per day based on data loading time."
+    }
+   }
+  },
+  "UserDefinedFunctionResource": {
+   "id": "UserDefinedFunctionResource",
+   "type": "object",
+   "properties": {
+    "inlineCode": {
+     "type": "string",
+     "description": "[Pick one] An inline resource that contains code for a user-defined function (UDF). Providing a inline code resource is equivalent to providing a URI for a file containing the same code."
+    },
+    "resourceUri": {
+     "type": "string",
+     "description": "[Pick one] A code resource to load from a Google Cloud Storage URI (gs://bucket/path)."
+    }
+   }
+  },
+  "ViewDefinition": {
+   "id": "ViewDefinition",
+   "type": "object",
+   "properties": {
+    "query": {
+     "type": "string",
+     "description": "[Required] A query that BigQuery executes when the view is referenced."
+    },
+    "useLegacySql": {
+     "type": "boolean",
+     "description": "[Experimental] Specifies whether to use BigQuery's legacy SQL for this view. The default value is true. If set to false, the view will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ Queries and views that reference this view must use the same flag value."
+    },
+    "userDefinedFunctionResources": {
+     "type": "array",
+     "description": "[Experimental] Describes user-defined function resources used in the query.",
+     "items": {
+      "$ref": "UserDefinedFunctionResource"
+     }
+    }
+   }
+  }
+ },
+ "resources": {
+  "datasets": {
+   "methods": {
+    "delete": {
+     "id": "bigquery.datasets.delete",
+     "path": "projects/{projectId}/datasets/{datasetId}",
+     "httpMethod": "DELETE",
+     "description": "Deletes the dataset specified by the datasetId value. Before you can delete a dataset, you must delete all its tables, either manually or by specifying deleteContents. Immediately after deletion, you can create another dataset with the same name.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of dataset being deleted",
+       "required": true,
+       "location": "path"
+      },
+      "deleteContents": {
+       "type": "boolean",
+       "description": "If True, delete all the tables in the dataset. If False and the dataset contains tables, the request will fail. Default is False",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the dataset being deleted",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "get": {
+     "id": "bigquery.datasets.get",
+     "path": "projects/{projectId}/datasets/{datasetId}",
+     "httpMethod": "GET",
+     "description": "Returns the dataset specified by datasetID.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the requested dataset",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the requested dataset",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "response": {
+      "$ref": "Dataset"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "insert": {
+     "id": "bigquery.datasets.insert",
+     "path": "projects/{projectId}/datasets",
+     "httpMethod": "POST",
+     "description": "Creates a new empty dataset.",
+     "parameters": {
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the new dataset",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId"
+     ],
+     "request": {
+      "$ref": "Dataset"
+     },
+     "response": {
+      "$ref": "Dataset"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "list": {
+     "id": "bigquery.datasets.list",
+     "path": "projects/{projectId}/datasets",
+     "httpMethod": "GET",
+     "description": "Lists all datasets in the specified project to which you have been granted the READER dataset role.",
+     "parameters": {
+      "all": {
+       "type": "boolean",
+       "description": "Whether to list all datasets, including hidden ones",
+       "location": "query"
+      },
+      "filter": {
+       "type": "string",
+       "description": "An expression for filtering the results of the request by label. The syntax is \"labels.[:]\". Multiple filters can be ANDed together by connecting with a space. Example: \"labels.department:receiving labels.active\". See https://cloud.google.com/bigquery/docs/labeling-datasets#filtering_datasets_using_labels for details.",
+       "location": "query"
+      },
+      "maxResults": {
+       "type": "integer",
+       "description": "The maximum number of results to return",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, to request the next page of results",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the datasets to be listed",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId"
+     ],
+     "response": {
+      "$ref": "DatasetList"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "patch": {
+     "id": "bigquery.datasets.patch",
+     "path": "projects/{projectId}/datasets/{datasetId}",
+     "httpMethod": "PATCH",
+     "description": "Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource. This method supports patch semantics.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the dataset being updated",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the dataset being updated",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "request": {
+      "$ref": "Dataset"
+     },
+     "response": {
+      "$ref": "Dataset"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "update": {
+     "id": "bigquery.datasets.update",
+     "path": "projects/{projectId}/datasets/{datasetId}",
+     "httpMethod": "PUT",
+     "description": "Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the dataset being updated",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the dataset being updated",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "request": {
+      "$ref": "Dataset"
+     },
+     "response": {
+      "$ref": "Dataset"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    }
+   }
+  },
+  "jobs": {
+   "methods": {
+    "cancel": {
+     "id": "bigquery.jobs.cancel",
+     "path": "project/{projectId}/jobs/{jobId}/cancel",
+     "httpMethod": "POST",
+     "description": "Requests that a job be cancelled. This call will return immediately, and the client will need to poll for the job status to see if the cancel completed successfully. Cancelled jobs may still incur costs.",
+     "parameters": {
+      "jobId": {
+       "type": "string",
+       "description": "[Required] Job ID of the job to cancel",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "[Required] Project ID of the job to cancel",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "jobId"
+     ],
+     "response": {
+      "$ref": "JobCancelResponse"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "get": {
+     "id": "bigquery.jobs.get",
+     "path": "projects/{projectId}/jobs/{jobId}",
+     "httpMethod": "GET",
+     "description": "Returns information about a specific job. Job information is available for a six month period after creation. Requires that you're the person who ran the job, or have the Is Owner project role.",
+     "parameters": {
+      "jobId": {
+       "type": "string",
+       "description": "[Required] Job ID of the requested job",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "[Required] Project ID of the requested job",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "jobId"
+     ],
+     "response": {
+      "$ref": "Job"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "getQueryResults": {
+     "id": "bigquery.jobs.getQueryResults",
+     "path": "projects/{projectId}/queries/{jobId}",
+     "httpMethod": "GET",
+     "description": "Retrieves the results of a query job.",
+     "parameters": {
+      "jobId": {
+       "type": "string",
+       "description": "[Required] Job ID of the query job",
+       "required": true,
+       "location": "path"
+      },
+      "maxResults": {
+       "type": "integer",
+       "description": "Maximum number of results to read",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, to request the next page of results",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "[Required] Project ID of the query job",
+       "required": true,
+       "location": "path"
+      },
+      "startIndex": {
+       "type": "string",
+       "description": "Zero-based index of the starting row",
+       "format": "uint64",
+       "location": "query"
+      },
+      "timeoutMs": {
+       "type": "integer",
+       "description": "How long to wait for the query to complete, in milliseconds, before returning. Default is 10 seconds. If the timeout passes before the job completes, the 'jobComplete' field in the response will be false",
+       "format": "uint32",
+       "location": "query"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "jobId"
+     ],
+     "response": {
+      "$ref": "GetQueryResultsResponse"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "insert": {
+     "id": "bigquery.jobs.insert",
+     "path": "projects/{projectId}/jobs",
+     "httpMethod": "POST",
+     "description": "Starts a new asynchronous job. Requires the Can View project role.",
+     "parameters": {
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the project that will be billed for the job",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId"
+     ],
+     "request": {
+      "$ref": "Job"
+     },
+     "response": {
+      "$ref": "Job"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/devstorage.full_control",
+      "https://www.googleapis.com/auth/devstorage.read_only",
+      "https://www.googleapis.com/auth/devstorage.read_write"
+     ],
+     "supportsMediaUpload": true,
+     "mediaUpload": {
+      "accept": [
+       "*/*"
+      ],
+      "protocols": {
+       "simple": {
+        "multipart": true,
+        "path": "/upload/bigquery/v2/projects/{projectId}/jobs"
+       },
+       "resumable": {
+        "multipart": true,
+        "path": "/resumable/upload/bigquery/v2/projects/{projectId}/jobs"
+       }
+      }
+     }
+    },
+    "list": {
+     "id": "bigquery.jobs.list",
+     "path": "projects/{projectId}/jobs",
+     "httpMethod": "GET",
+     "description": "Lists all jobs that you started in the specified project. Job information is available for a six month period after creation. The job list is sorted in reverse chronological order, by job creation time. Requires the Can View project role, or the Is Owner project role if you set the allUsers property.",
+     "parameters": {
+      "allUsers": {
+       "type": "boolean",
+       "description": "Whether to display jobs owned by all users in the project. Default false",
+       "location": "query"
+      },
+      "maxResults": {
+       "type": "integer",
+       "description": "Maximum number of results to return",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, to request the next page of results",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the jobs to list",
+       "required": true,
+       "location": "path"
+      },
+      "projection": {
+       "type": "string",
+       "description": "Restrict information returned to a set of selected fields",
+       "enum": [
+        "full",
+        "minimal"
+       ],
+       "enumDescriptions": [
+        "Includes all job data",
+        "Does not include the job configuration"
+       ],
+       "location": "query"
+      },
+      "stateFilter": {
+       "type": "string",
+       "description": "Filter for job state",
+       "enum": [
+        "done",
+        "pending",
+        "running"
+       ],
+       "enumDescriptions": [
+        "Finished jobs",
+        "Pending jobs",
+        "Running jobs"
+       ],
+       "repeated": true,
+       "location": "query"
+      }
+     },
+     "parameterOrder": [
+      "projectId"
+     ],
+     "response": {
+      "$ref": "JobList"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "query": {
+     "id": "bigquery.jobs.query",
+     "path": "projects/{projectId}/queries",
+     "httpMethod": "POST",
+     "description": "Runs a BigQuery SQL query synchronously and returns query results if the query completes within a specified timeout.",
+     "parameters": {
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the project billed for the query",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId"
+     ],
+     "request": {
+      "$ref": "QueryRequest"
+     },
+     "response": {
+      "$ref": "QueryResponse"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    }
+   }
+  },
+  "projects": {
+   "methods": {
+    "list": {
+     "id": "bigquery.projects.list",
+     "path": "projects",
+     "httpMethod": "GET",
+     "description": "Lists all projects to which you have been granted any project role.",
+     "parameters": {
+      "maxResults": {
+       "type": "integer",
+       "description": "Maximum number of results to return",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, to request the next page of results",
+       "location": "query"
+      }
+     },
+     "response": {
+      "$ref": "ProjectList"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    }
+   }
+  },
+  "tabledata": {
+   "methods": {
+    "insertAll": {
+     "id": "bigquery.tabledata.insertAll",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}/insertAll",
+     "httpMethod": "POST",
+     "description": "Streams data into BigQuery one record at a time without needing to run a load job. Requires the WRITER dataset role.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the destination table.",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the destination table.",
+       "required": true,
+       "location": "path"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the destination table.",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "request": {
+      "$ref": "TableDataInsertAllRequest"
+     },
+     "response": {
+      "$ref": "TableDataInsertAllResponse"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/bigquery.insertdata",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "list": {
+     "id": "bigquery.tabledata.list",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}/data",
+     "httpMethod": "GET",
+     "description": "Retrieves table data from a specified set of rows. Requires the READER dataset role.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the table to read",
+       "required": true,
+       "location": "path"
+      },
+      "maxResults": {
+       "type": "integer",
+       "description": "Maximum number of results to return",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, identifying the result set",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the table to read",
+       "required": true,
+       "location": "path"
+      },
+      "startIndex": {
+       "type": "string",
+       "description": "Zero-based index of the starting row to read",
+       "format": "uint64",
+       "location": "query"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the table to read",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "response": {
+      "$ref": "TableDataList"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    }
+   }
+  },
+  "tables": {
+   "methods": {
+    "delete": {
+     "id": "bigquery.tables.delete",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}",
+     "httpMethod": "DELETE",
+     "description": "Deletes the table specified by tableId from the dataset. If the table contains data, all the data will be deleted.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the table to delete",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the table to delete",
+       "required": true,
+       "location": "path"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the table to delete",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "get": {
+     "id": "bigquery.tables.get",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}",
+     "httpMethod": "GET",
+     "description": "Gets the specified table resource by table ID. This method does not return the data in the table, it only returns the table resource, which describes the structure of this table.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the requested table",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the requested table",
+       "required": true,
+       "location": "path"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the requested table",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "response": {
+      "$ref": "Table"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "insert": {
+     "id": "bigquery.tables.insert",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables",
+     "httpMethod": "POST",
+     "description": "Creates a new, empty table in the dataset.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the new table",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the new table",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "request": {
+      "$ref": "Table"
+     },
+     "response": {
+      "$ref": "Table"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "list": {
+     "id": "bigquery.tables.list",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables",
+     "httpMethod": "GET",
+     "description": "Lists all tables in the specified dataset. Requires the READER dataset role.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the tables to list",
+       "required": true,
+       "location": "path"
+      },
+      "maxResults": {
+       "type": "integer",
+       "description": "Maximum number of results to return",
+       "format": "uint32",
+       "location": "query"
+      },
+      "pageToken": {
+       "type": "string",
+       "description": "Page token, returned by a previous call, to request the next page of results",
+       "location": "query"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the tables to list",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId"
+     ],
+     "response": {
+      "$ref": "TableList"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform",
+      "https://www.googleapis.com/auth/cloud-platform.read-only"
+     ]
+    },
+    "patch": {
+     "id": "bigquery.tables.patch",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}",
+     "httpMethod": "PATCH",
+     "description": "Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource. This method supports patch semantics.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the table to update",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the table to update",
+       "required": true,
+       "location": "path"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the table to update",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "request": {
+      "$ref": "Table"
+     },
+     "response": {
+      "$ref": "Table"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    },
+    "update": {
+     "id": "bigquery.tables.update",
+     "path": "projects/{projectId}/datasets/{datasetId}/tables/{tableId}",
+     "httpMethod": "PUT",
+     "description": "Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource.",
+     "parameters": {
+      "datasetId": {
+       "type": "string",
+       "description": "Dataset ID of the table to update",
+       "required": true,
+       "location": "path"
+      },
+      "projectId": {
+       "type": "string",
+       "description": "Project ID of the table to update",
+       "required": true,
+       "location": "path"
+      },
+      "tableId": {
+       "type": "string",
+       "description": "Table ID of the table to update",
+       "required": true,
+       "location": "path"
+      }
+     },
+     "parameterOrder": [
+      "projectId",
+      "datasetId",
+      "tableId"
+     ],
+     "request": {
+      "$ref": "Table"
+     },
+     "response": {
+      "$ref": "Table"
+     },
+     "scopes": [
+      "https://www.googleapis.com/auth/bigquery",
+      "https://www.googleapis.com/auth/cloud-platform"
+     ]
+    }
+   }
+  }
+ }
+}
diff --git a/samples/bigquery_sample/bigquery_v2/__init__.py b/samples/bigquery_sample/bigquery_v2/__init__.py
new file mode 100644
index 0000000..2816da8
--- /dev/null
+++ b/samples/bigquery_sample/bigquery_v2/__init__.py
@@ -0,0 +1,5 @@
+"""Package marker file."""
+
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)
diff --git a/samples/bigquery_sample/bigquery_v2/bigquery_v2.py b/samples/bigquery_sample/bigquery_v2/bigquery_v2.py
new file mode 100644
index 0000000..7cd69b5
--- /dev/null
+++ b/samples/bigquery_sample/bigquery_v2/bigquery_v2.py
@@ -0,0 +1,1096 @@
+#!/usr/bin/env python
+"""CLI for bigquery, version v2."""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+import code
+import os
+import platform
+import sys
+
+from apitools.base.protorpclite import message_types
+from apitools.base.protorpclite import messages
+
+from google.apputils import appcommands
+import gflags as flags
+
+import apitools.base.py as apitools_base
+from apitools.base.py import cli as apitools_base_cli
+import bigquery_v2_client as client_lib
+import bigquery_v2_messages as messages
+
+
+def _DeclareBigqueryFlags():
+  """Declare global flags in an idempotent way."""
+  if 'api_endpoint' in flags.FLAGS:
+    return
+  flags.DEFINE_string(
+      'api_endpoint',
+      u'https://www.googleapis.com/bigquery/v2/',
+      'URL of the API endpoint to use.',
+      short_name='bigquery_url')
+  flags.DEFINE_string(
+      'history_file',
+      u'~/.bigquery.v2.history',
+      'File with interactive shell history.')
+  flags.DEFINE_multistring(
+      'add_header', [],
+      'Additional http headers (as key=value strings). '
+      'Can be specified multiple times.')
+  flags.DEFINE_string(
+      'service_account_json_keyfile', '',
+      'Filename for a JSON service account key downloaded'
+      ' from the Developer Console.')
+  flags.DEFINE_enum(
+      'alt',
+      u'json',
+      [u'json'],
+      u'Data format for the response.')
+  flags.DEFINE_string(
+      'fields',
+      None,
+      u'Selector specifying which fields to include in a partial response.')
+  flags.DEFINE_string(
+      'key',
+      None,
+      u'API key. Your API key identifies your project and provides you with '
+      u'API access, quota, and reports. Required unless you provide an OAuth '
+      u'2.0 token.')
+  flags.DEFINE_string(
+      'oauth_token',
+      None,
+      u'OAuth 2.0 token for the current user.')
+  flags.DEFINE_boolean(
+      'prettyPrint',
+      'True',
+      u'Returns response with indentations and line breaks.')
+  flags.DEFINE_string(
+      'quotaUser',
+      None,
+      u'Available to use for quota purposes for server-side applications. Can'
+      u' be any arbitrary string assigned to a user, but should not exceed 40'
+      u' characters. Overrides userIp if both are provided.')
+  flags.DEFINE_string(
+      'trace',
+      None,
+      'A tracing token of the form "token:" to include in api '
+      'requests.')
+  flags.DEFINE_string(
+      'userIp',
+      None,
+      u'IP address of the site where the request originates. Use this if you '
+      u'want to enforce per-user limits.')
+
+
+FLAGS = flags.FLAGS
+apitools_base_cli.DeclareBaseFlags()
+_DeclareBigqueryFlags()
+
+
+def GetGlobalParamsFromFlags():
+  """Return a StandardQueryParameters based on flags."""
+  result = messages.StandardQueryParameters()
+  if FLAGS['alt'].present:
+    result.alt = messages.StandardQueryParameters.AltValueValuesEnum(FLAGS.alt)
+  if FLAGS['fields'].present:
+    result.fields = FLAGS.fields.decode('utf8')
+  if FLAGS['key'].present:
+    result.key = FLAGS.key.decode('utf8')
+  if FLAGS['oauth_token'].present:
+    result.oauth_token = FLAGS.oauth_token.decode('utf8')
+  if FLAGS['prettyPrint'].present:
+    result.prettyPrint = FLAGS.prettyPrint
+  if FLAGS['quotaUser'].present:
+    result.quotaUser = FLAGS.quotaUser.decode('utf8')
+  if FLAGS['trace'].present:
+    result.trace = FLAGS.trace.decode('utf8')
+  if FLAGS['userIp'].present:
+    result.userIp = FLAGS.userIp.decode('utf8')
+  return result
+
+
+def GetClientFromFlags():
+  """Return a client object, configured from flags."""
+  log_request = FLAGS.log_request or FLAGS.log_request_response
+  log_response = FLAGS.log_response or FLAGS.log_request_response
+  api_endpoint = apitools_base.NormalizeApiEndpoint(FLAGS.api_endpoint)
+  additional_http_headers = dict(x.split('=', 1) for x in FLAGS.add_header)
+  credentials_args = {
+      'service_account_json_keyfile': os.path.expanduser(FLAGS.service_account_json_keyfile)
+  }
+  try:
+    client = client_lib.BigqueryV2(
+        api_endpoint, log_request=log_request,
+        log_response=log_response,
+        credentials_args=credentials_args,
+        additional_http_headers=additional_http_headers)
+  except apitools_base.CredentialsError as e:
+    print 'Error creating credentials: %s' % e
+    sys.exit(1)
+  return client
+
+
+class PyShell(appcommands.Cmd):
+
+  def Run(self, _):
+    """Run an interactive python shell with the client."""
+    client = GetClientFromFlags()
+    params = GetGlobalParamsFromFlags()
+    for field in params.all_fields():
+      value = params.get_assigned_value(field.name)
+      if value != field.default:
+        client.AddGlobalParam(field.name, value)
+    banner = """
+           == bigquery interactive console ==
+                 client: a bigquery client
+          apitools_base: base apitools module
+         messages: the generated messages module
+    """
+    local_vars = {
+        'apitools_base': apitools_base,
+        'client': client,
+        'client_lib': client_lib,
+        'messages': messages,
+    }
+    if platform.system() == 'Linux':
+      console = apitools_base_cli.ConsoleWithReadline(
+          local_vars, histfile=FLAGS.history_file)
+    else:
+      console = code.InteractiveConsole(local_vars)
+    try:
+      console.interact(banner)
+    except SystemExit as e:
+      return e.code
+
+
+class DatasetsDelete(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.Delete."""
+
+  usage = """datasets_delete  """
+
+  def __init__(self, name, fv):
+    super(DatasetsDelete, self).__init__(name, fv)
+    flags.DEFINE_boolean(
+        'deleteContents',
+        None,
+        u'If True, delete all the tables in the dataset. If False and the '
+        u'dataset contains tables, the request will fail. Default is False',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Deletes the dataset specified by the datasetId value. Before you can
+    delete a dataset, you must delete all its tables, either manually or by
+    specifying deleteContents. Immediately after deletion, you can create
+    another dataset with the same name.
+
+    Args:
+      projectId: Project ID of the dataset being deleted
+      datasetId: Dataset ID of dataset being deleted
+
+    Flags:
+      deleteContents: If True, delete all the tables in the dataset. If False
+        and the dataset contains tables, the request will fail. Default is
+        False
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsDeleteRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    if FLAGS['deleteContents'].present:
+      request.deleteContents = FLAGS.deleteContents
+    result = client.datasets.Delete(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class DatasetsGet(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.Get."""
+
+  usage = """datasets_get  """
+
+  def __init__(self, name, fv):
+    super(DatasetsGet, self).__init__(name, fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Returns the dataset specified by datasetID.
+
+    Args:
+      projectId: Project ID of the requested dataset
+      datasetId: Dataset ID of the requested dataset
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsGetRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    result = client.datasets.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class DatasetsInsert(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.Insert."""
+
+  usage = """datasets_insert """
+
+  def __init__(self, name, fv):
+    super(DatasetsInsert, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'dataset',
+        None,
+        u'A Dataset resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId):
+    """Creates a new empty dataset.
+
+    Args:
+      projectId: Project ID of the new dataset
+
+    Flags:
+      dataset: A Dataset resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsInsertRequest(
+        projectId=projectId.decode('utf8'),
+        )
+    if FLAGS['dataset'].present:
+      request.dataset = apitools_base.JsonToMessage(messages.Dataset, FLAGS.dataset)
+    result = client.datasets.Insert(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class DatasetsList(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.List."""
+
+  usage = """datasets_list """
+
+  def __init__(self, name, fv):
+    super(DatasetsList, self).__init__(name, fv)
+    flags.DEFINE_boolean(
+        'all',
+        None,
+        u'Whether to list all datasets, including hidden ones',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'filter',
+        None,
+        u'An expression for filtering the results of the request by label. '
+        u'The syntax is "labels.[:]". Multiple filters can be ANDed together '
+        u'by connecting with a space. Example: "labels.department:receiving '
+        u'labels.active". See https://cloud.google.com/bigquery/docs'
+        u'/labeling-datasets#filtering_datasets_using_labels for details.',
+        flag_values=fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'The maximum number of results to return',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, to request the next page '
+        u'of results',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId):
+    """Lists all datasets in the specified project to which you have been
+    granted the READER dataset role.
+
+    Args:
+      projectId: Project ID of the datasets to be listed
+
+    Flags:
+      all: Whether to list all datasets, including hidden ones
+      filter: An expression for filtering the results of the request by label.
+        The syntax is "labels.[:]". Multiple filters can be ANDed together by
+        connecting with a space. Example: "labels.department:receiving
+        labels.active". See https://cloud.google.com/bigquery/docs/labeling-
+        datasets#filtering_datasets_using_labels for details.
+      maxResults: The maximum number of results to return
+      pageToken: Page token, returned by a previous call, to request the next
+        page of results
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsListRequest(
+        projectId=projectId.decode('utf8'),
+        )
+    if FLAGS['all'].present:
+      request.all = FLAGS.all
+    if FLAGS['filter'].present:
+      request.filter = FLAGS.filter.decode('utf8')
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    result = client.datasets.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class DatasetsPatch(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.Patch."""
+
+  usage = """datasets_patch  """
+
+  def __init__(self, name, fv):
+    super(DatasetsPatch, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'dataset',
+        None,
+        u'A Dataset resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Updates information in an existing dataset. The update method replaces
+    the entire dataset resource, whereas the patch method only replaces fields
+    that are provided in the submitted dataset resource. This method supports
+    patch semantics.
+
+    Args:
+      projectId: Project ID of the dataset being updated
+      datasetId: Dataset ID of the dataset being updated
+
+    Flags:
+      dataset: A Dataset resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsPatchRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    if FLAGS['dataset'].present:
+      request.dataset = apitools_base.JsonToMessage(messages.Dataset, FLAGS.dataset)
+    result = client.datasets.Patch(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class DatasetsUpdate(apitools_base_cli.NewCmd):
+  """Command wrapping datasets.Update."""
+
+  usage = """datasets_update  """
+
+  def __init__(self, name, fv):
+    super(DatasetsUpdate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'dataset',
+        None,
+        u'A Dataset resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Updates information in an existing dataset. The update method replaces
+    the entire dataset resource, whereas the patch method only replaces fields
+    that are provided in the submitted dataset resource.
+
+    Args:
+      projectId: Project ID of the dataset being updated
+      datasetId: Dataset ID of the dataset being updated
+
+    Flags:
+      dataset: A Dataset resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryDatasetsUpdateRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    if FLAGS['dataset'].present:
+      request.dataset = apitools_base.JsonToMessage(messages.Dataset, FLAGS.dataset)
+    result = client.datasets.Update(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsCancel(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.Cancel."""
+
+  usage = """jobs_cancel  """
+
+  def __init__(self, name, fv):
+    super(JobsCancel, self).__init__(name, fv)
+
+  def RunWithArgs(self, projectId, jobId):
+    """Requests that a job be cancelled. This call will return immediately,
+    and the client will need to poll for the job status to see if the cancel
+    completed successfully. Cancelled jobs may still incur costs.
+
+    Args:
+      projectId: [Required] Project ID of the job to cancel
+      jobId: [Required] Job ID of the job to cancel
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsCancelRequest(
+        projectId=projectId.decode('utf8'),
+        jobId=jobId.decode('utf8'),
+        )
+    result = client.jobs.Cancel(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsGet(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.Get."""
+
+  usage = """jobs_get  """
+
+  def __init__(self, name, fv):
+    super(JobsGet, self).__init__(name, fv)
+
+  def RunWithArgs(self, projectId, jobId):
+    """Returns information about a specific job. Job information is available
+    for a six month period after creation. Requires that you're the person who
+    ran the job, or have the Is Owner project role.
+
+    Args:
+      projectId: [Required] Project ID of the requested job
+      jobId: [Required] Job ID of the requested job
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsGetRequest(
+        projectId=projectId.decode('utf8'),
+        jobId=jobId.decode('utf8'),
+        )
+    result = client.jobs.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsGetQueryResults(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.GetQueryResults."""
+
+  usage = """jobs_getQueryResults  """
+
+  def __init__(self, name, fv):
+    super(JobsGetQueryResults, self).__init__(name, fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'Maximum number of results to read',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, to request the next page '
+        u'of results',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'startIndex',
+        None,
+        u'Zero-based index of the starting row',
+        flag_values=fv)
+    flags.DEFINE_integer(
+        'timeoutMs',
+        None,
+        u'How long to wait for the query to complete, in milliseconds, before'
+        u' returning. Default is 10 seconds. If the timeout passes before the'
+        u" job completes, the 'jobComplete' field in the response will be "
+        u'false',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, jobId):
+    """Retrieves the results of a query job.
+
+    Args:
+      projectId: [Required] Project ID of the query job
+      jobId: [Required] Job ID of the query job
+
+    Flags:
+      maxResults: Maximum number of results to read
+      pageToken: Page token, returned by a previous call, to request the next
+        page of results
+      startIndex: Zero-based index of the starting row
+      timeoutMs: How long to wait for the query to complete, in milliseconds,
+        before returning. Default is 10 seconds. If the timeout passes before
+        the job completes, the 'jobComplete' field in the response will be
+        false
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsGetQueryResultsRequest(
+        projectId=projectId.decode('utf8'),
+        jobId=jobId.decode('utf8'),
+        )
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    if FLAGS['startIndex'].present:
+      request.startIndex = int(FLAGS.startIndex)
+    if FLAGS['timeoutMs'].present:
+      request.timeoutMs = FLAGS.timeoutMs
+    result = client.jobs.GetQueryResults(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsInsert(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.Insert."""
+
+  usage = """jobs_insert """
+
+  def __init__(self, name, fv):
+    super(JobsInsert, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'job',
+        None,
+        u'A Job resource to be passed as the request body.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'upload_filename',
+        '',
+        'Filename to use for upload.',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'upload_mime_type',
+        '',
+        'MIME type to use for the upload. Only needed if the extension on '
+        '--upload_filename does not determine the correct (or any) MIME '
+        'type.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId):
+    """Starts a new asynchronous job. Requires the Can View project role.
+
+    Args:
+      projectId: Project ID of the project that will be billed for the job
+
+    Flags:
+      job: A Job resource to be passed as the request body.
+      upload_filename: Filename to use for upload.
+      upload_mime_type: MIME type to use for the upload. Only needed if the
+        extension on --upload_filename does not determine the correct (or any)
+        MIME type.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsInsertRequest(
+        projectId=projectId.decode('utf8'),
+        )
+    if FLAGS['job'].present:
+      request.job = apitools_base.JsonToMessage(messages.Job, FLAGS.job)
+    upload = None
+    if FLAGS.upload_filename:
+      upload = apitools_base.Upload.FromFile(
+          FLAGS.upload_filename, FLAGS.upload_mime_type,
+          progress_callback=apitools_base.UploadProgressPrinter,
+          finish_callback=apitools_base.UploadCompletePrinter)
+    result = client.jobs.Insert(
+        request, global_params=global_params, upload=upload)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsList(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.List."""
+
+  usage = """jobs_list """
+
+  def __init__(self, name, fv):
+    super(JobsList, self).__init__(name, fv)
+    flags.DEFINE_boolean(
+        'allUsers',
+        None,
+        u'Whether to display jobs owned by all users in the project. Default '
+        u'false',
+        flag_values=fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'Maximum number of results to return',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, to request the next page '
+        u'of results',
+        flag_values=fv)
+    flags.DEFINE_enum(
+        'projection',
+        u'full',
+        [u'full', u'minimal'],
+        u'Restrict information returned to a set of selected fields',
+        flag_values=fv)
+    flags.DEFINE_enum(
+        'stateFilter',
+        u'done',
+        [u'done', u'pending', u'running'],
+        u'Filter for job state',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId):
+    """Lists all jobs that you started in the specified project. Job
+    information is available for a six month period after creation. The job
+    list is sorted in reverse chronological order, by job creation time.
+    Requires the Can View project role, or the Is Owner project role if you
+    set the allUsers property.
+
+    Args:
+      projectId: Project ID of the jobs to list
+
+    Flags:
+      allUsers: Whether to display jobs owned by all users in the project.
+        Default false
+      maxResults: Maximum number of results to return
+      pageToken: Page token, returned by a previous call, to request the next
+        page of results
+      projection: Restrict information returned to a set of selected fields
+      stateFilter: Filter for job state
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsListRequest(
+        projectId=projectId.decode('utf8'),
+        )
+    if FLAGS['allUsers'].present:
+      request.allUsers = FLAGS.allUsers
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    if FLAGS['projection'].present:
+      request.projection = messages.BigqueryJobsListRequest.ProjectionValueValuesEnum(FLAGS.projection)
+    if FLAGS['stateFilter'].present:
+      request.stateFilter = [messages.BigqueryJobsListRequest.StateFilterValueValuesEnum(x) for x in FLAGS.stateFilter]
+    result = client.jobs.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class JobsQuery(apitools_base_cli.NewCmd):
+  """Command wrapping jobs.Query."""
+
+  usage = """jobs_query """
+
+  def __init__(self, name, fv):
+    super(JobsQuery, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'queryRequest',
+        None,
+        u'A QueryRequest resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId):
+    """Runs a BigQuery SQL query synchronously and returns query results if
+    the query completes within a specified timeout.
+
+    Args:
+      projectId: Project ID of the project billed for the query
+
+    Flags:
+      queryRequest: A QueryRequest resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryJobsQueryRequest(
+        projectId=projectId.decode('utf8'),
+        )
+    if FLAGS['queryRequest'].present:
+      request.queryRequest = apitools_base.JsonToMessage(messages.QueryRequest, FLAGS.queryRequest)
+    result = client.jobs.Query(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class ProjectsList(apitools_base_cli.NewCmd):
+  """Command wrapping projects.List."""
+
+  usage = """projects_list"""
+
+  def __init__(self, name, fv):
+    super(ProjectsList, self).__init__(name, fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'Maximum number of results to return',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, to request the next page '
+        u'of results',
+        flag_values=fv)
+
+  def RunWithArgs(self):
+    """Lists all projects to which you have been granted any project role.
+
+    Flags:
+      maxResults: Maximum number of results to return
+      pageToken: Page token, returned by a previous call, to request the next
+        page of results
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryProjectsListRequest(
+        )
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    result = client.projects.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TabledataInsertAll(apitools_base_cli.NewCmd):
+  """Command wrapping tabledata.InsertAll."""
+
+  usage = """tabledata_insertAll   """
+
+  def __init__(self, name, fv):
+    super(TabledataInsertAll, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'tableDataInsertAllRequest',
+        None,
+        u'A TableDataInsertAllRequest resource to be passed as the request '
+        u'body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Streams data into BigQuery one record at a time without needing to run
+    a load job. Requires the WRITER dataset role.
+
+    Args:
+      projectId: Project ID of the destination table.
+      datasetId: Dataset ID of the destination table.
+      tableId: Table ID of the destination table.
+
+    Flags:
+      tableDataInsertAllRequest: A TableDataInsertAllRequest resource to be
+        passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTabledataInsertAllRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    if FLAGS['tableDataInsertAllRequest'].present:
+      request.tableDataInsertAllRequest = apitools_base.JsonToMessage(messages.TableDataInsertAllRequest, FLAGS.tableDataInsertAllRequest)
+    result = client.tabledata.InsertAll(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TabledataList(apitools_base_cli.NewCmd):
+  """Command wrapping tabledata.List."""
+
+  usage = """tabledata_list   """
+
+  def __init__(self, name, fv):
+    super(TabledataList, self).__init__(name, fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'Maximum number of results to return',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, identifying the result set',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'startIndex',
+        None,
+        u'Zero-based index of the starting row to read',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Retrieves table data from a specified set of rows. Requires the READER
+    dataset role.
+
+    Args:
+      projectId: Project ID of the table to read
+      datasetId: Dataset ID of the table to read
+      tableId: Table ID of the table to read
+
+    Flags:
+      maxResults: Maximum number of results to return
+      pageToken: Page token, returned by a previous call, identifying the
+        result set
+      startIndex: Zero-based index of the starting row to read
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTabledataListRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    if FLAGS['startIndex'].present:
+      request.startIndex = int(FLAGS.startIndex)
+    result = client.tabledata.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesDelete(apitools_base_cli.NewCmd):
+  """Command wrapping tables.Delete."""
+
+  usage = """tables_delete   """
+
+  def __init__(self, name, fv):
+    super(TablesDelete, self).__init__(name, fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Deletes the table specified by tableId from the dataset. If the table
+    contains data, all the data will be deleted.
+
+    Args:
+      projectId: Project ID of the table to delete
+      datasetId: Dataset ID of the table to delete
+      tableId: Table ID of the table to delete
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesDeleteRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    result = client.tables.Delete(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesGet(apitools_base_cli.NewCmd):
+  """Command wrapping tables.Get."""
+
+  usage = """tables_get   """
+
+  def __init__(self, name, fv):
+    super(TablesGet, self).__init__(name, fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Gets the specified table resource by table ID. This method does not
+    return the data in the table, it only returns the table resource, which
+    describes the structure of this table.
+
+    Args:
+      projectId: Project ID of the requested table
+      datasetId: Dataset ID of the requested table
+      tableId: Table ID of the requested table
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesGetRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    result = client.tables.Get(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesInsert(apitools_base_cli.NewCmd):
+  """Command wrapping tables.Insert."""
+
+  usage = """tables_insert  """
+
+  def __init__(self, name, fv):
+    super(TablesInsert, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'table',
+        None,
+        u'A Table resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Creates a new, empty table in the dataset.
+
+    Args:
+      projectId: Project ID of the new table
+      datasetId: Dataset ID of the new table
+
+    Flags:
+      table: A Table resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesInsertRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    if FLAGS['table'].present:
+      request.table = apitools_base.JsonToMessage(messages.Table, FLAGS.table)
+    result = client.tables.Insert(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesList(apitools_base_cli.NewCmd):
+  """Command wrapping tables.List."""
+
+  usage = """tables_list  """
+
+  def __init__(self, name, fv):
+    super(TablesList, self).__init__(name, fv)
+    flags.DEFINE_integer(
+        'maxResults',
+        None,
+        u'Maximum number of results to return',
+        flag_values=fv)
+    flags.DEFINE_string(
+        'pageToken',
+        None,
+        u'Page token, returned by a previous call, to request the next page '
+        u'of results',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId):
+    """Lists all tables in the specified dataset. Requires the READER dataset
+    role.
+
+    Args:
+      projectId: Project ID of the tables to list
+      datasetId: Dataset ID of the tables to list
+
+    Flags:
+      maxResults: Maximum number of results to return
+      pageToken: Page token, returned by a previous call, to request the next
+        page of results
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesListRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        )
+    if FLAGS['maxResults'].present:
+      request.maxResults = FLAGS.maxResults
+    if FLAGS['pageToken'].present:
+      request.pageToken = FLAGS.pageToken.decode('utf8')
+    result = client.tables.List(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesPatch(apitools_base_cli.NewCmd):
+  """Command wrapping tables.Patch."""
+
+  usage = """tables_patch   """
+
+  def __init__(self, name, fv):
+    super(TablesPatch, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'table',
+        None,
+        u'A Table resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Updates information in an existing table. The update method replaces
+    the entire table resource, whereas the patch method only replaces fields
+    that are provided in the submitted table resource. This method supports
+    patch semantics.
+
+    Args:
+      projectId: Project ID of the table to update
+      datasetId: Dataset ID of the table to update
+      tableId: Table ID of the table to update
+
+    Flags:
+      table: A Table resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesPatchRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    if FLAGS['table'].present:
+      request.table = apitools_base.JsonToMessage(messages.Table, FLAGS.table)
+    result = client.tables.Patch(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+class TablesUpdate(apitools_base_cli.NewCmd):
+  """Command wrapping tables.Update."""
+
+  usage = """tables_update   """
+
+  def __init__(self, name, fv):
+    super(TablesUpdate, self).__init__(name, fv)
+    flags.DEFINE_string(
+        'table',
+        None,
+        u'A Table resource to be passed as the request body.',
+        flag_values=fv)
+
+  def RunWithArgs(self, projectId, datasetId, tableId):
+    """Updates information in an existing table. The update method replaces
+    the entire table resource, whereas the patch method only replaces fields
+    that are provided in the submitted table resource.
+
+    Args:
+      projectId: Project ID of the table to update
+      datasetId: Dataset ID of the table to update
+      tableId: Table ID of the table to update
+
+    Flags:
+      table: A Table resource to be passed as the request body.
+    """
+    client = GetClientFromFlags()
+    global_params = GetGlobalParamsFromFlags()
+    request = messages.BigqueryTablesUpdateRequest(
+        projectId=projectId.decode('utf8'),
+        datasetId=datasetId.decode('utf8'),
+        tableId=tableId.decode('utf8'),
+        )
+    if FLAGS['table'].present:
+      request.table = apitools_base.JsonToMessage(messages.Table, FLAGS.table)
+    result = client.tables.Update(
+        request, global_params=global_params)
+    print apitools_base_cli.FormatOutput(result)
+
+
+def main(_):
+  appcommands.AddCmd('pyshell', PyShell)
+  appcommands.AddCmd('datasets_delete', DatasetsDelete)
+  appcommands.AddCmd('datasets_get', DatasetsGet)
+  appcommands.AddCmd('datasets_insert', DatasetsInsert)
+  appcommands.AddCmd('datasets_list', DatasetsList)
+  appcommands.AddCmd('datasets_patch', DatasetsPatch)
+  appcommands.AddCmd('datasets_update', DatasetsUpdate)
+  appcommands.AddCmd('jobs_cancel', JobsCancel)
+  appcommands.AddCmd('jobs_get', JobsGet)
+  appcommands.AddCmd('jobs_getQueryResults', JobsGetQueryResults)
+  appcommands.AddCmd('jobs_insert', JobsInsert)
+  appcommands.AddCmd('jobs_list', JobsList)
+  appcommands.AddCmd('jobs_query', JobsQuery)
+  appcommands.AddCmd('projects_list', ProjectsList)
+  appcommands.AddCmd('tabledata_insertAll', TabledataInsertAll)
+  appcommands.AddCmd('tabledata_list', TabledataList)
+  appcommands.AddCmd('tables_delete', TablesDelete)
+  appcommands.AddCmd('tables_get', TablesGet)
+  appcommands.AddCmd('tables_insert', TablesInsert)
+  appcommands.AddCmd('tables_list', TablesList)
+  appcommands.AddCmd('tables_patch', TablesPatch)
+  appcommands.AddCmd('tables_update', TablesUpdate)
+
+  apitools_base_cli.SetupLogger()
+  if hasattr(appcommands, 'SetDefaultCommand'):
+    appcommands.SetDefaultCommand('pyshell')
+
+
+run_main = apitools_base_cli.run_main
+
+if __name__ == '__main__':
+  appcommands.Run()
diff --git a/samples/bigquery_sample/bigquery_v2/bigquery_v2_client.py b/samples/bigquery_sample/bigquery_v2/bigquery_v2_client.py
new file mode 100644
index 0000000..363f470
--- /dev/null
+++ b/samples/bigquery_sample/bigquery_v2/bigquery_v2_client.py
@@ -0,0 +1,649 @@
+"""Generated client library for bigquery version v2."""
+# NOTE: This file is autogenerated and should not be edited by hand.
+from apitools.base.py import base_api
+from samples.bigquery_sample.bigquery_v2 import bigquery_v2_messages as messages
+
+
+class BigqueryV2(base_api.BaseApiClient):
+  """Generated client library for service bigquery version v2."""
+
+  MESSAGES_MODULE = messages
+  BASE_URL = u'https://www.googleapis.com/bigquery/v2/'
+
+  _PACKAGE = u'bigquery'
+  _SCOPES = [u'https://www.googleapis.com/auth/bigquery', u'https://www.googleapis.com/auth/bigquery.insertdata', u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write']
+  _VERSION = u'v2'
+  _CLIENT_ID = '1042881264118.apps.googleusercontent.com'
+  _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+  _USER_AGENT = 'x_Tw5K8nnjoRAqULM9PFAC2b'
+  _CLIENT_CLASS_NAME = u'BigqueryV2'
+  _URL_VERSION = u'v2'
+  _API_KEY = None
+
+  def __init__(self, url='', credentials=None,
+               get_credentials=True, http=None, model=None,
+               log_request=False, log_response=False,
+               credentials_args=None, default_global_params=None,
+               additional_http_headers=None):
+    """Create a new bigquery handle."""
+    url = url or self.BASE_URL
+    super(BigqueryV2, self).__init__(
+        url, credentials=credentials,
+        get_credentials=get_credentials, http=http, model=model,
+        log_request=log_request, log_response=log_response,
+        credentials_args=credentials_args,
+        default_global_params=default_global_params,
+        additional_http_headers=additional_http_headers)
+    self.datasets = self.DatasetsService(self)
+    self.jobs = self.JobsService(self)
+    self.projects = self.ProjectsService(self)
+    self.tabledata = self.TabledataService(self)
+    self.tables = self.TablesService(self)
+
+  class DatasetsService(base_api.BaseApiService):
+    """Service class for the datasets resource."""
+
+    _NAME = u'datasets'
+
+    def __init__(self, client):
+      super(BigqueryV2.DatasetsService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def Delete(self, request, global_params=None):
+      """Deletes the dataset specified by the datasetId value. Before you can delete a dataset, you must delete all its tables, either manually or by specifying deleteContents. Immediately after deletion, you can create another dataset with the same name.
+
+      Args:
+        request: (BigqueryDatasetsDeleteRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (BigqueryDatasetsDeleteResponse) The response message.
+      """
+      config = self.GetMethodConfig('Delete')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Delete.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'DELETE',
+        method_id=u'bigquery.datasets.delete',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[u'deleteContents'],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}',
+        request_field='',
+        request_type_name=u'BigqueryDatasetsDeleteRequest',
+        response_type_name=u'BigqueryDatasetsDeleteResponse',
+        supports_download=False,
+    )
+
+    def Get(self, request, global_params=None):
+      """Returns the dataset specified by datasetID.
+
+      Args:
+        request: (BigqueryDatasetsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Dataset) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.datasets.get',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}',
+        request_field='',
+        request_type_name=u'BigqueryDatasetsGetRequest',
+        response_type_name=u'Dataset',
+        supports_download=False,
+    )
+
+    def Insert(self, request, global_params=None):
+      """Creates a new empty dataset.
+
+      Args:
+        request: (BigqueryDatasetsInsertRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Dataset) The response message.
+      """
+      config = self.GetMethodConfig('Insert')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Insert.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.datasets.insert',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets',
+        request_field=u'dataset',
+        request_type_name=u'BigqueryDatasetsInsertRequest',
+        response_type_name=u'Dataset',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      """Lists all datasets in the specified project to which you have been granted the READER dataset role.
+
+      Args:
+        request: (BigqueryDatasetsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (DatasetList) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.datasets.list',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[u'all', u'filter', u'maxResults', u'pageToken'],
+        relative_path=u'projects/{projectId}/datasets',
+        request_field='',
+        request_type_name=u'BigqueryDatasetsListRequest',
+        response_type_name=u'DatasetList',
+        supports_download=False,
+    )
+
+    def Patch(self, request, global_params=None):
+      """Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource. This method supports patch semantics.
+
+      Args:
+        request: (BigqueryDatasetsPatchRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Dataset) The response message.
+      """
+      config = self.GetMethodConfig('Patch')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Patch.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'bigquery.datasets.patch',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}',
+        request_field=u'dataset',
+        request_type_name=u'BigqueryDatasetsPatchRequest',
+        response_type_name=u'Dataset',
+        supports_download=False,
+    )
+
+    def Update(self, request, global_params=None):
+      """Updates information in an existing dataset. The update method replaces the entire dataset resource, whereas the patch method only replaces fields that are provided in the submitted dataset resource.
+
+      Args:
+        request: (BigqueryDatasetsUpdateRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Dataset) The response message.
+      """
+      config = self.GetMethodConfig('Update')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Update.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'bigquery.datasets.update',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}',
+        request_field=u'dataset',
+        request_type_name=u'BigqueryDatasetsUpdateRequest',
+        response_type_name=u'Dataset',
+        supports_download=False,
+    )
+
+  class JobsService(base_api.BaseApiService):
+    """Service class for the jobs resource."""
+
+    _NAME = u'jobs'
+
+    def __init__(self, client):
+      super(BigqueryV2.JobsService, self).__init__(client)
+      self._upload_configs = {
+          'Insert': base_api.ApiUploadInfo(
+              accept=['*/*'],
+              max_size=None,
+              resumable_multipart=True,
+              resumable_path=u'/resumable/upload/bigquery/v2/projects/{projectId}/jobs',
+              simple_multipart=True,
+              simple_path=u'/upload/bigquery/v2/projects/{projectId}/jobs',
+          ),
+          }
+
+    def Cancel(self, request, global_params=None):
+      """Requests that a job be cancelled. This call will return immediately, and the client will need to poll for the job status to see if the cancel completed successfully. Cancelled jobs may still incur costs.
+
+      Args:
+        request: (BigqueryJobsCancelRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (JobCancelResponse) The response message.
+      """
+      config = self.GetMethodConfig('Cancel')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Cancel.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.jobs.cancel',
+        ordered_params=[u'projectId', u'jobId'],
+        path_params=[u'jobId', u'projectId'],
+        query_params=[],
+        relative_path=u'project/{projectId}/jobs/{jobId}/cancel',
+        request_field='',
+        request_type_name=u'BigqueryJobsCancelRequest',
+        response_type_name=u'JobCancelResponse',
+        supports_download=False,
+    )
+
+    def Get(self, request, global_params=None):
+      """Returns information about a specific job. Job information is available for a six month period after creation. Requires that you're the person who ran the job, or have the Is Owner project role.
+
+      Args:
+        request: (BigqueryJobsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Job) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.jobs.get',
+        ordered_params=[u'projectId', u'jobId'],
+        path_params=[u'jobId', u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/jobs/{jobId}',
+        request_field='',
+        request_type_name=u'BigqueryJobsGetRequest',
+        response_type_name=u'Job',
+        supports_download=False,
+    )
+
+    def GetQueryResults(self, request, global_params=None):
+      """Retrieves the results of a query job.
+
+      Args:
+        request: (BigqueryJobsGetQueryResultsRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (GetQueryResultsResponse) The response message.
+      """
+      config = self.GetMethodConfig('GetQueryResults')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    GetQueryResults.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.jobs.getQueryResults',
+        ordered_params=[u'projectId', u'jobId'],
+        path_params=[u'jobId', u'projectId'],
+        query_params=[u'maxResults', u'pageToken', u'startIndex', u'timeoutMs'],
+        relative_path=u'projects/{projectId}/queries/{jobId}',
+        request_field='',
+        request_type_name=u'BigqueryJobsGetQueryResultsRequest',
+        response_type_name=u'GetQueryResultsResponse',
+        supports_download=False,
+    )
+
+    def Insert(self, request, global_params=None, upload=None):
+      """Starts a new asynchronous job. Requires the Can View project role.
+
+      Args:
+        request: (BigqueryJobsInsertRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+        upload: (Upload, default: None) If present, upload
+            this stream with the request.
+      Returns:
+        (Job) The response message.
+      """
+      config = self.GetMethodConfig('Insert')
+      upload_config = self.GetUploadConfig('Insert')
+      return self._RunMethod(
+          config, request, global_params=global_params,
+          upload=upload, upload_config=upload_config)
+
+    Insert.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.jobs.insert',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/jobs',
+        request_field=u'job',
+        request_type_name=u'BigqueryJobsInsertRequest',
+        response_type_name=u'Job',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      """Lists all jobs that you started in the specified project. Job information is available for a six month period after creation. The job list is sorted in reverse chronological order, by job creation time. Requires the Can View project role, or the Is Owner project role if you set the allUsers property.
+
+      Args:
+        request: (BigqueryJobsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (JobList) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.jobs.list',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[u'allUsers', u'maxResults', u'pageToken', u'projection', u'stateFilter'],
+        relative_path=u'projects/{projectId}/jobs',
+        request_field='',
+        request_type_name=u'BigqueryJobsListRequest',
+        response_type_name=u'JobList',
+        supports_download=False,
+    )
+
+    def Query(self, request, global_params=None):
+      """Runs a BigQuery SQL query synchronously and returns query results if the query completes within a specified timeout.
+
+      Args:
+        request: (BigqueryJobsQueryRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (QueryResponse) The response message.
+      """
+      config = self.GetMethodConfig('Query')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Query.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.jobs.query',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/queries',
+        request_field=u'queryRequest',
+        request_type_name=u'BigqueryJobsQueryRequest',
+        response_type_name=u'QueryResponse',
+        supports_download=False,
+    )
+
+  class ProjectsService(base_api.BaseApiService):
+    """Service class for the projects resource."""
+
+    _NAME = u'projects'
+
+    def __init__(self, client):
+      super(BigqueryV2.ProjectsService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def List(self, request, global_params=None):
+      """Lists all projects to which you have been granted any project role.
+
+      Args:
+        request: (BigqueryProjectsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ProjectList) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.projects.list',
+        ordered_params=[],
+        path_params=[],
+        query_params=[u'maxResults', u'pageToken'],
+        relative_path=u'projects',
+        request_field='',
+        request_type_name=u'BigqueryProjectsListRequest',
+        response_type_name=u'ProjectList',
+        supports_download=False,
+    )
+
+  class TabledataService(base_api.BaseApiService):
+    """Service class for the tabledata resource."""
+
+    _NAME = u'tabledata'
+
+    def __init__(self, client):
+      super(BigqueryV2.TabledataService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def InsertAll(self, request, global_params=None):
+      """Streams data into BigQuery one record at a time without needing to run a load job. Requires the WRITER dataset role.
+
+      Args:
+        request: (BigqueryTabledataInsertAllRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (TableDataInsertAllResponse) The response message.
+      """
+      config = self.GetMethodConfig('InsertAll')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    InsertAll.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.tabledata.insertAll',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}/insertAll',
+        request_field=u'tableDataInsertAllRequest',
+        request_type_name=u'BigqueryTabledataInsertAllRequest',
+        response_type_name=u'TableDataInsertAllResponse',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      """Retrieves table data from a specified set of rows. Requires the READER dataset role.
+
+      Args:
+        request: (BigqueryTabledataListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (TableDataList) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.tabledata.list',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[u'maxResults', u'pageToken', u'startIndex'],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}/data',
+        request_field='',
+        request_type_name=u'BigqueryTabledataListRequest',
+        response_type_name=u'TableDataList',
+        supports_download=False,
+    )
+
+  class TablesService(base_api.BaseApiService):
+    """Service class for the tables resource."""
+
+    _NAME = u'tables'
+
+    def __init__(self, client):
+      super(BigqueryV2.TablesService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def Delete(self, request, global_params=None):
+      """Deletes the table specified by tableId from the dataset. If the table contains data, all the data will be deleted.
+
+      Args:
+        request: (BigqueryTablesDeleteRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (BigqueryTablesDeleteResponse) The response message.
+      """
+      config = self.GetMethodConfig('Delete')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Delete.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'DELETE',
+        method_id=u'bigquery.tables.delete',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}',
+        request_field='',
+        request_type_name=u'BigqueryTablesDeleteRequest',
+        response_type_name=u'BigqueryTablesDeleteResponse',
+        supports_download=False,
+    )
+
+    def Get(self, request, global_params=None):
+      """Gets the specified table resource by table ID. This method does not return the data in the table, it only returns the table resource, which describes the structure of this table.
+
+      Args:
+        request: (BigqueryTablesGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Table) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.tables.get',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}',
+        request_field='',
+        request_type_name=u'BigqueryTablesGetRequest',
+        response_type_name=u'Table',
+        supports_download=False,
+    )
+
+    def Insert(self, request, global_params=None):
+      """Creates a new, empty table in the dataset.
+
+      Args:
+        request: (BigqueryTablesInsertRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Table) The response message.
+      """
+      config = self.GetMethodConfig('Insert')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Insert.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'POST',
+        method_id=u'bigquery.tables.insert',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables',
+        request_field=u'table',
+        request_type_name=u'BigqueryTablesInsertRequest',
+        response_type_name=u'Table',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      """Lists all tables in the specified dataset. Requires the READER dataset role.
+
+      Args:
+        request: (BigqueryTablesListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (TableList) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'bigquery.tables.list',
+        ordered_params=[u'projectId', u'datasetId'],
+        path_params=[u'datasetId', u'projectId'],
+        query_params=[u'maxResults', u'pageToken'],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables',
+        request_field='',
+        request_type_name=u'BigqueryTablesListRequest',
+        response_type_name=u'TableList',
+        supports_download=False,
+    )
+
+    def Patch(self, request, global_params=None):
+      """Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource. This method supports patch semantics.
+
+      Args:
+        request: (BigqueryTablesPatchRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Table) The response message.
+      """
+      config = self.GetMethodConfig('Patch')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Patch.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PATCH',
+        method_id=u'bigquery.tables.patch',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}',
+        request_field=u'table',
+        request_type_name=u'BigqueryTablesPatchRequest',
+        response_type_name=u'Table',
+        supports_download=False,
+    )
+
+    def Update(self, request, global_params=None):
+      """Updates information in an existing table. The update method replaces the entire table resource, whereas the patch method only replaces fields that are provided in the submitted table resource.
+
+      Args:
+        request: (BigqueryTablesUpdateRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Table) The response message.
+      """
+      config = self.GetMethodConfig('Update')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Update.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'PUT',
+        method_id=u'bigquery.tables.update',
+        ordered_params=[u'projectId', u'datasetId', u'tableId'],
+        path_params=[u'datasetId', u'projectId', u'tableId'],
+        query_params=[],
+        relative_path=u'projects/{projectId}/datasets/{datasetId}/tables/{tableId}',
+        request_field=u'table',
+        request_type_name=u'BigqueryTablesUpdateRequest',
+        response_type_name=u'Table',
+        supports_download=False,
+    )
diff --git a/samples/bigquery_sample/bigquery_v2/bigquery_v2_messages.py b/samples/bigquery_sample/bigquery_v2/bigquery_v2_messages.py
new file mode 100644
index 0000000..9b68f9c
--- /dev/null
+++ b/samples/bigquery_sample/bigquery_v2/bigquery_v2_messages.py
@@ -0,0 +1,2050 @@
+"""Generated message classes for bigquery version v2.
+
+A data platform for customers to create, manage, share and query data.
+"""
+# NOTE: This file is autogenerated and should not be edited by hand.
+
+from apitools.base.protorpclite import messages as _messages
+from apitools.base.py import encoding
+from apitools.base.py import extra_types
+
+
+package = 'bigquery'
+
+
+class BigqueryDatasetsDeleteRequest(_messages.Message):
+  """A BigqueryDatasetsDeleteRequest object.
+
+  Fields:
+    datasetId: Dataset ID of dataset being deleted
+    deleteContents: If True, delete all the tables in the dataset. If False
+      and the dataset contains tables, the request will fail. Default is False
+    projectId: Project ID of the dataset being deleted
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  deleteContents = _messages.BooleanField(2)
+  projectId = _messages.StringField(3, required=True)
+
+
+class BigqueryDatasetsDeleteResponse(_messages.Message):
+  """An empty BigqueryDatasetsDelete response."""
+
+
+class BigqueryDatasetsGetRequest(_messages.Message):
+  """A BigqueryDatasetsGetRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the requested dataset
+    projectId: Project ID of the requested dataset
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+
+
+class BigqueryDatasetsInsertRequest(_messages.Message):
+  """A BigqueryDatasetsInsertRequest object.
+
+  Fields:
+    dataset: A Dataset resource to be passed as the request body.
+    projectId: Project ID of the new dataset
+  """
+
+  dataset = _messages.MessageField('Dataset', 1)
+  projectId = _messages.StringField(2, required=True)
+
+
+class BigqueryDatasetsListRequest(_messages.Message):
+  """A BigqueryDatasetsListRequest object.
+
+  Fields:
+    all: Whether to list all datasets, including hidden ones
+    filter: An expression for filtering the results of the request by label.
+      The syntax is "labels.[:]". Multiple filters can be ANDed together by
+      connecting with a space. Example: "labels.department:receiving
+      labels.active". See https://cloud.google.com/bigquery/docs/labeling-
+      datasets#filtering_datasets_using_labels for details.
+    maxResults: The maximum number of results to return
+    pageToken: Page token, returned by a previous call, to request the next
+      page of results
+    projectId: Project ID of the datasets to be listed
+  """
+
+  all = _messages.BooleanField(1)
+  filter = _messages.StringField(2)
+  maxResults = _messages.IntegerField(3, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(4)
+  projectId = _messages.StringField(5, required=True)
+
+
+class BigqueryDatasetsPatchRequest(_messages.Message):
+  """A BigqueryDatasetsPatchRequest object.
+
+  Fields:
+    dataset: A Dataset resource to be passed as the request body.
+    datasetId: Dataset ID of the dataset being updated
+    projectId: Project ID of the dataset being updated
+  """
+
+  dataset = _messages.MessageField('Dataset', 1)
+  datasetId = _messages.StringField(2, required=True)
+  projectId = _messages.StringField(3, required=True)
+
+
+class BigqueryDatasetsUpdateRequest(_messages.Message):
+  """A BigqueryDatasetsUpdateRequest object.
+
+  Fields:
+    dataset: A Dataset resource to be passed as the request body.
+    datasetId: Dataset ID of the dataset being updated
+    projectId: Project ID of the dataset being updated
+  """
+
+  dataset = _messages.MessageField('Dataset', 1)
+  datasetId = _messages.StringField(2, required=True)
+  projectId = _messages.StringField(3, required=True)
+
+
+class BigqueryJobsCancelRequest(_messages.Message):
+  """A BigqueryJobsCancelRequest object.
+
+  Fields:
+    jobId: [Required] Job ID of the job to cancel
+    projectId: [Required] Project ID of the job to cancel
+  """
+
+  jobId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+
+
+class BigqueryJobsGetQueryResultsRequest(_messages.Message):
+  """A BigqueryJobsGetQueryResultsRequest object.
+
+  Fields:
+    jobId: [Required] Job ID of the query job
+    maxResults: Maximum number of results to read
+    pageToken: Page token, returned by a previous call, to request the next
+      page of results
+    projectId: [Required] Project ID of the query job
+    startIndex: Zero-based index of the starting row
+    timeoutMs: How long to wait for the query to complete, in milliseconds,
+      before returning. Default is 10 seconds. If the timeout passes before
+      the job completes, the 'jobComplete' field in the response will be false
+  """
+
+  jobId = _messages.StringField(1, required=True)
+  maxResults = _messages.IntegerField(2, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(3)
+  projectId = _messages.StringField(4, required=True)
+  startIndex = _messages.IntegerField(5, variant=_messages.Variant.UINT64)
+  timeoutMs = _messages.IntegerField(6, variant=_messages.Variant.UINT32)
+
+
+class BigqueryJobsGetRequest(_messages.Message):
+  """A BigqueryJobsGetRequest object.
+
+  Fields:
+    jobId: [Required] Job ID of the requested job
+    projectId: [Required] Project ID of the requested job
+  """
+
+  jobId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+
+
+class BigqueryJobsInsertRequest(_messages.Message):
+  """A BigqueryJobsInsertRequest object.
+
+  Fields:
+    job: A Job resource to be passed as the request body.
+    projectId: Project ID of the project that will be billed for the job
+  """
+
+  job = _messages.MessageField('Job', 1)
+  projectId = _messages.StringField(2, required=True)
+
+
+class BigqueryJobsListRequest(_messages.Message):
+  """A BigqueryJobsListRequest object.
+
+  Enums:
+    ProjectionValueValuesEnum: Restrict information returned to a set of
+      selected fields
+    StateFilterValueValuesEnum: Filter for job state
+
+  Fields:
+    allUsers: Whether to display jobs owned by all users in the project.
+      Default false
+    maxResults: Maximum number of results to return
+    pageToken: Page token, returned by a previous call, to request the next
+      page of results
+    projectId: Project ID of the jobs to list
+    projection: Restrict information returned to a set of selected fields
+    stateFilter: Filter for job state
+  """
+
+  class ProjectionValueValuesEnum(_messages.Enum):
+    """Restrict information returned to a set of selected fields
+
+    Values:
+      full: Includes all job data
+      minimal: Does not include the job configuration
+    """
+    full = 0
+    minimal = 1
+
+  class StateFilterValueValuesEnum(_messages.Enum):
+    """Filter for job state
+
+    Values:
+      done: Finished jobs
+      pending: Pending jobs
+      running: Running jobs
+    """
+    done = 0
+    pending = 1
+    running = 2
+
+  allUsers = _messages.BooleanField(1)
+  maxResults = _messages.IntegerField(2, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(3)
+  projectId = _messages.StringField(4, required=True)
+  projection = _messages.EnumField('ProjectionValueValuesEnum', 5)
+  stateFilter = _messages.EnumField('StateFilterValueValuesEnum', 6, repeated=True)
+
+
+class BigqueryJobsQueryRequest(_messages.Message):
+  """A BigqueryJobsQueryRequest object.
+
+  Fields:
+    projectId: Project ID of the project billed for the query
+    queryRequest: A QueryRequest resource to be passed as the request body.
+  """
+
+  projectId = _messages.StringField(1, required=True)
+  queryRequest = _messages.MessageField('QueryRequest', 2)
+
+
+class BigqueryProjectsListRequest(_messages.Message):
+  """A BigqueryProjectsListRequest object.
+
+  Fields:
+    maxResults: Maximum number of results to return
+    pageToken: Page token, returned by a previous call, to request the next
+      page of results
+  """
+
+  maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(2)
+
+
+class BigqueryTabledataInsertAllRequest(_messages.Message):
+  """A BigqueryTabledataInsertAllRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the destination table.
+    projectId: Project ID of the destination table.
+    tableDataInsertAllRequest: A TableDataInsertAllRequest resource to be
+      passed as the request body.
+    tableId: Table ID of the destination table.
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  tableDataInsertAllRequest = _messages.MessageField('TableDataInsertAllRequest', 3)
+  tableId = _messages.StringField(4, required=True)
+
+
+class BigqueryTabledataListRequest(_messages.Message):
+  """A BigqueryTabledataListRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the table to read
+    maxResults: Maximum number of results to return
+    pageToken: Page token, returned by a previous call, identifying the result
+      set
+    projectId: Project ID of the table to read
+    startIndex: Zero-based index of the starting row to read
+    tableId: Table ID of the table to read
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  maxResults = _messages.IntegerField(2, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(3)
+  projectId = _messages.StringField(4, required=True)
+  startIndex = _messages.IntegerField(5, variant=_messages.Variant.UINT64)
+  tableId = _messages.StringField(6, required=True)
+
+
+class BigqueryTablesDeleteRequest(_messages.Message):
+  """A BigqueryTablesDeleteRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the table to delete
+    projectId: Project ID of the table to delete
+    tableId: Table ID of the table to delete
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  tableId = _messages.StringField(3, required=True)
+
+
+class BigqueryTablesDeleteResponse(_messages.Message):
+  """An empty BigqueryTablesDelete response."""
+
+
+class BigqueryTablesGetRequest(_messages.Message):
+  """A BigqueryTablesGetRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the requested table
+    projectId: Project ID of the requested table
+    tableId: Table ID of the requested table
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  tableId = _messages.StringField(3, required=True)
+
+
+class BigqueryTablesInsertRequest(_messages.Message):
+  """A BigqueryTablesInsertRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the new table
+    projectId: Project ID of the new table
+    table: A Table resource to be passed as the request body.
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  table = _messages.MessageField('Table', 3)
+
+
+class BigqueryTablesListRequest(_messages.Message):
+  """A BigqueryTablesListRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the tables to list
+    maxResults: Maximum number of results to return
+    pageToken: Page token, returned by a previous call, to request the next
+      page of results
+    projectId: Project ID of the tables to list
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  maxResults = _messages.IntegerField(2, variant=_messages.Variant.UINT32)
+  pageToken = _messages.StringField(3)
+  projectId = _messages.StringField(4, required=True)
+
+
+class BigqueryTablesPatchRequest(_messages.Message):
+  """A BigqueryTablesPatchRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the table to update
+    projectId: Project ID of the table to update
+    table: A Table resource to be passed as the request body.
+    tableId: Table ID of the table to update
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  table = _messages.MessageField('Table', 3)
+  tableId = _messages.StringField(4, required=True)
+
+
+class BigqueryTablesUpdateRequest(_messages.Message):
+  """A BigqueryTablesUpdateRequest object.
+
+  Fields:
+    datasetId: Dataset ID of the table to update
+    projectId: Project ID of the table to update
+    table: A Table resource to be passed as the request body.
+    tableId: Table ID of the table to update
+  """
+
+  datasetId = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  table = _messages.MessageField('Table', 3)
+  tableId = _messages.StringField(4, required=True)
+
+
+class BigtableColumn(_messages.Message):
+  """A BigtableColumn object.
+
+  Fields:
+    encoding: [Optional] The encoding of the values when the type is not
+      STRING. Acceptable encoding values are: TEXT - indicates values are
+      alphanumeric text strings. BINARY - indicates values are encoded using
+      HBase Bytes.toBytes family of functions. 'encoding' can also be set at
+      the column family level. However, the setting at this level takes
+      precedence if 'encoding' is set at both levels.
+    fieldName: [Optional] If the qualifier is not a valid BigQuery field
+      identifier i.e. does not match [a-zA-Z][a-zA-Z0-9_]*, a valid identifier
+      must be provided as the column field name and is used as field name in
+      queries.
+    onlyReadLatest: [Optional] If this is set, only the latest version of
+      value in this column are exposed. 'onlyReadLatest' can also be set at
+      the column family level. However, the setting at this level takes
+      precedence if 'onlyReadLatest' is set at both levels.
+    qualifierEncoded: [Required] Qualifier of the column. Columns in the
+      parent column family that has this exact qualifier are exposed as .
+      field. If the qualifier is valid UTF-8 string, it can be specified in
+      the qualifier_string field. Otherwise, a base-64 encoded value must be
+      set to qualifier_encoded. The column field name is the same as the
+      column qualifier. However, if the qualifier is not a valid BigQuery
+      field identifier i.e. does not match [a-zA-Z][a-zA-Z0-9_]*, a valid
+      identifier must be provided as field_name.
+    qualifierString: A string attribute.
+    type: [Optional] The type to convert the value in cells of this column.
+      The values are expected to be encoded using HBase Bytes.toBytes function
+      when using the BINARY encoding value. Following BigQuery types are
+      allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN Default
+      type is BYTES. 'type' can also be set at the column family level.
+      However, the setting at this level takes precedence if 'type' is set at
+      both levels.
+  """
+
+  encoding = _messages.StringField(1)
+  fieldName = _messages.StringField(2)
+  onlyReadLatest = _messages.BooleanField(3)
+  qualifierEncoded = _messages.BytesField(4)
+  qualifierString = _messages.StringField(5)
+  type = _messages.StringField(6)
+
+
+class BigtableColumnFamily(_messages.Message):
+  """A BigtableColumnFamily object.
+
+  Fields:
+    columns: [Optional] Lists of columns that should be exposed as individual
+      fields as opposed to a list of (column name, value) pairs. All columns
+      whose qualifier matches a qualifier in this list can be accessed as ..
+      Other columns can be accessed as a list through .Column field.
+    encoding: [Optional] The encoding of the values when the type is not
+      STRING. Acceptable encoding values are: TEXT - indicates values are
+      alphanumeric text strings. BINARY - indicates values are encoded using
+      HBase Bytes.toBytes family of functions. This can be overridden for a
+      specific column by listing that column in 'columns' and specifying an
+      encoding for it.
+    familyId: Identifier of the column family.
+    onlyReadLatest: [Optional] If this is set only the latest version of value
+      are exposed for all columns in this column family. This can be
+      overridden for a specific column by listing that column in 'columns' and
+      specifying a different setting for that column.
+    type: [Optional] The type to convert the value in cells of this column
+      family. The values are expected to be encoded using HBase Bytes.toBytes
+      function when using the BINARY encoding value. Following BigQuery types
+      are allowed (case-sensitive) - BYTES STRING INTEGER FLOAT BOOLEAN
+      Default type is BYTES. This can be overridden for a specific column by
+      listing that column in 'columns' and specifying a type for it.
+  """
+
+  columns = _messages.MessageField('BigtableColumn', 1, repeated=True)
+  encoding = _messages.StringField(2)
+  familyId = _messages.StringField(3)
+  onlyReadLatest = _messages.BooleanField(4)
+  type = _messages.StringField(5)
+
+
+class BigtableOptions(_messages.Message):
+  """A BigtableOptions object.
+
+  Fields:
+    columnFamilies: [Optional] List of column families to expose in the table
+      schema along with their types. This list restricts the column families
+      that can be referenced in queries and specifies their value types. You
+      can use this list to do type conversions - see the 'type' field for more
+      details. If you leave this list empty, all column families are present
+      in the table schema and their values are read as BYTES. During a query
+      only the column families referenced in that query are read from
+      Bigtable.
+    ignoreUnspecifiedColumnFamilies: [Optional] If field is true, then the
+      column families that are not specified in columnFamilies list are not
+      exposed in the table schema. Otherwise, they are read with BYTES type
+      values. The default value is false.
+    readRowkeyAsString: [Optional] If field is true, then the rowkey column
+      families will be read and converted to string. Otherwise they are read
+      with BYTES type values and users need to manually cast them with CAST if
+      necessary. The default value is false.
+  """
+
+  columnFamilies = _messages.MessageField('BigtableColumnFamily', 1, repeated=True)
+  ignoreUnspecifiedColumnFamilies = _messages.BooleanField(2)
+  readRowkeyAsString = _messages.BooleanField(3)
+
+
+class CsvOptions(_messages.Message):
+  """A CsvOptions object.
+
+  Fields:
+    allowJaggedRows: [Optional] Indicates if BigQuery should accept rows that
+      are missing trailing optional columns. If true, BigQuery treats missing
+      trailing columns as null values. If false, records with missing trailing
+      columns are treated as bad records, and if there are too many bad
+      records, an invalid error is returned in the job result. The default
+      value is false.
+    allowQuotedNewlines: [Optional] Indicates if BigQuery should allow quoted
+      data sections that contain newline characters in a CSV file. The default
+      value is false.
+    encoding: [Optional] The character encoding of the data. The supported
+      values are UTF-8 or ISO-8859-1. The default value is UTF-8. BigQuery
+      decodes the data after the raw, binary data has been split using the
+      values of the quote and fieldDelimiter properties.
+    fieldDelimiter: [Optional] The separator for fields in a CSV file.
+      BigQuery converts the string to ISO-8859-1 encoding, and then uses the
+      first byte of the encoded string to split the data in its raw, binary
+      state. BigQuery also supports the escape sequence "\t" to specify a tab
+      separator. The default value is a comma (',').
+    quote: [Optional] The value that is used to quote data sections in a CSV
+      file. BigQuery converts the string to ISO-8859-1 encoding, and then uses
+      the first byte of the encoded string to split the data in its raw,
+      binary state. The default value is a double-quote ('"'). If your data
+      does not contain quoted sections, set the property value to an empty
+      string. If your data contains quoted newline characters, you must also
+      set the allowQuotedNewlines property to true.
+    skipLeadingRows: [Optional] The number of rows at the top of a CSV file
+      that BigQuery will skip when reading the data. The default value is 0.
+      This property is useful if you have header rows in the file that should
+      be skipped.
+  """
+
+  allowJaggedRows = _messages.BooleanField(1)
+  allowQuotedNewlines = _messages.BooleanField(2)
+  encoding = _messages.StringField(3)
+  fieldDelimiter = _messages.StringField(4)
+  quote = _messages.StringField(5, default=u'"')
+  skipLeadingRows = _messages.IntegerField(6)
+
+
+class Dataset(_messages.Message):
+  """A Dataset object.
+
+  Messages:
+    AccessValueListEntry: A AccessValueListEntry object.
+    LabelsValue: [Experimental] The labels associated with this dataset. You
+      can use these to organize and group your datasets. You can set this
+      property when inserting or updating a dataset. Label keys and values can
+      be no longer than 63 characters, can only contain letters, numeric
+      characters, underscores and dashes. International characters are
+      allowed. Label values are optional. Label keys must start with a letter
+      and must be unique within a dataset. Both keys and values are
+      additionally constrained to be <= 128 bytes in size.
+
+  Fields:
+    access: [Optional] An array of objects that define dataset access for one
+      or more entities. You can set this property when inserting or updating a
+      dataset in order to control who is allowed to access the data. If
+      unspecified at dataset creation time, BigQuery adds default dataset
+      access for the following entities: access.specialGroup: projectReaders;
+      access.role: READER; access.specialGroup: projectWriters; access.role:
+      WRITER; access.specialGroup: projectOwners; access.role: OWNER;
+      access.userByEmail: [dataset creator email]; access.role: OWNER;
+    creationTime: [Output-only] The time when this dataset was created, in
+      milliseconds since the epoch.
+    datasetReference: [Required] A reference that identifies the dataset.
+    defaultTableExpirationMs: [Optional] The default lifetime of all tables in
+      the dataset, in milliseconds. The minimum value is 3600000 milliseconds
+      (one hour). Once this property is set, all newly-created tables in the
+      dataset will have an expirationTime property set to the creation time
+      plus the value in this property, and changing the value will only affect
+      new tables, not existing ones. When the expirationTime for a given table
+      is reached, that table will be deleted automatically. If a table's
+      expirationTime is modified or removed before the table expires, or if
+      you provide an explicit expirationTime when creating a table, that value
+      takes precedence over the default expiration time indicated by this
+      property.
+    description: [Optional] A user-friendly description of the dataset.
+    etag: [Output-only] A hash of the resource.
+    friendlyName: [Optional] A descriptive name for the dataset.
+    id: [Output-only] The fully-qualified unique name of the dataset in the
+      format projectId:datasetId. The dataset name without the project name is
+      given in the datasetId field. When creating a new dataset, leave this
+      field blank, and instead specify the datasetId field.
+    kind: [Output-only] The resource type.
+    labels: [Experimental] The labels associated with this dataset. You can
+      use these to organize and group your datasets. You can set this property
+      when inserting or updating a dataset. Label keys and values can be no
+      longer than 63 characters, can only contain letters, numeric characters,
+      underscores and dashes. International characters are allowed. Label
+      values are optional. Label keys must start with a letter and must be
+      unique within a dataset. Both keys and values are additionally
+      constrained to be <= 128 bytes in size.
+    lastModifiedTime: [Output-only] The date when this dataset or any of its
+      tables was last modified, in milliseconds since the epoch.
+    location: [Experimental] The geographic location where the dataset should
+      reside. Possible values include EU and US. The default value is US.
+    selfLink: [Output-only] A URL that can be used to access the resource
+      again. You can use this URL in Get or Update requests to the resource.
+  """
+
+  class AccessValueListEntry(_messages.Message):
+    """A AccessValueListEntry object.
+
+    Fields:
+      domain: [Pick one] A domain to grant access to. Any users signed in with
+        the domain specified will be granted the specified access. Example:
+        "example.com".
+      groupByEmail: [Pick one] An email address of a Google Group to grant
+        access to.
+      role: [Required] Describes the rights granted to the user specified by
+        the other member of the access object. The following string values are
+        supported: READER, WRITER, OWNER.
+      specialGroup: [Pick one] A special group to grant access to. Possible
+        values include: projectOwners: Owners of the enclosing project.
+        projectReaders: Readers of the enclosing project. projectWriters:
+        Writers of the enclosing project. allAuthenticatedUsers: All
+        authenticated BigQuery users.
+      userByEmail: [Pick one] An email address of a user to grant access to.
+        For example: fred@example.com.
+      view: [Pick one] A view from a different dataset to grant access to.
+        Queries executed against that view will have read access to tables in
+        this dataset. The role field is not required when this field is set.
+        If that view is updated by any user, access to the view needs to be
+        granted again via an update operation.
+    """
+
+    domain = _messages.StringField(1)
+    groupByEmail = _messages.StringField(2)
+    role = _messages.StringField(3)
+    specialGroup = _messages.StringField(4)
+    userByEmail = _messages.StringField(5)
+    view = _messages.MessageField('TableReference', 6)
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class LabelsValue(_messages.Message):
+    """[Experimental] The labels associated with this dataset. You can use
+    these to organize and group your datasets. You can set this property when
+    inserting or updating a dataset. Label keys and values can be no longer
+    than 63 characters, can only contain letters, numeric characters,
+    underscores and dashes. International characters are allowed. Label values
+    are optional. Label keys must start with a letter and must be unique
+    within a dataset. Both keys and values are additionally constrained to be
+    <= 128 bytes in size.
+
+    Messages:
+      AdditionalProperty: An additional property for a LabelsValue object.
+
+    Fields:
+      additionalProperties: Additional properties of type LabelsValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a LabelsValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A string attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.StringField(2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  access = _messages.MessageField('AccessValueListEntry', 1, repeated=True)
+  creationTime = _messages.IntegerField(2)
+  datasetReference = _messages.MessageField('DatasetReference', 3)
+  defaultTableExpirationMs = _messages.IntegerField(4)
+  description = _messages.StringField(5)
+  etag = _messages.StringField(6)
+  friendlyName = _messages.StringField(7)
+  id = _messages.StringField(8)
+  kind = _messages.StringField(9, default=u'bigquery#dataset')
+  labels = _messages.MessageField('LabelsValue', 10)
+  lastModifiedTime = _messages.IntegerField(11)
+  location = _messages.StringField(12)
+  selfLink = _messages.StringField(13)
+
+
+class DatasetList(_messages.Message):
+  """A DatasetList object.
+
+  Messages:
+    DatasetsValueListEntry: A DatasetsValueListEntry object.
+
+  Fields:
+    datasets: An array of the dataset resources in the project. Each resource
+      contains basic information. For full information about a particular
+      dataset resource, use the Datasets: get method. This property is omitted
+      when there are no datasets in the project.
+    etag: A hash value of the results page. You can use this property to
+      determine if the page has changed since the last request.
+    kind: The list type. This property always returns the value
+      "bigquery#datasetList".
+    nextPageToken: A token that can be used to request the next results page.
+      This property is omitted on the final results page.
+  """
+
+  class DatasetsValueListEntry(_messages.Message):
+    """A DatasetsValueListEntry object.
+
+    Messages:
+      LabelsValue: [Experimental] The labels associated with this dataset. You
+        can use these to organize and group your datasets.
+
+    Fields:
+      datasetReference: The dataset reference. Use this property to access
+        specific parts of the dataset's ID, such as project ID or dataset ID.
+      friendlyName: A descriptive name for the dataset, if one exists.
+      id: The fully-qualified, unique, opaque ID of the dataset.
+      kind: The resource type. This property always returns the value
+        "bigquery#dataset".
+      labels: [Experimental] The labels associated with this dataset. You can
+        use these to organize and group your datasets.
+    """
+
+    @encoding.MapUnrecognizedFields('additionalProperties')
+    class LabelsValue(_messages.Message):
+      """[Experimental] The labels associated with this dataset. You can use
+      these to organize and group your datasets.
+
+      Messages:
+        AdditionalProperty: An additional property for a LabelsValue object.
+
+      Fields:
+        additionalProperties: Additional properties of type LabelsValue
+      """
+
+      class AdditionalProperty(_messages.Message):
+        """An additional property for a LabelsValue object.
+
+        Fields:
+          key: Name of the additional property.
+          value: A string attribute.
+        """
+
+        key = _messages.StringField(1)
+        value = _messages.StringField(2)
+
+      additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+    datasetReference = _messages.MessageField('DatasetReference', 1)
+    friendlyName = _messages.StringField(2)
+    id = _messages.StringField(3)
+    kind = _messages.StringField(4, default=u'bigquery#dataset')
+    labels = _messages.MessageField('LabelsValue', 5)
+
+  datasets = _messages.MessageField('DatasetsValueListEntry', 1, repeated=True)
+  etag = _messages.StringField(2)
+  kind = _messages.StringField(3, default=u'bigquery#datasetList')
+  nextPageToken = _messages.StringField(4)
+
+
+class DatasetReference(_messages.Message):
+  """A DatasetReference object.
+
+  Fields:
+    datasetId: [Required] A unique ID for this dataset, without the project
+      name. The ID must contain only letters (a-z, A-Z), numbers (0-9), or
+      underscores (_). The maximum length is 1,024 characters.
+    projectId: [Optional] The ID of the project containing this dataset.
+  """
+
+  datasetId = _messages.StringField(1)
+  projectId = _messages.StringField(2)
+
+
+class ErrorProto(_messages.Message):
+  """A ErrorProto object.
+
+  Fields:
+    debugInfo: Debugging information. This property is internal to Google and
+      should not be used.
+    location: Specifies where the error occurred, if present.
+    message: A human-readable description of the error.
+    reason: A short error code that summarizes the error.
+  """
+
+  debugInfo = _messages.StringField(1)
+  location = _messages.StringField(2)
+  message = _messages.StringField(3)
+  reason = _messages.StringField(4)
+
+
+class ExplainQueryStage(_messages.Message):
+  """A ExplainQueryStage object.
+
+  Fields:
+    computeRatioAvg: Relative amount of time the average shard spent on CPU-
+      bound tasks.
+    computeRatioMax: Relative amount of time the slowest shard spent on CPU-
+      bound tasks.
+    id: Unique ID for stage within plan.
+    name: Human-readable name for stage.
+    readRatioAvg: Relative amount of time the average shard spent reading
+      input.
+    readRatioMax: Relative amount of time the slowest shard spent reading
+      input.
+    recordsRead: Number of records read into the stage.
+    recordsWritten: Number of records written by the stage.
+    steps: List of operations within the stage in dependency order
+      (approximately chronological).
+    waitRatioAvg: Relative amount of time the average shard spent waiting to
+      be scheduled.
+    waitRatioMax: Relative amount of time the slowest shard spent waiting to
+      be scheduled.
+    writeRatioAvg: Relative amount of time the average shard spent on writing
+      output.
+    writeRatioMax: Relative amount of time the slowest shard spent on writing
+      output.
+  """
+
+  computeRatioAvg = _messages.FloatField(1)
+  computeRatioMax = _messages.FloatField(2)
+  id = _messages.IntegerField(3)
+  name = _messages.StringField(4)
+  readRatioAvg = _messages.FloatField(5)
+  readRatioMax = _messages.FloatField(6)
+  recordsRead = _messages.IntegerField(7)
+  recordsWritten = _messages.IntegerField(8)
+  steps = _messages.MessageField('ExplainQueryStep', 9, repeated=True)
+  waitRatioAvg = _messages.FloatField(10)
+  waitRatioMax = _messages.FloatField(11)
+  writeRatioAvg = _messages.FloatField(12)
+  writeRatioMax = _messages.FloatField(13)
+
+
+class ExplainQueryStep(_messages.Message):
+  """A ExplainQueryStep object.
+
+  Fields:
+    kind: Machine-readable operation type.
+    substeps: Human-readable stage descriptions.
+  """
+
+  kind = _messages.StringField(1)
+  substeps = _messages.StringField(2, repeated=True)
+
+
+class ExternalDataConfiguration(_messages.Message):
+  """A ExternalDataConfiguration object.
+
+  Fields:
+    autodetect: [Experimental] Try to detect schema and format options
+      automatically. Any option specified explicitly will be honored.
+    bigtableOptions: [Optional] Additional options if sourceFormat is set to
+      BIGTABLE.
+    compression: [Optional] The compression type of the data source. Possible
+      values include GZIP and NONE. The default value is NONE. This setting is
+      ignored for Google Cloud Bigtable, Google Cloud Datastore backups and
+      Avro formats.
+    csvOptions: Additional properties to set if sourceFormat is set to CSV.
+    googleSheetsOptions: [Optional] Additional options if sourceFormat is set
+      to GOOGLE_SHEETS.
+    ignoreUnknownValues: [Optional] Indicates if BigQuery should allow extra
+      values that are not represented in the table schema. If true, the extra
+      values are ignored. If false, records with extra columns are treated as
+      bad records, and if there are too many bad records, an invalid error is
+      returned in the job result. The default value is false. The sourceFormat
+      property determines what BigQuery treats as an extra value: CSV:
+      Trailing columns JSON: Named values that don't match any column names
+      Google Cloud Bigtable: This setting is ignored. Google Cloud Datastore
+      backups: This setting is ignored. Avro: This setting is ignored.
+    maxBadRecords: [Optional] The maximum number of bad records that BigQuery
+      can ignore when reading data. If the number of bad records exceeds this
+      value, an invalid error is returned in the job result. The default value
+      is 0, which requires that all records are valid. This setting is ignored
+      for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
+      formats.
+    schema: [Optional] The schema for the data. Schema is required for CSV and
+      JSON formats. Schema is disallowed for Google Cloud Bigtable, Cloud
+      Datastore backups, and Avro formats.
+    sourceFormat: [Required] The data format. For CSV files, specify "CSV".
+      For Google sheets, specify "GOOGLE_SHEETS". For newline-delimited JSON,
+      specify "NEWLINE_DELIMITED_JSON". For Avro files, specify "AVRO". For
+      Google Cloud Datastore backups, specify "DATASTORE_BACKUP".
+      [Experimental] For Google Cloud Bigtable, specify "BIGTABLE". Please
+      note that reading from Google Cloud Bigtable is experimental and has to
+      be enabled for your project. Please contact Google Cloud Support to
+      enable this for your project.
+    sourceUris: [Required] The fully-qualified URIs that point to your data in
+      Google Cloud. For Google Cloud Storage URIs: Each URI can contain one
+      '*' wildcard character and it must come after the 'bucket' name. Size
+      limits related to load jobs apply to external data sources. For Google
+      Cloud Bigtable URIs: Exactly one URI can be specified and it has be a
+      fully specified and valid HTTPS URL for a Google Cloud Bigtable table.
+      For Google Cloud Datastore backups, exactly one URI can be specified,
+      and it must end with '.backup_info'. Also, the '*' wildcard character is
+      not allowed.
+  """
+
+  autodetect = _messages.BooleanField(1)
+  bigtableOptions = _messages.MessageField('BigtableOptions', 2)
+  compression = _messages.StringField(3)
+  csvOptions = _messages.MessageField('CsvOptions', 4)
+  googleSheetsOptions = _messages.MessageField('GoogleSheetsOptions', 5)
+  ignoreUnknownValues = _messages.BooleanField(6)
+  maxBadRecords = _messages.IntegerField(7, variant=_messages.Variant.INT32)
+  schema = _messages.MessageField('TableSchema', 8)
+  sourceFormat = _messages.StringField(9)
+  sourceUris = _messages.StringField(10, repeated=True)
+
+
+class GetQueryResultsResponse(_messages.Message):
+  """A GetQueryResultsResponse object.
+
+  Fields:
+    cacheHit: Whether the query result was fetched from the query cache.
+    errors: [Output-only] All errors and warnings encountered during the
+      running of the job. Errors here do not necessarily mean that the job has
+      completed or was unsuccessful.
+    etag: A hash of this response.
+    jobComplete: Whether the query has completed or not. If rows or totalRows
+      are present, this will always be true. If this is false, totalRows will
+      not be available.
+    jobReference: Reference to the BigQuery Job that was created to run the
+      query. This field will be present even if the original request timed
+      out, in which case GetQueryResults can be used to read the results once
+      the query has completed. Since this API only returns the first page of
+      results, subsequent pages can be fetched via the same mechanism
+      (GetQueryResults).
+    kind: The resource type of the response.
+    numDmlAffectedRows: [Output-only, Experimental] The number of rows
+      affected by a DML statement. Present only for DML statements INSERT,
+      UPDATE or DELETE.
+    pageToken: A token used for paging results.
+    rows: An object with as many results as can be contained within the
+      maximum permitted reply size. To get any additional rows, you can call
+      GetQueryResults and specify the jobReference returned above. Present
+      only when the query completes successfully.
+    schema: The schema of the results. Present only when the query completes
+      successfully.
+    totalBytesProcessed: The total number of bytes processed for this query.
+    totalRows: The total number of rows in the complete query result set,
+      which can be more than the number of rows in this single page of
+      results. Present only when the query completes successfully.
+  """
+
+  cacheHit = _messages.BooleanField(1)
+  errors = _messages.MessageField('ErrorProto', 2, repeated=True)
+  etag = _messages.StringField(3)
+  jobComplete = _messages.BooleanField(4)
+  jobReference = _messages.MessageField('JobReference', 5)
+  kind = _messages.StringField(6, default=u'bigquery#getQueryResultsResponse')
+  numDmlAffectedRows = _messages.IntegerField(7)
+  pageToken = _messages.StringField(8)
+  rows = _messages.MessageField('TableRow', 9, repeated=True)
+  schema = _messages.MessageField('TableSchema', 10)
+  totalBytesProcessed = _messages.IntegerField(11)
+  totalRows = _messages.IntegerField(12, variant=_messages.Variant.UINT64)
+
+
+class GoogleSheetsOptions(_messages.Message):
+  """A GoogleSheetsOptions object.
+
+  Fields:
+    skipLeadingRows: [Optional] The number of rows at the top of a sheet that
+      BigQuery will skip when reading the data. The default value is 0. This
+      property is useful if you have header rows that should be skipped. When
+      autodetect is on, behavior is the following: * skipLeadingRows
+      unspecified - Autodetect tries to detect headers in the first row. If
+      they are not detected, the row is read as data. Otherwise data is read
+      starting from the second row. * skipLeadingRows is 0 - Instructs
+      autodetect that there are no headers and data should be read starting
+      from the first row. * skipLeadingRows = N > 0 - Autodetect skips N-1
+      rows and tries to detect headers in row N. If headers are not detected,
+      row N is just skipped. Otherwise row N is used to extract column names
+      for the detected schema.
+  """
+
+  skipLeadingRows = _messages.IntegerField(1)
+
+
+class Job(_messages.Message):
+  """A Job object.
+
+  Fields:
+    configuration: [Required] Describes the job configuration.
+    etag: [Output-only] A hash of this resource.
+    id: [Output-only] Opaque ID field of the job
+    jobReference: [Optional] Reference describing the unique-per-user name of
+      the job.
+    kind: [Output-only] The type of the resource.
+    selfLink: [Output-only] A URL that can be used to access this resource
+      again.
+    statistics: [Output-only] Information about the job, including starting
+      time and ending time of the job.
+    status: [Output-only] The status of this job. Examine this value when
+      polling an asynchronous job to see if the job is complete.
+    user_email: [Output-only] Email address of the user who ran the job.
+  """
+
+  configuration = _messages.MessageField('JobConfiguration', 1)
+  etag = _messages.StringField(2)
+  id = _messages.StringField(3)
+  jobReference = _messages.MessageField('JobReference', 4)
+  kind = _messages.StringField(5, default=u'bigquery#job')
+  selfLink = _messages.StringField(6)
+  statistics = _messages.MessageField('JobStatistics', 7)
+  status = _messages.MessageField('JobStatus', 8)
+  user_email = _messages.StringField(9)
+
+
+class JobCancelResponse(_messages.Message):
+  """A JobCancelResponse object.
+
+  Fields:
+    job: The final state of the job.
+    kind: The resource type of the response.
+  """
+
+  job = _messages.MessageField('Job', 1)
+  kind = _messages.StringField(2, default=u'bigquery#jobCancelResponse')
+
+
+class JobConfiguration(_messages.Message):
+  """A JobConfiguration object.
+
+  Fields:
+    copy: [Pick one] Copies a table.
+    dryRun: [Optional] If set, don't actually run this job. A valid query will
+      return a mostly empty response with some processing statistics, while an
+      invalid query will return the same error it would if it wasn't a dry
+      run. Behavior of non-query jobs is undefined.
+    extract: [Pick one] Configures an extract job.
+    load: [Pick one] Configures a load job.
+    query: [Pick one] Configures a query job.
+  """
+
+  copy = _messages.MessageField('JobConfigurationTableCopy', 1)
+  dryRun = _messages.BooleanField(2)
+  extract = _messages.MessageField('JobConfigurationExtract', 3)
+  load = _messages.MessageField('JobConfigurationLoad', 4)
+  query = _messages.MessageField('JobConfigurationQuery', 5)
+
+
+class JobConfigurationExtract(_messages.Message):
+  """A JobConfigurationExtract object.
+
+  Fields:
+    compression: [Optional] The compression type to use for exported files.
+      Possible values include GZIP and NONE. The default value is NONE.
+    destinationFormat: [Optional] The exported file format. Possible values
+      include CSV, NEWLINE_DELIMITED_JSON and AVRO. The default value is CSV.
+      Tables with nested or repeated fields cannot be exported as CSV.
+    destinationUri: [Pick one] DEPRECATED: Use destinationUris instead,
+      passing only one URI as necessary. The fully-qualified Google Cloud
+      Storage URI where the extracted table should be written.
+    destinationUris: [Pick one] A list of fully-qualified Google Cloud Storage
+      URIs where the extracted table should be written.
+    fieldDelimiter: [Optional] Delimiter to use between fields in the exported
+      data. Default is ','
+    printHeader: [Optional] Whether to print out a header row in the results.
+      Default is true.
+    sourceTable: [Required] A reference to the table being exported.
+  """
+
+  compression = _messages.StringField(1)
+  destinationFormat = _messages.StringField(2)
+  destinationUri = _messages.StringField(3)
+  destinationUris = _messages.StringField(4, repeated=True)
+  fieldDelimiter = _messages.StringField(5)
+  printHeader = _messages.BooleanField(6, default=True)
+  sourceTable = _messages.MessageField('TableReference', 7)
+
+
+class JobConfigurationLoad(_messages.Message):
+  """A JobConfigurationLoad object.
+
+  Fields:
+    allowJaggedRows: [Optional] Accept rows that are missing trailing optional
+      columns. The missing values are treated as nulls. If false, records with
+      missing trailing columns are treated as bad records, and if there are
+      too many bad records, an invalid error is returned in the job result.
+      The default value is false. Only applicable to CSV, ignored for other
+      formats.
+    allowQuotedNewlines: Indicates if BigQuery should allow quoted data
+      sections that contain newline characters in a CSV file. The default
+      value is false.
+    autodetect: [Experimental] Indicates if we should automatically infer the
+      options and schema for CSV and JSON sources.
+    createDisposition: [Optional] Specifies whether the job is allowed to
+      create new tables. The following values are supported: CREATE_IF_NEEDED:
+      If the table does not exist, BigQuery creates the table. CREATE_NEVER:
+      The table must already exist. If it does not, a 'notFound' error is
+      returned in the job result. The default value is CREATE_IF_NEEDED.
+      Creation, truncation and append actions occur as one atomic update upon
+      job completion.
+    destinationTable: [Required] The destination table to load the data into.
+    encoding: [Optional] The character encoding of the data. The supported
+      values are UTF-8 or ISO-8859-1. The default value is UTF-8. BigQuery
+      decodes the data after the raw, binary data has been split using the
+      values of the quote and fieldDelimiter properties.
+    fieldDelimiter: [Optional] The separator for fields in a CSV file. The
+      separator can be any ISO-8859-1 single-byte character. To use a
+      character in the range 128-255, you must encode the character as UTF8.
+      BigQuery converts the string to ISO-8859-1 encoding, and then uses the
+      first byte of the encoded string to split the data in its raw, binary
+      state. BigQuery also supports the escape sequence "\t" to specify a tab
+      separator. The default value is a comma (',').
+    ignoreUnknownValues: [Optional] Indicates if BigQuery should allow extra
+      values that are not represented in the table schema. If true, the extra
+      values are ignored. If false, records with extra columns are treated as
+      bad records, and if there are too many bad records, an invalid error is
+      returned in the job result. The default value is false. The sourceFormat
+      property determines what BigQuery treats as an extra value: CSV:
+      Trailing columns JSON: Named values that don't match any column names
+    maxBadRecords: [Optional] The maximum number of bad records that BigQuery
+      can ignore when running the job. If the number of bad records exceeds
+      this value, an invalid error is returned in the job result. The default
+      value is 0, which requires that all records are valid.
+    projectionFields: [Experimental] If sourceFormat is set to
+      "DATASTORE_BACKUP", indicates which entity properties to load into
+      BigQuery from a Cloud Datastore backup. Property names are case
+      sensitive and must be top-level properties. If no properties are
+      specified, BigQuery loads all properties. If any named property isn't
+      found in the Cloud Datastore backup, an invalid error is returned in the
+      job result.
+    quote: [Optional] The value that is used to quote data sections in a CSV
+      file. BigQuery converts the string to ISO-8859-1 encoding, and then uses
+      the first byte of the encoded string to split the data in its raw,
+      binary state. The default value is a double-quote ('"'). If your data
+      does not contain quoted sections, set the property value to an empty
+      string. If your data contains quoted newline characters, you must also
+      set the allowQuotedNewlines property to true.
+    schema: [Optional] The schema for the destination table. The schema can be
+      omitted if the destination table already exists, or if you're loading
+      data from Google Cloud Datastore.
+    schemaInline: [Deprecated] The inline schema. For CSV schemas, specify as
+      "Field1:Type1[,Field2:Type2]*". For example, "foo:STRING, bar:INTEGER,
+      baz:FLOAT".
+    schemaInlineFormat: [Deprecated] The format of the schemaInline property.
+    schemaUpdateOptions: [Experimental] Allows the schema of the desitination
+      table to be updated as a side effect of the load job. Schema update
+      options are supported in two cases: when writeDisposition is
+      WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the
+      destination table is a partition of a table, specified by partition
+      decorators. For normal tables, WRITE_TRUNCATE will always overwrite the
+      schema. One or more of the following values are specified:
+      ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema.
+      ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original
+      schema to nullable.
+    skipLeadingRows: [Optional] The number of rows at the top of a CSV file
+      that BigQuery will skip when loading the data. The default value is 0.
+      This property is useful if you have header rows in the file that should
+      be skipped.
+    sourceFormat: [Optional] The format of the data files. For CSV files,
+      specify "CSV". For datastore backups, specify "DATASTORE_BACKUP". For
+      newline-delimited JSON, specify "NEWLINE_DELIMITED_JSON". For Avro,
+      specify "AVRO". The default value is CSV.
+    sourceUris: [Required] The fully-qualified URIs that point to your data in
+      Google Cloud Storage. Each URI can contain one '*' wildcard character
+      and it must come after the 'bucket' name.
+    writeDisposition: [Optional] Specifies the action that occurs if the
+      destination table already exists. The following values are supported:
+      WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the
+      table data. WRITE_APPEND: If the table already exists, BigQuery appends
+      the data to the table. WRITE_EMPTY: If the table already exists and
+      contains data, a 'duplicate' error is returned in the job result. The
+      default value is WRITE_APPEND. Each action is atomic and only occurs if
+      BigQuery is able to complete the job successfully. Creation, truncation
+      and append actions occur as one atomic update upon job completion.
+  """
+
+  allowJaggedRows = _messages.BooleanField(1)
+  allowQuotedNewlines = _messages.BooleanField(2)
+  autodetect = _messages.BooleanField(3)
+  createDisposition = _messages.StringField(4)
+  destinationTable = _messages.MessageField('TableReference', 5)
+  encoding = _messages.StringField(6)
+  fieldDelimiter = _messages.StringField(7)
+  ignoreUnknownValues = _messages.BooleanField(8)
+  maxBadRecords = _messages.IntegerField(9, variant=_messages.Variant.INT32)
+  projectionFields = _messages.StringField(10, repeated=True)
+  quote = _messages.StringField(11, default=u'"')
+  schema = _messages.MessageField('TableSchema', 12)
+  schemaInline = _messages.StringField(13)
+  schemaInlineFormat = _messages.StringField(14)
+  schemaUpdateOptions = _messages.StringField(15, repeated=True)
+  skipLeadingRows = _messages.IntegerField(16, variant=_messages.Variant.INT32)
+  sourceFormat = _messages.StringField(17)
+  sourceUris = _messages.StringField(18, repeated=True)
+  writeDisposition = _messages.StringField(19)
+
+
+class JobConfigurationQuery(_messages.Message):
+  """A JobConfigurationQuery object.
+
+  Messages:
+    TableDefinitionsValue: [Optional] If querying an external data source
+      outside of BigQuery, describes the data format, location and other
+      properties of the data source. By defining these properties, the data
+      source can then be queried as if it were a standard BigQuery table.
+
+  Fields:
+    allowLargeResults: If true, allows the query to produce arbitrarily large
+      result tables at a slight cost in performance. Requires destinationTable
+      to be set.
+    createDisposition: [Optional] Specifies whether the job is allowed to
+      create new tables. The following values are supported: CREATE_IF_NEEDED:
+      If the table does not exist, BigQuery creates the table. CREATE_NEVER:
+      The table must already exist. If it does not, a 'notFound' error is
+      returned in the job result. The default value is CREATE_IF_NEEDED.
+      Creation, truncation and append actions occur as one atomic update upon
+      job completion.
+    defaultDataset: [Optional] Specifies the default dataset to use for
+      unqualified table names in the query.
+    destinationTable: [Optional] Describes the table where the query results
+      should be stored. If not present, a new table will be created to store
+      the results.
+    flattenResults: [Optional] Flattens all nested and repeated fields in the
+      query results. The default value is true. allowLargeResults must be true
+      if this is set to false.
+    maximumBillingTier: [Optional] Limits the billing tier for this job.
+      Queries that have resource usage beyond this tier will fail (without
+      incurring a charge). If unspecified, this will be set to your project
+      default.
+    maximumBytesBilled: [Optional] Limits the bytes billed for this job.
+      Queries that will have bytes billed beyond this limit will fail (without
+      incurring a charge). If unspecified, this will be set to your project
+      default.
+    preserveNulls: [Deprecated] This property is deprecated.
+    priority: [Optional] Specifies a priority for the query. Possible values
+      include INTERACTIVE and BATCH. The default value is INTERACTIVE.
+    query: [Required] BigQuery SQL query to execute.
+    schemaUpdateOptions: [Experimental] Allows the schema of the desitination
+      table to be updated as a side effect of the query job. Schema update
+      options are supported in two cases: when writeDisposition is
+      WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the
+      destination table is a partition of a table, specified by partition
+      decorators. For normal tables, WRITE_TRUNCATE will always overwrite the
+      schema. One or more of the following values are specified:
+      ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema.
+      ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original
+      schema to nullable.
+    tableDefinitions: [Optional] If querying an external data source outside
+      of BigQuery, describes the data format, location and other properties of
+      the data source. By defining these properties, the data source can then
+      be queried as if it were a standard BigQuery table.
+    useLegacySql: [Experimental] Specifies whether to use BigQuery's legacy
+      SQL dialect for this query. The default value is true. If set to false,
+      the query will use BigQuery's standard SQL:
+      https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is
+      set to false, the values of allowLargeResults and flattenResults are
+      ignored; query will be run as if allowLargeResults is true and
+      flattenResults is false.
+    useQueryCache: [Optional] Whether to look for the result in the query
+      cache. The query cache is a best-effort cache that will be flushed
+      whenever tables in the query are modified. Moreover, the query cache is
+      only available when a query does not have a destination table specified.
+      The default value is true.
+    userDefinedFunctionResources: [Experimental] Describes user-defined
+      function resources used in the query.
+    writeDisposition: [Optional] Specifies the action that occurs if the
+      destination table already exists. The following values are supported:
+      WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the
+      table data. WRITE_APPEND: If the table already exists, BigQuery appends
+      the data to the table. WRITE_EMPTY: If the table already exists and
+      contains data, a 'duplicate' error is returned in the job result. The
+      default value is WRITE_EMPTY. Each action is atomic and only occurs if
+      BigQuery is able to complete the job successfully. Creation, truncation
+      and append actions occur as one atomic update upon job completion.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class TableDefinitionsValue(_messages.Message):
+    """[Optional] If querying an external data source outside of BigQuery,
+    describes the data format, location and other properties of the data
+    source. By defining these properties, the data source can then be queried
+    as if it were a standard BigQuery table.
+
+    Messages:
+      AdditionalProperty: An additional property for a TableDefinitionsValue
+        object.
+
+    Fields:
+      additionalProperties: Additional properties of type
+        TableDefinitionsValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a TableDefinitionsValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A ExternalDataConfiguration attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.MessageField('ExternalDataConfiguration', 2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  allowLargeResults = _messages.BooleanField(1)
+  createDisposition = _messages.StringField(2)
+  defaultDataset = _messages.MessageField('DatasetReference', 3)
+  destinationTable = _messages.MessageField('TableReference', 4)
+  flattenResults = _messages.BooleanField(5, default=True)
+  maximumBillingTier = _messages.IntegerField(6, variant=_messages.Variant.INT32, default=1)
+  maximumBytesBilled = _messages.IntegerField(7)
+  preserveNulls = _messages.BooleanField(8)
+  priority = _messages.StringField(9)
+  query = _messages.StringField(10)
+  schemaUpdateOptions = _messages.StringField(11, repeated=True)
+  tableDefinitions = _messages.MessageField('TableDefinitionsValue', 12)
+  useLegacySql = _messages.BooleanField(13)
+  useQueryCache = _messages.BooleanField(14, default=True)
+  userDefinedFunctionResources = _messages.MessageField('UserDefinedFunctionResource', 15, repeated=True)
+  writeDisposition = _messages.StringField(16)
+
+
+class JobConfigurationTableCopy(_messages.Message):
+  """A JobConfigurationTableCopy object.
+
+  Fields:
+    createDisposition: [Optional] Specifies whether the job is allowed to
+      create new tables. The following values are supported: CREATE_IF_NEEDED:
+      If the table does not exist, BigQuery creates the table. CREATE_NEVER:
+      The table must already exist. If it does not, a 'notFound' error is
+      returned in the job result. The default value is CREATE_IF_NEEDED.
+      Creation, truncation and append actions occur as one atomic update upon
+      job completion.
+    destinationTable: [Required] The destination table
+    sourceTable: [Pick one] Source table to copy.
+    sourceTables: [Pick one] Source tables to copy.
+    writeDisposition: [Optional] Specifies the action that occurs if the
+      destination table already exists. The following values are supported:
+      WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the
+      table data. WRITE_APPEND: If the table already exists, BigQuery appends
+      the data to the table. WRITE_EMPTY: If the table already exists and
+      contains data, a 'duplicate' error is returned in the job result. The
+      default value is WRITE_EMPTY. Each action is atomic and only occurs if
+      BigQuery is able to complete the job successfully. Creation, truncation
+      and append actions occur as one atomic update upon job completion.
+  """
+
+  createDisposition = _messages.StringField(1)
+  destinationTable = _messages.MessageField('TableReference', 2)
+  sourceTable = _messages.MessageField('TableReference', 3)
+  sourceTables = _messages.MessageField('TableReference', 4, repeated=True)
+  writeDisposition = _messages.StringField(5)
+
+
+class JobList(_messages.Message):
+  """A JobList object.
+
+  Messages:
+    JobsValueListEntry: A JobsValueListEntry object.
+
+  Fields:
+    etag: A hash of this page of results.
+    jobs: List of jobs that were requested.
+    kind: The resource type of the response.
+    nextPageToken: A token to request the next page of results.
+  """
+
+  class JobsValueListEntry(_messages.Message):
+    """A JobsValueListEntry object.
+
+    Fields:
+      configuration: [Full-projection-only] Specifies the job configuration.
+      errorResult: A result object that will be present only if the job has
+        failed.
+      id: Unique opaque ID of the job.
+      jobReference: Job reference uniquely identifying the job.
+      kind: The resource type.
+      state: Running state of the job. When the state is DONE, errorResult can
+        be checked to determine whether the job succeeded or failed.
+      statistics: [Output-only] Information about the job, including starting
+        time and ending time of the job.
+      status: [Full-projection-only] Describes the state of the job.
+      user_email: [Full-projection-only] Email address of the user who ran the
+        job.
+    """
+
+    configuration = _messages.MessageField('JobConfiguration', 1)
+    errorResult = _messages.MessageField('ErrorProto', 2)
+    id = _messages.StringField(3)
+    jobReference = _messages.MessageField('JobReference', 4)
+    kind = _messages.StringField(5, default=u'bigquery#job')
+    state = _messages.StringField(6)
+    statistics = _messages.MessageField('JobStatistics', 7)
+    status = _messages.MessageField('JobStatus', 8)
+    user_email = _messages.StringField(9)
+
+  etag = _messages.StringField(1)
+  jobs = _messages.MessageField('JobsValueListEntry', 2, repeated=True)
+  kind = _messages.StringField(3, default=u'bigquery#jobList')
+  nextPageToken = _messages.StringField(4)
+
+
+class JobReference(_messages.Message):
+  """A JobReference object.
+
+  Fields:
+    jobId: [Required] The ID of the job. The ID must contain only letters
+      (a-z, A-Z), numbers (0-9), underscores (_), or dashes (-). The maximum
+      length is 1,024 characters.
+    projectId: [Required] The ID of the project containing this job.
+  """
+
+  jobId = _messages.StringField(1)
+  projectId = _messages.StringField(2)
+
+
+class JobStatistics(_messages.Message):
+  """A JobStatistics object.
+
+  Fields:
+    creationTime: [Output-only] Creation time of this job, in milliseconds
+      since the epoch. This field will be present on all jobs.
+    endTime: [Output-only] End time of this job, in milliseconds since the
+      epoch. This field will be present whenever a job is in the DONE state.
+    extract: [Output-only] Statistics for an extract job.
+    load: [Output-only] Statistics for a load job.
+    query: [Output-only] Statistics for a query job.
+    startTime: [Output-only] Start time of this job, in milliseconds since the
+      epoch. This field will be present when the job transitions from the
+      PENDING state to either RUNNING or DONE.
+    totalBytesProcessed: [Output-only] [Deprecated] Use the bytes processed in
+      the query statistics instead.
+  """
+
+  creationTime = _messages.IntegerField(1)
+  endTime = _messages.IntegerField(2)
+  extract = _messages.MessageField('JobStatistics4', 3)
+  load = _messages.MessageField('JobStatistics3', 4)
+  query = _messages.MessageField('JobStatistics2', 5)
+  startTime = _messages.IntegerField(6)
+  totalBytesProcessed = _messages.IntegerField(7)
+
+
+class JobStatistics2(_messages.Message):
+  """A JobStatistics2 object.
+
+  Fields:
+    billingTier: [Output-only] Billing tier for the job.
+    cacheHit: [Output-only] Whether the query result was fetched from the
+      query cache.
+    numDmlAffectedRows: [Output-only, Experimental] The number of rows
+      affected by a DML statement. Present only for DML statements INSERT,
+      UPDATE or DELETE.
+    queryPlan: [Output-only, Experimental] Describes execution plan for the
+      query.
+    referencedTables: [Output-only, Experimental] Referenced tables for the
+      job. Queries that reference more than 50 tables will not have a complete
+      list.
+    schema: [Output-only, Experimental] The schema of the results. Present
+      only for successful dry run of non-legacy SQL queries.
+    totalBytesBilled: [Output-only] Total bytes billed for the job.
+    totalBytesProcessed: [Output-only] Total bytes processed for the job.
+  """
+
+  billingTier = _messages.IntegerField(1, variant=_messages.Variant.INT32)
+  cacheHit = _messages.BooleanField(2)
+  numDmlAffectedRows = _messages.IntegerField(3)
+  queryPlan = _messages.MessageField('ExplainQueryStage', 4, repeated=True)
+  referencedTables = _messages.MessageField('TableReference', 5, repeated=True)
+  schema = _messages.MessageField('TableSchema', 6)
+  totalBytesBilled = _messages.IntegerField(7)
+  totalBytesProcessed = _messages.IntegerField(8)
+
+
+class JobStatistics3(_messages.Message):
+  """A JobStatistics3 object.
+
+  Fields:
+    inputFileBytes: [Output-only] Number of bytes of source data in a load
+      job.
+    inputFiles: [Output-only] Number of source files in a load job.
+    outputBytes: [Output-only] Size of the loaded data in bytes. Note that
+      while a load job is in the running state, this value may change.
+    outputRows: [Output-only] Number of rows imported in a load job. Note that
+      while an import job is in the running state, this value may change.
+  """
+
+  inputFileBytes = _messages.IntegerField(1)
+  inputFiles = _messages.IntegerField(2)
+  outputBytes = _messages.IntegerField(3)
+  outputRows = _messages.IntegerField(4)
+
+
+class JobStatistics4(_messages.Message):
+  """A JobStatistics4 object.
+
+  Fields:
+    destinationUriFileCounts: [Output-only] Number of files per destination
+      URI or URI pattern specified in the extract configuration. These values
+      will be in the same order as the URIs specified in the 'destinationUris'
+      field.
+  """
+
+  destinationUriFileCounts = _messages.IntegerField(1, repeated=True)
+
+
+class JobStatus(_messages.Message):
+  """A JobStatus object.
+
+  Fields:
+    errorResult: [Output-only] Final error result of the job. If present,
+      indicates that the job has completed and was unsuccessful.
+    errors: [Output-only] All errors encountered during the running of the
+      job. Errors here do not necessarily mean that the job has completed or
+      was unsuccessful.
+    state: [Output-only] Running state of the job.
+  """
+
+  errorResult = _messages.MessageField('ErrorProto', 1)
+  errors = _messages.MessageField('ErrorProto', 2, repeated=True)
+  state = _messages.StringField(3)
+
+
+@encoding.MapUnrecognizedFields('additionalProperties')
+class JsonObject(_messages.Message):
+  """Represents a single JSON object.
+
+  Messages:
+    AdditionalProperty: An additional property for a JsonObject object.
+
+  Fields:
+    additionalProperties: Additional properties of type JsonObject
+  """
+
+  class AdditionalProperty(_messages.Message):
+    """An additional property for a JsonObject object.
+
+    Fields:
+      key: Name of the additional property.
+      value: A JsonValue attribute.
+    """
+
+    key = _messages.StringField(1)
+    value = _messages.MessageField('JsonValue', 2)
+
+  additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+
+JsonValue = extra_types.JsonValue
+
+
+class ProjectList(_messages.Message):
+  """A ProjectList object.
+
+  Messages:
+    ProjectsValueListEntry: A ProjectsValueListEntry object.
+
+  Fields:
+    etag: A hash of the page of results
+    kind: The type of list.
+    nextPageToken: A token to request the next page of results.
+    projects: Projects to which you have at least READ access.
+    totalItems: The total number of projects in the list.
+  """
+
+  class ProjectsValueListEntry(_messages.Message):
+    """A ProjectsValueListEntry object.
+
+    Fields:
+      friendlyName: A descriptive name for this project.
+      id: An opaque ID of this project.
+      kind: The resource type.
+      numericId: The numeric ID of this project.
+      projectReference: A unique reference to this project.
+    """
+
+    friendlyName = _messages.StringField(1)
+    id = _messages.StringField(2)
+    kind = _messages.StringField(3, default=u'bigquery#project')
+    numericId = _messages.IntegerField(4, variant=_messages.Variant.UINT64)
+    projectReference = _messages.MessageField('ProjectReference', 5)
+
+  etag = _messages.StringField(1)
+  kind = _messages.StringField(2, default=u'bigquery#projectList')
+  nextPageToken = _messages.StringField(3)
+  projects = _messages.MessageField('ProjectsValueListEntry', 4, repeated=True)
+  totalItems = _messages.IntegerField(5, variant=_messages.Variant.INT32)
+
+
+class ProjectReference(_messages.Message):
+  """A ProjectReference object.
+
+  Fields:
+    projectId: [Required] ID of the project. Can be either the numeric ID or
+      the assigned ID of the project.
+  """
+
+  projectId = _messages.StringField(1)
+
+
+class QueryRequest(_messages.Message):
+  """A QueryRequest object.
+
+  Fields:
+    defaultDataset: [Optional] Specifies the default datasetId and projectId
+      to assume for any unqualified table names in the query. If not set, all
+      table names in the query string must be qualified in the format
+      'datasetId.tableId'.
+    dryRun: [Optional] If set to true, BigQuery doesn't run the job. Instead,
+      if the query is valid, BigQuery returns statistics about the job such as
+      how many bytes would be processed. If the query is invalid, an error
+      returns. The default value is false.
+    kind: The resource type of the request.
+    maxResults: [Optional] The maximum number of rows of data to return per
+      page of results. Setting this flag to a small value such as 1000 and
+      then paging through results might improve reliability when the query
+      result set is large. In addition to this limit, responses are also
+      limited to 10 MB. By default, there is no maximum row count, and only
+      the byte limit applies.
+    preserveNulls: [Deprecated] This property is deprecated.
+    query: [Required] A query string, following the BigQuery query syntax, of
+      the query to execute. Example: "SELECT count(f1) FROM
+      [myProjectId:myDatasetId.myTableId]".
+    timeoutMs: [Optional] How long to wait for the query to complete, in
+      milliseconds, before the request times out and returns. Note that this
+      is only a timeout for the request, not the query. If the query takes
+      longer to run than the timeout value, the call returns without any
+      results and with the 'jobComplete' flag set to false. You can call
+      GetQueryResults() to wait for the query to complete and read the
+      results. The default value is 10000 milliseconds (10 seconds).
+    useLegacySql: [Experimental] Specifies whether to use BigQuery's legacy
+      SQL dialect for this query. The default value is true. If set to false,
+      the query will use BigQuery's standard SQL:
+      https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is
+      set to false, the values of allowLargeResults and flattenResults are
+      ignored; query will be run as if allowLargeResults is true and
+      flattenResults is false.
+    useQueryCache: [Optional] Whether to look for the result in the query
+      cache. The query cache is a best-effort cache that will be flushed
+      whenever tables in the query are modified. The default value is true.
+  """
+
+  defaultDataset = _messages.MessageField('DatasetReference', 1)
+  dryRun = _messages.BooleanField(2)
+  kind = _messages.StringField(3, default=u'bigquery#queryRequest')
+  maxResults = _messages.IntegerField(4, variant=_messages.Variant.UINT32)
+  preserveNulls = _messages.BooleanField(5)
+  query = _messages.StringField(6)
+  timeoutMs = _messages.IntegerField(7, variant=_messages.Variant.UINT32)
+  useLegacySql = _messages.BooleanField(8)
+  useQueryCache = _messages.BooleanField(9, default=True)
+
+
+class QueryResponse(_messages.Message):
+  """A QueryResponse object.
+
+  Fields:
+    cacheHit: Whether the query result was fetched from the query cache.
+    errors: [Output-only] All errors and warnings encountered during the
+      running of the job. Errors here do not necessarily mean that the job has
+      completed or was unsuccessful.
+    jobComplete: Whether the query has completed or not. If rows or totalRows
+      are present, this will always be true. If this is false, totalRows will
+      not be available.
+    jobReference: Reference to the Job that was created to run the query. This
+      field will be present even if the original request timed out, in which
+      case GetQueryResults can be used to read the results once the query has
+      completed. Since this API only returns the first page of results,
+      subsequent pages can be fetched via the same mechanism
+      (GetQueryResults).
+    kind: The resource type.
+    numDmlAffectedRows: [Output-only, Experimental] The number of rows
+      affected by a DML statement. Present only for DML statements INSERT,
+      UPDATE or DELETE.
+    pageToken: A token used for paging results.
+    rows: An object with as many results as can be contained within the
+      maximum permitted reply size. To get any additional rows, you can call
+      GetQueryResults and specify the jobReference returned above.
+    schema: The schema of the results. Present only when the query completes
+      successfully.
+    totalBytesProcessed: The total number of bytes processed for this query.
+      If this query was a dry run, this is the number of bytes that would be
+      processed if the query were run.
+    totalRows: The total number of rows in the complete query result set,
+      which can be more than the number of rows in this single page of
+      results.
+  """
+
+  cacheHit = _messages.BooleanField(1)
+  errors = _messages.MessageField('ErrorProto', 2, repeated=True)
+  jobComplete = _messages.BooleanField(3)
+  jobReference = _messages.MessageField('JobReference', 4)
+  kind = _messages.StringField(5, default=u'bigquery#queryResponse')
+  numDmlAffectedRows = _messages.IntegerField(6)
+  pageToken = _messages.StringField(7)
+  rows = _messages.MessageField('TableRow', 8, repeated=True)
+  schema = _messages.MessageField('TableSchema', 9)
+  totalBytesProcessed = _messages.IntegerField(10)
+  totalRows = _messages.IntegerField(11, variant=_messages.Variant.UINT64)
+
+
+class StandardQueryParameters(_messages.Message):
+  """Query parameters accepted by all methods.
+
+  Enums:
+    AltValueValuesEnum: Data format for the response.
+
+  Fields:
+    alt: Data format for the response.
+    fields: Selector specifying which fields to include in a partial response.
+    key: API key. Your API key identifies your project and provides you with
+      API access, quota, and reports. Required unless you provide an OAuth 2.0
+      token.
+    oauth_token: OAuth 2.0 token for the current user.
+    prettyPrint: Returns response with indentations and line breaks.
+    quotaUser: Available to use for quota purposes for server-side
+      applications. Can be any arbitrary string assigned to a user, but should
+      not exceed 40 characters. Overrides userIp if both are provided.
+    trace: A tracing token of the form "token:" to include in api
+      requests.
+    userIp: IP address of the site where the request originates. Use this if
+      you want to enforce per-user limits.
+  """
+
+  class AltValueValuesEnum(_messages.Enum):
+    """Data format for the response.
+
+    Values:
+      json: Responses with Content-Type of application/json
+    """
+    json = 0
+
+  alt = _messages.EnumField('AltValueValuesEnum', 1, default=u'json')
+  fields = _messages.StringField(2)
+  key = _messages.StringField(3)
+  oauth_token = _messages.StringField(4)
+  prettyPrint = _messages.BooleanField(5, default=True)
+  quotaUser = _messages.StringField(6)
+  trace = _messages.StringField(7)
+  userIp = _messages.StringField(8)
+
+
+class Streamingbuffer(_messages.Message):
+  """A Streamingbuffer object.
+
+  Fields:
+    estimatedBytes: [Output-only] A lower-bound estimate of the number of
+      bytes currently in the streaming buffer.
+    estimatedRows: [Output-only] A lower-bound estimate of the number of rows
+      currently in the streaming buffer.
+    oldestEntryTime: [Output-only] Contains the timestamp of the oldest entry
+      in the streaming buffer, in milliseconds since the epoch, if the
+      streaming buffer is available.
+  """
+
+  estimatedBytes = _messages.IntegerField(1, variant=_messages.Variant.UINT64)
+  estimatedRows = _messages.IntegerField(2, variant=_messages.Variant.UINT64)
+  oldestEntryTime = _messages.IntegerField(3, variant=_messages.Variant.UINT64)
+
+
+class Table(_messages.Message):
+  """A Table object.
+
+  Fields:
+    creationTime: [Output-only] The time when this table was created, in
+      milliseconds since the epoch.
+    description: [Optional] A user-friendly description of this table.
+    etag: [Output-only] A hash of this resource.
+    expirationTime: [Optional] The time when this table expires, in
+      milliseconds since the epoch. If not present, the table will persist
+      indefinitely. Expired tables will be deleted and their storage
+      reclaimed.
+    externalDataConfiguration: [Optional] Describes the data format, location,
+      and other properties of a table stored outside of BigQuery. By defining
+      these properties, the data source can then be queried as if it were a
+      standard BigQuery table.
+    friendlyName: [Optional] A descriptive name for this table.
+    id: [Output-only] An opaque ID uniquely identifying the table.
+    kind: [Output-only] The type of the resource.
+    lastModifiedTime: [Output-only] The time when this table was last
+      modified, in milliseconds since the epoch.
+    location: [Output-only] The geographic location where the table resides.
+      This value is inherited from the dataset.
+    numBytes: [Output-only] The size of this table in bytes, excluding any
+      data in the streaming buffer.
+    numLongTermBytes: [Output-only] The number of bytes in the table that are
+      considered "long-term storage".
+    numRows: [Output-only] The number of rows of data in this table, excluding
+      any data in the streaming buffer.
+    schema: [Optional] Describes the schema of this table.
+    selfLink: [Output-only] A URL that can be used to access this resource
+      again.
+    streamingBuffer: [Output-only] Contains information regarding this table's
+      streaming buffer, if one is present. This field will be absent if the
+      table is not being streamed to or if there is no data in the streaming
+      buffer.
+    tableReference: [Required] Reference describing the ID of this table.
+    timePartitioning: [Experimental] If specified, configures time-based
+      partitioning for this table.
+    type: [Output-only] Describes the table type. The following values are
+      supported: TABLE: A normal BigQuery table. VIEW: A virtual table defined
+      by a SQL query. EXTERNAL: A table that references data stored in an
+      external storage system, such as Google Cloud Storage. The default value
+      is TABLE.
+    view: [Optional] The view definition.
+  """
+
+  creationTime = _messages.IntegerField(1)
+  description = _messages.StringField(2)
+  etag = _messages.StringField(3)
+  expirationTime = _messages.IntegerField(4)
+  externalDataConfiguration = _messages.MessageField('ExternalDataConfiguration', 5)
+  friendlyName = _messages.StringField(6)
+  id = _messages.StringField(7)
+  kind = _messages.StringField(8, default=u'bigquery#table')
+  lastModifiedTime = _messages.IntegerField(9, variant=_messages.Variant.UINT64)
+  location = _messages.StringField(10)
+  numBytes = _messages.IntegerField(11)
+  numLongTermBytes = _messages.IntegerField(12)
+  numRows = _messages.IntegerField(13, variant=_messages.Variant.UINT64)
+  schema = _messages.MessageField('TableSchema', 14)
+  selfLink = _messages.StringField(15)
+  streamingBuffer = _messages.MessageField('Streamingbuffer', 16)
+  tableReference = _messages.MessageField('TableReference', 17)
+  timePartitioning = _messages.MessageField('TimePartitioning', 18)
+  type = _messages.StringField(19)
+  view = _messages.MessageField('ViewDefinition', 20)
+
+
+class TableCell(_messages.Message):
+  """A TableCell object.
+
+  Fields:
+    v: A extra_types.JsonValue attribute.
+  """
+
+  v = _messages.MessageField('extra_types.JsonValue', 1)
+
+
+class TableDataInsertAllRequest(_messages.Message):
+  """A TableDataInsertAllRequest object.
+
+  Messages:
+    RowsValueListEntry: A RowsValueListEntry object.
+
+  Fields:
+    ignoreUnknownValues: [Optional] Accept rows that contain values that do
+      not match the schema. The unknown values are ignored. Default is false,
+      which treats unknown values as errors.
+    kind: The resource type of the response.
+    rows: The rows to insert.
+    skipInvalidRows: [Optional] Insert all valid rows of a request, even if
+      invalid rows exist. The default value is false, which causes the entire
+      request to fail if any invalid rows exist.
+    templateSuffix: [Experimental] If specified, treats the destination table
+      as a base template, and inserts the rows into an instance table named
+      "{destination}{templateSuffix}". BigQuery will manage creation of the
+      instance table, using the schema of the base template table. See
+      https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-
+      tables for considerations when working with templates tables.
+  """
+
+  class RowsValueListEntry(_messages.Message):
+    """A RowsValueListEntry object.
+
+    Fields:
+      insertId: [Optional] A unique ID for each row. BigQuery uses this
+        property to detect duplicate insertion requests on a best-effort
+        basis.
+      json: [Required] A JSON object that contains a row of data. The object's
+        properties and values must match the destination table's schema.
+    """
+
+    insertId = _messages.StringField(1)
+    json = _messages.MessageField('JsonObject', 2)
+
+  ignoreUnknownValues = _messages.BooleanField(1)
+  kind = _messages.StringField(2, default=u'bigquery#tableDataInsertAllRequest')
+  rows = _messages.MessageField('RowsValueListEntry', 3, repeated=True)
+  skipInvalidRows = _messages.BooleanField(4)
+  templateSuffix = _messages.StringField(5)
+
+
+class TableDataInsertAllResponse(_messages.Message):
+  """A TableDataInsertAllResponse object.
+
+  Messages:
+    InsertErrorsValueListEntry: A InsertErrorsValueListEntry object.
+
+  Fields:
+    insertErrors: An array of errors for rows that were not inserted.
+    kind: The resource type of the response.
+  """
+
+  class InsertErrorsValueListEntry(_messages.Message):
+    """A InsertErrorsValueListEntry object.
+
+    Fields:
+      errors: Error information for the row indicated by the index property.
+      index: The index of the row that error applies to.
+    """
+
+    errors = _messages.MessageField('ErrorProto', 1, repeated=True)
+    index = _messages.IntegerField(2, variant=_messages.Variant.UINT32)
+
+  insertErrors = _messages.MessageField('InsertErrorsValueListEntry', 1, repeated=True)
+  kind = _messages.StringField(2, default=u'bigquery#tableDataInsertAllResponse')
+
+
+class TableDataList(_messages.Message):
+  """A TableDataList object.
+
+  Fields:
+    etag: A hash of this page of results.
+    kind: The resource type of the response.
+    pageToken: A token used for paging results. Providing this token instead
+      of the startIndex parameter can help you retrieve stable results when an
+      underlying table is changing.
+    rows: Rows of results.
+    totalRows: The total number of rows in the complete table.
+  """
+
+  etag = _messages.StringField(1)
+  kind = _messages.StringField(2, default=u'bigquery#tableDataList')
+  pageToken = _messages.StringField(3)
+  rows = _messages.MessageField('TableRow', 4, repeated=True)
+  totalRows = _messages.IntegerField(5)
+
+
+class TableFieldSchema(_messages.Message):
+  """A TableFieldSchema object.
+
+  Fields:
+    description: [Optional] The field description. The maximum length is 16K
+      characters.
+    fields: [Optional] Describes the nested schema fields if the type property
+      is set to RECORD.
+    mode: [Optional] The field mode. Possible values include NULLABLE,
+      REQUIRED and REPEATED. The default value is NULLABLE.
+    name: [Required] The field name. The name must contain only letters (a-z,
+      A-Z), numbers (0-9), or underscores (_), and must start with a letter or
+      underscore. The maximum length is 128 characters.
+    type: [Required] The field data type. Possible values include STRING,
+      BYTES, INTEGER, FLOAT, BOOLEAN, TIMESTAMP or RECORD (where RECORD
+      indicates that the field contains a nested schema).
+  """
+
+  description = _messages.StringField(1)
+  fields = _messages.MessageField('TableFieldSchema', 2, repeated=True)
+  mode = _messages.StringField(3)
+  name = _messages.StringField(4)
+  type = _messages.StringField(5)
+
+
+class TableList(_messages.Message):
+  """A TableList object.
+
+  Messages:
+    TablesValueListEntry: A TablesValueListEntry object.
+
+  Fields:
+    etag: A hash of this page of results.
+    kind: The type of list.
+    nextPageToken: A token to request the next page of results.
+    tables: Tables in the requested dataset.
+    totalItems: The total number of tables in the dataset.
+  """
+
+  class TablesValueListEntry(_messages.Message):
+    """A TablesValueListEntry object.
+
+    Fields:
+      friendlyName: The user-friendly name for this table.
+      id: An opaque ID of the table
+      kind: The resource type.
+      tableReference: A reference uniquely identifying the table.
+      type: The type of table. Possible values are: TABLE, VIEW.
+    """
+
+    friendlyName = _messages.StringField(1)
+    id = _messages.StringField(2)
+    kind = _messages.StringField(3, default=u'bigquery#table')
+    tableReference = _messages.MessageField('TableReference', 4)
+    type = _messages.StringField(5)
+
+  etag = _messages.StringField(1)
+  kind = _messages.StringField(2, default=u'bigquery#tableList')
+  nextPageToken = _messages.StringField(3)
+  tables = _messages.MessageField('TablesValueListEntry', 4, repeated=True)
+  totalItems = _messages.IntegerField(5, variant=_messages.Variant.INT32)
+
+
+class TableReference(_messages.Message):
+  """A TableReference object.
+
+  Fields:
+    datasetId: [Required] The ID of the dataset containing this table.
+    projectId: [Required] The ID of the project containing this table.
+    tableId: [Required] The ID of the table. The ID must contain only letters
+      (a-z, A-Z), numbers (0-9), or underscores (_). The maximum length is
+      1,024 characters.
+  """
+
+  datasetId = _messages.StringField(1)
+  projectId = _messages.StringField(2)
+  tableId = _messages.StringField(3)
+
+
+class TableRow(_messages.Message):
+  """A TableRow object.
+
+  Fields:
+    f: Represents a single row in the result set, consisting of one or more
+      fields.
+  """
+
+  f = _messages.MessageField('TableCell', 1, repeated=True)
+
+
+class TableSchema(_messages.Message):
+  """A TableSchema object.
+
+  Fields:
+    fields: Describes the fields in a table.
+  """
+
+  fields = _messages.MessageField('TableFieldSchema', 1, repeated=True)
+
+
+class TimePartitioning(_messages.Message):
+  """A TimePartitioning object.
+
+  Fields:
+    expirationMs: [Optional] Number of milliseconds for which to keep the
+      storage for a partition.
+    type: [Required] The only type supported is DAY, which will generate one
+      partition per day based on data loading time.
+  """
+
+  expirationMs = _messages.IntegerField(1)
+  type = _messages.StringField(2)
+
+
+class UserDefinedFunctionResource(_messages.Message):
+  """A UserDefinedFunctionResource object.
+
+  Fields:
+    inlineCode: [Pick one] An inline resource that contains code for a user-
+      defined function (UDF). Providing a inline code resource is equivalent
+      to providing a URI for a file containing the same code.
+    resourceUri: [Pick one] A code resource to load from a Google Cloud
+      Storage URI (gs://bucket/path).
+  """
+
+  inlineCode = _messages.StringField(1)
+  resourceUri = _messages.StringField(2)
+
+
+class ViewDefinition(_messages.Message):
+  """A ViewDefinition object.
+
+  Fields:
+    query: [Required] A query that BigQuery executes when the view is
+      referenced.
+    useLegacySql: [Experimental] Specifies whether to use BigQuery's legacy
+      SQL for this view. The default value is true. If set to false, the view
+      will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-
+      reference/ Queries and views that reference this view must use the same
+      flag value.
+    userDefinedFunctionResources: [Experimental] Describes user-defined
+      function resources used in the query.
+  """
+
+  query = _messages.StringField(1)
+  useLegacySql = _messages.BooleanField(2)
+  userDefinedFunctionResources = _messages.MessageField('UserDefinedFunctionResource', 3, repeated=True)
+
+
diff --git a/samples/regenerate_samples.py b/samples/regenerate_samples.py
index 8ef07ec..9d41795 100644
--- a/samples/regenerate_samples.py
+++ b/samples/regenerate_samples.py
@@ -20,6 +20,7 @@ import subprocess
 _GEN_CLIENT_BINARY = 'gen_client'
 
 _SAMPLES = [
+    'bigquery_sample/bigquery_v2.json',
     'dns_sample/dns_v1.json',
     'iam_sample/iam_v1.json',
     'fusiontables_sample/fusiontables_v1.json',
diff --git a/samples/uptodate_check_test.py b/samples/uptodate_check_test.py
index 73fcf3b..6fbea9c 100644
--- a/samples/uptodate_check_test.py
+++ b/samples/uptodate_check_test.py
@@ -67,6 +67,9 @@ class ClientGenCliTest(unittest2.TestCase):
                         api_name, prefix, expected_file)),
                     _GetContent(os.path.join(tmp_dir_path, expected_file)))
 
+    def testGenClient_BigqueryDoc(self):
+        self._CheckGeneratedFiles('bigquery', 'v2')
+
     def testGenClient_DnsDoc(self):
         self._CheckGeneratedFiles('dns', 'v1')
 
-- 
GitLab


From 513f539096e3331b0cb3000e5fa5e736292adf40 Mon Sep 17 00:00:00 2001
From: Charles Chen 
Date: Fri, 14 Oct 2016 13:31:14 -0700
Subject: [PATCH 263/295] Pin oauth2client to avoid using version 4.0.0

The 4.0.0 version of oauth2client removes locked_file.
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2175b4c..6e16d35 100644
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@ except ImportError:
 # Python version and OS.
 REQUIRED_PACKAGES = [
     'httplib2>=0.8',
-    'oauth2client>=1.5.2',
+    'oauth2client>=1.5.2,<4.0.0dev',
     'setuptools>=18.5',
     'six>=1.9.0',
     ]
-- 
GitLab


From c3a73509d9aff3f29d3b904d5a22e210a463b309 Mon Sep 17 00:00:00 2001
From: Charles Chen 
Date: Fri, 14 Oct 2016 13:45:47 -0700
Subject: [PATCH 264/295] Update apitools version to 0.5.5.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6e16d35..f1e06d5 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.4'
+_APITOOLS_VERSION = '0.5.5'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From a18c0af8c6f066f413b9b3640bdf42986ad2d150 Mon Sep 17 00:00:00 2001
From: przemekwitek 
Date: Wed, 2 Nov 2016 15:30:02 +0100
Subject: [PATCH 265/295] Fix '_IncludeFields' method.

Fix '_IncludeFields' method so that it leaves siblings of the cleared fields
unaffected.
---
 apitools/base/py/encoding.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py
index 7eec985..182f166 100644
--- a/apitools/base/py/encoding.py
+++ b/apitools/base/py/encoding.py
@@ -230,8 +230,7 @@ def _GetField(message, field_path):
 
 def _SetField(dictblob, field_path, value):
     for field in field_path[:-1]:
-        dictblob[field] = {}
-        dictblob = dictblob[field]
+        dictblob = dictblob.setdefault(field, {})
     dictblob[field_path[-1]] = value
 
 
-- 
GitLab


From 529614ae4fb84a37752d3eb072fe7ad8f2168cc9 Mon Sep 17 00:00:00 2001
From: przemekwitek 
Date: Wed, 2 Nov 2016 15:35:01 +0100
Subject: [PATCH 266/295] Fix unit test for the '_IncludeFields' method.

---
 apitools/base/py/encoding_test.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py
index d306aae..3564ca0 100644
--- a/apitools/base/py/encoding_test.py
+++ b/apitools/base/py/encoding_test.py
@@ -203,9 +203,12 @@ class EncodingTest(unittest2.TestCase):
         self.assertEqual(
             '{"nested": {"nested": null}}',
             encoding.MessageToJson(msg, include_fields=['nested.nested']))
-        self.assertEqual(
-            '{"nested": {"nested_list": []}}',
-            encoding.MessageToJson(msg, include_fields=['nested.nested_list']))
+        # When clearing 'nested.nested_list', its sibling ('nested.nested')
+        # should remain unaffected.
+        self.assertIn(
+            encoding.MessageToJson(msg, include_fields=['nested.nested_list']),
+            ['{"nested": {"nested": {}, "nested_list": []}}',
+             '{"nested": {"nested_list": [], "nested": {}}}'])
         self.assertEqual(
             '{"nested": {"nested": {"additional_properties": []}}}',
             encoding.MessageToJson(
-- 
GitLab


From 3f14c9197c6c727bfdafb802d6b1dd1962d910ad Mon Sep 17 00:00:00 2001
From: Craig Citro 
Date: Wed, 16 Nov 2016 23:47:32 -0800
Subject: [PATCH 267/295] Fix a nasty encoding corner case with JsonValue.

The json encoding code in protorpc had a bit of custom logic to handle the
case of repeated vs. non-repeated fields with the same line of code. However,
in the case of a JsonValue (aka `any` type in a discovery doc), it's possible
for a non-repeated value to itself be a list. In this case, we'd always
deserialize the whole list, and then ... only use the last value.

The fix here is to actually simplify the code a bit, and split the logic based
on whether the field is repeated, *not* whether the value is itself a list.

Also added a simple test.
---
 apitools/base/protorpclite/protojson.py | 22 ++++++++++------------
 apitools/base/py/extra_types_test.py    | 12 ++++++++++++
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py
index 5f611f4..c14c0c7 100644
--- a/apitools/base/protorpclite/protojson.py
+++ b/apitools/base/protorpclite/protojson.py
@@ -277,22 +277,20 @@ class ProtoJson(object):
                         'No variant found for unrecognized field: %s', key)
                 continue
 
-            # Normalize values in to a list.
-            if isinstance(value, list):
-                if not value:
-                    continue
-            else:
-                value = [value]
-
-            valid_value = []
-            for item in value:
-                valid_value.append(self.decode_field(field, item))
+            # This is just for consistency with the old behavior.
+            if value == []:
+                continue
 
             if field.repeated:
-                _ = getattr(message, field.name)
+                # This should be unnecessary? Or in fact become an error.
+                if not isinstance(value, list):
+                    value = [value]
+                valid_value = [self.decode_field(field, item)
+                               for item in value]
                 setattr(message, field.name, valid_value)
             else:
-                setattr(message, field.name, valid_value[-1])
+                setattr(message, field.name, self.decode_field(field, value))
+
         return message
 
     def decode_field(self, field, value):
diff --git a/apitools/base/py/extra_types_test.py b/apitools/base/py/extra_types_test.py
index d31e195..7e37f7c 100644
--- a/apitools/base/py/extra_types_test.py
+++ b/apitools/base/py/extra_types_test.py
@@ -129,6 +129,18 @@ class ExtraTypesTest(unittest2.TestCase):
         self.assertEqual(json_value, encoding.MessageToJson(value))
         self.assertEqual(json_obj, encoding.MessageToJson(obj))
 
+    def testJsonValueAsFieldTranslation(self):
+        class HasJsonValueMsg(messages.Message):
+            some_value = messages.MessageField(extra_types.JsonValue, 1)
+
+        msg_json = '{"some_value": [1, 2, 3]}'
+        msg = HasJsonValueMsg(
+            some_value=encoding.PyValueToMessage(
+                extra_types.JsonValue, [1, 2, 3]))
+        self.assertEqual(msg,
+                         encoding.JsonToMessage(HasJsonValueMsg, msg_json))
+        self.assertEqual(msg_json, encoding.MessageToJson(msg))
+
     def testDateField(self):
 
         class DateMsg(messages.Message):
-- 
GitLab


From 1351f3297a0c58bf2f4cae347ab74873bdf04817 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Sat, 3 Dec 2016 08:35:46 -0500
Subject: [PATCH 268/295] Update apitools version to 0.5.6.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f1e06d5..3ab0bec 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.5'
+_APITOOLS_VERSION = '0.5.6'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From 7d3b0982802e40ca76b4b6f1a34cb00624535d7c Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Mon, 19 Dec 2016 12:19:44 -0500
Subject: [PATCH 269/295] Allow tests to run without apitools package setup.

---
 apitools/gen/client_generation_test.py |  3 ++-
 apitools/gen/gen_client_lib.py         | 10 ++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/apitools/gen/client_generation_test.py b/apitools/gen/client_generation_test.py
index ae72ee0..5e7932a 100644
--- a/apitools/gen/client_generation_test.py
+++ b/apitools/gen/client_generation_test.py
@@ -22,6 +22,7 @@ import tempfile
 
 import unittest2
 
+from apitools.gen import gen_client
 from apitools.gen import test_utils
 
 
@@ -55,7 +56,7 @@ class ClientGenerationTest(unittest2.TestCase):
                 ]
                 logging.info('Testing API %s with command line: %s',
                              api, ' '.join(args))
-                retcode = subprocess.call(args)
+                retcode = gen_client.main(args)
                 if retcode == 128:
                     logging.error('Failed to fetch discovery doc, continuing.')
                     continue
diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py
index f9abfc6..c93f8a2 100644
--- a/apitools/gen/gen_client_lib.py
+++ b/apitools/gen/gen_client_lib.py
@@ -30,8 +30,14 @@ from apitools.gen import util
 
 def _ApitoolsVersion():
     """Returns version of the currently installed google-apitools package."""
-    import pkg_resources
-    return pkg_resources.get_distribution('google-apitools').version
+    try:
+        import pkg_resources
+    except ImportError:
+        return 'X.X.X'
+    try:
+        return pkg_resources.get_distribution('google-apitools').version
+    except pkg_resources.DistributionNotFound:
+        return 'X.X.X'
 
 
 def _StandardQueryParametersSchema(discovery_doc):
-- 
GitLab


From c589eb1534d83456a54c82322f3d229cd66d80cd Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Sat, 17 Dec 2016 17:39:12 -0500
Subject: [PATCH 270/295] Add limit support for limiting size of batch request.

Some apis impose limit on batch requests.
See for example:
  https://cloud.google.com/compute/docs/api/how-tos/batch
---
 apitools/base/py/__init__.py   |  1 +
 apitools/base/py/batch.py      | 34 ++++++++++++-------
 apitools/base/py/batch_test.py | 62 ++++++++++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 13 deletions(-)

diff --git a/apitools/base/py/__init__.py b/apitools/base/py/__init__.py
index 393aa14..f0003e2 100644
--- a/apitools/base/py/__init__.py
+++ b/apitools/base/py/__init__.py
@@ -17,6 +17,7 @@
 """Top-level imports for apitools base files."""
 
 # pylint:disable=wildcard-import
+# pylint:disable=redefined-builtin
 from apitools.base.py.base_api import *
 from apitools.base.py.batch import *
 from apitools.base.py.credentials_lib import *
diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py
index 61cfb25..95a8f7a 100644
--- a/apitools/base/py/batch.py
+++ b/apitools/base/py/batch.py
@@ -28,6 +28,7 @@ import uuid
 import six
 from six.moves import http_client
 from six.moves import urllib_parse
+from six.moves import range  # pylint: disable=redefined-builtin
 
 from apitools.base.py import exceptions
 from apitools.base.py import http_wrapper
@@ -180,7 +181,8 @@ class BatchApiRequest(object):
             http_request, self.retryable_codes, service, method_config)
         self.api_requests.append(api_request)
 
-    def Execute(self, http, sleep_between_polls=5, max_retries=5):
+    def Execute(self, http, sleep_between_polls=5, max_retries=5,
+                max_batch_size=None):
         """Execute all of the requests in the batch.
 
         Args:
@@ -190,33 +192,39 @@ class BatchApiRequest(object):
           max_retries: Max retries. Any requests that have not succeeded by
               this number of retries simply report the last response or
               exception, whatever it happened to be.
+          max_batch_size: int, if specified requests will be split in batches
+              of given size.
 
         Returns:
           List of ApiCalls.
         """
         requests = [request for request in self.api_requests
                     if not request.terminal_state]
+        batch_size = max_batch_size or len(requests)
 
         for attempt in range(max_retries):
             if attempt:
                 time.sleep(sleep_between_polls)
 
-            # Create a batch_http_request object and populate it with
-            # incomplete requests.
-            batch_http_request = BatchHttpRequest(batch_url=self.batch_url)
-            for request in requests:
-                batch_http_request.Add(
-                    request.http_request, request.HandleResponse)
-            batch_http_request.Execute(http)
+            for i in range(0, len(requests), batch_size):
+                # Create a batch_http_request object and populate it with
+                # incomplete requests.
+                batch_http_request = BatchHttpRequest(batch_url=self.batch_url)
+                for request in itertools.islice(requests,
+                                                i, i + batch_size):
+                    batch_http_request.Add(
+                        request.http_request, request.HandleResponse)
+                batch_http_request.Execute(http)
+
+                if hasattr(http.request, 'credentials'):
+                    if any(request.authorization_failed
+                           for request in itertools.islice(requests,
+                                                           i, i + batch_size)):
+                        http.request.credentials.refresh(http)
 
             # Collect retryable requests.
             requests = [request for request in self.api_requests if not
                         request.terminal_state]
-
-            if hasattr(http.request, 'credentials'):
-                if any(request.authorization_failed for request in requests):
-                    http.request.credentials.refresh(http)
-
             if not requests:
                 break
 
diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py
index b966d5a..3140f60 100644
--- a/apitools/base/py/batch_test.py
+++ b/apitools/base/py/batch_test.py
@@ -19,6 +19,7 @@ import textwrap
 
 import mock
 from six.moves import http_client
+from six.moves import range  # pylint:disable=redefined-builtin
 from six.moves.urllib import parse
 import unittest2
 
@@ -196,6 +197,67 @@ class BatchTest(unittest2.TestCase):
             self.assertEqual('content', response.content)
             self.assertEqual(desired_url, response.request_url)
 
+    def _MakeResponse(self, number_of_parts):
+        return http_wrapper.Response(
+            info={
+                'status': '200',
+                'content-type': 'multipart/mixed; boundary="boundary"',
+            },
+            content='--boundary\n' + '--boundary\n'.join(
+                textwrap.dedent("""\
+                    content-type: text/plain
+                    content-id: 
+
+                    HTTP/1.1 200 OK
+                    response {0} content
+
+                    """)
+                .format(i) for i in range(number_of_parts)) + '--boundary--',
+            request_url=None,
+        )
+
+    def _MakeSampleRequest(self, url, name):
+        return http_wrapper.Request(url, 'POST', {
+            'content-type': 'multipart/mixed; boundary="None"',
+            'content-length': 80,
+        }, '{0} {1}'.format(name, 'x' * (79 - len(name))))
+
+    def testMultipleRequestInBatchWithMax(self):
+        mock_service = FakeService()
+
+        desired_url = 'https://www.example.com'
+        batch_api_request = batch.BatchApiRequest(batch_url=desired_url)
+
+        number_of_requests = 10
+        max_batch_size = 3
+        for i in range(number_of_requests):
+            batch_api_request.Add(
+                mock_service, 'unused', None,
+                {'desired_request': self._MakeSampleRequest(
+                    desired_url, 'Sample-{0}'.format(i))})
+
+        responses = []
+        for i in range(0, number_of_requests, max_batch_size):
+            responses.append(
+                self._MakeResponse(
+                    min(number_of_requests - i, max_batch_size)))
+        with mock.patch.object(http_wrapper, 'MakeRequest',
+                               autospec=True) as mock_request:
+            self.__ConfigureMock(
+                mock_request,
+                expected_request=http_wrapper.Request(desired_url, 'POST', {
+                    'content-type': 'multipart/mixed; boundary="None"',
+                    'content-length': 1142,
+                }, 'x' * 1142),
+                response=responses)
+            api_request_responses = batch_api_request.Execute(
+                FakeHttp(), max_batch_size=max_batch_size)
+
+        self.assertEqual(number_of_requests, len(api_request_responses))
+        self.assertEqual(
+            -(-number_of_requests // max_batch_size),
+            mock_request.call_count)
+
     def testRefreshOnAuthFailure(self):
         mock_service = FakeService()
 
-- 
GitLab


From 3cfaf2c2a6cae66d00b9e10e8fcd1e26a94967c2 Mon Sep 17 00:00:00 2001
From: Travis Hobrla 
Date: Wed, 18 Jan 2017 10:57:11 -0800
Subject: [PATCH 271/295] Treat exceptions accessing GCE credential cache file
 as a cache miss

This fixes an issue where problems accessing the cache file on
the filesystem (for example, in a container with no mounted
filesystem) would cause apitools to abort entirely, as opposed to
simply not caching the credentials.

Fixes https://github.com/google/apitools/issues/139
---
 apitools/base/py/credentials_lib.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index efffe8c..c0210c3 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -302,6 +302,9 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
                             if (creds['scopes'] in
                                     (None, cached_creds['scopes'])):
                                 scopes = cached_creds['scopes']
+                except:  # pylint: disable=bare-except
+                    # Treat exceptions as a cache miss.
+                    pass
                 finally:
                     cache_file.unlock_and_close()
         return scopes
@@ -331,6 +334,9 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
                         # If it's not locked, the locking process will
                         # write the same data to the file, so just
                         # continue.
+                except:  # pylint: disable=bare-except
+                    # Treat exceptions as a cache miss.
+                    pass
                 finally:
                     cache_file.unlock_and_close()
 
-- 
GitLab


From b204d3a7c7d298863b080fd45a6088a7a07a9c7b Mon Sep 17 00:00:00 2001
From: Travis Hobrla 
Date: Wed, 18 Jan 2017 12:37:04 -0800
Subject: [PATCH 272/295] Don't swallow KeyboardInterrupts

---
 apitools/base/py/credentials_lib.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index c0210c3..c9ca485 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -302,6 +302,8 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
                             if (creds['scopes'] in
                                     (None, cached_creds['scopes'])):
                                 scopes = cached_creds['scopes']
+                except KeyboardInterrupt:
+                    raise
                 except:  # pylint: disable=bare-except
                     # Treat exceptions as a cache miss.
                     pass
@@ -334,6 +336,8 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
                         # If it's not locked, the locking process will
                         # write the same data to the file, so just
                         # continue.
+                except KeyboardInterrupt:
+                    raise
                 except:  # pylint: disable=bare-except
                     # Treat exceptions as a cache miss.
                     pass
-- 
GitLab


From 5cf5daa7f3904d762604570df7dec0e6c7aa91ef Mon Sep 17 00:00:00 2001
From: Chamikara Jayalath 
Date: Sun, 29 Jan 2017 14:44:42 -0800
Subject: [PATCH 273/295] Updates encoding.py to not to set/reset global
 logger.

Removes a warning log entry that caused above to be added.
---
 apitools/base/protorpclite/protojson.py |  3 ---
 apitools/base/py/encoding.py            | 14 +++-----------
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py
index c14c0c7..0c6c92c 100644
--- a/apitools/base/protorpclite/protojson.py
+++ b/apitools/base/protorpclite/protojson.py
@@ -272,9 +272,6 @@ class ProtoJson(object):
                 variant = self.__find_variant(value)
                 if variant:
                     message.set_unrecognized_field(key, value, variant)
-                else:
-                    logging.warning(
-                        'No variant found for unrecognized field: %s', key)
                 continue
 
             # This is just for consistency with the old behavior.
diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py
index 182f166..baf6537 100644
--- a/apitools/base/py/encoding.py
+++ b/apitools/base/py/encoding.py
@@ -20,7 +20,6 @@ import base64
 import collections
 import datetime
 import json
-import logging
 import os
 import sys
 
@@ -276,16 +275,9 @@ class _ProtoJsonApiTools(protojson.ProtoJson):
         if message_type in _CUSTOM_MESSAGE_CODECS:
             return _CUSTOM_MESSAGE_CODECS[
                 message_type].decoder(encoded_message)
-        # We turn off the default logging in protorpc. We may want to
-        # remove this later.
-        old_level = logging.getLogger().level
-        logging.getLogger().setLevel(logging.ERROR)
-        try:
-            result = _DecodeCustomFieldNames(message_type, encoded_message)
-            result = super(_ProtoJsonApiTools, self).decode_message(
-                message_type, result)
-        finally:
-            logging.getLogger().setLevel(old_level)
+        result = _DecodeCustomFieldNames(message_type, encoded_message)
+        result = super(_ProtoJsonApiTools, self).decode_message(
+            message_type, result)
         result = _ProcessUnknownEnums(result, encoded_message)
         result = _ProcessUnknownMessages(result, encoded_message)
         return _DecodeUnknownFields(result, encoded_message)
-- 
GitLab


From 03b7abba8ad4869ce2bd8a25202725d1028de675 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 31 Jan 2017 16:15:31 -0500
Subject: [PATCH 274/295] Update apitools version to 0.5.7.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3ab0bec..23e1a9c 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.6'
+_APITOOLS_VERSION = '0.5.7'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From 806831578e3574f6eea926ad9099a12c0589fcb8 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Mon, 27 Feb 2017 15:00:06 -0500
Subject: [PATCH 275/295] Add batch request callback hook

---
 apitools/base/py/batch.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py
index 95a8f7a..e9690bf 100644
--- a/apitools/base/py/batch.py
+++ b/apitools/base/py/batch.py
@@ -182,7 +182,7 @@ class BatchApiRequest(object):
         self.api_requests.append(api_request)
 
     def Execute(self, http, sleep_between_polls=5, max_retries=5,
-                max_batch_size=None):
+                max_batch_size=None, batch_request_callback=None):
         """Execute all of the requests in the batch.
 
         Args:
@@ -209,7 +209,10 @@ class BatchApiRequest(object):
             for i in range(0, len(requests), batch_size):
                 # Create a batch_http_request object and populate it with
                 # incomplete requests.
-                batch_http_request = BatchHttpRequest(batch_url=self.batch_url)
+                batch_http_request = BatchHttpRequest(
+                    batch_url=self.batch_url,
+                    callback=batch_request_callback
+                )
                 for request in itertools.islice(requests,
                                                 i, i + batch_size):
                     batch_http_request.Add(
-- 
GitLab


From d69c9b03cc9592aa2cf418cc62242a6040670db6 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Mon, 27 Feb 2017 17:54:47 -0500
Subject: [PATCH 276/295] Augment test to use callback

---
 apitools/base/py/batch_test.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py
index 3140f60..e27ee81 100644
--- a/apitools/base/py/batch_test.py
+++ b/apitools/base/py/batch_test.py
@@ -150,9 +150,17 @@ class BatchTest(unittest2.TestCase):
                                   exceptions.HttpError)
 
     def testSingleRequestInBatch(self):
+        desired_url = 'https://www.example.com'
+
+        callback_result = None
+        def _Callback(response, exception):
+            self.assertEqual({'status': '200'}, response.info)
+            self.assertEqual('content', response.content)
+            self.assertEqual(desired_url, response.request_url)
+            self.assertIsNone(exception)
+
         mock_service = FakeService()
 
-        desired_url = 'https://www.example.com'
         batch_api_request = batch.BatchApiRequest(batch_url=desired_url)
         # The request to be added. The actual request sent will be somewhat
         # larger, as this is added to a batch.
@@ -185,7 +193,8 @@ class BatchTest(unittest2.TestCase):
                 'desired_request': desired_request,
             })
 
-            api_request_responses = batch_api_request.Execute(FakeHttp())
+            api_request_responses = batch_api_request.Execute(
+                FakeHttp(), batch_request_callback=_Callback)
 
             self.assertEqual(1, len(api_request_responses))
             self.assertEqual(1, mock_request.call_count)
-- 
GitLab


From 33235a1d2e93a9362f98398462750739ea60b330 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Mon, 27 Feb 2017 18:04:14 -0500
Subject: [PATCH 277/295] Test fails if callback not run

---
 apitools/base/py/batch_test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py
index e27ee81..d684b25 100644
--- a/apitools/base/py/batch_test.py
+++ b/apitools/base/py/batch_test.py
@@ -152,12 +152,13 @@ class BatchTest(unittest2.TestCase):
     def testSingleRequestInBatch(self):
         desired_url = 'https://www.example.com'
 
-        callback_result = None
+        callback_was_called = []
         def _Callback(response, exception):
             self.assertEqual({'status': '200'}, response.info)
             self.assertEqual('content', response.content)
             self.assertEqual(desired_url, response.request_url)
             self.assertIsNone(exception)
+            callback_was_called.append(1)
 
         mock_service = FakeService()
 
@@ -205,6 +206,7 @@ class BatchTest(unittest2.TestCase):
             self.assertEqual({'status': '200'}, response.info)
             self.assertEqual('content', response.content)
             self.assertEqual(desired_url, response.request_url)
+        self.assertEquals(1, len(callback_was_called))
 
     def _MakeResponse(self, number_of_parts):
         return http_wrapper.Response(
-- 
GitLab


From 4e88a3db52620e358af85a2f403465cdfbcb7715 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Tue, 28 Feb 2017 13:58:04 -0500
Subject: [PATCH 278/295] Add args comment

---
 apitools/base/py/batch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apitools/base/py/batch.py b/apitools/base/py/batch.py
index e9690bf..f925ccf 100644
--- a/apitools/base/py/batch.py
+++ b/apitools/base/py/batch.py
@@ -194,6 +194,8 @@ class BatchApiRequest(object):
               exception, whatever it happened to be.
           max_batch_size: int, if specified requests will be split in batches
               of given size.
+          batch_request_callback: function of (http_response, exception) passed
+              to BatchHttpRequest which will be run on any given results.
 
         Returns:
           List of ApiCalls.
-- 
GitLab


From 814de48d664231311228293a12db7977e9f411b4 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Thu, 2 Mar 2017 12:59:47 -0500
Subject: [PATCH 279/295] Fix whitespace issue

---
 apitools/base/py/batch_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py
index d684b25..bd9b120 100644
--- a/apitools/base/py/batch_test.py
+++ b/apitools/base/py/batch_test.py
@@ -153,6 +153,7 @@ class BatchTest(unittest2.TestCase):
         desired_url = 'https://www.example.com'
 
         callback_was_called = []
+
         def _Callback(response, exception):
             self.assertEqual({'status': '200'}, response.info)
             self.assertEqual('content', response.content)
-- 
GitLab


From 1baacc65b1e80d92e2756f658481f2f62143bf98 Mon Sep 17 00:00:00 2001
From: Jonathan Coveney 
Date: Thu, 2 Mar 2017 12:43:09 -0500
Subject: [PATCH 280/295] Update from 0.5.7 to 0.5.8

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 23e1a9c..b49ae0c 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.7'
+_APITOOLS_VERSION = '0.5.8'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From 1aa43065350cf5460233ed422338e2a8d3d94b70 Mon Sep 17 00:00:00 2001
From: Craig Citro 
Date: Sat, 25 Mar 2017 19:47:53 -0700
Subject: [PATCH 281/295] Add the Metadata-Flavor header for GCE detection.
 (#148)

This is required for GCE detection: newer versions of the metadata service
will reject requests without this header, meaning that GCE detection will
fail.
---
 apitools/base/py/util.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py
index 65a1510..0986ebd 100644
--- a/apitools/base/py/util.py
+++ b/apitools/base/py/util.py
@@ -63,7 +63,8 @@ def DetectGce():
     """
     try:
         o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open(
-            urllib_request.Request('http://metadata.google.internal'))
+            urllib_request.Request('http://metadata.google.internal',
+                                   headers={'Metadata-Flavor': 'Google'}))
     except urllib_error.URLError:
         return False
     return (o.getcode() == http_client.OK and
-- 
GitLab


From e8ab5d97fdba9596f7c89ff011c5ffa3b78e76ee Mon Sep 17 00:00:00 2001
From: Craig Citro 
Date: Sat, 25 Mar 2017 20:32:18 -0700
Subject: [PATCH 282/295] Allow customizing the GCE metadata service address
 via an env var. (#149)

The goal here is to make it possible for a user of a binary that depends on
this library (eg the google cloud SDK) to be able to customize where it looks
for the GCE metadata service. (An adventurous user can already customize the
GCE metadata service location via the existing global vars in this library.)
---
 apitools/base/py/credentials_lib.py | 7 ++++---
 apitools/base/py/util.py            | 6 ++++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index c9ca485..669e508 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -209,10 +209,11 @@ def _EnsureFileExists(filename):
 def _GceMetadataRequest(relative_url, use_metadata_ip=False):
     """Request the given url from the GCE metadata service."""
     if use_metadata_ip:
-        base_url = 'http://169.254.169.254/'
+        base_url = os.environ.get('GCE_METADATA_IP', '169.254.169.254')
     else:
-        base_url = 'http://metadata.google.internal/'
-    url = base_url + 'computeMetadata/v1/' + relative_url
+        base_url = os.environ.get(
+            'GCE_METADATA_ROOT', 'metadata.google.internal')
+    url = 'http://' + base_url + '/computeMetadata/v1/' + relative_url
     # Extra header requirement can be found here:
     # https://developers.google.com/compute/docs/metadata
     headers = {'Metadata-Flavor': 'Google'}
diff --git a/apitools/base/py/util.py b/apitools/base/py/util.py
index 0986ebd..112259e 100644
--- a/apitools/base/py/util.py
+++ b/apitools/base/py/util.py
@@ -61,10 +61,12 @@ def DetectGce():
     Returns:
       True iff we're running on a GCE instance.
     """
+    metadata_url = 'http://{}'.format(
+        os.environ.get('GCE_METADATA_ROOT', 'metadata.google.internal'))
     try:
         o = urllib_request.build_opener(urllib_request.ProxyHandler({})).open(
-            urllib_request.Request('http://metadata.google.internal',
-                                   headers={'Metadata-Flavor': 'Google'}))
+            urllib_request.Request(
+                metadata_url, headers={'Metadata-Flavor': 'Google'}))
     except urllib_error.URLError:
         return False
     return (o.getcode() == http_client.OK and
-- 
GitLab


From 035bc85224c8d166df7dd7cb6696cdb2d20ef7f0 Mon Sep 17 00:00:00 2001
From: Craig Citro 
Date: Mon, 3 Apr 2017 13:00:45 -0700
Subject: [PATCH 283/295] Fix a bug with ADC being None.

Unsurprisingly, None doesn't have a `create_scoped` method. :)
---
 apitools/base/py/credentials_lib.py      | 2 ++
 apitools/base/py/credentials_lib_test.py | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index 669e508..6648454 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -656,6 +656,8 @@ def _GetApplicationDefaultCredentials(
     # cloud-platform, our scopes are a subset of cloud scopes, and the
     # ADC will work.
     cp = 'https://www.googleapis.com/auth/cloud-platform'
+    if credentials is None:
+        return None
     if not isinstance(credentials, gc) or cp in scopes:
         return credentials.create_scoped(scopes)
     return None
diff --git a/apitools/base/py/credentials_lib_test.py b/apitools/base/py/credentials_lib_test.py
index 3238fca..1bf5aa7 100644
--- a/apitools/base/py/credentials_lib_test.py
+++ b/apitools/base/py/credentials_lib_test.py
@@ -80,6 +80,13 @@ class CredentialsLibTest(unittest2.TestCase):
             # The urllib module does weird things with header case.
             self.assertEqual('Google', req.get_header('Metadata-flavor'))
 
+    def testGetAdcNone(self):
+        # Tests that we correctly return None when ADC aren't present in
+        # the well-known file.
+        creds = credentials_lib._GetApplicationDefaultCredentials(
+            client_info={'scope': ''})
+        self.assertIsNone(creds)
+
 
 class TestGetRunFlowFlags(unittest2.TestCase):
 
-- 
GitLab


From 9f6062c9ab780b7ba2b3481432adf7d7f512cfde Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Mon, 17 Apr 2017 11:00:07 -0400
Subject: [PATCH 284/295] Process schemas in predictable order in codegen.

---
 apitools/gen/gen_client_lib.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apitools/gen/gen_client_lib.py b/apitools/gen/gen_client_lib.py
index c93f8a2..b910f0f 100644
--- a/apitools/gen/gen_client_lib.py
+++ b/apitools/gen/gen_client_lib.py
@@ -90,7 +90,7 @@ class DescriptorGenerator(object):
             self.__root_package, self.__base_files_package,
             self.__protorpc_package)
         schemas = self.__discovery_doc.get('schemas', {})
-        for schema_name, schema in schemas.items():
+        for schema_name, schema in sorted(schemas.items()):
             self.__message_registry.AddDescriptorFromSchema(
                 schema_name, schema)
 
-- 
GitLab


From 77f3625295cc2bea1870d98eb66e22b51375e7d2 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Tue, 18 Apr 2017 16:17:04 -0400
Subject: [PATCH 285/295] Fix new lint warnings/errors.

---
 apitools/base/protorpclite/messages.py      | 19 +++++-------
 apitools/base/protorpclite/messages_test.py |  2 --
 apitools/base/protorpclite/protojson.py     |  4 +--
 apitools/base/protorpclite/test_util.py     |  3 +-
 apitools/base/protorpclite/util.py          |  3 +-
 apitools/base/py/app2.py                    | 34 ++++++++++-----------
 apitools/base/py/base_api.py                | 10 +++---
 apitools/base/py/base_cli.py                | 10 +++---
 apitools/base/py/batch_test.py              |  3 +-
 apitools/base/py/credentials_lib.py         |  1 +
 apitools/base/py/encoding_test.py           |  1 -
 apitools/base/py/testing/mock.py            |  3 +-
 apitools/base/py/transfer.py                | 14 +++------
 apitools/gen/message_registry.py            |  3 +-
 setup.py                                    |  2 +-
 15 files changed, 48 insertions(+), 64 deletions(-)

diff --git a/apitools/base/protorpclite/messages.py b/apitools/base/protorpclite/messages.py
index ade2168..df59d18 100644
--- a/apitools/base/protorpclite/messages.py
+++ b/apitools/base/protorpclite/messages.py
@@ -169,6 +169,7 @@ FIRST_RESERVED_FIELD_NUMBER = 19000
 LAST_RESERVED_FIELD_NUMBER = 19999
 
 
+# pylint: disable=no-value-for-parameter
 class _DefinitionClass(type):
     """Base meta-class used for definition meta-classes.
 
@@ -250,8 +251,7 @@ class _DefinitionClass(type):
         outer_definition_name = cls.outer_definition_name()
         if outer_definition_name is None:
             return six.text_type(cls.__name__)
-        else:
-            return u'%s.%s' % (outer_definition_name, cls.__name__)
+        return u'%s.%s' % (outer_definition_name, cls.__name__)
 
     def outer_definition_name(cls):
         """Helper method for creating outer definition name.
@@ -264,8 +264,7 @@ class _DefinitionClass(type):
         outer_definition = cls.message_definition()
         if not outer_definition:
             return util.get_package_for_module(cls.__module__)
-        else:
-            return outer_definition.definition_name()
+        return outer_definition.definition_name()
 
     def definition_package(cls):
         """Helper method for creating creating the package of a definition.
@@ -276,8 +275,7 @@ class _DefinitionClass(type):
         outer_definition = cls.message_definition()
         if not outer_definition:
             return util.get_package_for_module(cls.__module__)
-        else:
-            return outer_definition.definition_package()
+        return outer_definition.definition_package()
 
 
 class _EnumClass(_DefinitionClass):
@@ -1103,8 +1101,7 @@ class FieldList(list):
         message_class = self.__field.message_definition()
         if message_class is None:
             return self.__field, None, None
-        else:
-            return None, message_class, self.__field.number
+        return None, message_class, self.__field.number
 
     def __setstate__(self, state):
         """Enable unpickling.
@@ -1299,8 +1296,7 @@ class Field(six.with_metaclass(_FieldMeta, object)):
             if self.repeated:
                 value = FieldList(self, value)
             else:
-                value = (  # pylint: disable=redefined-variable-type
-                    self.validate(value))
+                value = self.validate(value)
             message_instance._Message__tags[self.number] = value
 
     def __get__(self, message_instance, message_class):
@@ -1310,8 +1306,7 @@ class Field(six.with_metaclass(_FieldMeta, object)):
         result = message_instance._Message__tags.get(self.number)
         if result is None:
             return self.default
-        else:
-            return result
+        return result
 
     def validate_element(self, value):
         """Validate single element of field.
diff --git a/apitools/base/protorpclite/messages_test.py b/apitools/base/protorpclite/messages_test.py
index 04ce871..78fe76e 100644
--- a/apitools/base/protorpclite/messages_test.py
+++ b/apitools/base/protorpclite/messages_test.py
@@ -851,7 +851,6 @@ class FieldTest(test_util.TestCase):
         field = messages.FloatField(1)
         self.assertEquals(type(field.validate_element(12)), float)
         self.assertEquals(type(field.validate_element(12.0)), float)
-        # pylint: disable=redefined-variable-type
         field = messages.IntegerField(1)
         self.assertEquals(type(field.validate_element(12)), int)
         self.assertRaises(messages.ValidationError,
@@ -1659,7 +1658,6 @@ class MessageTest(test_util.TestCase):
             messages.ValidationError,
             "Field val is repeated. Found: ",
             setattr, message, 'val', SubMessage())
-        # pylint: disable=redefined-variable-type
         message.val = [SubMessage()]
         message_field.validate(message)
 
diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py
index 0c6c92c..781afff 100644
--- a/apitools/base/protorpclite/protojson.py
+++ b/apitools/base/protorpclite/protojson.py
@@ -125,8 +125,8 @@ class MessageJSONEncoder(json.JSONEncoder):
                     unknown_key)
                 result[unknown_key] = unrecognized_field
             return result
-        else:
-            return super(MessageJSONEncoder, self).default(value)
+
+        return super(MessageJSONEncoder, self).default(value)
 
 
 class ProtoJson(object):
diff --git a/apitools/base/protorpclite/test_util.py b/apitools/base/protorpclite/test_util.py
index f6bc2de..a86cfc7 100644
--- a/apitools/base/protorpclite/test_util.py
+++ b/apitools/base/protorpclite/test_util.py
@@ -641,5 +641,4 @@ def get_module_name(module_attribute):
         module_file = inspect.getfile(module_attribute)
         default = os.path.basename(module_file).split('.')[0]
         return default
-    else:
-        return module_attribute.__module__
+    return module_attribute.__module__
diff --git a/apitools/base/protorpclite/util.py b/apitools/base/protorpclite/util.py
index f6147e9..7a7797d 100644
--- a/apitools/base/protorpclite/util.py
+++ b/apitools/base/protorpclite/util.py
@@ -189,8 +189,7 @@ def get_package_for_module(module):
                 split_name = os.path.splitext(base_name)
                 if len(split_name) == 1:
                     return six.text_type(base_name)
-                else:
-                    return u'.'.join(split_name[:-1])
+                return u'.'.join(split_name[:-1])
 
         return six.text_type(module.__name__)
 
diff --git a/apitools/base/py/app2.py b/apitools/base/py/app2.py
index 9aba591..c0ea9e0 100644
--- a/apitools/base/py/app2.py
+++ b/apitools/base/py/app2.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 """Appcommands-compatible command class with extra fixins."""
+from __future__ import absolute_import
 from __future__ import print_function
 
 import cmd
@@ -25,11 +26,11 @@ import sys
 import traceback
 import types
 
-import six
-
+import gflags as flags
 from google.apputils import app
 from google.apputils import appcommands
-import gflags as flags
+import six
+
 
 __all__ = [
     'NewCmd',
@@ -50,8 +51,7 @@ def _SafeMakeAscii(s):
         return s.encode('ascii')
     elif isinstance(s, str):
         return s.decode('ascii')
-    else:
-        return six.text_type(s).encode('ascii', 'backslashreplace')
+    return six.text_type(s).encode('ascii', 'backslashreplace')
 
 
 class NewCmd(appcommands.Cmd):
@@ -91,8 +91,7 @@ class NewCmd(appcommands.Cmd):
     def _GetFlag(self, flagname):
         if flagname in self._command_flags:
             return self._command_flags[flagname]
-        else:
-            return None
+        return None
 
     def Run(self, argv):
         """Run this command.
@@ -129,8 +128,7 @@ class NewCmd(appcommands.Cmd):
 
         if self._debug_mode:
             return self.RunDebug(args, {})
-        else:
-            return self.RunSafely(args, {})
+        return self.RunSafely(args, {})
 
     def RunCmdLoop(self, argv):
         """Hook for use in cmd.Cmd-based command shells."""
@@ -220,7 +218,7 @@ class CommandLoop(cmd.Cmd):
     def last_return_code(self):
         return self._last_return_code
 
-    def _set_prompt(self):
+    def _set_prompt(self):  # pylint: disable=invalid-name
         self.prompt = self._default_prompt
 
     def do_EOF(self, *unused_args):  # pylint: disable=invalid-name
@@ -306,11 +304,14 @@ class CommandLoop(cmd.Cmd):
         names.remove('do_EOF')
         return names
 
-    def do_help(self, command_name):
+    def do_help(self, arg):
         """Print the help for command_name (if present) or general help."""
 
+        command_name = arg
+
         # TODO(craigcitro): Add command-specific flags.
         def FormatOneCmd(name, command, command_names):
+            """Format one command."""
             indent_size = appcommands.GetMaxCommandLength() + 3
             if len(command_names) > 1:
                 indent = ' ' * indent_size
@@ -322,12 +323,11 @@ class CommandLoop(cmd.Cmd):
                 first_line = '%-*s%s' % (indent_size,
                                          name + ':', first_help_line)
                 return '\n'.join((first_line, rest))
-            else:
-                default_indent = '  '
-                return '\n' + flags.TextWrap(
-                    command.CommandGetHelp('', cmd_names=command_names),
-                    indent=default_indent,
-                    firstline_indent=default_indent) + '\n'
+            default_indent = '  '
+            return '\n' + flags.TextWrap(
+                command.CommandGetHelp('', cmd_names=command_names),
+                indent=default_indent,
+                firstline_indent=default_indent) + '\n'
 
         if not command_name:
             print('\nHelp for commands:\n')
diff --git a/apitools/base/py/base_api.py b/apitools/base/py/base_api.py
index 56aae3d..98836c9 100644
--- a/apitools/base/py/base_api.py
+++ b/apitools/base/py/base_api.py
@@ -340,6 +340,7 @@ class BaseApiClient(object):
     @property
     def _default_global_params(self):
         if self.__default_global_params is None:
+            # pylint: disable=not-callable
             self.__default_global_params = self.params_type()
         return self.__default_global_params
 
@@ -605,11 +606,10 @@ class BaseApiService(object):
                 request_url=http_response.request_url)
         if self.__client.response_type_model == 'json':
             return http_response.content
-        else:
-            response_type = _LoadClass(method_config.response_type_name,
-                                       self.__client.MESSAGES_MODULE)
-            return self.__client.DeserializeMessage(
-                response_type, http_response.content)
+        response_type = _LoadClass(method_config.response_type_name,
+                                   self.__client.MESSAGES_MODULE)
+        return self.__client.DeserializeMessage(
+            response_type, http_response.content)
 
     def __SetBaseHeaders(self, http_request, client):
         """Fill in the basic headers on http_request."""
diff --git a/apitools/base/py/base_cli.py b/apitools/base/py/base_cli.py
index d6fe67f..2527e64 100644
--- a/apitools/base/py/base_cli.py
+++ b/apitools/base/py/base_cli.py
@@ -16,6 +16,8 @@
 
 """Base script for generated CLI."""
 
+from __future__ import absolute_import
+
 import atexit
 import code
 import logging
@@ -24,8 +26,8 @@ import readline
 import rlcompleter
 import sys
 
-from google.apputils import appcommands
 import gflags as flags
+from google.apputils import appcommands
 
 from apitools.base.py import encoding
 from apitools.base.py import exceptions
@@ -97,15 +99,13 @@ class _SmartCompleter(rlcompleter.Completer):
         if ('(' in readline.get_line_buffer() or
                 not callable(val)):
             return word
-        else:
-            return word + '('
+        return word + '('
 
     def complete(self, text, state):
         if not readline.get_line_buffer().strip():
             if not state:
                 return '  '
-            else:
-                return None
+            return None
         return rlcompleter.Completer.complete(self, text, state)
 
 
diff --git a/apitools/base/py/batch_test.py b/apitools/base/py/batch_test.py
index bd9b120..9bf9dd0 100644
--- a/apitools/base/py/batch_test.py
+++ b/apitools/base/py/batch_test.py
@@ -94,8 +94,7 @@ class BatchTest(unittest2.TestCase):
             self.assertEqual(expected_request.http_method, request.http_method)
             if isinstance(response, list):
                 return response.pop(0)
-            else:
-                return response
+            return response
 
         mock_request.side_effect = CheckRequest
 
diff --git a/apitools/base/py/credentials_lib.py b/apitools/base/py/credentials_lib.py
index 6648454..913c144 100644
--- a/apitools/base/py/credentials_lib.py
+++ b/apitools/base/py/credentials_lib.py
@@ -379,6 +379,7 @@ class GceAssertionCredentials(gce.AppAssertionCredentials):
         return util.NormalizeScopes(scope.strip()
                                     for scope in response.readlines())
 
+    # pylint: disable=arguments-differ
     def _refresh(self, do_request):
         """Refresh self.access_token.
 
diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py
index 3564ca0..f1a3887 100644
--- a/apitools/base/py/encoding_test.py
+++ b/apitools/base/py/encoding_test.py
@@ -198,7 +198,6 @@ class EncodingTest(unittest2.TestCase):
             '{"nested": {"additional_properties": []}}',
             encoding.MessageToJson(
                 msg, include_fields=['nested.additional_properties']))
-        # pylint: disable=redefined-variable-type
         msg = ExtraNestedMessage(nested=msg)
         self.assertEqual(
             '{"nested": {"nested": null}}',
diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py
index 6642dc8..e9e78e7 100644
--- a/apitools/base/py/testing/mock.py
+++ b/apitools/base/py/testing/mock.py
@@ -245,8 +245,7 @@ class _MockedMethod(object):
 def _MakeMockedService(api_name, collection_name,
                        mock_client, service, real_service):
     class MockedService(base_api.BaseApiService):
-        def __init__(self, real_client):
-            super(MockedService, self).__init__(real_client)
+        pass
 
     for method in service.GetMethodsList():
         real_method = None
diff --git a/apitools/base/py/transfer.py b/apitools/base/py/transfer.py
index 29bbbcc..9fb63a8 100644
--- a/apitools/base/py/transfer.py
+++ b/apitools/base/py/transfer.py
@@ -271,9 +271,8 @@ class Download(_Transfer):
     def __str__(self):
         if not self.initialized:
             return 'Download (uninitialized)'
-        else:
-            return 'Download with %d/%s bytes transferred from url %s' % (
-                self.progress, self.total_size, self.url)
+        return 'Download with %d/%s bytes transferred from url %s' % (
+            self.progress, self.total_size, self.url)
 
     def ConfigureRequest(self, http_request, url_builder):
         url_builder.query_params['alt'] = 'media'
@@ -648,9 +647,8 @@ class Upload(_Transfer):
     def __str__(self):
         if not self.initialized:
             return 'Upload (uninitialized)'
-        else:
-            return 'Upload with %d/%s bytes transferred for url %s' % (
-                self.progress, self.total_size or '???', self.url)
+        return 'Upload with %d/%s bytes transferred for url %s' % (
+            self.progress, self.total_size or '???', self.url)
 
     @property
     def strategy(self):
@@ -850,8 +848,7 @@ class Upload(_Transfer):
         # go ahead and pump the bytes now.
         if self.auto_transfer:
             return self.StreamInChunks()
-        else:
-            return http_response
+        return http_response
 
     def __GetLastByte(self, range_header):
         _, _, end = range_header.partition('-')
@@ -992,7 +989,6 @@ class Upload(_Transfer):
             # https://code.google.com/p/httplib2/issues/detail?id=176 which can
             # cause httplib2 to skip bytes on 401's for file objects.
             # Rework this solution to be more general.
-            # pylint: disable=redefined-variable-type
             body_stream = body_stream.read(self.chunksize)
         else:
             end = min(start + self.chunksize, self.total_size)
diff --git a/apitools/gen/message_registry.py b/apitools/gen/message_registry.py
index a7e9a92..4f004de 100644
--- a/apitools/gen/message_registry.py
+++ b/apitools/gen/message_registry.py
@@ -441,8 +441,7 @@ class MessageRegistry(object):
                     entry_name_hint, items.get('items'), parent_name)
                 return TypeInfo(type_name=entry_type_name,
                                 variant=messages.Variant.MESSAGE)
-            else:
-                return self.__GetTypeInfo(items, entry_name_hint)
+            return self.__GetTypeInfo(items, entry_name_hint)
         elif type_name == 'any':
             self.__AddImport('from %s import extra_types' %
                              self.__base_files_package)
diff --git a/setup.py b/setup.py
index b49ae0c..017f13d 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ except ImportError:
 REQUIRED_PACKAGES = [
     'httplib2>=0.8',
     'oauth2client>=1.5.2,<4.0.0dev',
-    'setuptools>=18.5',
+    'setuptools==18.5',
     'six>=1.9.0',
     ]
 
-- 
GitLab


From bb0b8832da897053f82eea85cd80df21d6f8cc2f Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Thu, 20 Apr 2017 14:24:09 -0400
Subject: [PATCH 286/295] Update for v0.5.9 release.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 017f13d..b035cdc 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.8'
+_APITOOLS_VERSION = '0.5.9'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From ec03d83eb4fd251537e4a12e22a7adb984ea6d65 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj 
Date: Tue, 25 Apr 2017 11:04:42 -0700
Subject: [PATCH 287/295] Upgrade setup tools to the latest version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index b035cdc..6423503 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ except ImportError:
 REQUIRED_PACKAGES = [
     'httplib2>=0.8',
     'oauth2client>=1.5.2,<4.0.0dev',
-    'setuptools==18.5',
+    'setuptools>=35.0.1',
     'six>=1.9.0',
     ]
 
-- 
GitLab


From 3add89c891c39d7985484eb22cb4f4774d531f2a Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj 
Date: Tue, 25 Apr 2017 11:24:11 -0700
Subject: [PATCH 288/295] Empty list is a valid response for repeated fields

---
 apitools/base/protorpclite/protojson.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/apitools/base/protorpclite/protojson.py b/apitools/base/protorpclite/protojson.py
index 781afff..4c87cf4 100644
--- a/apitools/base/protorpclite/protojson.py
+++ b/apitools/base/protorpclite/protojson.py
@@ -274,10 +274,6 @@ class ProtoJson(object):
                     message.set_unrecognized_field(key, value, variant)
                 continue
 
-            # This is just for consistency with the old behavior.
-            if value == []:
-                continue
-
             if field.repeated:
                 # This should be unnecessary? Or in fact become an error.
                 if not isinstance(value, list):
@@ -286,6 +282,9 @@ class ProtoJson(object):
                                for item in value]
                 setattr(message, field.name, valid_value)
             else:
+                # This is just for consistency with the old behavior.
+                if value == []:
+                    continue
                 setattr(message, field.name, self.decode_field(field, value))
 
         return message
-- 
GitLab


From d34466c18870bce9e5629b6594f403db3061a6e8 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj 
Date: Tue, 25 Apr 2017 12:16:35 -0700
Subject: [PATCH 289/295] Add unittest for repeated list fields being parsed
 correctly

---
 apitools/base/protorpclite/protojson_test.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/apitools/base/protorpclite/protojson_test.py b/apitools/base/protorpclite/protojson_test.py
index a349710..4e4702a 100644
--- a/apitools/base/protorpclite/protojson_test.py
+++ b/apitools/base/protorpclite/protojson_test.py
@@ -401,6 +401,16 @@ class ProtojsonTest(test_util.TestCase,
             MyMessage, '{"a_repeated_custom": [1, 2, 3]}')
         self.assertEquals(MyMessage(a_repeated_custom=[1, 2, 3]), message)
 
+    def testDecodeRepeatedEmpty(self):
+        message = protojson.decode_message(
+            MyMessage, '{"a_repeated": []}')
+        self.assertEquals(MyMessage(a_repeated=[]), message)
+
+    def testDecodeNone(self):
+        message = protojson.decode_message(
+            MyMessage, '{"an_integer": []}')
+        self.assertEquals(MyMessage(an_integer=None), message)
+
     def testDecodeBadBase64BytesField(self):
         """Test decoding improperly encoded base64 bytes value."""
         self.assertRaisesWithRegexpMatch(
-- 
GitLab


From 22d5cd19a03c09ee69c36e135db8bd63b05e78de Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj 
Date: Tue, 25 Apr 2017 13:11:57 -0700
Subject: [PATCH 290/295] Remove extra dependencies on setuptools and six

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6423503..86e66f9 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,6 @@ except ImportError:
 REQUIRED_PACKAGES = [
     'httplib2>=0.8',
     'oauth2client>=1.5.2,<4.0.0dev',
-    'setuptools>=35.0.1',
     'six>=1.9.0',
     ]
 
-- 
GitLab


From 91d8c9f4eacdb661b7537e7363cc0c510eadddab Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj 
Date: Tue, 25 Apr 2017 13:40:09 -0700
Subject: [PATCH 291/295] Update version to 0.5.10 for release

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6423503..caca9ee 100644
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.9'
+_APITOOLS_VERSION = '0.5.10'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab


From b21f6d78b3624dcbbc0ddbb7108e80adc08e6ccf Mon Sep 17 00:00:00 2001
From: Mark Pellegrini 
Date: Fri, 28 Apr 2017 16:05:02 -0400
Subject: [PATCH 292/295] Fixing _MockedMethod so that you can call
 client.GetMethodsList() when a client is mocked out using
 mock.Client().Mock().

---
 apitools/base/py/testing/mock.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apitools/base/py/testing/mock.py b/apitools/base/py/testing/mock.py
index e9e78e7..89adca5 100644
--- a/apitools/base/py/testing/mock.py
+++ b/apitools/base/py/testing/mock.py
@@ -186,6 +186,7 @@ class _MockedMethod(object):
     """A mocked API service method."""
 
     def __init__(self, key, mocked_client, real_method):
+        self.__name__ = real_method.__name__
         self.__key = key
         self.__mocked_client = mocked_client
         self.__real_method = real_method
-- 
GitLab


From d6db0445f628ec9d7fbd08e06ea62a99b6ef16ac Mon Sep 17 00:00:00 2001
From: Zachary Newman 
Date: Fri, 28 Apr 2017 16:14:18 -0400
Subject: [PATCH 293/295] Add function to encode AdditionalProperty messages

It takes in a dict-like object, optionally sorts the items, and returns
a populated message.
---
 apitools/base/py/encoding.py      | 11 +++++++++
 apitools/base/py/encoding_test.py | 40 +++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py
index baf6537..6e38bbe 100644
--- a/apitools/base/py/encoding.py
+++ b/apitools/base/py/encoding.py
@@ -117,6 +117,17 @@ def MessageToDict(message):
     """Convert the given message to a dictionary."""
     return json.loads(MessageToJson(message))
 
+def DictToProtoMap(properties, additional_property_type, sort_items=False):
+    """Convert the given dictionary to an AdditionalProperty message."""
+    items = properties.items()
+    if sort_items:
+        items = sorted(items)
+    map_ = []
+    for key, value in items:
+        map_.append(additional_property_type.AdditionalProperty(
+            key=key, value=value))
+    return additional_property_type(additional_properties=map_)
+
 
 def PyValueToMessage(message_type, value):
     """Convert the given python value to a message of type message_type."""
diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py
index f1a3887..f9eb1fc 100644
--- a/apitools/base/py/encoding_test.py
+++ b/apitools/base/py/encoding_test.py
@@ -463,3 +463,43 @@ class EncodingTest(unittest2.TestCase):
         encoded_msg = '{"a": [{"one": 1}]}'
         msg = encoding.JsonToMessage(RepeatedJsonValueMessage, encoded_msg)
         self.assertEqual(encoded_msg, encoding.MessageToJson(msg))
+
+    def testDictToProtoMap(self):
+        dict_ = {'key': 'value'}
+
+        encoded_msg = encoding.DictToProtoMap(dict_,
+                                              AdditionalPropertiesMessage)
+        expected_msg = AdditionalPropertiesMessage()
+        expected_msg.additional_properties = [
+            AdditionalPropertiesMessage.AdditionalProperty(
+                key='key', value='value')
+        ]
+        self.assertEqual(encoded_msg, expected_msg)
+
+    def testDictToProtoMapSorted(self):
+        tuples = [('key{0:02}'.format(i), 'value') for i in range(100)]
+        dict_ = dict(tuples)
+
+        encoded_msg = encoding.DictToProtoMap(dict_,
+                                              AdditionalPropertiesMessage,
+                                              sort_items=True)
+        expected_msg = AdditionalPropertiesMessage()
+        expected_msg.additional_properties = [
+            AdditionalPropertiesMessage.AdditionalProperty(
+                key=key, value=value)
+            for key, value in tuples
+        ]
+        self.assertEqual(encoded_msg, expected_msg)
+
+
+    def testDictToProtoMapNumeric(self):
+        dict_ = {'key': 1}
+
+        encoded_msg = encoding.DictToProtoMap(dict_,
+                                              AdditionalIntPropertiesMessage)
+        expected_msg = AdditionalIntPropertiesMessage()
+        expected_msg.additional_properties = [
+            AdditionalIntPropertiesMessage.AdditionalProperty(
+                key='key', value=1)
+        ]
+        self.assertEqual(encoded_msg, expected_msg)
-- 
GitLab


From 4b1cf184e628efe79a4592a9091385a96dba0170 Mon Sep 17 00:00:00 2001
From: Zachary Newman 
Date: Fri, 28 Apr 2017 16:28:08 -0400
Subject: [PATCH 294/295] Fix up lint errors

---
 apitools/base/py/encoding.py      | 1 +
 apitools/base/py/encoding_test.py | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/apitools/base/py/encoding.py b/apitools/base/py/encoding.py
index 6e38bbe..598f6e6 100644
--- a/apitools/base/py/encoding.py
+++ b/apitools/base/py/encoding.py
@@ -117,6 +117,7 @@ def MessageToDict(message):
     """Convert the given message to a dictionary."""
     return json.loads(MessageToJson(message))
 
+
 def DictToProtoMap(properties, additional_property_type, sort_items=False):
     """Convert the given dictionary to an AdditionalProperty message."""
     items = properties.items()
diff --git a/apitools/base/py/encoding_test.py b/apitools/base/py/encoding_test.py
index f9eb1fc..cb6bfe5 100644
--- a/apitools/base/py/encoding_test.py
+++ b/apitools/base/py/encoding_test.py
@@ -491,7 +491,6 @@ class EncodingTest(unittest2.TestCase):
         ]
         self.assertEqual(encoded_msg, expected_msg)
 
-
     def testDictToProtoMapNumeric(self):
         dict_ = {'key': 1}
 
-- 
GitLab


From 4de8ac93e119c3556bb0e96c45842abf1a6b6540 Mon Sep 17 00:00:00 2001
From: "Arthur D. Cherba" 
Date: Mon, 8 May 2017 11:25:43 -0400
Subject: [PATCH 295/295] Update for v0.5.11 release.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f127fe6..9667697 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,7 @@ py_version = platform.python_version()
 if py_version < '2.7':
     REQUIRED_PACKAGES.append('argparse>=1.2.1')
 
-_APITOOLS_VERSION = '0.5.10'
+_APITOOLS_VERSION = '0.5.11'
 
 with open('README.rst') as fileobj:
     README = fileobj.read()
-- 
GitLab