diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000000000000000000000000000000000000..34e433b7dfc1575aef36d0fff300d8a5a30b4d4c --- /dev/null +++ b/Android.bp @@ -0,0 +1,43 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +python_binary_host { + name: "external_updater", + main: "external_updater.py", + srcs: [ + "*.py", + "metadata.proto", + ], + libs: [ + "python-symbol", + "libprotobuf-python", + ], + proto: { + canonical_path_from_root: false, + }, + data: [ + "update_package.sh", + ], + version: { + py2: { + enabled: false, + embedded_launcher: false, + }, + py3: { + enabled: true, + embedded_launcher: false, + }, + }, +} + diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f52b2bd167cd9dbcbd5cc64d95efc356a11f1ea9 --- /dev/null +++ b/PREUPLOAD.cfg @@ -0,0 +1,5 @@ +[Builtin Hooks] +pylint = true + +[Builtin Hooks Options] +pylint = --executable-path pylint3 ${PREUPLOAD_FILES} diff --git a/archive_utils.py b/archive_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..91007a751bdfd4965ba28667081cad1ebbe47d9a --- /dev/null +++ b/archive_utils.py @@ -0,0 +1,130 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions to process archive files.""" + +import os +import tempfile +import tarfile +import urllib.parse +import zipfile + + +class ZipFileWithPermission(zipfile.ZipFile): + """Subclassing Zipfile to preserve file permission. + + See https://bugs.python.org/issue15795 + """ + + def extract(self, member, path=None, pwd=None): + ret_val = super().extract(member, path, pwd) + + if not isinstance(member, zipfile.ZipInfo): + member = self.getinfo(member) + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(ret_val, attr) + return ret_val + + +def unzip(archive_path, target_path): + """Extracts zip file to a path. + + Args: + archive_path: Path to the zip file. + target_path: Path to extract files to. + """ + + with ZipFileWithPermission(archive_path) as zfile: + zfile.extractall(target_path) + + +def untar(archive_path, target_path): + """Extracts tar file to a path. + + Args: + archive_path: Path to the tar file. + target_path: Path to extract files to. + """ + + with tarfile.open(archive_path, mode='r') as tfile: + tfile.extractall(target_path) + + +ARCHIVE_TYPES = { + '.zip': unzip, + '.tar.gz': untar, + '.tar.bz2': untar, + '.tar.xz': untar, +} + + +def is_supported_archive(url): + """Checks whether the url points to a supported archive.""" + return get_extract_func(url) is not None + + +def get_extract_func(url): + """Gets the function to extract an archive. + + Args: + url: The url to the archive file. + + Returns: + A function to extract the archive. None if not found. + """ + + parsed_url = urllib.parse.urlparse(url) + filename = os.path.basename(parsed_url.path) + for ext, func in ARCHIVE_TYPES.items(): + if filename.endswith(ext): + return func + return None + + +def download_and_extract(url): + """Downloads and extracts an archive file to a temporary directory. + + Args: + url: Url to download. + + Returns: + Path to the temporary directory. + """ + + print('Downloading {}'.format(url)) + archive_file, _headers = urllib.request.urlretrieve(url) + + temporary_dir = tempfile.mkdtemp() + print('Extracting {} to {}'.format(archive_file, temporary_dir)) + get_extract_func(url)(archive_file, temporary_dir) + + return temporary_dir + + +def find_archive_root(path): + """Finds the real root of an extracted archive. + + Sometimes archives has additional layers of directories. This function tries + to guess the right 'root' path by entering all single sub-directories. + + Args: + path: Path to the extracted archive. + + Returns: + The root path we found. + """ + for root, dirs, files in os.walk(path): + if files or len(dirs) > 1: + return root + return path diff --git a/external_updater.py b/external_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..4200676ff84edf240b9f89084253f568005d2671 --- /dev/null +++ b/external_updater.py @@ -0,0 +1,197 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A commandline tool to check and update packages in external/ + +Example usage: +updater.sh checkall +updater.sh update kotlinc +""" + +import argparse +import os + +from google.protobuf import text_format # pylint: disable=import-error + +import fileutils +from github_archive_updater import GithubArchiveUpdater +import updater_utils + + +UPDATERS = [GithubArchiveUpdater] + + +def color_string(string, color): + """Changes the color of a string when print to terminal.""" + colors = { + 'SUCCESS': '\x1b[92m', + 'FAILED': '\x1b[91m', + } + end_color = '\033[0m' + return colors[color] + string + end_color + + +def build_updater(proj_path): + """Build updater for a project specified by proj_path. + + Reads and parses METADATA file. And builds updater based on the information. + + Args: + proj_path: Absolute or relative path to the project. + + Returns: + The updater object built. None if there's any error. + """ + + proj_path = fileutils.get_absolute_project_path(proj_path) + try: + metadata = fileutils.read_metadata(proj_path) + except text_format.ParseError as err: + print('{} {}.'.format(color_string('Invalid metadata file:', 'FAILED'), + err)) + return None + + try: + updater = updater_utils.create_updater(metadata, proj_path, UPDATERS) + except ValueError: + print(color_string('No supported URL.', 'FAILED')) + return None + return updater + + +def check_update(proj_path): + """Checks updates for a project. Prints result on console. + + Args: + proj_path: Absolute or relative path to the project. + """ + + print( + '{} {}. '.format(color_string('Checking', 'SUCCESS'), + fileutils.get_relative_project_path(proj_path)), + end='') + updater = build_updater(proj_path) + if updater is None: + return + try: + latest = updater.get_latest_version() + current = updater.get_current_version() + except IOError as e: + print('{} {}.'.format(color_string('Failed to check latest version.', + 'FAILED'), + e)) + return + + if current != latest: + print('{} Current version: {}. Latest version: {}.'. format( + color_string('New version found.', 'SUCCESS'), current, latest)) + else: + print('No new version. Current version: {}.'.format(latest)) + + +def check(args): + """Handler for check command.""" + + check_update(args.path) + + +def update(args): + """Handler for update command.""" + + updater = build_updater(args.path) + if updater is None: + return + try: + latest = updater.get_latest_version() + current = updater.get_current_version() + except IOError as e: + print('{} {}.'.format( + color_string('Failed to check latest version.', + 'FAILED'), + e)) + return + + if current == latest and not args.force: + print( + '{} for {}. Current version {} is latest. ' + 'Use --force to update anyway.'.format( + color_string( + 'Nothing to update', + 'FAILED'), + args.path, + current)) + return + + print('{} from version {} to version {}.{}'.format( + color_string('Updating', 'SUCCESS'), args.path, current, latest)) + + updater.update() + + +def checkall(args): + """Handler for checkall command.""" + for root, _dirs, files in os.walk(args.path): + if fileutils.METADATA_FILENAME in files: + check_update(root) + + +def parse_args(): + """Parses commandline arguments.""" + + parser = argparse.ArgumentParser( + description='Check updates for third party projects in external/.') + subparsers = parser.add_subparsers(dest='cmd') + subparsers.required = True + + # Creates parser for check command. + check_parser = subparsers.add_parser( + 'check', help='Check update for one project.') + check_parser.add_argument( + 'path', + help='Path of the project. ' + 'Relative paths will be resolved from external/.') + check_parser.set_defaults(func=check) + + # Creates parser for checkall command. + checkall_parser = subparsers.add_parser( + 'checkall', help='Check update for all projects.') + checkall_parser.add_argument( + '--path', + default=fileutils.EXTERNAL_PATH, + help='Starting path for all projects. Default to external/.') + checkall_parser.set_defaults(func=checkall) + + # Creates parser for update command. + update_parser = subparsers.add_parser('update', help='Update one project.') + update_parser.add_argument( + 'path', + help='Path of the project. ' + 'Relative paths will be resolved from external/.') + update_parser.add_argument( + '--force', + help='Run update even if there\'s no new version.', + action='store_true') + update_parser.set_defaults(func=update) + + return parser.parse_args() + + +def main(): + """The main entry.""" + + args = parse_args() + args.func(args) + + +if __name__ == '__main__': + main() diff --git a/fileutils.py b/fileutils.py new file mode 100644 index 0000000000000000000000000000000000000000..e0a0f66f18c3d49cfd472e01d63ab63ffb3fae5c --- /dev/null +++ b/fileutils.py @@ -0,0 +1,86 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tool functions to deal with files.""" + +import datetime +import os + +from google.protobuf import text_format # pylint: disable=import-error + +import metadata_pb2 # pylint: disable=import-error + +ANDROID_TOP = os.environ.get('ANDROID_BUILD_TOP', os.getcwd()) +EXTERNAL_PATH = os.path.join(ANDROID_TOP, 'external/') + +METADATA_FILENAME = 'METADATA' + + +def get_absolute_project_path(project_path): + """Gets absolute path of a project. + + Path resolution starts from external/. + """ + return os.path.join(EXTERNAL_PATH, project_path) + + +def get_metadata_path(project_path): + """Gets the absolute path of METADATA for a project.""" + return os.path.join( + get_absolute_project_path(project_path), METADATA_FILENAME) + + +def get_relative_project_path(project_path): + """Gets the relative path of a project starting from external/.""" + project_path = get_absolute_project_path(project_path) + return os.path.relpath(project_path, EXTERNAL_PATH) + + +def read_metadata(proj_path): + """Reads and parses METADATA file for a project. + + Args: + proj_path: Path to the project. + + Returns: + Parsed MetaData proto. + + Raises: + text_format.ParseError: Occurred when the METADATA file is invalid. + FileNotFoundError: Occurred when METADATA file is not found. + """ + + with open(get_metadata_path(proj_path), 'r') as metadata_file: + metadata = metadata_file.read() + return text_format.Parse(metadata, metadata_pb2.MetaData()) + + +def write_metadata(proj_path, metadata): + """Writes updated METADATA file for a project. + + This function updates last_upgrade_date in metadata and write to the project + directory. + + Args: + proj_path: Path to the project. + metadata: The MetaData proto to write. + """ + + date = metadata.third_party.last_upgrade_date + now = datetime.datetime.now() + date.year = now.year + date.month = now.month + date.day = now.day + text_metadata = text_format.MessageToString(metadata) + with open(get_metadata_path(proj_path), 'w') as metadata_file: + metadata_file.write(text_metadata) diff --git a/github_archive_updater.py b/github_archive_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..7f0bb770994c6f3a25df65cb1d97df0e9e5a1514 --- /dev/null +++ b/github_archive_updater.py @@ -0,0 +1,111 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module to update packages from GitHub archive.""" + + +import json +import re +import shutil +import urllib.request + +import archive_utils +import fileutils +import metadata_pb2 # pylint: disable=import-error +import updater_utils + +GITHUB_URL_PATTERN = (r'^https:\/\/github.com\/([-\w]+)\/([-\w]+)\/' + + r'(releases\/download\/|archive\/)') +GITHUB_URL_RE = re.compile(GITHUB_URL_PATTERN) + + +class GithubArchiveUpdater(): + """Updater for archives from GitHub. + + This updater supports release archives in GitHub. Version is determined by + release name in GitHub. + """ + + VERSION_FIELD = 'tag_name' + + def __init__(self, url, proj_path, metadata): + self.proj_path = proj_path + self.metadata = metadata + self.old_url = url + self.owner = None + self.repo = None + self.data = None + self._parse_url(url) + + def _parse_url(self, url): + if url.type != metadata_pb2.URL.ARCHIVE: + raise ValueError('Only archive url from Github is supported.') + match = GITHUB_URL_RE.match(url.value) + if match is None: + raise ValueError('Url format is not supported.') + try: + self.owner, self.repo = match.group(1, 2) + except IndexError: + raise ValueError('Url format is not supported.') + + def get_latest_version(self): + """Checks upstream and returns the latest version name we found.""" + + url = 'https://api.github.com/repos/{}/{}/releases/latest'.format( + self.owner, self.repo) + with urllib.request.urlopen(url) as request: + self.data = json.loads(request.read().decode()) + return self.data[self.VERSION_FIELD] + + def get_current_version(self): + """Returns the latest version name recorded in METADATA.""" + return self.metadata.third_party.version + + def _write_metadata(self, url, path): + updated_metadata = metadata_pb2.MetaData() + updated_metadata.CopyFrom(self.metadata) + updated_metadata.third_party.version = self.data[self.VERSION_FIELD] + for metadata_url in updated_metadata.third_party.url: + if metadata_url == self.old_url: + metadata_url.value = url + fileutils.write_metadata(path, updated_metadata) + + def update(self): + """Updates the package. + + Has to call get_latest_version() before this function. + """ + + supported_assets = [ + a for a in self.data['assets'] + if archive_utils.is_supported_archive(a['browser_download_url'])] + + # Finds the minimum sized archive to download. + minimum_asset = min( + supported_assets, key=lambda asset: asset['size'], default=None) + if minimum_asset is not None: + latest_url = minimum_asset.get('browser_download_url') + else: + # Guess the tarball url for source code. + latest_url = 'https://github.com/{}/{}/archive/{}.tar.gz'.format( + self.owner, self.repo, self.data.get('tag_name')) + + temporary_dir = None + try: + temporary_dir = archive_utils.download_and_extract(latest_url) + package_dir = archive_utils.find_archive_root(temporary_dir) + self._write_metadata(latest_url, package_dir) + updater_utils.replace_package(package_dir, self.proj_path) + finally: + shutil.rmtree(temporary_dir, ignore_errors=True) + urllib.request.urlcleanup() diff --git a/metadata.proto b/metadata.proto new file mode 100644 index 0000000000000000000000000000000000000000..2362c2e2a5d34085eb0ab0b385315b29974ccfd4 --- /dev/null +++ b/metadata.proto @@ -0,0 +1,69 @@ +// copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A proto definition used to parse METADATA file in third party projects. + +// This proto will only contain fields and values the updater cares about. +// It is not intended to be the formal definition of METADATA file. + +syntax = "proto3"; + +package external_updater; + +message MetaData { + string name = 1; + string description = 3; + ThirdPartyMetaData third_party = 13; +} + +enum LicenseType { + UNKNOWN = 0; + BY_EXCEPTION_ONLY = 1; + NOTICE = 2; + PERMISSIVE = 3; + RECIPROCAL = 4; + RESTRICTED_IF_STATICALLY_LINKED = 5; + RESTRICTED = 6; + UNENCUMBERED = 7; +} + +message ThirdPartyMetaData { + repeated URL url = 1; + string version = 2; + LicenseType license_type = 4; + Date last_upgrade_date = 10; +} + +message URL { + enum Type { + UNKNOWN = 0; + HOMEPAGE = 1; + ARCHIVE = 2; + GIT = 3; + SVN = 7; + HG = 8; + DARCS = 9; + OTHER = 11; + } + + Type type = 1; + + string value = 2; +} + +message Date { + int32 year = 1; + int32 month = 2; + int32 day = 3; +} diff --git a/update_package.sh b/update_package.sh new file mode 100644 index 0000000000000000000000000000000000000000..8b9629a6f6d5f97ee25471fc535c562d47d9d601 --- /dev/null +++ b/update_package.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2007 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used by external_updater to replace a package. Don't +# invoke directly + +cd $1 + +# Copies all files we want to reserve. +cp -a -n $2/Android.bp $1/ 2> /dev/null +cp -a -n $2/Android.mk $1/ 2> /dev/null +cp -a -n $2/LICENSE $1/ 2> /dev/null +cp -a -n $2/NOTICE $1/ 2> /dev/null +cp -a -n $2/MODULE_LICENSE_* $1/ 2> /dev/null +cp -a -n $2/METADATA $1/ 2> /dev/null +cp -a -n $2/.git $1/ 2> /dev/null +cp -a -n $2/.gitignore $1/ 2> /dev/null +cp -a -n $2/patches $1/ 2> /dev/null +cp -a -n $2/post_update.sh $1/ 2> /dev/null + +# Applies all patches +for p in $1/patches/*.diff +do + [ -e "$p" ] || continue + echo Applying $p + patch -p1 -d $1 < $p; +done + +if [ -f $1/post_update.sh ] +then + echo Running post update script + $1/post_update.sh $1 $2 +fi + +# Swap old and new. +rm -rf $2 +mv $1 $2 diff --git a/updater.sh b/updater.sh new file mode 100755 index 0000000000000000000000000000000000000000..2a3da75381596e88e3e81e8d90d8d5913f4e4832 --- /dev/null +++ b/updater.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (C) 2007 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cd $(dirname "$0")/../.. +source build/envsetup.sh +mmma tools/external_updater +out/soong/host/linux-x86/bin/external_updater $@ diff --git a/updater_utils.py b/updater_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e9d9620e7f304c88009d947ba4644bee89f3a806 --- /dev/null +++ b/updater_utils.py @@ -0,0 +1,57 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Helper functions for updaters.""" + +import os +import subprocess +import sys + + +def create_updater(metadata, proj_path, updaters): + """Creates corresponding updater object for a project. + + Args: + metadata: Parsed proto for METADATA file. + proj_path: Absolute path for the project. + + Returns: + An updater object. + + Raises: + ValueError: Occurred when there's no updater for all urls. + """ + for url in metadata.third_party.url: + for updater in updaters: + try: + return updater(url, proj_path, metadata) + except ValueError: + pass + + raise ValueError('No supported URL.') + + +def replace_package(source_dir, target_dir): + """Invokes a shell script to prepare and update a project. + + Args: + source_dir: Path to the new downloaded and extracted package. + target_dir: The path to the project in Android source tree. + """ + + print('Updating {} using {}.'.format(target_dir, source_dir)) + script_path = os.path.join( + os.path.dirname( + sys.argv[0]), + 'update_package.sh') + subprocess.check_call(['bash', script_path, source_dir, target_dir])