Initial checkin for external updater am: dfdda475

am: ba632336

Change-Id: Ie8038f9979535e8f95dd090ed9927a33dde76da8

Android.bp

+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+python_binary_host {
+    name: "external_updater",
+    main: "external_updater.py",
+    srcs: [
+        "*.py",
+        "metadata.proto",
+    ],
+    libs: [
+        "python-symbol",
+        "libprotobuf-python",
+    ],
+    proto: {
+        canonical_path_from_root: false,
+    },
+    data: [
+        "update_package.sh",
+    ],
+    version: {
+        py2: {
+            enabled: false,
+            embedded_launcher: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: false,
+        },
+    },
+}
+

PREUPLOAD.cfg

+[Builtin Hooks]
+pylint = true
+
+[Builtin Hooks Options]
+pylint = --executable-path pylint3 ${PREUPLOAD_FILES}

archive_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Functions to process archive files."""
+
+import os
+import tempfile
+import tarfile
+import urllib.parse
+import zipfile
+
+
+class ZipFileWithPermission(zipfile.ZipFile):
+    """Subclassing Zipfile to preserve file permission.
+
+    See https://bugs.python.org/issue15795
+    """
+
+    def extract(self, member, path=None, pwd=None):
+        ret_val = super().extract(member, path, pwd)
+
+        if not isinstance(member, zipfile.ZipInfo):
+            member = self.getinfo(member)
+        attr = member.external_attr >> 16
+        if attr != 0:
+            os.chmod(ret_val, attr)
+        return ret_val
+
+
+def unzip(archive_path, target_path):
+    """Extracts zip file to a path.
+
+    Args:
+        archive_path: Path to the zip file.
+        target_path: Path to extract files to.
+    """
+
+    with ZipFileWithPermission(archive_path) as zfile:
+        zfile.extractall(target_path)
+
+
+def untar(archive_path, target_path):
+    """Extracts tar file to a path.
+
+    Args:
+        archive_path: Path to the tar file.
+        target_path: Path to extract files to.
+    """
+
+    with tarfile.open(archive_path, mode='r') as tfile:
+        tfile.extractall(target_path)
+
+
+ARCHIVE_TYPES = {
+    '.zip': unzip,
+    '.tar.gz': untar,
+    '.tar.bz2': untar,
+    '.tar.xz': untar,
+}
+
+
+def is_supported_archive(url):
+    """Checks whether the url points to a supported archive."""
+    return get_extract_func(url) is not None
+
+
+def get_extract_func(url):
+    """Gets the function to extract an archive.
+
+    Args:
+        url: The url to the archive file.
+
+    Returns:
+        A function to extract the archive. None if not found.
+    """
+
+    parsed_url = urllib.parse.urlparse(url)
+    filename = os.path.basename(parsed_url.path)
+    for ext, func in ARCHIVE_TYPES.items():
+        if filename.endswith(ext):
+            return func
+    return None
+
+
+def download_and_extract(url):
+    """Downloads and extracts an archive file to a temporary directory.
+
+    Args:
+        url: Url to download.
+
+    Returns:
+        Path to the temporary directory.
+    """
+
+    print('Downloading {}'.format(url))
+    archive_file, _headers = urllib.request.urlretrieve(url)
+
+    temporary_dir = tempfile.mkdtemp()
+    print('Extracting {} to {}'.format(archive_file, temporary_dir))
+    get_extract_func(url)(archive_file, temporary_dir)
+
+    return temporary_dir
+
+
+def find_archive_root(path):
+    """Finds the real root of an extracted archive.
+
+    Sometimes archives has additional layers of directories. This function tries
+    to guess the right 'root' path by entering all single sub-directories.
+
+    Args:
+        path: Path to the extracted archive.
+
+    Returns:
+        The root path we found.
+    """
+    for root, dirs, files in os.walk(path):
+        if files or len(dirs) > 1:
+            return root
+    return path

external_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A commandline tool to check and update packages in external/
+
+Example usage:
+updater.sh checkall
+updater.sh update kotlinc
+"""
+
+import argparse
+import os
+
+from google.protobuf import text_format    # pylint: disable=import-error
+
+import fileutils
+from github_archive_updater import GithubArchiveUpdater
+import updater_utils
+
+
+UPDATERS = [GithubArchiveUpdater]
+
+
+def color_string(string, color):
+    """Changes the color of a string when print to terminal."""
+    colors = {
+        'SUCCESS': '\x1b[92m',
+        'FAILED': '\x1b[91m',
+    }
+    end_color = '\033[0m'
+    return colors[color] + string + end_color
+
+
+def build_updater(proj_path):
+    """Build updater for a project specified by proj_path.
+
+    Reads and parses METADATA file. And builds updater based on the information.
+
+    Args:
+      proj_path: Absolute or relative path to the project.
+
+    Returns:
+      The updater object built. None if there's any error.
+    """
+
+    proj_path = fileutils.get_absolute_project_path(proj_path)
+    try:
+        metadata = fileutils.read_metadata(proj_path)
+    except text_format.ParseError as err:
+        print('{} {}.'.format(color_string('Invalid metadata file:', 'FAILED'),
+                              err))
+        return None
+
+    try:
+        updater = updater_utils.create_updater(metadata, proj_path, UPDATERS)
+    except ValueError:
+        print(color_string('No supported URL.', 'FAILED'))
+        return None
+    return updater
+
+
+def check_update(proj_path):
+    """Checks updates for a project. Prints result on console.
+
+    Args:
+      proj_path: Absolute or relative path to the project.
+    """
+
+    print(
+        '{} {}. '.format(color_string('Checking', 'SUCCESS'),
+                         fileutils.get_relative_project_path(proj_path)),
+        end='')
+    updater = build_updater(proj_path)
+    if updater is None:
+        return
+    try:
+        latest = updater.get_latest_version()
+        current = updater.get_current_version()
+    except IOError as e:
+        print('{} {}.'.format(color_string('Failed to check latest version.',
+                                           'FAILED'),
+                              e))
+        return
+
+    if current != latest:
+        print('{} Current version: {}. Latest version: {}.'. format(
+            color_string('New version found.', 'SUCCESS'), current, latest))
+    else:
+        print('No new version. Current version: {}.'.format(latest))
+
+
+def check(args):
+    """Handler for check command."""
+
+    check_update(args.path)
+
+
+def update(args):
+    """Handler for update command."""
+
+    updater = build_updater(args.path)
+    if updater is None:
+        return
+    try:
+        latest = updater.get_latest_version()
+        current = updater.get_current_version()
+    except IOError as e:
+        print('{} {}.'.format(
+            color_string('Failed to check latest version.',
+                         'FAILED'),
+            e))
+        return
+
+    if current == latest and not args.force:
+        print(
+            '{} for {}. Current version {} is latest. '
+            'Use --force to update anyway.'.format(
+                color_string(
+                    'Nothing to update',
+                    'FAILED'),
+                args.path,
+                current))
+        return
+
+    print('{} from version {} to version {}.{}'.format(
+        color_string('Updating', 'SUCCESS'), args.path, current, latest))
+
+    updater.update()
+
+
+def checkall(args):
+    """Handler for checkall command."""
+    for root, _dirs, files in os.walk(args.path):
+        if fileutils.METADATA_FILENAME in files:
+            check_update(root)
+
+
+def parse_args():
+    """Parses commandline arguments."""
+
+    parser = argparse.ArgumentParser(
+        description='Check updates for third party projects in external/.')
+    subparsers = parser.add_subparsers(dest='cmd')
+    subparsers.required = True
+
+    # Creates parser for check command.
+    check_parser = subparsers.add_parser(
+        'check', help='Check update for one project.')
+    check_parser.add_argument(
+        'path',
+        help='Path of the project. '
+        'Relative paths will be resolved from external/.')
+    check_parser.set_defaults(func=check)
+
+    # Creates parser for checkall command.
+    checkall_parser = subparsers.add_parser(
+        'checkall', help='Check update for all projects.')
+    checkall_parser.add_argument(
+        '--path',
+        default=fileutils.EXTERNAL_PATH,
+        help='Starting path for all projects. Default to external/.')
+    checkall_parser.set_defaults(func=checkall)
+
+    # Creates parser for update command.
+    update_parser = subparsers.add_parser('update', help='Update one project.')
+    update_parser.add_argument(
+        'path',
+        help='Path of the project. '
+        'Relative paths will be resolved from external/.')
+    update_parser.add_argument(
+        '--force',
+        help='Run update even if there\'s no new version.',
+        action='store_true')
+    update_parser.set_defaults(func=update)
+
+    return parser.parse_args()
+
+
+def main():
+    """The main entry."""
+
+    args = parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

fileutils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tool functions to deal with files."""
+
+import datetime
+import os
+
+from google.protobuf import text_format    # pylint: disable=import-error
+
+import metadata_pb2    # pylint: disable=import-error
+
+ANDROID_TOP = os.environ.get('ANDROID_BUILD_TOP', os.getcwd())
+EXTERNAL_PATH = os.path.join(ANDROID_TOP, 'external/')
+
+METADATA_FILENAME = 'METADATA'
+
+
+def get_absolute_project_path(project_path):
+    """Gets absolute path of a project.
+
+    Path resolution starts from external/.
+    """
+    return os.path.join(EXTERNAL_PATH, project_path)
+
+
+def get_metadata_path(project_path):
+    """Gets the absolute path of METADATA for a project."""
+    return os.path.join(
+        get_absolute_project_path(project_path), METADATA_FILENAME)
+
+
+def get_relative_project_path(project_path):
+    """Gets the relative path of a project starting from external/."""
+    project_path = get_absolute_project_path(project_path)
+    return os.path.relpath(project_path, EXTERNAL_PATH)
+
+
+def read_metadata(proj_path):
+    """Reads and parses METADATA file for a project.
+
+    Args:
+      proj_path: Path to the project.
+
+    Returns:
+      Parsed MetaData proto.
+
+    Raises:
+      text_format.ParseError: Occurred when the METADATA file is invalid.
+      FileNotFoundError: Occurred when METADATA file is not found.
+    """
+
+    with open(get_metadata_path(proj_path), 'r') as metadata_file:
+        metadata = metadata_file.read()
+        return text_format.Parse(metadata, metadata_pb2.MetaData())
+
+
+def write_metadata(proj_path, metadata):
+    """Writes updated METADATA file for a project.
+
+    This function updates last_upgrade_date in metadata and write to the project
+    directory.
+
+    Args:
+      proj_path: Path to the project.
+      metadata: The MetaData proto to write.
+    """
+
+    date = metadata.third_party.last_upgrade_date
+    now = datetime.datetime.now()
+    date.year = now.year
+    date.month = now.month
+    date.day = now.day
+    text_metadata = text_format.MessageToString(metadata)
+    with open(get_metadata_path(proj_path), 'w') as metadata_file:
+        metadata_file.write(text_metadata)

github_archive_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to update packages from GitHub archive."""
+
+
+import json
+import re
+import shutil
+import urllib.request
+
+import archive_utils
+import fileutils
+import metadata_pb2    # pylint: disable=import-error
+import updater_utils
+
+GITHUB_URL_PATTERN = (r'^https:\/\/github.com\/([-\w]+)\/([-\w]+)\/' +
+                      r'(releases\/download\/|archive\/)')
+GITHUB_URL_RE = re.compile(GITHUB_URL_PATTERN)
+
+
+class GithubArchiveUpdater():
+    """Updater for archives from GitHub.
+
+    This updater supports release archives in GitHub. Version is determined by
+    release name in GitHub.
+    """
+
+    VERSION_FIELD = 'tag_name'
+
+    def __init__(self, url, proj_path, metadata):
+        self.proj_path = proj_path
+        self.metadata = metadata
+        self.old_url = url
+        self.owner = None
+        self.repo = None
+        self.data = None
+        self._parse_url(url)
+
+    def _parse_url(self, url):
+        if url.type != metadata_pb2.URL.ARCHIVE:
+            raise ValueError('Only archive url from Github is supported.')
+        match = GITHUB_URL_RE.match(url.value)
+        if match is None:
+            raise ValueError('Url format is not supported.')
+        try:
+            self.owner, self.repo = match.group(1, 2)
+        except IndexError:
+            raise ValueError('Url format is not supported.')
+
+    def get_latest_version(self):
+        """Checks upstream and returns the latest version name we found."""
+
+        url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
+            self.owner, self.repo)
+        with urllib.request.urlopen(url) as request:
+            self.data = json.loads(request.read().decode())
+        return self.data[self.VERSION_FIELD]
+
+    def get_current_version(self):
+        """Returns the latest version name recorded in METADATA."""
+        return self.metadata.third_party.version
+
+    def _write_metadata(self, url, path):
+        updated_metadata = metadata_pb2.MetaData()
+        updated_metadata.CopyFrom(self.metadata)
+        updated_metadata.third_party.version = self.data[self.VERSION_FIELD]
+        for metadata_url in updated_metadata.third_party.url:
+            if metadata_url == self.old_url:
+                metadata_url.value = url
+        fileutils.write_metadata(path, updated_metadata)
+
+    def update(self):
+        """Updates the package.
+
+        Has to call get_latest_version() before this function.
+        """
+
+        supported_assets = [
+            a for a in self.data['assets']
+            if archive_utils.is_supported_archive(a['browser_download_url'])]
+
+        # Finds the minimum sized archive to download.
+        minimum_asset = min(
+            supported_assets, key=lambda asset: asset['size'], default=None)
+        if minimum_asset is not None:
+            latest_url = minimum_asset.get('browser_download_url')
+        else:
+            # Guess the tarball url for source code.
+            latest_url = 'https://github.com/{}/{}/archive/{}.tar.gz'.format(
+                self.owner, self.repo, self.data.get('tag_name'))
+
+        temporary_dir = None
+        try:
+            temporary_dir = archive_utils.download_and_extract(latest_url)
+            package_dir = archive_utils.find_archive_root(temporary_dir)
+            self._write_metadata(latest_url, package_dir)
+            updater_utils.replace_package(package_dir, self.proj_path)
+        finally:
+            shutil.rmtree(temporary_dir, ignore_errors=True)
+            urllib.request.urlcleanup()

metadata.proto

+// copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A proto definition used to parse METADATA file in third party projects.
+
+// This proto will only contain fields and values the updater cares about.
+// It is not intended to be the formal definition of METADATA file.
+
+syntax = "proto3";
+
+package external_updater;
+
+message MetaData {
+  string name = 1;
+  string description = 3;
+  ThirdPartyMetaData third_party = 13;
+}
+
+enum LicenseType {
+  UNKNOWN = 0;
+  BY_EXCEPTION_ONLY = 1;
+  NOTICE = 2;
+  PERMISSIVE = 3;
+  RECIPROCAL = 4;
+  RESTRICTED_IF_STATICALLY_LINKED = 5;
+  RESTRICTED = 6;
+  UNENCUMBERED = 7;
+}
+
+message ThirdPartyMetaData {
+  repeated URL url = 1;
+  string version = 2;
+  LicenseType license_type = 4;
+  Date last_upgrade_date = 10;
+}
+
+message URL {
+  enum Type {
+    UNKNOWN = 0;
+    HOMEPAGE = 1;
+    ARCHIVE = 2;
+    GIT = 3;
+    SVN = 7;
+    HG = 8;
+    DARCS = 9;
+    OTHER = 11;
+  }
+
+  Type type = 1;
+
+  string value = 2;
+}
+
+message Date {
+  int32 year = 1;
+  int32 month = 2;
+  int32 day = 3;
+}

update_package.sh

+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used by external_updater to replace a package. Don't
+# invoke directly
+
+cd $1
+
+# Copies all files we want to reserve.
+cp -a -n $2/Android.bp       $1/  2> /dev/null
+cp -a -n $2/Android.mk       $1/  2> /dev/null
+cp -a -n $2/LICENSE          $1/  2> /dev/null
+cp -a -n $2/NOTICE           $1/  2> /dev/null
+cp -a -n $2/MODULE_LICENSE_* $1/  2> /dev/null
+cp -a -n $2/METADATA         $1/  2> /dev/null
+cp -a -n $2/.git             $1/  2> /dev/null
+cp -a -n $2/.gitignore       $1/  2> /dev/null
+cp -a -n $2/patches          $1/  2> /dev/null
+cp -a -n $2/post_update.sh   $1/  2> /dev/null
+
+# Applies all patches
+for p in $1/patches/*.diff
+do
+  [ -e "$p" ] || continue
+  echo Applying $p
+  patch -p1 -d $1 < $p;
+done
+
+if [ -f $1/post_update.sh ]
+then
+  echo Running post update script
+  $1/post_update.sh $1 $2
+fi
+
+# Swap old and new.
+rm -rf $2
+mv $1 $2

updater.sh

+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd $(dirname "$0")/../..
+source build/envsetup.sh
+mmma tools/external_updater
+out/soong/host/linux-x86/bin/external_updater $@

updater_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper functions for updaters."""
+
+import os
+import subprocess
+import sys
+
+
+def create_updater(metadata, proj_path, updaters):
+    """Creates corresponding updater object for a project.
+
+    Args:
+      metadata: Parsed proto for METADATA file.
+      proj_path: Absolute path for the project.
+
+    Returns:
+      An updater object.
+
+    Raises:
+      ValueError: Occurred when there's no updater for all urls.
+    """
+    for url in metadata.third_party.url:
+        for updater in updaters:
+            try:
+                return updater(url, proj_path, metadata)
+            except ValueError:
+                pass
+
+    raise ValueError('No supported URL.')
+
+
+def replace_package(source_dir, target_dir):
+    """Invokes a shell script to prepare and update a project.
+
+    Args:
+      source_dir: Path to the new downloaded and extracted package.
+      target_dir: The path to the project in Android source tree.
+    """
+
+    print('Updating {} using {}.'.format(target_dir, source_dir))
+    script_path = os.path.join(
+        os.path.dirname(
+            sys.argv[0]),
+        'update_package.sh')
+    subprocess.check_call(['bash', script_path, source_dir, target_dir])