Merge android10-release

Change-Id: Ib6065d5935a6113643a72c5f86b6fbdb675f4e15

.gitignore

+__pycache__

Android.bp

+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+python_binary_host {
+    name: "external_updater",
+    main: "external_updater.py",
+    srcs: [
+        "external_updater.py",
+    ],
+    libs: [
+        "external_updater_lib",
+    ],
+}
+
+python_binary_host {
+    name: "external_updater_notifier",
+    main: "notifier.py",
+    srcs: [
+        "git_utils.py",
+        "notifier.py",
+    ],
+}
+
+python_library_host {
+    name: "external_updater_lib",
+    srcs: [
+        "archive_utils.py",
+        "fileutils.py",
+        "git_updater.py",
+        "git_utils.py",
+        "github_archive_updater.py",
+        "metadata.proto",
+        "updater_utils.py",
+    ],
+    libs: [
+        "python-symbol",
+        "libprotobuf-python",
+    ],
+    proto: {
+        canonical_path_from_root: false,
+    },
+    data: [
+        "update_package.sh",
+    ],
+    version: {
+        py2: {
+            enabled: false,
+            embedded_launcher: false,
+        },
+        py3: {
+            enabled: true,
+            embedded_launcher: false,
+        },
+    },
+}
+
+python_test_host {
+    name: "external_updater_test",
+    main: "external_updater_test.py",
+    srcs: [
+        "external_updater_test.py",
+    ],
+    libs: [
+        "external_updater_lib",
+    ],
+}

OWNERS

+# Default code reviewers picked from top 3 or more developers.
+# Please update this list if you find better candidates.
+hhb@google.com
+enh@google.com

PREUPLOAD.cfg

+[Builtin Hooks]
+pylint = true
+
+[Builtin Hooks Options]
+pylint = --executable-path pylint3 ${PREUPLOAD_FILES}

archive_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Functions to process archive files."""
+
+import os
+import tempfile
+import tarfile
+import urllib.parse
+import zipfile
+
+
+class ZipFileWithPermission(zipfile.ZipFile):
+    """Subclassing Zipfile to preserve file permission.
+
+    See https://bugs.python.org/issue15795
+    """
+
+    def _extract_member(self, member, targetpath, pwd):
+        ret_val = super()._extract_member(member, targetpath, pwd)
+
+        if not isinstance(member, zipfile.ZipInfo):
+            member = self.getinfo(member)
+        attr = member.external_attr >> 16
+        if attr != 0:
+            os.chmod(ret_val, attr)
+        return ret_val
+
+
+def unzip(archive_path, target_path):
+    """Extracts zip file to a path.
+
+    Args:
+        archive_path: Path to the zip file.
+        target_path: Path to extract files to.
+    """
+
+    with ZipFileWithPermission(archive_path) as zfile:
+        zfile.extractall(target_path)
+
+
+def untar(archive_path, target_path):
+    """Extracts tar file to a path.
+
+    Args:
+        archive_path: Path to the tar file.
+        target_path: Path to extract files to.
+    """
+
+    with tarfile.open(archive_path, mode='r') as tfile:
+        tfile.extractall(target_path)
+
+
+ARCHIVE_TYPES = {
+    '.zip': unzip,
+    '.tar.gz': untar,
+    '.tar.bz2': untar,
+    '.tar.xz': untar,
+}
+
+
+def is_supported_archive(url):
+    """Checks whether the url points to a supported archive."""
+    return get_extract_func(url) is not None
+
+
+def get_extract_func(url):
+    """Gets the function to extract an archive.
+
+    Args:
+        url: The url to the archive file.
+
+    Returns:
+        A function to extract the archive. None if not found.
+    """
+
+    parsed_url = urllib.parse.urlparse(url)
+    filename = os.path.basename(parsed_url.path)
+    for ext, func in ARCHIVE_TYPES.items():
+        if filename.endswith(ext):
+            return func
+    return None
+
+
+def download_and_extract(url):
+    """Downloads and extracts an archive file to a temporary directory.
+
+    Args:
+        url: Url to download.
+
+    Returns:
+        Path to the temporary directory.
+    """
+
+    print('Downloading {}'.format(url))
+    archive_file, _headers = urllib.request.urlretrieve(url)
+
+    temporary_dir = tempfile.mkdtemp()
+    print('Extracting {} to {}'.format(archive_file, temporary_dir))
+    get_extract_func(url)(archive_file, temporary_dir)
+
+    return temporary_dir
+
+
+def find_archive_root(path):
+    """Finds the real root of an extracted archive.
+
+    Sometimes archives has additional layers of directories. This function tries
+    to guess the right 'root' path by entering all single sub-directories.
+
+    Args:
+        path: Path to the extracted archive.
+
+    Returns:
+        The root path we found.
+    """
+    for root, dirs, files in os.walk(path):
+        if files or len(dirs) > 1:
+            return root
+    return path

external_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A commandline tool to check and update packages in external/
+
+Example usage:
+updater.sh checkall
+updater.sh update kotlinc
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import time
+
+from google.protobuf import text_format    # pylint: disable=import-error
+
+from git_updater import GitUpdater
+from github_archive_updater import GithubArchiveUpdater
+import fileutils
+import git_utils
+import updater_utils
+
+
+UPDATERS = [GithubArchiveUpdater, GitUpdater]
+
+
+def color_string(string, color):
+    """Changes the color of a string when print to terminal."""
+    colors = {
+        'FRESH': '\x1b[32m',
+        'STALE': '\x1b[31;1m',
+        'ERROR': '\x1b[31m',
+    }
+    end_color = '\033[0m'
+    return colors[color] + string + end_color
+
+
+def build_updater(proj_path):
+    """Build updater for a project specified by proj_path.
+
+    Reads and parses METADATA file. And builds updater based on the information.
+
+    Args:
+      proj_path: Absolute or relative path to the project.
+
+    Returns:
+      The updater object built. None if there's any error.
+    """
+
+    proj_path = fileutils.get_absolute_project_path(proj_path)
+    try:
+        metadata = fileutils.read_metadata(proj_path)
+    except text_format.ParseError as err:
+        print('{} {}.'.format(color_string('Invalid metadata file:', 'ERROR'),
+                              err))
+        return None
+
+    try:
+        updater = updater_utils.create_updater(metadata, proj_path, UPDATERS)
+    except ValueError:
+        print(color_string('No supported URL.', 'ERROR'))
+        return None
+    return updater
+
+
+def has_new_version(updater):
+    """Whether an updater found a new version."""
+    return updater.get_current_version() != updater.get_latest_version()
+
+
+def _message_for_calledprocesserror(error):
+    return '\n'.join([error.stdout.decode('utf-8'),
+                      error.stderr.decode('utf-8')])
+
+
+def check_update(proj_path):
+    """Checks updates for a project. Prints result on console.
+
+    Args:
+      proj_path: Absolute or relative path to the project.
+    """
+
+    print(
+        'Checking {}. '.format(fileutils.get_relative_project_path(proj_path)),
+        end='')
+    updater = build_updater(proj_path)
+    if updater is None:
+        return (None, 'Failed to create updater')
+    try:
+        updater.check()
+        if has_new_version(updater):
+            print(color_string(' Out of date!', 'STALE'))
+        else:
+            print(color_string(' Up to date.', 'FRESH'))
+        return (updater, None)
+    except (IOError, ValueError) as err:
+        print('{} {}.'.format(color_string('Failed.', 'ERROR'),
+                              err))
+        return (updater, str(err))
+    except subprocess.CalledProcessError as err:
+        msg = _message_for_calledprocesserror(err)
+        print('{}\n{}'.format(msg, color_string('Failed.', 'ERROR')))
+        return (updater, msg)
+
+
+def _process_update_result(path):
+    res = {}
+    updater, err = check_update(path)
+    if err is not None:
+        res['error'] = str(err)
+    else:
+        res['current'] = updater.get_current_version()
+        res['latest'] = updater.get_latest_version()
+    return res
+
+
+def _check_some(paths, delay):
+    results = {}
+    for path in paths:
+        relative_path = fileutils.get_relative_project_path(path)
+        results[relative_path] = _process_update_result(path)
+        time.sleep(delay)
+    return results
+
+
+def _check_all(delay):
+    results = {}
+    for path, dirs, files in os.walk(fileutils.EXTERNAL_PATH):
+        dirs.sort(key=lambda d: d.lower())
+        if fileutils.METADATA_FILENAME in files:
+            # Skip sub directories.
+            dirs[:] = []
+            relative_path = fileutils.get_relative_project_path(path)
+            results[relative_path] = _process_update_result(path)
+            time.sleep(delay)
+    return results
+
+
+def check(args):
+    """Handler for check command."""
+    if args.all:
+        results = _check_all(args.delay)
+    else:
+        results = _check_some(args.paths, args.delay)
+
+    if args.json_output is not None:
+        with open(args.json_output, 'w') as f:
+            json.dump(results, f, sort_keys=True, indent=4)
+
+
+def update(args):
+    """Handler for update command."""
+    try:
+        _do_update(args)
+    except subprocess.CalledProcessError as err:
+        msg = _message_for_calledprocesserror(err)
+        print(
+            '{}\n{}'.format(
+                msg,
+                color_string(
+                    'Failed to upgrade.',
+                    'ERROR')))
+
+
+TMP_BRANCH_NAME = 'tmp_auto_upgrade'
+
+
+def _do_update(args):
+    updater, err = check_update(args.path)
+    if updater is None:
+        return
+    if not has_new_version(updater) and not args.force:
+        return
+
+    full_path = fileutils.get_absolute_project_path(args.path)
+    if args.branch_and_commit:
+        git_utils.checkout(full_path, args.remote_name + '/master')
+        try:
+            git_utils.delete_branch(full_path, TMP_BRANCH_NAME)
+        except subprocess.CalledProcessError as err:
+            # Still continue if the branch doesn't exist.
+            pass
+        git_utils.start_branch(full_path, TMP_BRANCH_NAME)
+
+    updater.update()
+
+    if args.branch_and_commit:
+        msg = 'Upgrade {} to {}\n\nTest: None'.format(
+            args.path, updater.get_latest_version())
+        git_utils.add_file(full_path, '*')
+        git_utils.commit(full_path, msg)
+
+    if args.push_change:
+        git_utils.push(full_path, args.remote_name)
+
+    if args.branch_and_commit:
+        git_utils.checkout(full_path, args.remote_name + '/master')
+
+
+def parse_args():
+    """Parses commandline arguments."""
+
+    parser = argparse.ArgumentParser(
+        description='Check updates for third party projects in external/.')
+    subparsers = parser.add_subparsers(dest='cmd')
+    subparsers.required = True
+
+    # Creates parser for check command.
+    check_parser = subparsers.add_parser(
+        'check', help='Check update for one project.')
+    check_parser.add_argument(
+        'paths', nargs='*',
+        help='Paths of the project. '
+        'Relative paths will be resolved from external/.')
+    check_parser.add_argument(
+        '--json_output',
+        help='Path of a json file to write result to.')
+    check_parser.add_argument(
+        '--all', action='store_true',
+        help='If set, check updates for all supported projects.')
+    check_parser.add_argument(
+        '--delay', default=0, type=int,
+        help='Time in seconds to wait between checking two projects.')
+    check_parser.set_defaults(func=check)
+
+    # Creates parser for update command.
+    update_parser = subparsers.add_parser('update', help='Update one project.')
+    update_parser.add_argument(
+        'path',
+        help='Path of the project. '
+        'Relative paths will be resolved from external/.')
+    update_parser.add_argument(
+        '--force',
+        help='Run update even if there\'s no new version.',
+        action='store_true')
+    update_parser.add_argument(
+        '--branch_and_commit', action='store_true',
+        help='Starts a new branch and commit changes.')
+    update_parser.add_argument(
+        '--push_change', action='store_true',
+        help='Pushes change to Gerrit.')
+    update_parser.add_argument(
+        '--remote_name', default='aosp', required=False,
+        help='Upstream remote name.')
+    update_parser.set_defaults(func=update)
+
+    return parser.parse_args()
+
+
+def main():
+    """The main entry."""
+
+    args = parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

external_updater_test.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for external updater."""
+
+import unittest
+
+import github_archive_updater
+
+
+class ExternalUpdaterTest(unittest.TestCase):
+    """Unit tests for external updater."""
+
+    def test_url_selection(self):
+        """Tests that GithubArchiveUpdater can choose the right url."""
+        prefix = "https://github.com/author/project/"
+        urls = [
+            prefix + "releases/download/ver-1.0/ver-1.0-binary.zip",
+            prefix + "releases/download/ver-1.0/ver-1.0-binary.tar.gz",
+            prefix + "releases/download/ver-1.0/ver-1.0-src.zip",
+            prefix + "releases/download/ver-1.0/ver-1.0-src.tar.gz",
+            prefix + "archive/ver-1.0.zip",
+            prefix + "archive/ver-1.0.tar.gz",
+        ]
+
+        previous_url = prefix + "releases/download/ver-0.9/ver-0.9-src.tar.gz"
+        url = github_archive_updater.choose_best_url(urls, previous_url)
+        expected_url = prefix + "releases/download/ver-1.0/ver-1.0-src.tar.gz"
+        self.assertEqual(url, expected_url)
+
+        previous_url = prefix + "archive/ver-0.9.zip"
+        url = github_archive_updater.choose_best_url(urls, previous_url)
+        expected_url = prefix + "archive/ver-1.0.zip"
+        self.assertEqual(url, expected_url)
+
+
+if __name__ == '__main__':
+    unittest.main()

fileutils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tool functions to deal with files."""
+
+import datetime
+import os
+
+from google.protobuf import text_format    # pylint: disable=import-error
+
+import metadata_pb2    # pylint: disable=import-error
+
+ANDROID_TOP = os.environ.get('ANDROID_BUILD_TOP', os.getcwd())
+EXTERNAL_PATH = os.path.join(ANDROID_TOP, 'external/')
+
+METADATA_FILENAME = 'METADATA'
+
+
+def get_absolute_project_path(project_path):
+    """Gets absolute path of a project.
+
+    Path resolution starts from external/.
+    """
+    return os.path.join(EXTERNAL_PATH, project_path)
+
+
+def get_metadata_path(project_path):
+    """Gets the absolute path of METADATA for a project."""
+    return os.path.join(
+        get_absolute_project_path(project_path), METADATA_FILENAME)
+
+
+def get_relative_project_path(project_path):
+    """Gets the relative path of a project starting from external/."""
+    project_path = get_absolute_project_path(project_path)
+    return os.path.relpath(project_path, EXTERNAL_PATH)
+
+
+def read_metadata(proj_path):
+    """Reads and parses METADATA file for a project.
+
+    Args:
+      proj_path: Path to the project.
+
+    Returns:
+      Parsed MetaData proto.
+
+    Raises:
+      text_format.ParseError: Occurred when the METADATA file is invalid.
+      FileNotFoundError: Occurred when METADATA file is not found.
+    """
+
+    with open(get_metadata_path(proj_path), 'r') as metadata_file:
+        metadata = metadata_file.read()
+        return text_format.Parse(metadata, metadata_pb2.MetaData())
+
+
+def write_metadata(proj_path, metadata):
+    """Writes updated METADATA file for a project.
+
+    This function updates last_upgrade_date in metadata and write to the project
+    directory.
+
+    Args:
+      proj_path: Path to the project.
+      metadata: The MetaData proto to write.
+    """
+
+    date = metadata.third_party.last_upgrade_date
+    now = datetime.datetime.now()
+    date.year = now.year
+    date.month = now.month
+    date.day = now.day
+    text_metadata = text_format.MessageToString(metadata)
+    with open(get_metadata_path(proj_path), 'w') as metadata_file:
+        metadata_file.write(text_metadata)

git_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to check updates from Git upstream."""
+
+
+import datetime
+
+import fileutils
+import git_utils
+import metadata_pb2    # pylint: disable=import-error
+import updater_utils
+
+
+class GitUpdater():
+    """Updater for Git upstream."""
+
+    def __init__(self, url, proj_path, metadata):
+        if url.type != metadata_pb2.URL.GIT:
+            raise ValueError('Only support GIT upstream.')
+        self.proj_path = proj_path
+        self.metadata = metadata
+        self.upstream_url = url
+        self.upstream_remote_name = None
+        self.android_remote_name = None
+        self.new_version = None
+
+    def _setup_remote(self):
+        remotes = git_utils.list_remotes(self.proj_path)
+        for name, url in remotes.items():
+            if url == self.upstream_url.value:
+                self.upstream_remote_name = name
+
+            # Guess android remote name.
+            if '/platform/external/' in url:
+                self.android_remote_name = name
+
+        if self.upstream_remote_name is None:
+            self.upstream_remote_name = "update_origin"
+            git_utils.add_remote(self.proj_path, self.upstream_remote_name,
+                                 self.upstream_url.value)
+
+        git_utils.fetch(self.proj_path,
+                        [self.upstream_remote_name, self.android_remote_name])
+
+    def check(self):
+        """Checks upstream and returns whether a new version is available."""
+
+        self._setup_remote()
+        if git_utils.is_commit(self.metadata.third_party.version):
+            # Update to remote head.
+            self._check_head()
+        else:
+            # Update to latest version tag.
+            self._check_tag()
+
+    def get_current_version(self):
+        """Returns the latest version name recorded in METADATA."""
+        return self.metadata.third_party.version
+
+    def get_latest_version(self):
+        """Returns the latest version name in upstream."""
+        return self.new_version
+
+    def _check_tag(self):
+        tags = git_utils.list_remote_tags(self.proj_path,
+                                          self.upstream_remote_name)
+        current_ver = self.get_current_version()
+        self.new_version = updater_utils.get_latest_version(
+            current_ver, tags)
+        print('Current version: {}. Latest version: {}'.format(
+            current_ver, self.new_version), end='')
+
+    def _check_head(self):
+        commits = git_utils.get_commits_ahead(
+            self.proj_path, self.upstream_remote_name + '/master',
+            self.android_remote_name + '/master')
+
+        if not commits:
+            self.new_version = self.get_current_version()
+            return
+
+        self.new_version = commits[0]
+
+        commit_time = git_utils.get_commit_time(self.proj_path, commits[-1])
+        time_behind = datetime.datetime.now() - commit_time
+        print('{} commits ({} days) behind.'.format(
+            len(commits), time_behind.days), end='')
+
+    def _write_metadata(self, path):
+        updated_metadata = metadata_pb2.MetaData()
+        updated_metadata.CopyFrom(self.metadata)
+        updated_metadata.third_party.version = self.new_version
+        fileutils.write_metadata(path, updated_metadata)
+
+    def update(self):
+        """Updates the package.
+
+        Has to call check() before this function.
+        """
+        upstream_branch = self.upstream_remote_name + '/master'
+
+        print("Running `git merge {merge_branch}`..."
+              .format(merge_branch=self.new_version))
+        git_utils.merge(self.proj_path, self.new_version)
+        self._write_metadata(self.proj_path)
+        git_utils.add_file(self.proj_path, 'METADATA')

git_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''Helper functions to communicate with Git.'''
+
+import datetime
+import re
+import subprocess
+
+
+def _run(cmd, cwd, redirect=True):
+    """Runs a command with stdout and stderr redirected."""
+    out = subprocess.PIPE if redirect else None
+    return subprocess.run(cmd, stdout=out, stderr=out,
+                          check=True, cwd=cwd)
+
+
+def fetch(proj_path, remote_names):
+    """Runs git fetch.
+
+    Args:
+        proj_path: Path to Git repository.
+        remote_names: Array of string to specify remote names.
+    """
+    _run(['git', 'fetch', '--multiple'] + remote_names, cwd=proj_path)
+
+
+def add_remote(proj_path, name, url):
+    """Adds a git remote.
+
+    Args:
+        proj_path: Path to Git repository.
+        name: Name of the new remote.
+        url: Url of the new remote.
+    """
+    _run(['git', 'remote', 'add', name, url], cwd=proj_path)
+
+
+def list_remotes(proj_path):
+    """Lists all Git remotes.
+
+    Args:
+        proj_path: Path to Git repository.
+
+    Returns:
+        A dict from remote name to remote url.
+    """
+    out = _run(['git', 'remote', '-v'], proj_path)
+    lines = out.stdout.decode('utf-8').splitlines()
+    return dict([line.split()[0:2] for line in lines])
+
+
+def get_commits_ahead(proj_path, branch, base_branch):
+    """Lists commits in `branch` but not `base_branch`."""
+    out = _run(['git', 'rev-list', '--left-only', '--ancestry-path',
+                '{}...{}'.format(branch, base_branch)],
+               proj_path)
+    return out.stdout.decode('utf-8').splitlines()
+
+
+def get_commit_time(proj_path, commit):
+    """Gets commit time of one commit."""
+    out = _run(['git', 'show', '-s', '--format=%ct', commit], cwd=proj_path)
+    return datetime.datetime.fromtimestamp(int(out.stdout))
+
+
+def list_remote_branches(proj_path, remote_name):
+    """Lists all branches for a remote."""
+    out = _run(['git', 'branch', '-r'], cwd=proj_path)
+    lines = out.stdout.decode('utf-8').splitlines()
+    stripped = [line.strip() for line in lines]
+    remote_path = remote_name + '/'
+    remote_path_len = len(remote_path)
+    return [line[remote_path_len:] for line in stripped
+            if line.startswith(remote_path)]
+
+
+def _parse_remote_tag(line):
+    tag_prefix = 'refs/tags/'
+    tag_suffix = '^{}'
+    try:
+        line = line[line.index(tag_prefix):]
+    except ValueError:
+        return None
+    line = line[len(tag_prefix):]
+    if line.endswith(tag_suffix):
+        line = line[:-len(tag_suffix)]
+    return line
+
+
+def list_remote_tags(proj_path, remote_name):
+    """Lists all tags for a remote."""
+    out = _run(['git', "ls-remote", "--tags", remote_name],
+               cwd=proj_path)
+    lines = out.stdout.decode('utf-8').splitlines()
+    tags = [_parse_remote_tag(line) for line in lines]
+    return list(set(tags))
+
+
+COMMIT_PATTERN = r'^[a-f0-9]{40}$'
+COMMIT_RE = re.compile(COMMIT_PATTERN)
+
+
+def is_commit(commit):
+    """Whether a string looks like a SHA1 hash."""
+    return bool(COMMIT_RE.match(commit))
+
+
+def merge(proj_path, branch):
+    """Merges a branch."""
+    try:
+        out = _run(['git', 'merge', branch, '--no-commit'],
+                   cwd=proj_path)
+    except subprocess.CalledProcessError:
+        # Merge failed. Error is already written to console.
+        subprocess.run(['git', 'merge', '--abort'], cwd=proj_path)
+        raise
+
+
+def add_file(proj_path, file_name):
+    """Stages a file."""
+    _run(['git', 'add', file_name], cwd=proj_path)
+
+
+def delete_branch(proj_path, branch_name):
+    """Force delete a branch."""
+    _run(['git', 'branch', '-D', branch_name], cwd=proj_path)
+
+
+def start_branch(proj_path, branch_name):
+    """Starts a new repo branch."""
+    _run(['repo', 'start', branch_name], cwd=proj_path)
+
+
+def commit(proj_path, message):
+    """Commits changes."""
+    _run(['git', 'commit', '-m', message], cwd=proj_path)
+
+
+def checkout(proj_path, branch_name):
+    """Checkouts a branch."""
+    _run(['git', 'checkout', branch_name], cwd=proj_path)
+
+
+def push(proj_path, remote_name):
+    """Pushes change to remote."""
+    return _run(['git', 'push', remote_name, 'HEAD:refs/for/master'],
+                cwd=proj_path, redirect=False)

github_archive_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to update packages from GitHub archive."""
+
+
+import json
+import re
+import urllib.request
+
+import archive_utils
+import fileutils
+import git_utils
+import metadata_pb2    # pylint: disable=import-error
+import updater_utils
+
+GITHUB_URL_PATTERN = (r'^https:\/\/github.com\/([-\w]+)\/([-\w]+)\/' +
+                      r'(releases\/download\/|archive\/)')
+GITHUB_URL_RE = re.compile(GITHUB_URL_PATTERN)
+
+
+def _edit_distance(str1, str2):
+    prev = list(range(0, len(str2) + 1))
+    for i, chr1 in enumerate(str1):
+        cur = [i + 1]
+        for j, chr2 in enumerate(str2):
+            if chr1 == chr2:
+                cur.append(prev[j])
+            else:
+                cur.append(min(prev[j + 1], prev[j], cur[j]) + 1)
+        prev = cur
+    return prev[len(str2)]
+
+
+def choose_best_url(urls, previous_url):
+    """Returns the best url to download from a list of candidate urls.
+
+    This function calculates similarity between previous url and each of new
+    urls. And returns the one best matches previous url.
+
+    Similarity is measured by editing distance.
+
+    Args:
+        urls: Array of candidate urls.
+        previous_url: String of the url used previously.
+
+    Returns:
+        One url from `urls`.
+    """
+    return min(urls, default=None,
+               key=lambda url: _edit_distance(
+                   url, previous_url))
+
+
+class GithubArchiveUpdater():
+    """Updater for archives from GitHub.
+
+    This updater supports release archives in GitHub. Version is determined by
+    release name in GitHub.
+    """
+
+    VERSION_FIELD = 'tag_name'
+
+    def __init__(self, url, proj_path, metadata):
+        self.proj_path = proj_path
+        self.metadata = metadata
+        self.old_url = url
+        self.owner = None
+        self.repo = None
+        self.new_version = None
+        self.new_url = None
+        self._parse_url(url)
+
+    def _parse_url(self, url):
+        if url.type != metadata_pb2.URL.ARCHIVE:
+            raise ValueError('Only archive url from Github is supported.')
+        match = GITHUB_URL_RE.match(url.value)
+        if match is None:
+            raise ValueError('Url format is not supported.')
+        try:
+            self.owner, self.repo = match.group(1, 2)
+        except IndexError:
+            raise ValueError('Url format is not supported.')
+
+    def _fetch_latest_version(self):
+        """Checks upstream and gets the latest release tag."""
+
+        url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
+            self.owner, self.repo)
+        with urllib.request.urlopen(url) as request:
+            data = json.loads(request.read().decode())
+        self.new_version = data[self.VERSION_FIELD]
+
+        supported_assets = [
+            a['browser_download_url'] for a in data['assets']
+            if archive_utils.is_supported_archive(a['browser_download_url'])]
+
+        # Adds source code urls.
+        supported_assets.append(
+            'https://github.com/{}/{}/archive/{}.tar.gz'.format(
+                self.owner, self.repo, data.get('tag_name')))
+        supported_assets.append(
+            'https://github.com/{}/{}/archive/{}.zip'.format(
+                self.owner, self.repo, data.get('tag_name')))
+
+        self.new_url = choose_best_url(supported_assets, self.old_url.value)
+
+    def _fetch_latest_commit(self):
+        """Checks upstream and gets the latest commit to master."""
+
+        url = 'https://api.github.com/repos/{}/{}/commits/master'.format(
+            self.owner, self.repo)
+        with urllib.request.urlopen(url) as request:
+            data = json.loads(request.read().decode())
+        self.new_version = data['sha']
+        self.new_url = 'https://github.com/{}/{}/archive/{}.zip'.format(
+            self.owner, self.repo, self.new_version)
+
+    def get_current_version(self):
+        """Returns the latest version name recorded in METADATA."""
+        return self.metadata.third_party.version
+
+    def get_latest_version(self):
+        """Returns the latest version name in upstream."""
+        return self.new_version
+
+    def _write_metadata(self, url, path):
+        updated_metadata = metadata_pb2.MetaData()
+        updated_metadata.CopyFrom(self.metadata)
+        updated_metadata.third_party.version = self.new_version
+        for metadata_url in updated_metadata.third_party.url:
+            if metadata_url == self.old_url:
+                metadata_url.value = url
+        fileutils.write_metadata(path, updated_metadata)
+
+    def check(self):
+        """Checks update for package.
+
+        Returns True if a new version is available.
+        """
+        current = self.get_current_version()
+        if git_utils.is_commit(current):
+            self._fetch_latest_commit()
+        else:
+            self._fetch_latest_version()
+        print('Current version: {}. Latest version: {}'.format(
+            current, self.new_version), end='')
+
+    def update(self):
+        """Updates the package.
+
+        Has to call check() before this function.
+        """
+        temporary_dir = None
+        try:
+            temporary_dir = archive_utils.download_and_extract(self.new_url)
+            package_dir = archive_utils.find_archive_root(temporary_dir)
+            self._write_metadata(self.new_url, package_dir)
+            updater_utils.replace_package(package_dir, self.proj_path)
+        finally:
+            # Don't remove the temporary directory, or it'll be impossible
+            # to debug the failure...
+            # shutil.rmtree(temporary_dir, ignore_errors=True)
+            urllib.request.urlcleanup()

metadata.proto

+// copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// A proto definition used to parse METADATA file in third party projects.
+
+// This proto will only contain fields and values the updater cares about.
+// It is not intended to be the formal definition of METADATA file.
+
+syntax = "proto3";
+
+package external_updater;
+
+message MetaData {
+  string name = 1;
+  string description = 3;
+  ThirdPartyMetaData third_party = 13;
+}
+
+enum LicenseType {
+  UNKNOWN = 0;
+  BY_EXCEPTION_ONLY = 1;
+  NOTICE = 2;
+  PERMISSIVE = 3;
+  RECIPROCAL = 4;
+  RESTRICTED_IF_STATICALLY_LINKED = 5;
+  RESTRICTED = 6;
+  UNENCUMBERED = 7;
+}
+
+message ThirdPartyMetaData {
+  repeated URL url = 1;
+  string version = 2;
+  LicenseType license_type = 4;
+  Date last_upgrade_date = 10;
+}
+
+message URL {
+  enum Type {
+    UNKNOWN = 0;
+    HOMEPAGE = 1;
+    ARCHIVE = 2;
+    GIT = 3;
+    SVN = 7;
+    HG = 8;
+    DARCS = 9;
+    OTHER = 11;
+  }
+
+  Type type = 1;
+
+  string value = 2;
+}
+
+message Date {
+  int32 year = 1;
+  int32 month = 2;
+  int32 day = 3;
+}

notifier.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Send notification email if new version is found.
+
+Example usage:
+external_updater_notifier \
+    --history ~/updater/history \
+    --generate_change \
+    --recipients xxx@xxx.xxx \
+    googletest
+"""
+
+from datetime import timedelta, datetime
+import argparse
+import json
+import os
+import re
+import subprocess
+import time
+
+import git_utils
+
+def parse_args():
+    """Parses commandline arguments."""
+
+    parser = argparse.ArgumentParser(
+        description='Check updates for third party projects in external/.')
+    parser.add_argument(
+        '--history',
+        help='Path of history file. If doesn'
+        't exist, a new one will be created.')
+    parser.add_argument(
+        '--recipients',
+        help='Comma separated recipients of notification email.')
+    parser.add_argument(
+        '--generate_change',
+        help='If set, an upgrade change will be uploaded to Gerrit.',
+        action='store_true', required=False)
+    parser.add_argument(
+        'paths', nargs='*',
+        help='Paths of the project.')
+    parser.add_argument(
+        '--all', action='store_true',
+        help='Checks all projects.')
+
+    return parser.parse_args()
+
+
+CHANGE_URL_PATTERN = r'(https:\/\/[^\s]*android-review[^\s]*) Upgrade'
+CHANGE_URL_RE = re.compile(CHANGE_URL_PATTERN)
+
+
+def _send_email(proj, latest_ver, recipient, upgrade_log):
+    print('Sending email for {}: {}'.format(proj, latest_ver))
+    msg = "New version: {}".format(latest_ver)
+    match = CHANGE_URL_RE.search(upgrade_log)
+    if match is not None:
+        msg += '\n\nAn upgrade change is generated at:\n{}'.format(
+            match.group(1))
+
+    msg += '\n\n'
+    msg += upgrade_log
+
+    subprocess.run(['sendgmr', '--to=' + recipient,
+                    '--subject=' + proj], check=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                   input=msg, encoding='ascii')
+
+
+NOTIFIED_TIME_KEY_NAME = 'latest_notified_time'
+
+
+def _should_notify(latest_ver, proj_history):
+    if latest_ver in proj_history:
+        # Processed this version before.
+        return False
+
+    timestamp = proj_history.get(NOTIFIED_TIME_KEY_NAME, 0)
+    time_diff = datetime.today() - datetime.fromtimestamp(timestamp)
+    if git_utils.is_commit(latest_ver) and time_diff <= timedelta(days=30):
+        return False
+
+    return True
+
+
+def _process_results(args, history, results):
+    for proj, res in results.items():
+        if 'latest' not in res:
+            continue
+        latest_ver = res['latest']
+        current_ver = res['current']
+        if latest_ver == current_ver:
+            continue
+        proj_history = history.setdefault(proj, {})
+        if _should_notify(latest_ver, proj_history):
+            upgrade_log = _upgrade(proj) if args.generate_change else ""
+            try:
+                _send_email(proj, latest_ver, args.recipients, upgrade_log)
+                proj_history[latest_ver] = int(time.time())
+                proj_history[NOTIFIED_TIME_KEY_NAME] = int(time.time())
+            except subprocess.CalledProcessError as err:
+                msg = """Failed to send email for {} ({}).
+stdout: {}
+stderr: {}""".format(proj, latest_ver, err.stdout, err.stderr)
+                print(msg)
+
+
+RESULT_FILE_PATH = '/tmp/update_check_result.json'
+
+
+def send_notification(args):
+    """Compare results and send notification."""
+    results = {}
+    with open(RESULT_FILE_PATH, 'r') as f:
+        results = json.load(f)
+    history = {}
+    try:
+        with open(args.history, 'r') as f:
+            history = json.load(f)
+    except FileNotFoundError:
+        pass
+
+    _process_results(args, history, results)
+
+    with open(args.history, 'w') as f:
+        json.dump(history, f, sort_keys=True, indent=4)
+
+
+def _upgrade(proj):
+    out = subprocess.run(['out/soong/host/linux-x86/bin/external_updater',
+                          'update', '--branch_and_commit', '--push_change',
+                          proj],
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         cwd=os.environ['ANDROID_BUILD_TOP'])
+    stdout = out.stdout.decode('utf-8')
+    stderr = out.stderr.decode('utf-8')
+    return """
+====================
+|    Debug Info    |
+====================
+-=-=-=-=stdout=-=-=-=-
+{}
+
+-=-=-=-=stderr=-=-=-=-
+{}
+""".format(stdout, stderr)
+
+
+def _check_updates(args):
+    params = ['out/soong/host/linux-x86/bin/external_updater',
+              'check', '--json_output', RESULT_FILE_PATH,
+              '--delay', '30']
+    if args.all:
+        params.append('--all')
+    else:
+        params += args.paths
+
+    subprocess.run(params, cwd=os.environ['ANDROID_BUILD_TOP'])
+
+
+def main():
+    """The main entry."""
+
+    args = parse_args()
+    _check_updates(args)
+    send_notification(args)
+
+
+if __name__ == '__main__':
+    main()

update_package.sh

+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used by external_updater to replace a package. Don't
+# invoke directly.
+
+set -e
+
+tmp_dir=$1
+external_dir=$2
+
+echo "Entering $tmp_dir..."
+cd $tmp_dir
+
+function CopyIfPresent() {
+  if [ -e $external_dir/$1 ]; then
+    cp -a -n $external_dir/$1 .
+  fi
+}
+
+echo "Copying preserved files..."
+CopyIfPresent "Android.bp"
+CopyIfPresent "Android.mk"
+CopyIfPresent "CleanSpec.mk"
+CopyIfPresent "LICENSE"
+CopyIfPresent "NOTICE"
+cp -a -f -n $external_dir/MODULE_LICENSE_* .
+CopyIfPresent "METADATA"
+CopyIfPresent ".git"
+CopyIfPresent ".gitignore"
+CopyIfPresent "patches"
+CopyIfPresent "post_update.sh"
+CopyIfPresent "OWNERS"
+
+echo "Applying patches..."
+for p in $tmp_dir/patches/*.diff
+do
+  [ -e "$p" ] || continue
+  echo "Applying $p..."
+  patch -p1 -d $tmp_dir < $p;
+done
+
+if [ -f $tmp_dir/post_update.sh ]
+then
+  echo "Running post update script"
+  $tmp_dir/post_update.sh $tmp_dir $external_dir
+fi
+
+echo "Swapping old and new..."
+rm -rf $external_dir
+mv $tmp_dir $external_dir
+
+exit 0

updater.sh

+#!/bin/bash
+#
+# Copyright (C) 2007 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+cd $(dirname "$0")/../..
+source build/envsetup.sh
+lunch aosp_arm-eng
+mmma tools/external_updater
+out/soong/host/linux-x86/bin/external_updater $@

updater_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper functions for updaters."""
+
+import os
+import re
+import subprocess
+import sys
+
+
+def create_updater(metadata, proj_path, updaters):
+    """Creates corresponding updater object for a project.
+
+    Args:
+      metadata: Parsed proto for METADATA file.
+      proj_path: Absolute path for the project.
+
+    Returns:
+      An updater object.
+
+    Raises:
+      ValueError: Occurred when there's no updater for all urls.
+    """
+    for url in metadata.third_party.url:
+        for updater in updaters:
+            try:
+                return updater(url, proj_path, metadata)
+            except ValueError:
+                pass
+
+    raise ValueError('No supported URL.')
+
+
+def replace_package(source_dir, target_dir):
+    """Invokes a shell script to prepare and update a project.
+
+    Args:
+      source_dir: Path to the new downloaded and extracted package.
+      target_dir: The path to the project in Android source tree.
+    """
+
+    print('Updating {} using {}.'.format(target_dir, source_dir))
+    script_path = os.path.join(
+        os.path.dirname(
+            sys.argv[0]),
+        'update_package.sh')
+    subprocess.check_call(['bash', script_path, source_dir, target_dir])
+
+VERSION_SPLITTER_PATTERN = r'[\.\-_]'
+VERSION_PATTERN = (r'^(?P<prefix>[^\d]*)' +
+                   r'(?P<version>\d+(' + VERSION_SPLITTER_PATTERN + r'\d+)*)' +
+                   r'(?P<suffix>.*)$')
+VERSION_RE = re.compile(VERSION_PATTERN)
+VERSION_SPLITTER_RE = re.compile(VERSION_SPLITTER_PATTERN)
+
+
+def _parse_version(version):
+    match = VERSION_RE.match(version)
+    if match is None:
+        raise ValueError('Invalid version.')
+    try:
+        return match.group('prefix', 'version', 'suffix')
+    except IndexError:
+        raise ValueError('Invalid version.')
+
+
+def _match_and_get_version(prefix, suffix, version):
+    try:
+        version_prefix, version, version_suffix = _parse_version(version)
+    except ValueError:
+        return []
+
+    right_format = (version_prefix == prefix and version_suffix == suffix)
+
+    return [right_format] + [int(v) for v in VERSION_SPLITTER_RE.split(version)]
+
+
+def get_latest_version(current_version, version_list):
+    """Gets the latest version name from a list of versions.
+
+    The new version must have the same prefix and suffix with old version.
+    If no matched version is newer, current version name will be returned.
+    """
+    prefix, _, suffix = _parse_version(current_version)
+
+    latest = max(version_list,
+                 key=lambda ver: _match_and_get_version(
+                     prefix, suffix, ver),
+                 default=[])
+    if not latest:
+        raise ValueError('No matching version.')
+    return latest