Support GIT upstream am: 24950e79

am: ec6029e3

Change-Id: I3804c44d3f314e6cbb256c5b34095717ec7e30c2

.gitignore

+__pycache__

external_updater.py

@@ -20,15 +20,17 @@ updater.sh update kotlinc
 
 import argparse
 import os
+import subprocess
 
 from google.protobuf import text_format    # pylint: disable=import-error
 
 import fileutils
+from git_updater import GitUpdater
 from github_archive_updater import GithubArchiveUpdater
 import updater_utils
 
 
-UPDATERS = [GithubArchiveUpdater]
+UPDATERS = [GithubArchiveUpdater, GitUpdater]
 
 
 def color_string(string, color):
@@ -82,21 +84,28 @@ def check_update(proj_path):
         end='')
     updater = build_updater(proj_path)
     if updater is None:
-        return
+        return (None, None)
     try:
-        latest = updater.get_latest_version()
-        current = updater.get_current_version()
-    except IOError as e:
-        print('{} {}.'.format(color_string('Failed to check latest version.',
-                                           'FAILED'),
-                              e))
-        return
-
-    if current != latest:
-        print('{} Current version: {}. Latest version: {}.'. format(
-            color_string('New version found.', 'SUCCESS'), current, latest))
-    else:
-        print('No new version. Current version: {}.'.format(latest))
+        new_version = updater.check()
+        if new_version:
+            print(color_string(' New version found.', 'SUCCESS'))
+        else:
+            print(' No new version.')
+        return (updater, new_version)
+    except IOError as err:
+        print('{} {}.'.format(color_string('Failed.', 'FAILED'),
+                              err))
+        return (None, None)
+    except subprocess.CalledProcessError as err:
+        print(
+            '{} {}\nstdout: {}\nstderr: {}.'.format(
+                color_string(
+                    'Failed.',
+                    'FAILED'),
+                err,
+                err.stdout,
+                err.stderr))
+        return (None, None)
 
 
 def check(args):
@@ -108,33 +117,12 @@ def check(args):
 def update(args):
     """Handler for update command."""
 
-    updater = build_updater(args.path)
+    updater, new_version = check_update(args.path)
     if updater is None:
         return
-    try:
-        latest = updater.get_latest_version()
-        current = updater.get_current_version()
-    except IOError as e:
-        print('{} {}.'.format(
-            color_string('Failed to check latest version.',
-                         'FAILED'),
-            e))
+    if not new_version and not args.force:
         return
 
-    if current == latest and not args.force:
-        print(
-            '{} for {}. Current version {} is latest. '
-            'Use --force to update anyway.'.format(
-                color_string(
-                    'Nothing to update',
-                    'FAILED'),
-                args.path,
-                current))
-        return
-
-    print('{} from version {} to version {}.{}'.format(
-        color_string('Updating', 'SUCCESS'), args.path, current, latest))
-
     updater.update()
 
 

git_updater.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to check updates from Git upstream."""
+
+
+import datetime
+
+import fileutils
+import git_utils
+import metadata_pb2    # pylint: disable=import-error
+
+
+class GitUpdater():
+    """Updater for Git upstream."""
+
+    def __init__(self, url, proj_path, metadata):
+        if url.type != metadata_pb2.URL.GIT:
+            raise ValueError('Only support GIT upstream.')
+        self.proj_path = proj_path
+        self.metadata = metadata
+        self.upstream_url = url
+        self.upstream_remote_name = None
+        self.android_remote_name = None
+        self.latest_commit = None
+
+    def _setup_remote(self):
+        remotes = git_utils.list_remotes(self.proj_path)
+        for name, url in remotes.items():
+            if url == self.upstream_url.value:
+                self.upstream_remote_name = name
+
+            # Guess android remote name.
+            if '/platform/external/' in url:
+                self.android_remote_name = name
+
+        if self.upstream_remote_name is None:
+            self.upstream_remote_name = "update_origin"
+            git_utils.add_remote(self.proj_path, self.upstream_remote_name,
+                                 self.upstream_url.value)
+
+        git_utils.fetch(self.proj_path,
+                        [self.upstream_remote_name, self.android_remote_name])
+
+    def check(self):
+        """Checks upstream and returns whether a new version is available."""
+
+        self._setup_remote()
+        commits = git_utils.get_commits_ahead(
+            self.proj_path, self.upstream_remote_name + '/master',
+            self.android_remote_name + '/master')
+
+        if not commits:
+            return False
+
+        self.latest_commit = commits[0]
+        commit_time = git_utils.get_commit_time(self.proj_path, commits[-1])
+        time_behind = datetime.datetime.now() - commit_time
+        print('{} commits ({} days) behind.'.format(
+            len(commits), time_behind.days), end='')
+        return True
+
+    def _write_metadata(self, path):
+        updated_metadata = metadata_pb2.MetaData()
+        updated_metadata.CopyFrom(self.metadata)
+        updated_metadata.third_party.version = self.latest_commit
+        fileutils.write_metadata(path, updated_metadata)
+
+    def update(self):
+        """Updates the package.
+
+        Has to call check() before this function.
+        """
+        # See whether we have a local upstream.
+        branches = git_utils.list_remote_branches(
+            self.proj_path, self.android_remote_name)
+        upstreams = [
+            branch for branch in branches if branch.startswith('upstream-')]
+        if len(upstreams) == 1:
+            merge_branch = '{}/{}'.format(
+                self.android_remote_name, upstreams[0])
+        elif not upstreams:
+            merge_branch = 'update_origin/master'
+        else:
+            raise ValueError('Ambiguous upstream branch. ' + upstreams)
+
+        upstream_branch = self.upstream_remote_name + '/master'
+
+        commits = git_utils.get_commits_ahead(
+            self.proj_path, merge_branch, upstream_branch)
+        if commits:
+            print('Warning! {} is {} commits ahead of {}. {}'.format(
+                merge_branch, len(commits), upstream_branch, commits))
+
+        commits = git_utils.get_commits_ahead(
+            self.proj_path, upstream_branch, merge_branch)
+        if commits:
+            print('Warning! {} is {} commits behind of {}.'.format(
+                merge_branch, len(commits), upstream_branch))
+
+        self._write_metadata(self.proj_path)
+        print("""
+This tool only updates METADATA. Run the following command to update:
+    git merge {merge_branch}
+
+To check all local changes:
+    git diff {merge_branch} HEAD
+""".format(merge_branch=merge_branch))

git_utils.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''Helper functions to communicate with Git.'''
+
+import datetime
+import subprocess
+
+
+def _run(cmd, cwd):
+    """Runs a command with stdout and stderr redirected."""
+    return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                          check=True, cwd=cwd)
+
+def fetch(proj_path, remote_names):
+    """Runs git fetch.
+
+    Args:
+        proj_path: Path to Git repository.
+        remote_names: Array of string to specify remote names.
+    """
+    _run(['git', 'fetch', '--multiple'] + remote_names, cwd=proj_path)
+
+def add_remote(proj_path, name, url):
+    """Adds a git remote.
+
+    Args:
+        proj_path: Path to Git repository.
+        name: Name of the new remote.
+        url: Url of the new remote.
+    """
+    _run(['git', 'remote', 'add', name, url], cwd=proj_path)
+
+def list_remotes(proj_path):
+    """Lists all Git remotes.
+
+    Args:
+        proj_path: Path to Git repository.
+
+    Returns:
+        A dict from remote name to remote url.
+    """
+    out = _run(['git', 'remote', '-v'], proj_path)
+    lines = out.stdout.decode('utf-8').splitlines()
+    return dict([line.split()[0:2] for line in lines])
+
+def get_commits_ahead(proj_path, branch, base_branch):
+    """Lists commits in `branch` but not `base_branch`."""
+    out = _run(['git', 'rev-list', '--left-only',
+                '{}...{}'.format(branch, base_branch)],
+               proj_path)
+    return out.stdout.decode('utf-8').splitlines()
+
+def get_commit_time(proj_path, commit):
+    """Gets commit time of one commit."""
+    out = _run(['git', 'show', '-s', '--format=%ct', commit], cwd=proj_path)
+    return datetime.datetime.fromtimestamp(int(out.stdout))
+
+def list_remote_branches(proj_path, remote_name):
+    """Lists all branches for a remote."""
+    out = _run(['git', 'branch', '-r'], cwd=proj_path)
+    lines = out.stdout.decode('utf-8').splitlines()
+    stripped = [line.strip() for line in lines]
+    remote_path = remote_name + '/'
+    remote_path_len = len(remote_path)
+    return [line[remote_path_len:] for line in stripped
+            if line.startswith(remote_path)]

github_archive_updater.py

@@ -58,7 +58,7 @@ class GithubArchiveUpdater():
         except IndexError:
             raise ValueError('Url format is not supported.')
 
-    def get_latest_version(self):
+    def _get_latest_version(self):
         """Checks upstream and returns the latest version name we found."""
 
         url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
@@ -67,7 +67,7 @@ class GithubArchiveUpdater():
             self.data = json.loads(request.read().decode())
         return self.data[self.VERSION_FIELD]
 
-    def get_current_version(self):
+    def _get_current_version(self):
         """Returns the latest version name recorded in METADATA."""
         return self.metadata.third_party.version
 
@@ -80,10 +80,21 @@ class GithubArchiveUpdater():
                 metadata_url.value = url
         fileutils.write_metadata(path, updated_metadata)
 
+    def check(self):
+        """Checks update for package.
+
+        Returns True if a new version is available.
+        """
+        latest = self._get_latest_version()
+        current = self._get_current_version()
+        print('Current version: {}. Latest version: {}'.format(
+            current, latest), end='')
+        return current != latest
+
     def update(self):
         """Updates the package.
 
-        Has to call get_latest_version() before this function.
+        Has to call check() before this function.
         """
 
         supported_assets = [

updater.sh

@@ -16,5 +16,6 @@
 
 cd $(dirname "$0")/../..
 source build/envsetup.sh
+lunch aosp_arm-eng
 mmma tools/external_updater
 out/soong/host/linux-x86/bin/external_updater $@