[Updater] Send email when a new version is found

Bug: 109748616
Test: ./updater.sh check kotlinc googletest --json_output=/tmp/res.json
Test: out/soong/host/linux-x86/bin/external_updater_notifier \
          --result_file=/tmp/res.json \
          --history ~/updater/history \
          --recipients=hhb@google.com

Change-Id: I27a4c1c2604d38106a08ce3eee1bcd03fdce80d7

Android.bp

@@ -23,15 +23,24 @@ python_binary_host {
     ],
 }
 
+python_binary_host {
+    name: "external_updater_notifier",
+    main: "notifier.py",
+    srcs: [
+        "notifier.py",
+    ],
+}
+
 python_library_host {
     name: "external_updater_lib",
     srcs: [
-        "*.py",
+        "archive_utils.py",
+        "fileutils.py",
+        "git_updater.py",
+        "git_utils.py",
+        "github_archive_updater.py",
         "metadata.proto",
-    ],
-    exclude_srcs: [
-        "*_test.py",
-        "external_updater.py",
+        "updater_utils.py",
     ],
     libs: [
         "python-symbol",

external_updater.py

@@ -19,8 +19,10 @@ updater.sh update kotlinc
 """
 
 import argparse
+import json
 import os
 import subprocess
+import time
 
 from google.protobuf import text_format    # pylint: disable=import-error
 
@@ -72,6 +74,11 @@ def build_updater(proj_path):
     return updater
 
 
+def has_new_version(updater):
+    """Whether an updater found a new version."""
+    return updater.get_current_version() != updater.get_latest_version()
+
+
 def check_update(proj_path):
     """Checks updates for a project. Prints result on console.
 
@@ -84,56 +91,78 @@ def check_update(proj_path):
         end='')
     updater = build_updater(proj_path)
     if updater is None:
-        return (None, None)
+        return (None, 'Failed to create updater')
     try:
-        new_version = updater.check()
-        if new_version:
+        updater.check()
+        if has_new_version(updater):
             print(color_string(' Out of date!', 'STALE'))
         else:
             print(color_string(' Up to date.', 'FRESH'))
-        return (updater, new_version)
+        return (updater, None)
     except (IOError, ValueError) as err:
         print('{} {}.'.format(color_string('Failed.', 'ERROR'),
                               err))
-        return (None, None)
+        return (updater, str(err))
     except subprocess.CalledProcessError as err:
-        print(
-            '{} {}\nstdout: {}\nstderr: {}.'.format(
-                color_string(
-                    'Failed.',
-                    'ERROR'),
-                err,
-                err.stdout,
-                err.stderr))
-        return (None, None)
+        msg = 'stdout: {}\nstderr: {}.'.format(
+            err.stdout,
+            err.stderr)
+        print('{} {}.'.format(color_string('Failed.', 'ERROR'),
+                              msg))
+        return (updater, msg)
+
+
+def _process_update_result(path):
+    res = {}
+    updater, err = check_update(path)
+    if err is not None:
+        res['error'] = str(err)
+    else:
+        res['current'] = updater.get_current_version()
+        res['latest'] = updater.get_latest_version()
+    return res
+
+
+def _check_some(paths, delay):
+    results = {}
+    for path in paths:
+        results[path] = _process_update_result(path)
+        time.sleep(delay)
+    return results
+
+
+def _check_all(delay):
+    results = {}
+    for path, dirs, files in os.walk(args.path):
+        dirs.sort(key=lambda d: d.lower())
+        if fileutils.METADATA_FILENAME in files:
+            # Skip sub directories.
+            dirs[:] = []
+            results[path] = _process_update_result(path)
+        time.sleep(delay)
+    return results
 
 
 def check(args):
     """Handler for check command."""
+    if args.all:
+        results = _check_all(args.delay)
+    else:
+        results = _check_some(args.paths, args.delay)
 
-    check_update(args.path)
+    if args.json_output is not None:
+        with open(args.json_output, 'w') as f:
+            json.dump(results, f, sort_keys=True, indent=4)
 
 
 def update(args):
     """Handler for update command."""
 
-    updater, new_version = check_update(args.path)
+    updater, err = check_update(args.path)
     if updater is None:
         return
-    if not new_version and not args.force:
-        return
-
-    updater.update()
-
-
-def checkall(args):
-    """Handler for checkall command."""
-    for root, dirs, files in os.walk(args.path):
-        dirs.sort(key=lambda d: d.lower())
-        if fileutils.METADATA_FILENAME in files:
-            # Skip sub directories.
-            dirs[:] = []
-            check_update(root)
+    if has_new_version(updater) or args.force:
+        updater.update()
 
 
 def parse_args():
@@ -148,20 +177,20 @@ def parse_args():
     check_parser = subparsers.add_parser(
         'check', help='Check update for one project.')
     check_parser.add_argument(
-        'path',
-        help='Path of the project. '
+        'paths', nargs='*',
+        help='Paths of the project. '
         'Relative paths will be resolved from external/.')
+    check_parser.add_argument(
+        '--json_output',
+        help='Path of a json file to write result to.')
+    check_parser.add_argument(
+        '--all',
+        help='If set, check updates for all supported projects.')
+    check_parser.add_argument(
+        '--delay', default=0, type=int,
+        help='Time in seconds to wait between checking two projects.')
     check_parser.set_defaults(func=check)
 
-    # Creates parser for checkall command.
-    checkall_parser = subparsers.add_parser(
-        'checkall', help='Check update for all projects.')
-    checkall_parser.add_argument(
-        '--path',
-        default=fileutils.EXTERNAL_PATH,
-        help='Starting path for all projects. Default to external/.')
-    checkall_parser.set_defaults(func=checkall)
-
     # Creates parser for update command.
     update_parser = subparsers.add_parser('update', help='Update one project.')
     update_parser.add_argument(

git_updater.py

@@ -60,21 +60,28 @@ class GitUpdater():
         self._setup_remote()
         if git_utils.is_commit(self.metadata.third_party.version):
             # Update to remote head.
-            return self._check_head()
+            self._check_head()
+        else:
+            # Update to latest version tag.
+            self._check_tag()
+
+    def get_current_version(self):
+        """Returns the latest version name recorded in METADATA."""
+        return self.metadata.third_party.version
 
-        # Update to latest version tag.
-        return self._check_tag()
+    def get_latest_version(self):
+        """Returns the latest version name in upstream."""
+        return self.new_version
 
     def _check_tag(self):
         tags = git_utils.list_remote_tags(self.proj_path,
                                           self.upstream_remote_name)
-        current_ver = self.metadata.third_party.version
+        current_ver = self.get_current_version()
         self.new_version = updater_utils.get_latest_version(
             current_ver, tags)
         self.merge_from = self.new_version
         print('Current version: {}. Latest version: {}'.format(
             current_ver, self.new_version), end='')
-        return self.new_version != current_ver
 
     def _check_head(self):
         commits = git_utils.get_commits_ahead(
@@ -82,7 +89,8 @@ class GitUpdater():
             self.android_remote_name + '/master')
 
         if not commits:
-            return False
+            self.new_version = self.get_current_version()
+            return
 
         self.new_version = commits[0]
 
@@ -101,7 +109,6 @@ class GitUpdater():
         time_behind = datetime.datetime.now() - commit_time
         print('{} commits ({} days) behind.'.format(
             len(commits), time_behind.days), end='')
-        return True
 
     def _write_metadata(self, path):
         updated_metadata = metadata_pb2.MetaData()
@@ -129,4 +136,6 @@ class GitUpdater():
                 self.merge_from, len(commits), upstream_branch))
 
         self._write_metadata(self.proj_path)
+        print("Running `git merge {merge_branch}`..."
+              .format(merge_branch=self.merge_from))
         git_utils.merge(self.proj_path, self.merge_from)

github_archive_updater.py

@@ -92,7 +92,7 @@ class GithubArchiveUpdater():
         except IndexError:
             raise ValueError('Url format is not supported.')
 
-    def _get_latest_version(self):
+    def _fetch_latest_version(self):
         """Checks upstream and gets the latest release tag."""
 
         url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
@@ -115,7 +115,7 @@ class GithubArchiveUpdater():
 
         self.new_url = choose_best_url(supported_assets, self.old_url.value)
 
-    def _get_latest_commit(self):
+    def _fetch_latest_commit(self):
         """Checks upstream and gets the latest commit to master."""
 
         url = 'https://api.github.com/repos/{}/{}/commits/master'.format(
@@ -126,10 +126,14 @@ class GithubArchiveUpdater():
         self.new_url = 'https://github.com/{}/{}/archive/{}.zip'.format(
             self.owner, self.repo, self.new_version)
 
-    def _get_current_version(self):
+    def get_current_version(self):
         """Returns the latest version name recorded in METADATA."""
         return self.metadata.third_party.version
 
+    def get_latest_version(self):
+        """Returns the latest version name in upstream."""
+        return self.new_version
+
     def _write_metadata(self, url, path):
         updated_metadata = metadata_pb2.MetaData()
         updated_metadata.CopyFrom(self.metadata)
@@ -144,14 +148,13 @@ class GithubArchiveUpdater():
 
         Returns True if a new version is available.
         """
-        current = self._get_current_version()
+        current = self.get_current_version()
         if git_utils.is_commit(current):
-            self._get_latest_commit()
+            self._fetch_latest_commit()
         else:
-            self._get_latest_version()
+            self._fetch_latest_version()
         print('Current version: {}. Latest version: {}'.format(
             current, self.new_version), end='')
-        return current != self.new_version
 
     def update(self):
         """Updates the package.

notifier.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Send notification email if new version is found.
+
+Example usage:
+external_updater_notifier \
+    --result_file ~/updater/new_result \
+    --history ~/updater/history \
+    --recipients xxx@xxx.xxx
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import time
+
+
+def parse_args():
+    """Parses commandline arguments."""
+
+    parser = argparse.ArgumentParser(
+        description='Check updates for third party projects in external/.')
+    parser.add_argument(
+        '--result_file',
+        help='Json check result generated by external updater.')
+    parser.add_argument(
+        '--history',
+        help='Path of history file. If doesn'
+        't exist, a new one will be created.')
+    parser.add_argument(
+        '--recipients',
+        help='Comma separated recipients of notification email.')
+
+    return parser.parse_args()
+
+
+def _send_email(proj, latest_ver, recipient):
+    print('Sending email for {}: {}'.format(proj, latest_ver))
+    msg = """New version: {}
+
+To upgrade:
+    tools/external_updater/updater.sh update {}""".format(
+        latest_ver, proj)
+    subprocess.run(['sendgmr', '--to=' + recipient,
+                    '--subject=' + proj], check=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                   input=msg, encoding='ascii')
+
+
+def _process_results(history, results, recipient):
+    for proj, res in results.items():
+        if 'latest' not in res:
+            continue
+        latest_ver = res['latest']
+        current_ver = res['current']
+        if latest_ver == current_ver:
+            continue
+        proj_history = history.setdefault(proj, {})
+        if latest_ver not in proj_history:
+            try:
+                _send_email(proj, latest_ver, recipient)
+                proj_history[latest_ver] = int(time.time())
+            except subprocess.CalledProcessError as err:
+                msg = """Failed to send email for {} ({}).
+stdout: {}
+stderr: {}""".format(proj, latest_ver, err.stdout, err.stderr)
+                print(msg)
+
+
+def send_notification(args):
+    """Compare results and send notification."""
+    results = {}
+    with open(args.result_file, 'r') as f:
+        results = json.load(f)
+    history = {}
+    try:
+        with open(args.history, 'r') as f:
+            history = json.load(f)
+    except FileNotFoundError:
+        pass
+
+    _process_results(history, results, args.recipients)
+
+    with open(args.history, 'w') as f:
+        json.dump(history, f, sort_keys=True)
+
+
+def main():
+    """The main entry."""
+
+    args = parse_args()
+    send_notification(args)
+
+
+if __name__ == '__main__':
+    main()

update_package.sh

@@ -43,6 +43,7 @@ CopyIfPresent ".git"
 CopyIfPresent ".gitignore"
 CopyIfPresent "patches"
 CopyIfPresent "post_update.sh"
+CopyIfPresent "OWNERS"
 
 echo "Applying patches..."
 for p in $tmp_dir/patches/*.diff