From 39aaab6a6e058bbc00f9ac9650d9b943d9db3ca6 Mon Sep 17 00:00:00 2001
From: Haibo Huang <hhb@google.com>
Date: Fri, 25 Jan 2019 12:23:03 -0800
Subject: [PATCH] [Updater] Send email when a new version is found
Bug: 109748616
Test: ./updater.sh check kotlinc googletest --json_output=/tmp/res.json
Test: out/soong/host/linux-x86/bin/external_updater_notifier \
--result_file=/tmp/res.json \
--history ~/updater/history \
--recipients=hhb@google.com
Change-Id: I27a4c1c2604d38106a08ce3eee1bcd03fdce80d7
---
Android.bp | 19 +++++--
external_updater.py | 111 ++++++++++++++++++++++++--------------
git_updater.py | 23 +++++---
github_archive_updater.py | 17 +++---
notifier.py | 108 +++++++++++++++++++++++++++++++++++++
update_package.sh | 1 +
6 files changed, 219 insertions(+), 60 deletions(-)
create mode 100644 notifier.py
diff --git a/Android.bp b/Android.bp
index b10515d..328cf7a 100644
--- a/Android.bp
+++ b/Android.bp
@@ -23,15 +23,24 @@ python_binary_host {
],
}
+python_binary_host {
+ name: "external_updater_notifier",
+ main: "notifier.py",
+ srcs: [
+ "notifier.py",
+ ],
+}
+
python_library_host {
name: "external_updater_lib",
srcs: [
- "*.py",
+ "archive_utils.py",
+ "fileutils.py",
+ "git_updater.py",
+ "git_utils.py",
+ "github_archive_updater.py",
"metadata.proto",
- ],
- exclude_srcs: [
- "*_test.py",
- "external_updater.py",
+ "updater_utils.py",
],
libs: [
"python-symbol",
diff --git a/external_updater.py b/external_updater.py
index 314c1e6..0ee2e86 100644
--- a/external_updater.py
+++ b/external_updater.py
@@ -19,8 +19,10 @@ updater.sh update kotlinc
"""
import argparse
+import json
import os
import subprocess
+import time
from google.protobuf import text_format # pylint: disable=import-error
@@ -72,6 +74,11 @@ def build_updater(proj_path):
return updater
+def has_new_version(updater):
+ """Whether an updater found a new version."""
+ return updater.get_current_version() != updater.get_latest_version()
+
+
def check_update(proj_path):
"""Checks updates for a project. Prints result on console.
@@ -84,56 +91,78 @@ def check_update(proj_path):
end='')
updater = build_updater(proj_path)
if updater is None:
- return (None, None)
+ return (None, 'Failed to create updater')
try:
- new_version = updater.check()
- if new_version:
+ updater.check()
+ if has_new_version(updater):
print(color_string(' Out of date!', 'STALE'))
else:
print(color_string(' Up to date.', 'FRESH'))
- return (updater, new_version)
+ return (updater, None)
except (IOError, ValueError) as err:
print('{} {}.'.format(color_string('Failed.', 'ERROR'),
err))
- return (None, None)
+ return (updater, str(err))
except subprocess.CalledProcessError as err:
- print(
- '{} {}\nstdout: {}\nstderr: {}.'.format(
- color_string(
- 'Failed.',
- 'ERROR'),
- err,
- err.stdout,
- err.stderr))
- return (None, None)
+ msg = 'stdout: {}\nstderr: {}.'.format(
+ err.stdout,
+ err.stderr)
+ print('{} {}.'.format(color_string('Failed.', 'ERROR'),
+ msg))
+ return (updater, msg)
+
+
+def _process_update_result(path):
+ res = {}
+ updater, err = check_update(path)
+ if err is not None:
+ res['error'] = str(err)
+ else:
+ res['current'] = updater.get_current_version()
+ res['latest'] = updater.get_latest_version()
+ return res
+
+
+def _check_some(paths, delay):
+ results = {}
+ for path in paths:
+ results[path] = _process_update_result(path)
+ time.sleep(delay)
+ return results
+
+
+def _check_all(delay):
+ results = {}
+ for path, dirs, files in os.walk(args.path):
+ dirs.sort(key=lambda d: d.lower())
+ if fileutils.METADATA_FILENAME in files:
+ # Skip sub directories.
+ dirs[:] = []
+ results[path] = _process_update_result(path)
+ time.sleep(delay)
+ return results
def check(args):
"""Handler for check command."""
+ if args.all:
+ results = _check_all(args.delay)
+ else:
+ results = _check_some(args.paths, args.delay)
- check_update(args.path)
+ if args.json_output is not None:
+ with open(args.json_output, 'w') as f:
+ json.dump(results, f, sort_keys=True, indent=4)
def update(args):
"""Handler for update command."""
- updater, new_version = check_update(args.path)
+ updater, err = check_update(args.path)
if updater is None:
return
- if not new_version and not args.force:
- return
-
- updater.update()
-
-
-def checkall(args):
- """Handler for checkall command."""
- for root, dirs, files in os.walk(args.path):
- dirs.sort(key=lambda d: d.lower())
- if fileutils.METADATA_FILENAME in files:
- # Skip sub directories.
- dirs[:] = []
- check_update(root)
+ if has_new_version(updater) or args.force:
+ updater.update()
def parse_args():
@@ -148,20 +177,20 @@ def parse_args():
check_parser = subparsers.add_parser(
'check', help='Check update for one project.')
check_parser.add_argument(
- 'path',
- help='Path of the project. '
+ 'paths', nargs='*',
+ help='Paths of the project. '
'Relative paths will be resolved from external/.')
+ check_parser.add_argument(
+ '--json_output',
+ help='Path of a json file to write result to.')
+ check_parser.add_argument(
+ '--all',
+ help='If set, check updates for all supported projects.')
+ check_parser.add_argument(
+ '--delay', default=0, type=int,
+ help='Time in seconds to wait between checking two projects.')
check_parser.set_defaults(func=check)
- # Creates parser for checkall command.
- checkall_parser = subparsers.add_parser(
- 'checkall', help='Check update for all projects.')
- checkall_parser.add_argument(
- '--path',
- default=fileutils.EXTERNAL_PATH,
- help='Starting path for all projects. Default to external/.')
- checkall_parser.set_defaults(func=checkall)
-
# Creates parser for update command.
update_parser = subparsers.add_parser('update', help='Update one project.')
update_parser.add_argument(
diff --git a/git_updater.py b/git_updater.py
index 15f0360..81ef922 100644
--- a/git_updater.py
+++ b/git_updater.py
@@ -60,21 +60,28 @@ class GitUpdater():
self._setup_remote()
if git_utils.is_commit(self.metadata.third_party.version):
# Update to remote head.
- return self._check_head()
+ self._check_head()
+ else:
+ # Update to latest version tag.
+ self._check_tag()
+
+ def get_current_version(self):
+ """Returns the latest version name recorded in METADATA."""
+ return self.metadata.third_party.version
- # Update to latest version tag.
- return self._check_tag()
+ def get_latest_version(self):
+ """Returns the latest version name in upstream."""
+ return self.new_version
def _check_tag(self):
tags = git_utils.list_remote_tags(self.proj_path,
self.upstream_remote_name)
- current_ver = self.metadata.third_party.version
+ current_ver = self.get_current_version()
self.new_version = updater_utils.get_latest_version(
current_ver, tags)
self.merge_from = self.new_version
print('Current version: {}. Latest version: {}'.format(
current_ver, self.new_version), end='')
- return self.new_version != current_ver
def _check_head(self):
commits = git_utils.get_commits_ahead(
@@ -82,7 +89,8 @@ class GitUpdater():
self.android_remote_name + '/master')
if not commits:
- return False
+ self.new_version = self.get_current_version()
+ return
self.new_version = commits[0]
@@ -101,7 +109,6 @@ class GitUpdater():
time_behind = datetime.datetime.now() - commit_time
print('{} commits ({} days) behind.'.format(
len(commits), time_behind.days), end='')
- return True
def _write_metadata(self, path):
updated_metadata = metadata_pb2.MetaData()
@@ -129,4 +136,6 @@ class GitUpdater():
self.merge_from, len(commits), upstream_branch))
self._write_metadata(self.proj_path)
+ print("Running `git merge {merge_branch}`..."
+ .format(merge_branch=self.merge_from))
git_utils.merge(self.proj_path, self.merge_from)
diff --git a/github_archive_updater.py b/github_archive_updater.py
index bcb4bd5..f177b54 100644
--- a/github_archive_updater.py
+++ b/github_archive_updater.py
@@ -92,7 +92,7 @@ class GithubArchiveUpdater():
except IndexError:
raise ValueError('Url format is not supported.')
- def _get_latest_version(self):
+ def _fetch_latest_version(self):
"""Checks upstream and gets the latest release tag."""
url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
@@ -115,7 +115,7 @@ class GithubArchiveUpdater():
self.new_url = choose_best_url(supported_assets, self.old_url.value)
- def _get_latest_commit(self):
+ def _fetch_latest_commit(self):
"""Checks upstream and gets the latest commit to master."""
url = 'https://api.github.com/repos/{}/{}/commits/master'.format(
@@ -126,10 +126,14 @@ class GithubArchiveUpdater():
self.new_url = 'https://github.com/{}/{}/archive/{}.zip'.format(
self.owner, self.repo, self.new_version)
- def _get_current_version(self):
+ def get_current_version(self):
"""Returns the latest version name recorded in METADATA."""
return self.metadata.third_party.version
+ def get_latest_version(self):
+ """Returns the latest version name in upstream."""
+ return self.new_version
+
def _write_metadata(self, url, path):
updated_metadata = metadata_pb2.MetaData()
updated_metadata.CopyFrom(self.metadata)
@@ -144,14 +148,13 @@ class GithubArchiveUpdater():
Returns True if a new version is available.
"""
- current = self._get_current_version()
+ current = self.get_current_version()
if git_utils.is_commit(current):
- self._get_latest_commit()
+ self._fetch_latest_commit()
else:
- self._get_latest_version()
+ self._fetch_latest_version()
print('Current version: {}. Latest version: {}'.format(
current, self.new_version), end='')
- return current != self.new_version
def update(self):
"""Updates the package.
diff --git a/notifier.py b/notifier.py
new file mode 100644
index 0000000..d33dc65
--- /dev/null
+++ b/notifier.py
@@ -0,0 +1,108 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Send notification email if new version is found.
+
+Example usage:
+external_updater_notifier \
+ --result_file ~/updater/new_result \
+ --history ~/updater/history \
+ --recipients xxx@xxx.xxx
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import time
+
+
+def parse_args():
+ """Parses commandline arguments."""
+
+ parser = argparse.ArgumentParser(
+ description='Check updates for third party projects in external/.')
+ parser.add_argument(
+ '--result_file',
+ help='Json check result generated by external updater.')
+ parser.add_argument(
+ '--history',
+ help='Path of history file. If doesn'
+ 't exist, a new one will be created.')
+ parser.add_argument(
+ '--recipients',
+ help='Comma separated recipients of notification email.')
+
+ return parser.parse_args()
+
+
+def _send_email(proj, latest_ver, recipient):
+ print('Sending email for {}: {}'.format(proj, latest_ver))
+ msg = """New version: {}
+
+To upgrade:
+ tools/external_updater/updater.sh update {}""".format(
+ latest_ver, proj)
+ subprocess.run(['sendgmr', '--to=' + recipient,
+ '--subject=' + proj], check=True,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ input=msg, encoding='ascii')
+
+
+def _process_results(history, results, recipient):
+ for proj, res in results.items():
+ if 'latest' not in res:
+ continue
+ latest_ver = res['latest']
+ current_ver = res['current']
+ if latest_ver == current_ver:
+ continue
+ proj_history = history.setdefault(proj, {})
+ if latest_ver not in proj_history:
+ try:
+ _send_email(proj, latest_ver, recipient)
+ proj_history[latest_ver] = int(time.time())
+ except subprocess.CalledProcessError as err:
+ msg = """Failed to send email for {} ({}).
+stdout: {}
+stderr: {}""".format(proj, latest_ver, err.stdout, err.stderr)
+ print(msg)
+
+
+def send_notification(args):
+ """Compare results and send notification."""
+ results = {}
+ with open(args.result_file, 'r') as f:
+ results = json.load(f)
+ history = {}
+ try:
+ with open(args.history, 'r') as f:
+ history = json.load(f)
+ except FileNotFoundError:
+ pass
+
+ _process_results(history, results, args.recipients)
+
+ with open(args.history, 'w') as f:
+ json.dump(history, f, sort_keys=True)
+
+
+def main():
+ """The main entry."""
+
+ args = parse_args()
+ send_notification(args)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/update_package.sh b/update_package.sh
index 8d5e3c6..d7cfa42 100644
--- a/update_package.sh
+++ b/update_package.sh
@@ -43,6 +43,7 @@ CopyIfPresent ".git"
CopyIfPresent ".gitignore"
CopyIfPresent "patches"
CopyIfPresent "post_update.sh"
+CopyIfPresent "OWNERS"
echo "Applying patches..."
for p in $tmp_dir/patches/*.diff
--
GitLab