Snap for 4939473 from edf70831 to qt-release

Change-Id: I64f14bfbf3bf579e8d22e7cff5b0a1134930c0f4

Android.bp

@@ -15,10 +15,24 @@
 python_binary_host {
     name: "external_updater",
     main: "external_updater.py",
+    srcs: [
+        "external_updater.py",
+    ],
+    libs: [
+        "external_updater_lib",
+    ],
+}
+
+python_library_host {
+    name: "external_updater_lib",
     srcs: [
         "*.py",
         "metadata.proto",
     ],
+    exclude_srcs: [
+        "*_test.py",
+        "external_updater.py",
+    ],
     libs: [
         "python-symbol",
         "libprotobuf-python",
@@ -41,3 +55,13 @@ python_binary_host {
     },
 }
 
+python_test_host {
+    name: "external_updater_test",
+    main: "external_updater_test.py",
+    srcs: [
+        "external_updater_test.py",
+    ],
+    libs: [
+        "external_updater_lib",
+    ],
+}

external_updater_test.py

+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for external updater."""
+
+import unittest
+
+import github_archive_updater
+
+
+class ExternalUpdaterTest(unittest.TestCase):
+    """Unit tests for external updater."""
+
+    def test_url_selection(self):
+        """Tests that GithubArchiveUpdater can choose the right url."""
+        prefix = "https://github.com/author/project/"
+        urls = [
+            prefix + "releases/download/ver-1.0/ver-1.0-binary.zip",
+            prefix + "releases/download/ver-1.0/ver-1.0-binary.tar.gz",
+            prefix + "releases/download/ver-1.0/ver-1.0-src.zip",
+            prefix + "releases/download/ver-1.0/ver-1.0-src.tar.gz",
+            prefix + "archive/ver-1.0.zip",
+            prefix + "archive/ver-1.0.tar.gz",
+        ]
+
+        previous_url = prefix + "releases/download/ver-0.9/ver-0.9-src.tar.gz"
+        url = github_archive_updater.choose_best_url(urls, previous_url)
+        expected_url = prefix + "releases/download/ver-1.0/ver-1.0-src.tar.gz"
+        self.assertEqual(url, expected_url)
+
+        previous_url = prefix + "archive/ver-0.9.zip"
+        url = github_archive_updater.choose_best_url(urls, previous_url)
+        expected_url = prefix + "archive/ver-1.0.zip"
+        self.assertEqual(url, expected_url)
+
+
+if __name__ == '__main__':
+    unittest.main()

git_utils.py

@@ -22,6 +22,7 @@ def _run(cmd, cwd):
     return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                           check=True, cwd=cwd)
 
+
 def fetch(proj_path, remote_names):
     """Runs git fetch.
 
@@ -31,6 +32,7 @@ def fetch(proj_path, remote_names):
     """
     _run(['git', 'fetch', '--multiple'] + remote_names, cwd=proj_path)
 
+
 def add_remote(proj_path, name, url):
     """Adds a git remote.
 
@@ -41,6 +43,7 @@ def add_remote(proj_path, name, url):
     """
     _run(['git', 'remote', 'add', name, url], cwd=proj_path)
 
+
 def list_remotes(proj_path):
     """Lists all Git remotes.
 
@@ -54,6 +57,7 @@ def list_remotes(proj_path):
     lines = out.stdout.decode('utf-8').splitlines()
     return dict([line.split()[0:2] for line in lines])
 
+
 def get_commits_ahead(proj_path, branch, base_branch):
     """Lists commits in `branch` but not `base_branch`."""
     out = _run(['git', 'rev-list', '--left-only',
@@ -61,11 +65,13 @@ def get_commits_ahead(proj_path, branch, base_branch):
                proj_path)
     return out.stdout.decode('utf-8').splitlines()
 
+
 def get_commit_time(proj_path, commit):
     """Gets commit time of one commit."""
     out = _run(['git', 'show', '-s', '--format=%ct', commit], cwd=proj_path)
     return datetime.datetime.fromtimestamp(int(out.stdout))
 
+
 def list_remote_branches(proj_path, remote_name):
     """Lists all branches for a remote."""
     out = _run(['git', 'branch', '-r'], cwd=proj_path)

github_archive_updater.py

@@ -16,7 +16,6 @@
 
 import json
 import re
-import shutil
 import urllib.request
 
 import archive_utils
@@ -29,6 +28,39 @@ GITHUB_URL_PATTERN = (r'^https:\/\/github.com\/([-\w]+)\/([-\w]+)\/' +
 GITHUB_URL_RE = re.compile(GITHUB_URL_PATTERN)
 
 
+def _edit_distance(str1, str2):
+    prev = list(range(0, len(str2) + 1))
+    for i, chr1 in enumerate(str1):
+        cur = [i + 1]
+        for j, chr2 in enumerate(str2):
+            if chr1 == chr2:
+                cur.append(prev[j])
+            else:
+                cur.append(min(prev[j + 1], prev[j], cur[j]) + 1)
+        prev = cur
+    return prev[len(str2)]
+
+
+def choose_best_url(urls, previous_url):
+    """Returns the best url to download from a list of candidate urls.
+
+    This function calculates similarity between previous url and each of new
+    urls. And returns the one best matches previous url.
+
+    Similarity is measured by editing distance.
+
+    Args:
+        urls: Array of candidate urls.
+        previous_url: String of the url used previously.
+
+    Returns:
+        One url from `urls`.
+    """
+    return min(urls, default=None,
+               key=lambda url: _edit_distance(
+                   url, previous_url))
+
+
 class GithubArchiveUpdater():
     """Updater for archives from GitHub.
 
@@ -98,18 +130,18 @@ class GithubArchiveUpdater():
         """
 
         supported_assets = [
-            a for a in self.data['assets']
+            a['browser_download_url'] for a in self.data['assets']
             if archive_utils.is_supported_archive(a['browser_download_url'])]
 
-        # Finds the minimum sized archive to download.
-        minimum_asset = min(
-            supported_assets, key=lambda asset: asset['size'], default=None)
-        if minimum_asset is not None:
-            latest_url = minimum_asset.get('browser_download_url')
-        else:
-            # Guess the tarball url for source code.
-            latest_url = 'https://github.com/{}/{}/archive/{}.tar.gz'.format(
-                self.owner, self.repo, self.data.get('tag_name'))
+        # Adds source code urls.
+        supported_assets.append(
+            'https://github.com/{}/{}/archive/{}.tar.gz'.format(
+                self.owner, self.repo, self.data.get('tag_name')))
+        supported_assets.append(
+            'https://github.com/{}/{}/archive/{}.zip'.format(
+                self.owner, self.repo, self.data.get('tag_name')))
+
+        latest_url = choose_best_url(supported_assets, self.old_url.value)
 
         temporary_dir = None
         try:
@@ -118,5 +150,7 @@ class GithubArchiveUpdater():
             self._write_metadata(latest_url, package_dir)
             updater_utils.replace_package(package_dir, self.proj_path)
         finally:
-            shutil.rmtree(temporary_dir, ignore_errors=True)
+            # Don't remove the temporary directory, or it'll be impossible
+            # to debug the failure...
+            # shutil.rmtree(temporary_dir, ignore_errors=True)
             urllib.request.urlcleanup()

update_package.sh

@@ -15,36 +15,50 @@
 # limitations under the License.
 
 # This script is used by external_updater to replace a package. Don't
-# invoke directly
-
-cd $1
-
-# Copies all files we want to reserve.
-cp -a -n $2/Android.bp       $1/  2> /dev/null
-cp -a -n $2/Android.mk       $1/  2> /dev/null
-cp -a -n $2/LICENSE          $1/  2> /dev/null
-cp -a -n $2/NOTICE           $1/  2> /dev/null
-cp -a -n $2/MODULE_LICENSE_* $1/  2> /dev/null
-cp -a -n $2/METADATA         $1/  2> /dev/null
-cp -a -n $2/.git             $1/  2> /dev/null
-cp -a -n $2/.gitignore       $1/  2> /dev/null
-cp -a -n $2/patches          $1/  2> /dev/null
-cp -a -n $2/post_update.sh   $1/  2> /dev/null
-
-# Applies all patches
-for p in $1/patches/*.diff
+# invoke directly.
+
+set -e
+
+tmp_dir=$1
+external_dir=$2
+
+echo "Entering $tmp_dir..."
+cd $tmp_dir
+
+function CopyIfPresent() {
+  if [ -e $external_dir/$1 ]; then
+    cp -a -n $external_dir/$1 .
+  fi
+}
+
+echo "Copying preserved files..."
+CopyIfPresent "Android.bp"
+CopyIfPresent "Android.mk"
+CopyIfPresent "LICENSE"
+CopyIfPresent "NOTICE"
+cp -a -f -n $external_dir/MODULE_LICENSE_* .
+CopyIfPresent "METADATA"
+CopyIfPresent ".git"
+CopyIfPresent ".gitignore"
+CopyIfPresent "patches"
+CopyIfPresent "post_update.sh"
+
+echo "Applying patches..."
+for p in $tmp_dir/patches/*.diff
 do
   [ -e "$p" ] || continue
-  echo Applying $p
-  patch -p1 -d $1 < $p;
+  echo "Applying $p..."
+  patch -p1 -d $tmp_dir < $p;
 done
 
-if [ -f $1/post_update.sh ]
+if [ -f $tmp_dir/post_update.sh ]
 then
-  echo Running post update script
-  $1/post_update.sh $1 $2
+  echo "Running post update script"
+  $tmp_dir/post_update.sh $tmp_dir $external_dir
 fi
 
-# Swap old and new.
-rm -rf $2
-mv $1 $2
+echo "Swapping old and new..."
+rm -rf $external_dir
+mv $tmp_dir $external_dir
+
+exit 0

updater.sh

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 cd $(dirname "$0")/../..
 source build/envsetup.sh
 lunch aosp_arm-eng