Merge branch 'bug-fix-grab-bag' into 'master'

bug fix grab bag; and, complete 'build' JSON status

Closes #344

See merge request fdroid/fdroidserver!875

.gitignore

@@ -41,13 +41,19 @@ makebuildserver.config.py
 /tests/OBBMainTwoVersions.apk
 /tests/archive/categories.txt
 /tests/archive/icons*
+/tests/archive/index.css
+/tests/archive/index.html
 /tests/archive/index.jar
+/tests/archive/index.png
 /tests/archive/index_unsigned.jar
 /tests/archive/index.xml
 /tests/archive/index-v1.jar
 /tests/archive/index-v1.json
 /tests/metadata/org.videolan.vlc/en-US/icon*.png
+/tests/repo/index.css
+/tests/repo/index.html
 /tests/repo/index.jar
+/tests/repo/index.png
 /tests/repo/index_unsigned.jar
 /tests/repo/index-v1.jar
 /tests/repo/info.guardianproject.urzip/

MANIFEST.in

@@ -551,6 +551,13 @@ include tests/dump_internal_metadata_format.py
 include tests/exception.TestCase
 include tests/extra/manual-vmtools-test.py
 include tests/funding-usernames.yaml
+include tests/get_android_tools_versions/android-ndk/android-ndk-r21d/source.properties
+include tests/get_android_tools_versions/android-ndk/r11c/source.properties
+include tests/get_android_tools_versions/android-ndk/r17c/source.properties
+include tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties
+include tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties
+include tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties
+include tests/get_android_tools_versions/android-sdk/tools/source.properties
 include tests/getsig/getsig.java
 include tests/getsig/make.sh
 include tests/getsig/run.sh

examples/fdroid_fetchsrclibs.py

@@ -21,7 +21,7 @@ def main():
     options = parser.parse_args()
     common.options = options
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs)
+    allapps = metadata.read_metadata(pkgs, check_vcs=True)
     apps = common.read_app_args(options.appid, allapps, True)
     srclib_dir = os.path.join('build', 'srclib')
     os.makedirs(srclib_dir, exist_ok=True)

fdroidserver/build.py

@@ -83,7 +83,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
                 buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
                                                          'cat /home/vagrant/buildserverid'],
                                                         cwd='builder').strip().decode()
-                status_output['buildserverid'] = buildserverid
                 logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
                               .format(buildserverid=buildserverid))
             except Exception as e:
@@ -1013,7 +1012,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True)
+    allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True, check_vcs=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):
@@ -1266,12 +1265,14 @@ def main():
                     .format(id=fdroidserverid))
         if os.cpu_count():
             txt += "* host processors: %d\n" % os.cpu_count()
+            status_output['hostOsCpuCount'] = os.cpu_count()
         if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
             with open('/proc/meminfo') as fp:
                 for line in fp:
                     m = re.search(r'MemTotal:\s*([0-9].*)', line)
                     if m:
                         txt += "* host RAM: %s\n" % m.group(1)
+                        status_output['hostProcMeminfoMemTotal'] = m.group(1)
                         break
         fdroid_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
         buildserver_config = os.path.join(fdroid_path, 'makebuildserver.config.py')
@@ -1281,9 +1282,11 @@ def main():
                     m = re.search(r'cpus\s*=\s*([0-9].*)', line)
                     if m:
                         txt += "* guest processors: %s\n" % m.group(1)
+                        status_output['guestVagrantVmCpus'] = m.group(1)
                     m = re.search(r'memory\s*=\s*([0-9].*)', line)
                     if m:
                         txt += "* guest RAM: %s MB\n" % m.group(1)
+                        status_output['guestVagrantVmMemory'] = m.group(1)
         txt += "* successful builds: %d\n" % len(build_succeeded)
         txt += "* failed builds: %d\n" % len(failed_builds)
         txt += "\n\n"
@@ -1291,6 +1294,9 @@ def main():
         newpage = site.Pages['build']
         newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
 
+    if buildserverid:
+        status_output['buildserver'] = {'commitId': buildserverid}
+
     if not options.onserver:
         common.write_status_json(status_output)
 

fdroidserver/common.py

@@ -3821,7 +3821,7 @@ def get_android_tools_versions(ndk_path=None):
             with open(ndk_release_txt, 'r') as fp:
                 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
 
-    pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
+    pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE)
     for root, dirs, files in os.walk(sdk_path):
         if 'source.properties' in files:
             source_properties = os.path.join(root, 'source.properties')

fdroidserver/index.py

@@ -563,12 +563,25 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
             json.dump(output, fp, default=_index_encoder_default)
 
     if common.options.nosign:
+        _copy_to_local_copy_dir(repodir, index_file)
         logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
     else:
         signindex.config = common.config
         signindex.sign_index_v1(repodir, json_name)
 
 
+def _copy_to_local_copy_dir(repodir, f):
+    local_copy_dir = common.config.get('local_copy_dir', '')
+    if os.path.exists(local_copy_dir):
+        destdir = os.path.join(local_copy_dir, repodir)
+        if not os.path.exists(destdir):
+            os.mkdir(destdir)
+        shutil.copy2(f, destdir, follow_symlinks=False)
+    elif local_copy_dir:
+        raise FDroidException(_('"local_copy_dir" {path} does not exist!')
+                              .format(path=local_copy_dir))
+
+
 def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
     """Sorts the supplied list to ensure a deterministic sort order for
     package entries in the index file. This sort-order also expresses
@@ -577,7 +590,6 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
 
     :param packages: list of packages which need to be sorted before but into index file.
     """
-
     GROUP_DEV_SIGNED = 1
     GROUP_FDROID_SIGNED = 2
     GROUP_OTHER_SIGNED = 3
@@ -585,22 +597,22 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
     def v1_sort_keys(package):
         packageName = package.get('packageName', None)
 
-        sig = package.get('signer', None)
+        signer = package.get('signer', None)
 
-        dev_sig = common.metadata_find_developer_signature(packageName)
+        dev_signer = common.metadata_find_developer_signature(packageName)
         group = GROUP_OTHER_SIGNED
-        if dev_sig and dev_sig == sig:
+        if dev_signer and dev_signer == signer:
             group = GROUP_DEV_SIGNED
         else:
-            fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
+            fdroid_signer = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
-            if fdroidsig and fdroidsig == sig:
+            if fdroid_signer and fdroid_signer == signer:
                 group = GROUP_FDROID_SIGNED
 
         versionCode = None
         if package.get('versionCode', None):
             versionCode = -int(package['versionCode'])
 
-        return(packageName, group, sig, versionCode)
+        return(packageName, group, signer, versionCode)
 
     packages.sort(key=v1_sort_keys)
 
@@ -705,11 +717,11 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
                 if name_from_apk is None:
                     name_from_apk = apk.get('name')
         for versionCode, apksforver in apksbyversion.items():
-            fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
+            fdroid_signer = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
             fdroid_signed_apk = None
             name_match_apk = None
             for x in apksforver:
-                if fdroidsig and x.get('signer', None) == fdroidsig:
+                if fdroid_signer and x.get('signer', None) == fdroid_signer:
                     fdroid_signed_apk = x
                 if common.apk_release_filename.match(x.get('apkName', '')):
                     name_match_apk = x
@@ -926,6 +938,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
         # Sign the index...
         signed = os.path.join(repodir, 'index.jar')
         if common.options.nosign:
+            _copy_to_local_copy_dir(repodir, os.path.join(repodir, jar_output))
             # Remove old signed index if not signing
             if os.path.exists(signed):
                 os.remove(signed)

fdroidserver/metadata.py

@@ -541,7 +541,7 @@ def read_srclibs():
         srclibs[srclibname] = parse_yaml_srclib(metadatapath)
 
 
-def read_metadata(appids={}, refresh=True, sort_by_time=False):
+def read_metadata(appids={}, refresh=True, sort_by_time=False, check_vcs=False):
     """Return a list of App instances sorted newest first
 
     This reads all of the metadata files in a 'data' repository, then
@@ -597,7 +597,7 @@ def read_metadata(appids={}, refresh=True, sort_by_time=False):
         if appid in apps:
             _warn_or_exception(_("Found multiple metadata files for {appid}")
                                .format(appid=appid))
-        app = parse_metadata(metadatapath, appid in appids, refresh)
+        app = parse_metadata(metadatapath, check_vcs, refresh)
         check_metadata(app)
         apps[app.id] = app
 

fdroidserver/scanner.py

@@ -106,6 +106,9 @@ def scan_binary(apkfile):
     logging.info("Scanning APK for known non-free classes.")
     result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
     problems = 0
+    if result.returncode != 0:
+        problems += 1
+        logging.error(result.output)
     for suspect, regexp in CODE_SIGNATURES.items():
         matches = regexp.findall(result.output)
         if matches:

fdroidserver/update.py

@@ -1423,7 +1423,7 @@ def _get_apk_icons_src(apkfile, icon_name):
 
     """
     icons_src = dict()
-    density_re = re.compile(r'^res/(.*)/{}\.(png|xml)$'.format(icon_name))
+    density_re = re.compile(r'^res/(.*)/{}\.png$'.format(icon_name))
     with zipfile.ZipFile(apkfile) as zf:
         for filename in zf.namelist():
             m = density_re.match(filename)
@@ -1686,7 +1686,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
                                 .format(apkfilename=apkfile) + str(e))
 
         # extract icons from APK zip file
-        iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
+        iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
         try:
             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
         finally:
@@ -1773,8 +1773,6 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
         if m and m.group(4) == 'png':
             density = screen_resolutions[m.group(2)]
             pngs[m.group(3) + '/' + density] = m.group(0)
-
-    icon_type = None
     empty_densities = []
     for density in screen_densities:
         if density not in apk['icons_src']:
@@ -1782,7 +1780,7 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
             continue
         icon_src = apk['icons_src'][density]
         icon_dir = get_icon_dir(repo_dir, density)
-        icon_type = '.png'
+        icon_dest = os.path.join(icon_dir, icon_filename)
 
         # Extract the icon files per density
         if icon_src.endswith('.xml'):
@@ -1793,48 +1791,44 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
                     icon_src = name
             if icon_src.endswith('.xml'):
                 empty_densities.append(density)
-                icon_type = '.xml'
+                continue
-        icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
-
         try:
             with open(icon_dest, 'wb') as f:
                 f.write(get_icon_bytes(apkzip, icon_src))
-            apk['icons'][density] = icon_filename + icon_type
+            apk['icons'][density] = icon_filename
         except (zipfile.BadZipFile, ValueError, KeyError) as e:
             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
             del apk['icons_src'][density]
             empty_densities.append(density)
 
     # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
-    if '-1' in apk['icons_src']:
+    if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
         icon_src = apk['icons_src']['-1']
-        icon_type = icon_src[-4:]
+        icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
-        icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
         with open(icon_path, 'wb') as f:
             f.write(get_icon_bytes(apkzip, icon_src))
-        if icon_type == '.png':
+        im = None
-            im = None
+        try:
-            try:
+            im = Image.open(icon_path)
-                im = Image.open(icon_path)
+            dpi = px_to_dpi(im.size[0])
-                dpi = px_to_dpi(im.size[0])
+            for density in screen_densities:
-                for density in screen_densities:
+                if density in apk['icons']:
-                    if density in apk['icons']:
+                    break
-                        break
+                if density == screen_densities[-1] or dpi >= int(density):
-                    if density == screen_densities[-1] or dpi >= int(density):
+                    apk['icons'][density] = icon_filename
-                        apk['icons'][density] = icon_filename + icon_type
+                    shutil.move(icon_path,
-                        shutil.move(icon_path,
+                                os.path.join(get_icon_dir(repo_dir, density), icon_filename))
-                                    os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
+                    empty_densities.remove(density)
-                        empty_densities.remove(density)
+                    break
-                        break
+        except Exception as e:
-            except Exception as e:
+            logging.warning(_("Failed reading {path}: {error}")
-                logging.warning(_("Failed reading {path}: {error}")
+                            .format(path=icon_path, error=e))
-                                .format(path=icon_path, error=e))
+        finally:
-            finally:
+            if im and hasattr(im, 'close'):
-                if im and hasattr(im, 'close'):
+                im.close()
-                    im.close()
 
     if apk['icons']:
-        apk['icon'] = icon_filename + icon_type
+        apk['icon'] = icon_filename
 
     return empty_densities
 
@@ -1849,7 +1843,6 @@ def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
     :param repo_dir: The directory of the APK's repository
 
     """
-    icon_filename += '.png'
     # First try resizing down to not lose quality
     last_density = None
     for density in screen_densities:
@@ -2324,7 +2317,13 @@ def main():
                                            options.use_date_from_apk)
     cachechanged = cachechanged or fcachechanged
     apks += files
+    appid_has_apks = set()
+    appid_has_repo_files = set()
     for apk in apks:
+        if apk['apkName'].endswith('.apk'):
+            appid_has_apks.add(apk['packageName'])
+        else:
+            appid_has_repo_files.add(apk['packageName'])
         if apk['packageName'] not in apps:
             if options.create_metadata:
                 create_metadata_from_template(apk)
@@ -2343,6 +2342,15 @@ def main():
                 else:
                     logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
 
+    mismatch_errors = ''
+    for appid in appid_has_apks:
+        if appid in appid_has_repo_files:
+            appid_files = ', '.join(glob.glob(os.path.join('repo', appid + '_[0-9]*.*')))
+            mismatch_errors += (_('{appid} has both APKs and files: {files}')
+                                .format(appid=appid, files=appid_files)) + '\n'
+    if mismatch_errors:
+        raise FDroidException(mismatch_errors)
+
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:
         archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)

tests/common.TestCase

@@ -1627,6 +1627,21 @@ class CommonTest(unittest.TestCase):
                 {'AutoName': testvalue, 'id': 'nope'}]:
             self.assertEqual(testvalue, fdroidserver.common.get_app_display_name(app))
 
+    def test_get_android_tools_versions(self):
+        sdk_path = os.path.join(self.basedir, 'get_android_tools_versions')
+        fdroidserver.common.config = {'sdk_path': sdk_path}
+        components = fdroidserver.common.get_android_tools_versions()
+        expected = (
+            ('android-ndk/android-ndk-r21d', '21.3.6528147'),
+            ('android-ndk/r11c', '11.2.2725575'),
+            ('android-ndk/r17c', '17.2.4988734'),
+            ('android-sdk/patcher/v4', '1'),
+            ('android-sdk/platforms/android-30', '3'),
+            ('android-sdk/skiaparser/1', '6'),
+            ('android-sdk/tools', '26.1.1'),
+        )
+        self.assertSequenceEqual(expected, sorted(components))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/get_android_tools_versions/android-ndk/android-ndk-r21d/source.properties

@@ -0,0 +1,2 @@
+Pkg.Desc = Android NDK
+Pkg.Revision = 21.3.6528147

tests/get_android_tools_versions/android-ndk/r11c/source.properties

@@ -0,0 +1,2 @@
+Pkg.Desc = Android NDK
+Pkg.Revision = 11.2.2725575

tests/get_android_tools_versions/android-ndk/r17c/source.properties

@@ -0,0 +1,2 @@
+Pkg.Desc = Android NDK
+Pkg.Revision = 17.2.4988734

tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties

@@ -0,0 +1,18 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+Pkg.Revision=1
+Pkg.Path=patcher;v4
+Pkg.Desc=SDK Patch Applier v4

tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties

@@ -0,0 +1,9 @@
+Pkg.Desc=Android SDK Platform 11
+Pkg.UserSrc=false
+Platform.Version=11
+Platform.CodeName=
+Pkg.Revision=3
+AndroidVersion.ApiLevel=30
+Layoutlib.Api=15
+Layoutlib.Revision=1
+Platform.MinToolsRev=22

tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties

@@ -0,0 +1,3 @@
+Pkg.Revision=6
+Pkg.Path=skiaparser;1
+Pkg.Desc=Layout Inspector image server for API 29-30

tests/get_android_tools_versions/android-sdk/tools/source.properties

@@ -0,0 +1,6 @@
+Pkg.UserSrc=false
+Pkg.Revision=26.1.1
+Platform.MinPlatformToolsRev=20
+Pkg.Dependencies=emulator
+Pkg.Path=tools
+Pkg.Desc=Android SDK Tools

tests/index.TestCase

@@ -395,6 +395,15 @@ class IndexTest(unittest.TestCase):
             self.maxDiff = None
             self.assertEquals(css, pretty_css)
 
+    def test_v1_sort_packages_with_invalid(self):
+        i = [{'packageName': 'org.smssecure.smssecure',
+              'apkName': 'smssecure-custom.fake',
+              'signer': None,
+              'versionCode': 11111}]
+
+        fdroidserver.index.v1_sort_packages(
+            i, fdroidserver.common.load_stats_fdroid_signing_key_fingerprints())
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/repo/index-v1.json

@@ -54,7 +54,6 @@
       "name": "Caffeine Tile",
       "summary": "Test app for extracting icons when an XML one is default",
       "added": 1539129600000,
-      "icon": "info.zwanenburg.caffeinetile.4.xml",
       "packageName": "info.zwanenburg.caffeinetile",
       "lastUpdated": 1539129600000
     },

tests/repo/index.xml

@@ -53,7 +53,6 @@ Features:
 		<lastupdated>2018-10-10</lastupdated>
 		<name>Caffeine Tile</name>
 		<summary>Test app for extracting icons when an XML one is default</summary>
-		<icon>info.zwanenburg.caffeinetile.4.xml</icon>
 		<desc>No description available</desc>
 		<license>Unknown</license>
 		<categories>Development</categories>

tests/run-tests

@@ -735,8 +735,9 @@ cd $REPOROOT
 $fdroid init
 $fdroid update --create-metadata --verbose
 $fdroid readmeta
-$fdroid deploy --local-copy-dir=/tmp/fdroid
+LOCAL_COPY_DIR=`create_test_dir`/fdroid
-$fdroid deploy --local-copy-dir=/tmp/fdroid --verbose
+$fdroid deploy --local-copy-dir=$LOCAL_COPY_DIR
+$fdroid deploy --local-copy-dir=$LOCAL_COPY_DIR --verbose
 
 # now test the errors work
 set +e

tests/update.TestCase

@@ -707,8 +707,7 @@ class UpdateTest(unittest.TestCase):
 
         apk_info = fdroidserver.update.scan_apk('repo/info.zwanenburg.caffeinetile_4.apk')
         self.assertEqual(apk_info.get('versionName'), '1.3')
-        self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable/ic_coffee_on.xml',
+        self.assertEqual(apk_info['icons_src'], {})
-                                                 '-1': 'res/drawable/ic_coffee_on.xml'})
 
         apk_info = fdroidserver.update.scan_apk('repo/com.politedroid_6.apk')
         self.assertEqual(apk_info.get('versionName'), '1.5')