Merge branch 'polish-for-1.0' into 'master'

Polish for 1.0

Closes #421

See merge request fdroid/fdroidserver!388

buildserver/provision-android-sdk

@@ -74,12 +74,25 @@ y
 EOH
 
 mkdir -p $ANDROID_HOME/licenses/
+
 cat << EOF > $ANDROID_HOME/licenses/android-sdk-license
 
 8933bad161af4178b1185d1a37fbf41ea5269c55
+
 d56f5187479451eabf01fb78af6dfcb131a6481e
 EOF
-echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+79120722343a6f314e0719f863036c702b0e6b2a
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
 echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1"
 echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1"
 echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2"

fdroidserver/build.py

@@ -23,6 +23,7 @@ import shutil
 import glob
 import subprocess
 import re
+import resource
 import tarfile
 import traceback
 import time
@@ -1120,6 +1121,19 @@ def main():
     if not apps:
         raise FDroidException("No apps to process.")
 
+    # make sure enough open files are allowed to process everything
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    if len(apps) > soft:
+        try:
+            soft = len(apps) * 2
+            if soft > hard:
+                soft = hard
+            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+            logging.debug(_('Set open file limit to {integer}')
+                          .format(integer=soft))
+        except (OSError, ValueError) as e:
+            logging.warning(_('Setting open file limit failed: ') + str(e))
+
     if options.latest:
         for app in apps.values():
             for build in reversed(app.builds):

fdroidserver/common.py

@@ -40,7 +40,7 @@ import json
 import xml.etree.ElementTree as XMLElementTree
 
 from binascii import hexlify
-from datetime import datetime
+from datetime import datetime, timedelta
 from distutils.version import LooseVersion
 from queue import Queue
 from zipfile import ZipFile
@@ -444,17 +444,16 @@ def get_local_metadata_files():
     return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
 
 
-def read_pkg_args(args, allow_vercodes=False):
+def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
     """
-    :param args: arguments in the form of multiple appid:[vc] strings
+    :param appids: arguments in the form of multiple appid:[vc] strings
     :returns: a dictionary with the set of vercodes specified for each package
     """
-
     vercodes = {}
-    if not args:
+    if not appid_versionCode_pairs:
         return vercodes
 
-    for p in args:
+    for p in appid_versionCode_pairs:
         if allow_vercodes and ':' in p:
             package, vercode = p.split(':')
         else:
@@ -468,13 +467,17 @@ def read_pkg_args(args, allow_vercodes=False):
     return vercodes
 
 
-def read_app_args(args, allapps, allow_vercodes=False):
-    """
-    On top of what read_pkg_args does, this returns the whole app metadata, but
-    limiting the builds list to the builds matching the vercodes specified.
+def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+    """Build a list of App instances for processing
+
+    On top of what read_pkg_args does, this returns the whole app
+    metadata, but limiting the builds list to the builds matching the
+    appid_versionCode_pairs and vercodes specified.  If no appid_versionCode_pairs are specified, then
+    all App and Build instances are returned.
+
     """
 
-    vercodes = read_pkg_args(args, allow_vercodes)
+    vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
 
     if not vercodes:
         return allapps
@@ -1716,6 +1719,23 @@ def natural_key(s):
     return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
 
 
+def check_system_clock(dt_obj, path):
+    """Check if system clock is updated based on provided date
+
+    If an APK has files newer than the system time, suggest updating
+    the system clock.  This is useful for offline systems, used for
+    signing, which do not have another source of clock sync info. It
+    has to be more than 24 hours newer because ZIP/APK files do not
+    store timezone info
+
+    """
+    checkdt = dt_obj - timedelta(1)
+    if datetime.today() < checkdt:
+        logging.warning(_('System clock is older than date in {path}!').format(path=path)
+                        + '\n' + _('Set clock to that time using:') + '\n'
+                        + 'sudo date -s "' + str(dt_obj) + '"')
+
+
 class KnownApks:
     """permanent store of existing APKs with the date they were added
 
@@ -1744,6 +1764,7 @@ class KnownApks:
                         date = datetime.strptime(t[-1], '%Y-%m-%d')
                         filename = line[0:line.rfind(appid) - 1]
                         self.apks[filename] = (appid, date)
+                        check_system_clock(date, self.path)
         self.changed = False
 
     def writeifchanged(self):

fdroidserver/init.py

@@ -178,6 +178,7 @@ def main():
                              + '" does not exist, creating a new keystore there.')
     common.write_to_config(test_config, 'keystore', keystore)
     repo_keyalias = None
+    keydname = None
     if options.repo_keyalias:
         repo_keyalias = options.repo_keyalias
         common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
@@ -211,7 +212,16 @@ def main():
                                    flags=re.MULTILINE)
             with open('opensc-fdroid.cfg', 'w') as f:
                 f.write(opensc_fdroid)
-    elif not os.path.exists(keystore):
+    elif os.path.exists(keystore):
+        to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
+        if repo_keyalias:
+            to_set.remove('repo_keyalias')
+        if keydname:
+            to_set.remove('keydname')
+        logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore)
+                        + '\n' + _('Now set these in config.py:') + ' '
+                        + ', '.join(to_set) + '\n')
+    else:
         password = common.genpassword()
         c = dict(test_config)
         c['keystorepass'] = password

fdroidserver/nightly.py

@@ -155,6 +155,7 @@ def main():
         repo_url = repo_base + '/repo'
         git_mirror_path = os.path.join(repo_basedir, 'git-mirror')
         git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo')
+        git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata')
         if not os.path.isdir(git_mirror_repodir):
             logging.debug(_('cloning {url}').format(url=clone_url))
             try:
@@ -186,10 +187,10 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update README")
 
-        icon_path = os.path.join(repo_basedir, 'icon.png')
+        icon_path = os.path.join(git_mirror_path, 'icon.png')
         try:
             import qrcode
-            img = qrcode.make('Some data here')
+            img = qrcode.make(repo_url)
             with open(icon_path, 'wb') as fp:
                 fp.write(img)
         except Exception:
@@ -197,9 +198,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             shutil.copy(exampleicon, icon_path)
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update repo/website icon")
+        shutil.copy(icon_path, repo_basedir)
 
         os.chdir(repo_basedir)
-        common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+        if os.path.isdir(git_mirror_repodir):
+            common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+        if os.path.isdir(git_mirror_metadatadir):
+            common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -254,7 +259,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             except subprocess.CalledProcessError:
                 pass
 
-        subprocess.check_call(['fdroid', 'update', '--rename-apks', '--verbose'], cwd=repo_basedir)
+        subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
+                              cwd=repo_basedir)
+        common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/')
+        mirror_git_repo.git.add(all=True)
+        mirror_git_repo.index.commit("update app metadata")
         try:
             subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir)
         except subprocess.CalledProcessError:

fdroidserver/server.py

@@ -513,7 +513,7 @@ def upload_to_virustotal(repo_section, vt_apikey):
                             with open(outputfilename, 'w') as fp:
                                 json.dump(response, fp, indent=2, sort_keys=True)
 
-                        if response.get('positives') > 0:
+                        if response.get('positives', 0) > 0:
                             logging.warning(repofilename + ' has been flagged by virustotal '
                                             + str(response['positives']) + ' times:'
                                             + '\n\t' + response['permalink'])

fdroidserver/update.py

@@ -29,7 +29,7 @@ import zipfile
 import hashlib
 import pickle
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 from argparse import ArgumentParser
 
 import collections
@@ -1297,22 +1297,11 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
 
         apkzip = zipfile.ZipFile(apkfile, 'r')
 
-        # if an APK has files newer than the system time, suggest updating
-        # the system clock.  This is useful for offline systems, used for
-        # signing, which do not have another source of clock sync info. It
-        # has to be more than 24 hours newer because ZIP/APK files do not
-        # store timezone info
         manifest = apkzip.getinfo('AndroidManifest.xml')
         if manifest.date_time[1] == 0:  # month can't be zero
             logging.debug(_('AndroidManifest.xml has no date'))
         else:
-            dt_obj = datetime(*manifest.date_time)
-            checkdt = dt_obj - timedelta(1)
-            if datetime.today() < checkdt:
-                logging.warning('System clock is older than manifest in: '
-                                + apkfilename
-                                + '\nSet clock to that time using:\n'
-                                + 'sudo date -s "' + str(dt_obj) + '"')
+            common.check_system_clock(datetime(*manifest.date_time), apkfilename)
 
         # extract icons from APK zip file
         iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])

fdroidserver/vmtools.py

@@ -29,6 +29,8 @@ import textwrap
 from .common import FDroidException
 from logging import getLogger
 
+from fdroidserver import _
+
 logger = getLogger('fdroidserver-vmtools')
 
 
@@ -383,7 +385,9 @@ class LibvirtBuildVm(FDroidBuildVm):
             vol = storagePool.storageVolLookupByName(self.srvname + '.img')
             imagepath = vol.path()
             # TODO use a libvirt storage pool to ensure the img file is readable
-            _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
+            if not os.access(imagepath, os.R_OK):
+                logger.warning(_('Cannot read "{path}"!').format(path=imagepath))
+                _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
             shutil.copy2(imagepath, 'box.img')
             _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
             img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])

jenkins-build-all

@@ -31,7 +31,6 @@ else
         echo "No virtualization is used."
 fi
 sudo /bin/chmod -R a+rX /var/lib/libvirt/images
-ulimit -n 2048
 echo 'maximum allowed number of open file descriptors: ' `ulimit -n`
 ls -ld /var/lib/libvirt/images
 ls -l /var/lib/libvirt/images || echo no access

setup.py

@@ -89,6 +89,7 @@ setup(name='fdroidserver',
           'pyasn1-modules',
           'python-vagrant',
           'PyYAML',
+          'qrcode',
           'ruamel.yaml >= 0.13',
           'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
           'docker-py >= 1.9, < 2.0',