Merge branch 'polish-verification.f-droid.org-setup' into 'master'

Polish verification.f droid.org setup

See merge request fdroid/fdroidserver!509

.gitlab-ci.yml

@@ -121,24 +121,17 @@ pip_install:
     - fdroid readmeta
     - fdroid update --help
 
-pyup_io_safety_check:
+lint_format_safety_checks:
   image: alpine:3.7
   variables:
     LANG: C.UTF-8
   script:
-    - apk add --no-cache ca-certificates python3
+    - apk add --no-cache bash dash ca-certificates python3
     - python3 -m ensurepip
-    - pip3 install safety
-    - safety check --full-report
-
-pylint:
-  image: alpine:3.7
-  variables:
-    LANG: C.UTF-8
-  script:
-    - apk add --no-cache ca-certificates python3
-    - python3 -m ensurepip
-    - pip3 install pylint
+    - pip3 install pep8 pyflakes pylint safety
+    - export EXITVALUE=0
+    - ./hooks/pre-commit || export EXITVALUE=1
+    - safety check --full-report || export EXITVALUE=1
     - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
             fdroid
             makebuildserver
@@ -146,6 +139,8 @@ pylint:
             fdroidserver/*.py
             tests/*.py
             tests/*.TestCase
+        || export EXITVALUE=1
+    - exit $EXITVALUE
 
 fedora_latest:
   image: fedora:latest

fdroid

@@ -156,7 +156,7 @@ def main():
     # str(e) often doesn't contain a reason, so just show the backtrace
     except Exception as e:
         logging.critical(_("Unknown exception found!"))
-        raise
+        raise e
     sys.exit(0)
 
 

fdroidserver/common.py

@@ -2788,7 +2788,7 @@ def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
         htmlfile = logfilename + '.diffoscope.html'
         textfile = logfilename + '.diffoscope.txt'
         if subprocess.call([config['diffoscope'],
-                            '--max-report-size', '12345678', '--max-diff-block-lines', '100',
+                            '--max-report-size', '12345678', '--max-diff-block-lines', '128',
                             '--html', htmlfile, '--text', textfile,
                             absapk1, absapk2]) != 0:
             return("Failed to run diffoscope " + apk1)

fdroidserver/verify.py

@@ -40,6 +40,8 @@ def main():
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
     parser.add_argument("appid", nargs='*', help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
+    parser.add_argument("--reuse-remote-apk", action="store_true", default=False,
+                        help=_("Verify against locally cached copy rather than redownloading."))
     options = parser.parse_args()
 
     config = common.read_config(options)
@@ -74,18 +76,19 @@ def main():
             logging.info("Processing {apkfilename}".format(apkfilename=apkfilename))
 
             remoteapk = os.path.join(tmp_dir, apkfilename)
-            if os.path.exists(remoteapk):
-                os.remove(remoteapk)
-            url = 'https://f-droid.org/repo/' + apkfilename
-            logging.info("...retrieving " + url)
-            try:
-                net.download_file(url, dldir=tmp_dir)
-            except requests.exceptions.HTTPError as e:
+            if not options.reuse_remote_apk or not os.path.exists(remoteapk):
+                if os.path.exists(remoteapk):
+                    os.remove(remoteapk)
+                url = 'https://f-droid.org/repo/' + apkfilename
+                logging.info("...retrieving " + url)
                 try:
-                    net.download_file(url.replace('/repo', '/archive'), dldir=tmp_dir)
-                except requests.exceptions.HTTPError as e:
-                    raise FDroidException(_('Downloading {url} failed. {error}')
-                                          .format(url=url, error=e))
+                    net.download_file(url, dldir=tmp_dir)
+                except requests.exceptions.HTTPError:
+                    try:
+                        net.download_file(url.replace('/repo', '/archive'), dldir=tmp_dir)
+                    except requests.exceptions.HTTPError as e:
+                        raise FDroidException(_('Downloading {url} failed. {error}')
+                                              .format(url=url, error=e))
 
             compare_result = common.verify_apks(
                 remoteapk,
@@ -101,9 +104,11 @@ def main():
             logging.info("...NOT verified - {0}".format(e))
             notverified += 1
 
-    logging.info(_("Finished"))
-    logging.info("{0} successfully verified".format(verified))
-    logging.info("{0} NOT verified".format(notverified))
+    if verified > 0:
+        logging.info("{0} successfully verified".format(verified))
+    if notverified > 0:
+        logging.info("{0} NOT verified".format(notverified))
+    sys.exit(notverified)
 
 
 if __name__ == "__main__":

makebuildserver

@@ -74,7 +74,7 @@ config = {
 if os.path.isfile('/usr/bin/systemd-detect-virt'):
     try:
         virt = subprocess.check_output('/usr/bin/systemd-detect-virt').strip().decode('utf-8')
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         virt = 'none'
     if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
         logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')

tests/run-tests

@@ -447,6 +447,21 @@ test -e repo/com.politedroid_5.apk
 ! test -e repo/com.politedroid_6.apk
 
 
+#------------------------------------------------------------------------------#
+echo_header 'test that verify can succeed and fail'
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+test -d tmp || mkdir tmp
+test -d unsigned || mkdir unsigned
+cp $WORKSPACE/tests/repo/com.politedroid_6.apk tmp/
+cp $WORKSPACE/tests/repo/com.politedroid_6.apk unsigned/
+$fdroid verify --reuse-remote-apk --verbose com.politedroid
+# force a fail
+cp $WORKSPACE/tests/repo/com.politedroid_5.apk unsigned/com.politedroid_6.apk
+! $fdroid verify --reuse-remote-apk --verbose com.politedroid
+
+
 #------------------------------------------------------------------------------#
 echo_header 'test allowing disabled signatures in repo and archive'