Merge branch 'random-small-fixes' into 'master'

Random small fixes

Closes #222

See merge request !292

.gitlab-ci.yml

@@ -16,6 +16,7 @@ metadata_v0:
   script:
     - cd tests
     - cp dump_internal_metadata_format.py dump.py # since this isn't in old commits
+    - export GITCOMMIT=`git describe`
     - git checkout 0.7.0  # or any old commit of your choosing
     - cd ..
     - sed -i "s/'Author Email',/'Author Email',\n'Author Web Site',/" fdroidserver/metadata.py
@@ -24,7 +25,7 @@ metadata_v0:
     - ../tests/dump.py
     - cd ..
     - git reset --hard
-    - git checkout master
+    - git checkout $GITCOMMIT
     - cd fdroiddata
     - ../tests/dump.py
     - "sed -i -e '/AuthorWebSite/d'

fdroidserver/init.py

@@ -246,6 +246,6 @@ then run "fdroid update -c; fdroid update".  You might also want to edit
 "config.py" to set the URL, repo name, and more.  You should also set up
 a signing key (a temporary one might have been automatically generated).
 
-For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository
-and https://f-droid.org/manual/fdroid.html#Signing
+For more info: https://f-droid.org/docs/Setup_an_F-Droid_App_Repo
+and https://f-droid.org/docs/Signing_Process
 ''')

fdroidserver/lint.py

@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from argparse import ArgumentParser
+import glob
 import os
 import re
 import sys
@@ -378,6 +379,29 @@ def check_extlib_dir(apps):
         yield "Unused extlib at %s" % os.path.join(dir_path, path)
 
 
+def check_for_unsupported_metadata_files(basedir=""):
+    """Checks whether any non-metadata files are in metadata/"""
+
+    global config
+
+    return_value = False
+    formats = config['accepted_formats']
+    for f in glob.glob(basedir + 'metadata/*') + glob.glob(basedir + 'metadata/.*'):
+        if os.path.isdir(f):
+            exists = False
+            for t in formats:
+                exists = exists or os.path.exists(f + '.' + t)
+            if not exists:
+                print('"' + f + '/" has no matching metadata file!')
+                return_value = True
+        elif not os.path.splitext(f)[1][1:] in formats:
+            print('"' + f.replace(basedir, '')
+                  + '" is not a supported file format: (' + ','.join(formats) + ')')
+            return_value = True
+
+    return return_value
+
+
 def main():
 
     global config, options
@@ -398,7 +422,7 @@ def main():
     allapps = metadata.read_metadata(xref=True)
     apps = common.read_app_args(options.appid, allapps, False)
 
-    anywarns = False
+    anywarns = check_for_unsupported_metadata_files()
 
     apps_check_funcs = []
     if len(options.appid) == 0:

fdroidserver/update.py

@@ -37,7 +37,6 @@ from binascii import hexlify
 from PIL import Image
 import logging
 
-from . import btlog
 from . import common
 from . import index
 from . import metadata
@@ -471,13 +470,24 @@ def sha256sum(filename):
     return sha.hexdigest()
 
 
-def has_old_openssl(filename):
-    '''checks for known vulnerable openssl versions in the APK'''
+def has_known_vulnerability(filename):
+    """checks for known vulnerabilities in the APK
+
+    Checks OpenSSL .so files in the APK to see if they are a known vulnerable
+    version.  Google also enforces this:
+    https://support.google.com/faqs/answer/6376725?hl=en
+
+    Checks whether there are more than one classes.dex or AndroidManifest.xml
+    files, which is invalid and an essential part of the "Master Key" attack.
+
+    http://www.saurik.com/id/17
+    """
 
     # statically load this pattern
-    if not hasattr(has_old_openssl, "pattern"):
-        has_old_openssl.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
+    if not hasattr(has_known_vulnerability, "pattern"):
+        has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 
+    files_in_apk = set()
     with zipfile.ZipFile(filename) as zf:
         for name in zf.namelist():
             if name.endswith('libcrypto.so') or name.endswith('libssl.so'):
@@ -486,7 +496,7 @@ def has_old_openssl(filename):
                     chunk = lib.read(4096)
                     if chunk == b'':
                         break
-                    m = has_old_openssl.pattern.search(chunk)
+                    m = has_known_vulnerability.pattern.search(chunk)
                     if m:
                         version = m.group(1).decode('ascii')
                         if version.startswith('1.0.1') and version[5] >= 'r' \
@@ -496,6 +506,11 @@ def has_old_openssl(filename):
                             logging.warning('"%s" contains outdated %s (%s)', filename, name, version)
                             return True
                         break
+            elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
+                if name in files_in_apk:
+                    return True
+                files_in_apk.add(name)
+
     return False
 
 
@@ -1173,7 +1188,7 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
         if not common.verify_apk_signature(apkfile):
             return True, None, False
 
-        if has_old_openssl(apkfile):
+        if has_known_vulnerability(apkfile):
             apk['antiFeatures'].add('KnownVuln')
 
         apkzip = zipfile.ZipFile(apkfile, 'r')
@@ -1695,6 +1710,7 @@ def main():
 
     git_remote = config.get('binary_transparency_remote')
     if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
+        from . import btlog
         btlog.make_binary_transparency_log(repodirs)
 
     if config['update_stats']:

locale/fdroidserver.pot

@@ -327,7 +327,9 @@ msgid "Download logs we don't have"
 msgstr ""
 
 #: ../fdroidserver/stats.py:66
-msgid "Recalculate aggregate stats - use when changes "
+msgid ""
+"Recalculate aggregate stats - use when changes "
+"have been made that would invalidate old cached data."
 msgstr ""
 
 #: ../fdroidserver/stats.py:69

tests/lint.TestCase

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+import fdroidserver.common
+import fdroidserver.lint
+
+
+class LintTest(unittest.TestCase):
+    '''fdroidserver/lint.py'''
+
+    def test_check_for_unsupported_metadata_files(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['accepted_formats'] = ('txt', 'yml')
+        fdroidserver.common.config = config
+        fdroidserver.lint.config = config
+        self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files())
+
+        tmpdir = os.path.join(localmodule, '.testfiles')
+        tmptestsdir = tempfile.mkdtemp(prefix='test_check_for_unsupported_metadata_files-',
+                                       dir=tmpdir)
+        self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/'))
+        shutil.copytree(os.path.join(localmodule, 'tests', 'metadata'),
+                        os.path.join(tmptestsdir, 'metadata'),
+                        ignore=shutil.ignore_patterns('apk', 'dump', '*.json'))
+        self.assertFalse(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/'))
+        shutil.copy(os.path.join(localmodule, 'tests', 'metadata', 'org.adaway.json'),
+                    os.path.join(tmptestsdir, 'metadata'))
+        self.assertTrue(fdroidserver.lint.check_for_unsupported_metadata_files(tmptestsdir + '/'))
+
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.lint.options, args) = parser.parse_args(['--verbose'])
+    fdroidserver.common.options = fdroidserver.lint.options
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(LintTest))
+    unittest.main()