From 283f10dec1f8b9517c9a59b962860a28898a60d5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Oct 2020 10:02:05 +0200 Subject: [PATCH 1/5] index: generate repo icon if missing, and add tests --- fdroidserver/index.py | 17 +++++++++++++++-- tests/index.TestCase | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 6d247141..d89e5cb8 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -593,8 +593,21 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing # Copy the repo icon into the repo directory... icon_dir = os.path.join(repodir, 'icons') - iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon'])) - shutil.copyfile(common.config['repo_icon'], iconfilename) + repo_icon = common.config.get('repo_icon', common.default_config['repo_icon']) + iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon)) + if os.path.exists(repo_icon): + shutil.copyfile(common.config['repo_icon'], iconfilename) + else: + logging.warning(_('repo_icon %s does not exist, generating placeholder.') + % repo_icon) + os.makedirs(os.path.dirname(iconfilename), exist_ok=True) + try: + import qrcode + qrcode.make(common.config['repo_url']).save(iconfilename) + except Exception: + exampleicon = os.path.join(common.get_examples_dir(), + common.default_config['repo_icon']) + shutil.copy(exampleicon, iconfilename) def extract_pubkey(): diff --git a/tests/index.TestCase b/tests/index.TestCase index 5fb56f10..10a5b9a6 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import datetime import inspect import logging import optparse @@ -29,17 +30,25 @@ from testcommon import TmpCwd GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135' +class Options: + nosign = True + pretty = False + verbose = False + + class IndexTest(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.DEBUG) self.basedir = os.path.join(localmodule, 'tests') + os.chmod(os.path.join(self.basedir, 'config.py'), 0o600) self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) os.chdir(self.basedir) fdroidserver.common.config = None + fdroidserver.common.options = Options config = fdroidserver.common.read_config(fdroidserver.common.options) config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner') fdroidserver.common.config = config @@ -215,6 +224,29 @@ class IndexTest(unittest.TestCase): self.maxDiff = None self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2)) + def test_make_v0(self): + tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, + dir=self.tmpdir) + os.chdir(tmptestsdir) + os.mkdir('repo') + repo_icons_dir = os.path.join('repo', 'icons') + self.assertFalse(os.path.isdir(repo_icons_dir)) + repodict = { + 'address': 'https://example.com/fdroid/repo', + 'description': 'This is just a test', + 'icon': 'blahblah', + 'name': 'test', + 'timestamp': datetime.datetime.now(), + 'version': 12, + } + requestsdict = {'install': [], 'uninstall': []} + fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff' + fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, []) + self.assertTrue(os.path.isdir(repo_icons_dir)) + self.assertTrue(os.path.exists(os.path.join(repo_icons_dir, + fdroidserver.common.default_config['repo_icon']))) + self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml'))) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) @@ -222,7 +254,8 @@ if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") - (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + (options, args) = parser.parse_args() + Options.verbose = options.verbose newSuite = unittest.TestSuite() newSuite.addTest(unittest.makeSuite(IndexTest)) From 790b5a2888444e9f69c9dba8b724f18c40c81980 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Oct 2020 10:04:30 +0200 Subject: [PATCH 2/5] update: use "app" as dict not App instance in apply_info_from_latest_apk This allows update.apply_info_from_latest_apk() to be used as part of the API. This way "app" can be a dict or an App instance. --- fdroidserver/index.py | 2 +- fdroidserver/update.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index d89e5cb8..078b64fb 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -61,7 +61,7 @@ def make(apps, apks, repodir, archive): common.assert_config_keystore(common.config) # Historically the index has been sorted by App Name, so we enforce this ordering here - sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper()) + sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper()) sortedapps = collections.OrderedDict() for appid in sortedids: sortedapps[appid] = apps[appid] diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 0849442b..3f2903d1 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1971,26 +1971,26 @@ def apply_info_from_latest_apk(apps, apks): bestver = apk['versionCode'] bestapk = apk - if app.NoSourceSince: + if app['NoSourceSince']: apk['antiFeatures'].add('NoSourceSince') - if not app.added: + if not app['added']: logging.debug("Don't know when " + appid + " was added") - if not app.lastUpdated: + if not app['lastUpdated']: logging.debug("Don't know when " + appid + " was last updated") if bestver == UNSET_VERSION_CODE: - if app.Name is None: - app.Name = app.AutoName or appid - app.icon = None + if app['Name'] is None: + app['Name'] = app['AutoName'] or appid + app['icon'] = None logging.debug("Application " + appid + " has no packages") else: - if app.Name is None: - app.Name = bestapk['name'] - app.icon = bestapk['icon'] if 'icon' in bestapk else None - if app.CurrentVersionCode is None: - app.CurrentVersionCode = str(bestver) + if app['Name'] is None: + app['Name'] = bestapk['name'] + app['icon'] = bestapk['icon'] if 'icon' in bestapk else None + if app['CurrentVersionCode'] is None: + app['CurrentVersionCode'] = str(bestver) def make_categories_txt(repodir, categories): From 3c64996089376e5c10b7010d7c2bbe97ffe319b7 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Oct 2020 10:08:49 +0200 Subject: [PATCH 3/5] update: test if options is instantated before using attributes This makes it possible to use process_apks(), get_cache(), and anything calling disabled_algorithms_allowed() as an API without having to set options up beforehand. --- fdroidserver/update.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 3f2903d1..6ffb0973 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -123,7 +123,9 @@ def get_all_icon_dirs(repodir): def disabled_algorithms_allowed(): - return options.allow_disabled_algorithms or config['allow_disabled_algorithms'] + return ((options is not None and options.allow_disabled_algorithms) + or (config is not None and config['allow_disabled_algorithms']) + or common.default_config['allow_disabled_algorithms']) def status_update_json(apps, apks): @@ -521,7 +523,7 @@ def get_cache(): """ apkcachefile = get_cache_file() ada = disabled_algorithms_allowed() - if not options.clean and os.path.exists(apkcachefile): + if options is not None and not options.clean and os.path.exists(apkcachefile): with open(apkcachefile) as fp: apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict) if apkcache.get("METADATA_VERSION") != METADATA_VERSION \ @@ -1778,7 +1780,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False): for icon_dir in get_all_icon_dirs(repodir): if os.path.exists(icon_dir): - if options.clean: + if options is not None and options.clean: shutil.rmtree(icon_dir) os.makedirs(icon_dir) else: From 05cd8c68109e533ed0e57154c05e6a6067803b70 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Oct 2020 10:22:32 +0200 Subject: [PATCH 4/5] scanner: expose "usual suspects" patterns for use in an API --- fdroidserver/scanner.py | 85 +++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 2757526c..8230831e 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -40,6 +40,47 @@ json_per_build = DEFAULT_JSON_PER_BUILD MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^}]*}""", re.DOTALL) +CODE_SIGNATURES = { + # The `apkanalyzer dex packages` output looks like this: + # M d 1 1 93 + # The first column has P/C/M/F for package, class, method or field + # The second column has x/k/r/d for removed, kept, referenced and defined. + # We already filter for defined only in the apkanalyzer call. 'r' will be + # for things referenced but not distributed in the apk. + exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [ + r'(com\.google\.firebase[^\s]*)', + r'(com\.google\.android\.gms[^\s]*)', + r'(com\.google\.tagmanager[^\s]*)', + r'(com\.google\.analytics[^\s]*)', + r'(com\.android\.billing[^\s]*)', + ] +} + +# Common known non-free blobs (always lower case): +NON_FREE_GRADLE_LINES = { + exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ + r'flurryagent', + r'paypal.*mpl', + r'admob.*sdk.*android', + r'google.*ad.*view', + r'google.*admob', + r'google.*play.*services', + r'crittercism', + r'heyzap', + r'jpct.*ae', + r'youtube.*android.*player.*api', + r'bugsense', + r'crashlytics', + r'ouya.*sdk', + r'libspen23', + r'firebase', + r'''["']com.facebook.android['":]''', + r'cloudrail', + r'com.tencent.bugly', + r'appcenter-push', + ] +} + def get_gradle_compile_commands(build): compileCommands = ['compile', @@ -59,25 +100,10 @@ def get_gradle_compile_commands(build): def scan_binary(apkfile): - usual_suspects = { - # The `apkanalyzer dex packages` output looks like this: - # M d 1 1 93 - # The first column has P/C/M/F for package, class, method or field - # The second column has x/k/r/d for removed, kept, referenced and defined. - # We already filter for defined only in the apkanalyzer call. 'r' will be - # for things referenced but not distributed in the apk. - exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [ - r'(com\.google\.firebase[^\s]*)', - r'(com\.google\.android\.gms[^\s]*)', - r'(com\.google\.tagmanager[^\s]*)', - r'(com\.google\.analytics[^\s]*)', - r'(com\.android\.billing[^\s]*)', - ] - } logging.info("Scanning APK for known non-free classes.") result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False) problems = 0 - for suspect, regexp in usual_suspects.items(): + for suspect, regexp in CODE_SIGNATURES.items(): matches = regexp.findall(result.output) if matches: for m in set(matches): @@ -95,31 +121,6 @@ def scan_source(build_dir, build=metadata.Build()): count = 0 - # Common known non-free blobs (always lower case): - usual_suspects = { - exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [ - r'flurryagent', - r'paypal.*mpl', - r'admob.*sdk.*android', - r'google.*ad.*view', - r'google.*admob', - r'google.*play.*services', - r'crittercism', - r'heyzap', - r'jpct.*ae', - r'youtube.*android.*player.*api', - r'bugsense', - r'crashlytics', - r'ouya.*sdk', - r'libspen23', - r'firebase', - r'''["']com.facebook.android['":]''', - r'cloudrail', - r'com.tencent.bugly', - r'appcenter-push', - ] - } - whitelisted = [ 'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE 'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE @@ -130,7 +131,7 @@ def scan_source(build_dir, build=metadata.Build()): return any(wl in s for wl in whitelisted) def suspects_found(s): - for n, r in usual_suspects.items(): + for n, r in NON_FREE_GRADLE_LINES.items(): if r.match(s) and not is_whitelisted(s): yield n From 602cf30c1ee48365442fb51975f8e1e05c55013e Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 7 Oct 2020 18:40:03 +0200 Subject: [PATCH 5/5] update: fix bug where only last appid was added to antiFeatures status appid will never be present in `antiFeatures[af]`, so the entry was being reinitalized each time. --- fdroidserver/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6ffb0973..1708700a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -149,7 +149,7 @@ def status_update_json(apps, apks): antiFeatures = output['antiFeatures'] # JSON camelCase if af not in antiFeatures: antiFeatures[af] = dict() - if appid not in antiFeatures[af]: + if 'apps' not in antiFeatures[af]: antiFeatures[af]['apps'] = set() antiFeatures[af]['apps'].add(appid)