From 0a1793ab7f4757bd2a6ae84af557742b2131034a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 13 Jan 2021 09:08:22 +0100 Subject: [PATCH 1/4] update: write out cache as soon as possible When working or testing with large APK collections, oftentimes it can take hours to scan all the APKs. If there is a failure before the apkcache is written out, then all that work is lost since no cache is written out. This moves the final cache writing before writing the index and knownapks to make it more likely that the apkcache is successfully written. --- fdroidserver/update.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index ef1310c7..de1948fa 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2310,6 +2310,10 @@ def main(): # This will be done again (as part of apply_info_from_latest_apk) for repo and archive # separately later on, but it's fairly cheap anyway. read_names_from_apks(apps, apks + archapks) + + if cachechanged: + write_cache(apkcache) + # The added date currently comes from the oldest apk which might be in the archive. # So we need this populated at app level before continuing with only processing /repo # or /archive @@ -2349,9 +2353,6 @@ def main(): # Update known apks info... knownapks.writeifchanged() - if cachechanged: - write_cache(apkcache) - # Update the wiki... if options.wiki: logging.warning(_('wiki support is deprecated and will be removed in the next release!')) From c7fcfe3bfa777c065773dd71d2abbb3ceb5b3bf6 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 Jan 2021 14:43:37 +0100 Subject: [PATCH 2/4] update: fix setting current_version_file for make_current_version_link Before, it would never set current_version_file to the current APK since it first set current_version_code, then tested against CurrentVersionCode. So if there is only a single APK and its the CurrentVersionCode, then current_version_file would not get set. refs #772 --- fdroidserver/index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 9b2b68c9..f2cbac0f 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -466,10 +466,10 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing for apk in apklist: file_extension = common.get_file_extension(apk['apkName']) # find the APK for the "Current Version" - if current_version_code < apk['versionCode']: - current_version_code = apk['versionCode'] if current_version_code < int(app.CurrentVersionCode): current_version_file = apk['apkName'] + if current_version_code < apk['versionCode']: + current_version_code = apk['versionCode'] apkel = doc.createElement("package") apel.appendChild(apkel) From 0f6b638986017886d9129bf031e807b1de77abbb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 17 Nov 2020 23:49:04 +0100 Subject: [PATCH 3/4] common.get_app_display_name() for finding app names --- fdroidserver/checkupdates.py | 8 ++------ fdroidserver/common.py | 20 ++++++++++++++++++++ fdroidserver/index.py | 2 +- fdroidserver/lint.py | 2 +- fdroidserver/update.py | 8 +++++--- tests/common.TestCase | 15 +++++++++++++++ 6 files changed, 44 insertions(+), 11 deletions(-) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index fe9b63d8..460a7373 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -326,7 +326,7 @@ def try_init_submodules(app, last_build, vcs): try: vcs.initsubmodules() except NoSubmodulesException: - logging.info("No submodules present for {}".format(app.Name)) + logging.info("No submodules present for {}".format(_getappname(app))) # Return all directories under startdir that contain any of the manifest @@ -359,11 +359,7 @@ def possible_subdirs(app): def _getappname(app): - if app.Name: - return app.Name - if app.AutoName: - return app.AutoName - return app.id + return common.get_app_display_name(app) def _getcvname(app): diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1de841c1..153ab148 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3628,6 +3628,26 @@ def version_code_string_to_int(vercode): return int(vercode) +def get_app_display_name(app): + """get a human readable name for the app for logging and sorting + + When trying to find a localized name, this first tries en-US since + that his the historical language used for sorting. + + """ + if app.get('Name'): + return app['Name'] + if app.get('localized'): + localized = app['localized'].get('en-US') + if not localized: + for v in app['localized'].values(): + localized = v + break + if localized.get('name'): + return localized['name'] + return app.get('AutoName') or app['id'] + + def local_rsync(options, fromdir, todir): '''Rsync method for local to local copying of things diff --git a/fdroidserver/index.py b/fdroidserver/index.py index f2cbac0f..8c60027c 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -61,7 +61,7 @@ def make(apps, apks, repodir, archive): common.assert_config_keystore(common.config) # Historically the index has been sorted by App Name, so we enforce this ordering here - sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper()) + sortedids = sorted(apps, key=lambda appid: common.get_app_display_name(apps[appid]).upper()) sortedapps = collections.OrderedDict() for appid in sortedids: sortedapps[appid] = apps[appid] diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 61916e71..9e7aaeda 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -337,7 +337,7 @@ def check_duplicates(app): else: links_seen.add(v) - name = app.Name or app.AutoName + name = common.get_app_display_name(app) if app.Summary and name: if app.Summary.lower() == name.lower(): yield _("Summary '%s' is just the app's name") % app.Summary diff --git a/fdroidserver/update.py b/fdroidserver/update.py index de1948fa..5f42cdf9 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -207,9 +207,12 @@ def update_wiki(apps, apks): requiresroot = 'Yes' else: requiresroot = 'No' + + apppagename = common.get_app_display_name(app) + wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % ( appid, - app.Name, + apppagename, app.added.strftime('%Y-%m-%d') if app.added else '', app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '', app.SourceCode, @@ -338,7 +341,6 @@ def update_wiki(apps, apks): # Make a redirect from the name to the ID too, unless there's # already an existing page with the name and it isn't a redirect. noclobber = False - apppagename = app.Name for ch in '_{}:[]|': apppagename = apppagename.replace(ch, ' ') # Drop double spaces caused mostly by replacing ':' above @@ -2116,7 +2118,7 @@ def read_names_from_apks(apps, apks): if bestver == UNSET_VERSION_CODE: if app.Name is None: - app.Name = app.AutoName or appid + app.Name = common.get_app_display_name(app) app.icon = None else: if app.Name is None: diff --git a/tests/common.TestCase b/tests/common.TestCase index 253f8037..2f226dee 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1612,6 +1612,21 @@ class CommonTest(unittest.TestCase): self.assertEqual([dirtyfile, 'repo/status/running.json'], data['fdroiddata']['untrackedFiles']) + def test_get_app_display_name(self): + testvalue = 'WIN!' + for app in [ + {'Name': testvalue}, + {'AutoName': testvalue}, + {'id': testvalue}, + {'id': 'a', 'localized': {'de-AT': {'name': testvalue}}}, + {'id': 'a', 'localized': { + 'de-AT': {'name': 'nope'}, + 'en-US': {'name': testvalue}, + }}, + {'AutoName': 'ignore me', 'Name': testvalue, 'id': 'nope'}, + {'AutoName': testvalue, 'id': 'nope'}]: + self.assertEqual(testvalue, fdroidserver.common.get_app_display_name(app)) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) From fff59e519724c7f16bd3861de60297f80c71beeb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 13 Jan 2020 15:43:42 +0100 Subject: [PATCH 4/4] only use AutoName: in checkupdates AutoName: is only needed for the commit messages generated by checkupdates, and it makes the logic for localized names confusing. closes #654 refs #304 --- fdroidserver/common.py | 4 +- fdroidserver/index.py | 20 ++- fdroidserver/lint.py | 3 - fdroidserver/update.py | 56 ++++--- tests/index.TestCase | 68 ++++++++- .../metadata/info.guardianproject.checkey.yml | 1 + .../en-US/name.txt | 1 + .../ja-JP/name.txt | 1 + tests/repo/index-v1.json | 43 ++++-- tests/repo/index.xml | 2 +- tests/update.TestCase | 139 ++++++++++++++++++ 11 files changed, 289 insertions(+), 49 deletions(-) create mode 100644 tests/metadata/info.guardianproject.checkey/en-US/name.txt create mode 100644 tests/metadata/info.guardianproject.checkey/ja-JP/name.txt diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 153ab148..22a4f3ff 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3511,9 +3511,9 @@ def get_certificate(signature_block_file): def load_stats_fdroid_signing_key_fingerprints(): - """Load list of signing-key fingerprints stored by fdroid publish from file. + """Load signing-key fingerprints stored in file generated by fdroid publish. - :returns: list of dictionanryies containing the singing-key fingerprints. + :returns: dict containing the signing-key fingerprints. """ jar_file = os.path.join('stats', 'publishsigkeys.jar') if not os.path.isfile(jar_file): diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 8c60027c..95b46ed5 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -159,7 +159,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ if not v: continue if k in ('Builds', 'comments', 'metadatapath', - 'ArchivePolicy', 'AutoUpdateMode', 'MaintainerNotes', + 'ArchivePolicy', 'AutoName', 'AutoUpdateMode', 'MaintainerNotes', 'Provides', 'Repo', 'RepoType', 'RequiresRoot', 'UpdateCheckData', 'UpdateCheckIgnore', 'UpdateCheckMode', 'UpdateCheckName', 'NoSourceSince', 'VercodeOperation'): @@ -172,10 +172,6 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_ k = 'suggestedVersionCode' elif k == 'CurrentVersion': # TODO make SuggestedVersionName the canonical name k = 'suggestedVersionName' - elif k == 'AutoName': - if 'Name' not in apps[packageName]: - d['name'] = v - continue else: k = k[:1].lower() + k[1:] d[k] = v @@ -326,6 +322,8 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing value = localized[lang].get(lkey) if not value: value = default + if not value and name == 'name' and app.get('AutoName'): + value = app['AutoName'] el.appendChild(doc.createTextNode(value)) parent.appendChild(el) @@ -363,10 +361,13 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing # Get a list of the apks for this app... apklist = [] + name_from_apk = None apksbyversion = collections.defaultdict(lambda: []) for apk in apks: if apk.get('versionCode') and apk.get('packageName') == appid: apksbyversion[apk['versionCode']].append(apk) + if name_from_apk is None: + name_from_apk = apk.get('name') for versionCode, apksforver in apksbyversion.items(): fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer') fdroid_signed_apk = None @@ -398,7 +399,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing if app.lastUpdated: addElement('lastupdated', app.lastUpdated.strftime('%Y-%m-%d'), doc, apel) - addElementCheckLocalized('name', app, 'Name', doc, apel) + addElementCheckLocalized('name', app, 'Name', doc, apel, name_from_apk) addElementCheckLocalized('summary', app, 'Summary', doc, apel) if app.icon: @@ -543,7 +544,12 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing and common.config['make_current_version_link'] \ and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] - sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) + name = app.get(namefield) + if not name and namefield == 'Name': + name = app.get('localized', {}).get('en-US', {}).get('name') + if not name: + name = app.id + sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', name.encode('utf-8')) apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8') current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 9e7aaeda..99a62f31 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -323,9 +323,6 @@ def check_categories(app): def check_duplicates(app): - if app.Name and app.Name == app.AutoName: - yield _("Name '%s' is just the auto name - remove it") % app.Name - links_seen = set() for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']: v = app.get(f) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 5f42cdf9..80e092c3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1905,15 +1905,10 @@ def apply_info_from_latest_apk(apps, apks): logging.debug("Don't know when " + appid + " was last updated") if bestver == UNSET_VERSION_CODE: - - if app.get('Name') is None: - app['Name'] = app['AutoName'] or appid app['icon'] = None logging.debug("Application " + appid + " has no packages") else: - if app.get('Name') is None: - app['Name'] = bestapk['name'] - app['icon'] = bestapk['icon'] if 'icon' in bestapk else None + app.icon = bestapk['icon'] if 'icon' in bestapk else None if app.get('CurrentVersionCode') is None: app['CurrentVersionCode'] = str(bestver) @@ -2106,23 +2101,45 @@ def read_added_date_from_all_apks(apps, apks): app['lastUpdated'] = apk['added'] -def read_names_from_apks(apps, apks): - """This is a stripped down copy of apply_info_from_latest_apk that only parses app names""" +def insert_missing_app_names_from_apks(apps, apks): + """Use app name from APK if it is not set in the metadata + + Name -> localized -> from APK + + The name from the APK is set as the default name for the app if + there is no other default set, e.g. app['Name'] or + app['localized']['en-US']['name']. The en-US locale is defined in + the F-Droid ecosystem as the locale of last resort, as in the one + that should always be present. en-US is used since it is the + locale of the source strings. + + This should only be used for index v0 and v1. Later versions of + the index should be sorted by Application ID, since it is + guaranteed to always be there. Before, the index was stored by + the app name (aka ) to save the + website from having to sort the entries. That is no longer + relevant since the website switched from Wordpress to Jekyll. + + """ for appid, app in apps.items(): + if app.get('Name') is not None: + continue + if app.get('localized', {}).get('en-US', {}).get('name') is not None: + continue + bestver = UNSET_VERSION_CODE for apk in apks: if apk['packageName'] == appid: - if apk['versionCode'] > bestver: + if apk.get('name') and apk['versionCode'] > bestver: bestver = apk['versionCode'] bestapk = apk - if bestver == UNSET_VERSION_CODE: - if app.Name is None: - app.Name = common.get_app_display_name(app) - app.icon = None - else: - if app.Name is None: - app.Name = bestapk['name'] + if bestver != UNSET_VERSION_CODE: + if 'localized' not in app: + app['localized'] = {} + if 'en-US' not in app['localized']: + app['localized']['en-US'] = {} + app['localized']['en-US']['name'] = bestapk.get('name') def get_apps_with_packages(apps, apks): @@ -2161,6 +2178,7 @@ def prepare_apps(apps, apks, repodir): translate_per_build_anti_features(apps_with_packages, apks) if repodir == 'repo': insert_localized_app_metadata(apps_with_packages) + insert_missing_app_names_from_apks(apps_with_packages, apks) return apps_with_packages @@ -2307,12 +2325,6 @@ def main(): else: archapks = [] - # We need app.Name populated for all apps regardless of which repo they end up in - # for the old-style inter-app links, so let's do it before we do anything else. - # This will be done again (as part of apply_info_from_latest_apk) for repo and archive - # separately later on, but it's fairly cheap anyway. - read_names_from_apks(apps, apks + archapks) - if cachechanged: write_cache(apkcache) diff --git a/tests/index.TestCase b/tests/index.TestCase index 131b3a1a..16885dbf 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -247,7 +247,7 @@ class IndexTest(unittest.TestCase): self.maxDiff = None self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2)) - def test_make_v0(self): + def test_make_v0_repo_only(self): tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(tmptestsdir) @@ -264,7 +264,71 @@ class IndexTest(unittest.TestCase): } requestsdict = {'install': [], 'uninstall': []} fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff' - fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, []) + fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, {}) + self.assertTrue(os.path.isdir(repo_icons_dir)) + self.assertTrue(os.path.exists(os.path.join(repo_icons_dir, + fdroidserver.common.default_config['repo_icon']))) + self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml'))) + + def test_make_v0(self): + tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, + dir=self.tmpdir) + os.chdir(tmptestsdir) + os.mkdir('metadata') + os.mkdir('repo') + metadatafile = 'metadata/info.zwanenburg.caffeinetile.yml' + shutil.copy(os.path.join(self.basedir, metadatafile), + metadatafile) + repo_icons_dir = os.path.join('repo', 'icons') + self.assertFalse(os.path.isdir(repo_icons_dir)) + repodict = { + 'address': 'https://example.com/fdroid/repo', + 'description': 'This is just a test', + 'icon': 'blahblah', + 'name': 'test', + 'timestamp': datetime.datetime.now(), + 'version': 12, + } + app = fdroidserver.metadata.parse_metadata(metadatafile, False, False) + app['icon'] = 'info.zwanenburg.caffeinetile.4.xml' + app['CurrentVersionCode'] = '4' + apps = {app.id: app} + apk = { + 'hash': 'dbbdd7deadb038862f426b71efe4a64df8c3edf25d669e935f349510e16f65db', + 'hashType': 'sha256', + 'uses-permission': [ + [ + 'android.permission.WAKE_LOCK', + None + ] + ], + 'uses-permission-sdk-23': [], + 'features': [], + 'icons_src': { + '160': 'res/drawable/ic_coffee_on.xml', + '-1': 'res/drawable/ic_coffee_on.xml' + }, + 'icons': { + '160': 'info.zwanenburg.caffeinetile.4.xml' + }, + 'antiFeatures': [], + 'packageName': 'info.zwanenburg.caffeinetile', + 'versionCode': 4, + 'name': 'Caffeine Tile', + 'versionName': '1.3', + 'minSdkVersion': 24, + 'targetSdkVersion': 25, + 'sig': '03f9b2f848d22fd1d8d1331e8b1b486d', + 'signer': '51cfa5c8a743833ad89acf81cb755936876a5c8b8eca54d1ffdcec0cdca25d0e', + 'size': 11740, + 'apkName': 'info.zwanenburg.caffeinetile_4.apk', + 'icon': 'info.zwanenburg.caffeinetile.4.xml', + 'added': datetime.datetime.fromtimestamp(1539122400), + } + requestsdict = {'install': [], 'uninstall': []} + fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff' + fdroidserver.common.config['make_current_version_link'] = True + fdroidserver.index.make_v0(apps, [apk], 'repo', repodict, requestsdict, {}) self.assertTrue(os.path.isdir(repo_icons_dir)) self.assertTrue(os.path.exists(os.path.join(repo_icons_dir, fdroidserver.common.default_config['repo_icon']))) diff --git a/tests/metadata/info.guardianproject.checkey.yml b/tests/metadata/info.guardianproject.checkey.yml index 78b75320..7d933abd 100644 --- a/tests/metadata/info.guardianproject.checkey.yml +++ b/tests/metadata/info.guardianproject.checkey.yml @@ -9,6 +9,7 @@ Translation: https://www.transifex.com/otf/checkey Bitcoin: 1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk Liberapay: GuardianProject +Name: Checkey the app! AutoName: Checkey AutoUpdateMode: None diff --git a/tests/metadata/info.guardianproject.checkey/en-US/name.txt b/tests/metadata/info.guardianproject.checkey/en-US/name.txt new file mode 100644 index 00000000..09d725c6 --- /dev/null +++ b/tests/metadata/info.guardianproject.checkey/en-US/name.txt @@ -0,0 +1 @@ +Checkey: info on local apps diff --git a/tests/metadata/info.guardianproject.checkey/ja-JP/name.txt b/tests/metadata/info.guardianproject.checkey/ja-JP/name.txt new file mode 100644 index 00000000..c96d43b5 --- /dev/null +++ b/tests/metadata/info.guardianproject.checkey/ja-JP/name.txt @@ -0,0 +1 @@ +Checkey: ローカルアプリの情報 diff --git a/tests/repo/index-v1.json b/tests/repo/index-v1.json index e781d974..fe93947a 100644 --- a/tests/repo/index-v1.json +++ b/tests/repo/index-v1.json @@ -32,14 +32,18 @@ "flattrID": "cad90e036b975ed129a3ce80a0750466", "issueTracker": "https://gitlab.com/souch/SMSbypass/issues", "license": "GPL-3.0-only", - "name": "Battery level", "sourceCode": "https://gitlab.com/souch/SMSbypass/tree/HEAD", "summary": "Filter SMS and show them in a fake app", "webSite": "https://gitlab.com/souch/SMSbypass", "added": 1524700800000, "icon": "souch.smsbypass.9.png", "packageName": "souch.smsbypass", - "lastUpdated": 1524700800000 + "lastUpdated": 1524700800000, + "localized": { + "en-US": { + "name": "Battery level" + } + } }, { "categories": [ @@ -77,13 +81,17 @@ "donate": "https://f-droid.org/about", "issueTracker": "https://gitlab.com/fdroid/privileged-extension/issues", "license": "Apache-2.0", - "name": "fake.ota.update_1234", "sourceCode": "https://gitlab.com/fdroid/privileged-extension", "summary": "Tests whether OTA ZIP files are being include", "webSite": "https://f-droid.org", "added": 1457568000000, "packageName": "fake.ota.update", - "lastUpdated": 1457568000000 + "lastUpdated": 1457568000000, + "localized": { + "en-US": { + "name": "fake.ota.update_1234" + } + } }, { "categories": [ @@ -106,12 +114,16 @@ "suggestedVersionCode": "99999999", "liberapay": "12334", "license": "GPL-3.0-only", - "name": "OBB Main Old Version", "sourceCode": "https://github.com/eighthave/urzip", "added": 1388448000000, "icon": "obb.main.oldversion.1444412523.png", "packageName": "obb.main.oldversion", - "lastUpdated": 1388448000000 + "lastUpdated": 1388448000000, + "localized": { + "en-US": { + "name": "OBB Main Old Version" + } + } }, { "bitcoin": "1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk", @@ -120,12 +132,16 @@ ], "suggestedVersionCode": "99999999", "license": "GPL-3.0-only", - "name": "OBB Main Two Versions", "sourceCode": "https://github.com/eighthave/urzip", "added": 1444608000000, "icon": "obb.main.twoversions.1101617.png", "packageName": "obb.main.twoversions", - "lastUpdated": 1466380800000 + "lastUpdated": 1466380800000, + "localized": { + "en-US": { + "name": "OBB Main Two Versions" + } + } }, { "bitcoin": "1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk", @@ -134,7 +150,6 @@ ], "suggestedVersionCode": "99999999", "license": "GPL-3.0-only", - "name": "OBB Main/Patch Current", "sourceCode": "https://github.com/eighthave/urzip", "added": 1461369600000, "icon": "obb.mainpatch.current.1619.png", @@ -144,6 +159,7 @@ "en-US": { "featureGraphic": "featureGraphic_ffhLaojxbGAfu9ROe1MJgK5ux8d0OVc6b65nmvOBaTk=.png", "icon": "icon_WI0pkO3LsklrsTAnRr-OQSxkkoMY41lYe2-fAvXLiLg=.png", + "name": "OBB Main/Patch Current", "phoneScreenshots": [ "screenshot-main.png" ], @@ -165,13 +181,17 @@ "description": "Activates silent mode during calendar events.", "issueTracker": "https://github.com/miguelvps/PoliteDroid/issues", "license": "GPL-3.0-only", - "name": "Polite Droid", "sourceCode": "https://github.com/miguelvps/PoliteDroid", "summary": "Calendar tool", "added": 1498176000000, "icon": "com.politedroid.6.png", "packageName": "com.politedroid", - "lastUpdated": 1498176000000 + "lastUpdated": 1498176000000, + "localized": { + "en-US": { + "name": "Polite Droid" + } + } }, { "authorWebSite": "https://guardianproject.info", @@ -187,7 +207,6 @@ "issueTracker": "https://dev.guardianproject.info/projects/urzip/issues", "liberapayID": "9999999", "license": "GPL-3.0-only", - "name": "urzip-\u03c0\u00c7\u00c7\u03c0\u00c7\u00c7\u73b0\u4ee3\u6c49\u8bed\u901a\u7528\u5b57-\u0431\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438-\u0639\u0631\u0628\u064a1234", "openCollective": "f-droid-just-testing", "sourceCode": "https://github.com/guardianproject/urzip", "summary": "\u4e00\u4e2a\u5b9e\u7528\u5de5\u5177\uff0c\u83b7\u53d6\u5df2\u5b89\u88c5\u5728\u60a8\u7684\u8bbe\u5907\u4e0a\u7684\u5e94\u7528\u7684\u6709\u5173\u4fe1\u606f", diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 298fd022..9af45f96 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -371,7 +371,7 @@ APK is called F-Droid Privileged Extension. info.guardianproject.urzip 2016-06-23 2016-06-23 - urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 + title 一个实用工具，获取已安装在您的设备上的应用的有关信息 info.guardianproject.urzip.100.png It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始，一键触摸即可显示 APK 的指纹，并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接，让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。 diff --git a/tests/update.TestCase b/tests/update.TestCase index 4593f770..ffc8fa5f 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -2,6 +2,7 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 +import copy import git import glob import inspect @@ -69,6 +70,10 @@ class UpdateTest(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.INFO) + logging.getLogger('androguard.axml').setLevel(logging.INFO) + logging.getLogger('androguard.core.api_specific_resources').setLevel(logging.INFO) + from PIL import PngImagePlugin + logging.getLogger(PngImagePlugin.__name__).setLevel(logging.INFO) self.basedir = os.path.join(localmodule, 'tests') self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) if not os.path.exists(self.tmpdir): @@ -174,6 +179,140 @@ class UpdateTest(unittest.TestCase): elif packageName == 'eu.siacs.conversations': self.assertEqual('Conversations', app['localized']['en-US']['name']) + def test_name_title_scraping(self): + """metadata file --> fdroiddata localized files --> fastlane/triple-t in app source --> APK""" + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.update.config = config + os.chdir(os.path.join(localmodule, 'tests')) + fdroidserver.update.options = type('', (), {})() + fdroidserver.update.options.clean = True + fdroidserver.update.options.delete_unknown = True + fdroidserver.update.options.rename_apks = False + fdroidserver.update.options.allow_disabled_algorithms = False + + apps = fdroidserver.metadata.read_metadata() + knownapks = fdroidserver.common.KnownApks() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + fdroidserver.update.insert_localized_app_metadata(apps) + fdroidserver.update.apply_info_from_latest_apk(apps, apks) + app = apps['info.guardianproject.urzip'] + self.assertIsNone(app.Name) + self.assertTrue('localized' in app) + self.assertEqual('title', app['localized']['en-US']['name']) + app = apps['org.videolan.vlc'] + self.assertIsNone(app.Name) + self.assertTrue('localized' in app) + self.assertFalse('name' in app['localized']['en-US']) + app = apps['info.guardianproject.checkey'] + self.assertEqual('Checkey the app!', app.Name) + self.assertTrue('localized' in app) + self.assertEqual('Checkey: info on local apps', app['localized']['en-US']['name']) + self.assertEqual('Checkey: ローカルアプリの情報', app['localized']['ja-JP']['name']) + app = apps['org.adaway'] + self.assertIsNone(app.Name) + self.assertFalse('localized' in app) + app = apps['obb.main.twoversions'] + self.assertIsNone(app.Name) + self.assertFalse('localized' in app) + + def test_insert_missing_app_names_from_apks(self): + """en-US serves as the final, default, fallback value with index-v1""" + testvalue = 'TESTVALUE!' + apps = { + 'none': {}, + 'name': {'Name': testvalue}, + 'onlyapk': {'Name': None}, + 'autoname': {'AutoName': 'autoname', 'Name': None}, + 'onlylocalized': {'localized': {'en-US': {'name': testvalue}}}, + 'non_en_us_localized': {'localized': {'de-AT': {'name': 'leiwand'}}}, + 'apks': {}, + } + apks = [ + {'packageName': 'none', 'name': '', 'versionCode': 1}, + {'packageName': 'name', 'name': 'fromapk', 'versionCode': 1}, + {'packageName': 'onlyapk', 'name': testvalue, 'versionCode': 1}, + {'packageName': 'autoname', 'name': testvalue, 'versionCode': 1}, + {'packageName': 'onlylocalized', 'name': 'fromapk', 'versionCode': 1}, + {'packageName': 'non_en_us_localized', 'name': testvalue, 'versionCode': 0xcafe}, + {'packageName': 'apks', 'name': 'fromapk1', 'versionCode': 1}, + {'packageName': 'apks', 'name': 'fromapk2', 'versionCode': 2}, + {'packageName': 'apks', 'name': testvalue, 'versionCode': 3}, + ] + fdroidserver.update.insert_missing_app_names_from_apks(apps, apks) + for appid, app in apps.items(): + if appid == 'none': + self.assertIsNone(app.get('Name')) + self.assertIsNone(app.get('localized')) + elif appid == 'onlyapk': + self.assertIsNone(app.get('Name')) + self.assertEqual(testvalue, app['localized']['en-US']['name']) + elif appid == 'autoname': + self.assertIsNone(app.get('Name')) + self.assertEqual(testvalue, app['localized']['en-US']['name']) + elif appid == 'onlylocalized': + self.assertIsNone(app.get('Name')) + self.assertEqual(testvalue, app['localized']['en-US']['name']) + elif appid == 'non_en_us_localized': + self.assertIsNone(app.get('Name')) + self.assertEqual(testvalue, app['localized']['en-US']['name']) + elif appid == 'name': + self.assertEqual(testvalue, app['Name']) + self.assertIsNone(app.get('localized')) + elif appid == 'apks': + self.assertIsNone(app.get('Name')) + self.assertEqual(testvalue, app['localized']['en-US']['name']) + + def test_insert_missing_app_names_from_apks_from_repo(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + fdroidserver.common.config = config + fdroidserver.update.config = config + fdroidserver.update.options = type('', (), {})() + fdroidserver.update.options.clean = True + fdroidserver.update.options.delete_unknown = True + fdroidserver.update.options.rename_apks = False + fdroidserver.update.options.allow_disabled_algorithms = False + + apps = fdroidserver.metadata.read_metadata() + knownapks = fdroidserver.common.KnownApks() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + + appid = 'info.guardianproject.checkey' + testapps = {appid: copy.copy(apps[appid])} + self.assertEqual('Checkey the app!', testapps[appid]['Name']) + del(testapps[appid]['Name']) + fdroidserver.update.insert_missing_app_names_from_apks(testapps, apks) + self.assertIsNone(testapps[appid].get('Name')) + + repoapps = fdroidserver.update.prepare_apps(apps, apks, 'repo') + fdroidserver.update.insert_missing_app_names_from_apks(repoapps, apks) + self.assertIsNone(repoapps['com.politedroid']['Name']) + self.assertEqual('Polite Droid', + repoapps['com.politedroid']['localized']['en-US']['name']) + self.assertEqual('Duplicate Permisssions', repoapps['duplicate.permisssions']['Name']) + self.assertEqual('Caffeine Tile', repoapps['info.zwanenburg.caffeinetile']['Name']) + self.assertEqual('No minSdkVersion or targetSdkVersion', repoapps['no.min.target.sdk']['Name']) + self.assertIsNone(repoapps['obb.main.oldversion'].get('Name')) + self.assertEqual('OBB Main Old Version', + repoapps['obb.main.oldversion']['localized']['en-US']['name']) + self.assertIsNone(repoapps['obb.main.twoversions'].get('Name')) + self.assertEqual('OBB Main Two Versions', + repoapps['obb.main.twoversions']['localized']['en-US']['name']) + self.assertIsNone(repoapps['souch.smsbypass'].get('Name')) + self.assertEqual('Battery level', + repoapps['souch.smsbypass']['localized']['en-US']['name']) + self.assertIsNone(repoapps['info.guardianproject.urzip'].get('Name')) + self.assertEqual('title', + repoapps['info.guardianproject.urzip']['localized']['en-US']['name']) + self.assertIsNone(repoapps['obb.mainpatch.current'].get('Name')) + + del(repoapps['info.guardianproject.urzip']['localized']) + fdroidserver.update.insert_missing_app_names_from_apks(repoapps, apks) + self.assertEqual('urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234', + repoapps['info.guardianproject.urzip']['localized']['en-US']['name']) + def test_insert_triple_t_metadata(self): importer = os.path.join(self.basedir, 'tmp', 'importer') packageName = 'org.fdroid.ci.test.app'