diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 02d8aceb..8ced40c9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -93,8 +93,12 @@ default_config = { 'keystore': 'keystore.jks', 'smartcardoptions': [], 'char_limits': { + 'Author': 256, + 'Name': 30, 'Summary': 80, 'Description': 4000, + 'Video': 256, + 'WhatsNew': 500, }, 'keyaliases': {}, 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 1d978dcc..fe7e07f1 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -70,6 +70,7 @@ app_fields = set([ 'License', 'Author Name', 'Author Email', + 'Author Web Site', 'Web Site', 'Source Code', 'Issue Tracker', @@ -119,6 +120,7 @@ class App(dict): self.License = 'Unknown' self.AuthorName = None self.AuthorEmail = None + self.AuthorWebSite = None self.WebSite = '' self.SourceCode = '' self.IssueTracker = '' @@ -1199,6 +1201,7 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build): w_field_always('License') w_field_nonempty('Author Name') w_field_nonempty('Author Email') + w_field_nonempty('Author Web Site') w_field_always('Web Site') w_field_always('Source Code') w_field_always('Issue Tracker') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 271d334c..a97a40bf 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -66,6 +66,11 @@ all_screen_densities = ['0'] + screen_densities UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion']) UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion']) +ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg') +GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner') +SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots', + 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots') + def dpi_to_px(density): return (int(density) * 48) / 160 @@ -551,54 +556,189 @@ def insert_obbs(repodir, apps, apks): break -def insert_graphics(repodir, apps): - """Scans for screenshot PNG files in statically defined screenshots - directory and adds them to the app metadata. The screenshots and - graphic must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg" +def _get_localized_dict(app, locale): + '''get the dict to add localized store metadata to''' + if 'localized' not in app: + app['localized'] = collections.OrderedDict() + if locale not in app['localized']: + app['localized'][locale] = collections.OrderedDict() + return app['localized'][locale] + + +def _set_localized_text_entry(app, locale, key, f): + limit = config['char_limits'][key] + localized = _get_localized_dict(app, locale) + with open(f) as fp: + text = fp.read()[:limit] + if len(text) > 0: + localized[key] = text + + +def _set_author_entry(app, key, f): + limit = config['char_limits']['Author'] + with open(f) as fp: + text = fp.read()[:limit] + if len(text) > 0: + app[key] = text + + +def copy_triple_t_store_metadata(apps): + """Include store metadata from the app's source repo + + The Triple-T Gradle Play Publisher is a plugin that has a standard + file layout for all of the metadata and graphics that the Google + Play Store accepts. Since F-Droid has the git repo, it can just + pluck those files directly. This method reads any text files into + the app dict, then copies any graphics into the fdroid repo + directory structure. + + This needs to be run before insert_localized_app_metadata() so that + the graphics files that are copied into the fdroid repo get + properly indexed. + + https://github.com/Triple-T/gradle-play-publisher#upload-images + https://github.com/Triple-T/gradle-play-publisher#play-store-metadata + + """ + + if not os.path.isdir('build'): + return # nothing to do + + for packageName, app in apps.items(): + for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')): + logging.debug('Triple-T Gradle Play Publisher: ' + d) + for root, dirs, files in os.walk(d): + segments = root.split('/') + locale = segments[-2] + for f in files: + if f == 'fulldescription': + _set_localized_text_entry(app, locale, 'Description', + os.path.join(root, f)) + continue + elif f == 'shortdescription': + _set_localized_text_entry(app, locale, 'Summary', + os.path.join(root, f)) + continue + elif f == 'title': + _set_localized_text_entry(app, locale, 'Name', + os.path.join(root, f)) + continue + elif f == 'video': + _set_localized_text_entry(app, locale, 'Video', + os.path.join(root, f)) + continue + elif f == 'whatsnew': + _set_localized_text_entry(app, segments[-1], 'WhatsNew', + os.path.join(root, f)) + continue + elif f == 'contactEmail': + _set_author_entry(app, 'AuthorEmail', os.path.join(root, f)) + continue + elif f == 'contactPhone': + _set_author_entry(app, 'AuthorPhone', os.path.join(root, f)) + continue + elif f == 'contactWebsite': + _set_author_entry(app, 'AuthorWebSite', os.path.join(root, f)) + continue + + base, extension = common.get_extension(f) + dirname = os.path.basename(root) + if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: + if segments[-2] == 'listing': + locale = segments[-3] + else: + locale = segments[-2] + destdir = os.path.join('repo', packageName, locale) + os.makedirs(destdir, mode=0o755, exist_ok=True) + sourcefile = os.path.join(root, f) + destfile = os.path.join(destdir, dirname + '.' + extension) + logging.debug('copying ' + sourcefile + ' ' + destfile) + shutil.copy(sourcefile, destfile) + + +def insert_localized_app_metadata(apps): + """scans standard locations for graphics and localized text + + Scans for localized description files, store graphics, and + screenshot PNG files in statically defined screenshots directory + and adds them to the app metadata. The screenshots and graphic + must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg" and must be in the following layout: repo/packageName/locale/featureGraphic.png repo/packageName/locale/phoneScreenshots/1.png repo/packageName/locale/phoneScreenshots/2.png + The changelog files must be text files named with the versionCode + ending with ".txt" and must be in the following layout: + https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#changelogs-whats-new + + repo/packageName/locale/changelogs/12345.txt + + This will scan the each app's source repo then the metadata/ dir + for these standard locations of changelog files. If it finds + them, they will be added to the dict of all packages, with the + versions in the metadata/ folder taking precendence over the what + is in the app's source repo. + Where "packageName" is the app's packageName and "locale" is the locale of the graphics, e.g. what language they are in, using the IETF RFC5646 - format (en-US, fr-CA, es-MX, etc). This is following this pattern: + format (en-US, fr-CA, es-MX, etc). + + This will also scan the app's git for a fastlane folder, and the + metadata/ folder and the apps' source repos for standard locations + of graphic and screenshot files. If it finds them, it will copy + them into the repo. The fastlane files follow this pattern: https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots - This will also scan the metadata/ folder and the apps' source repos - for standard locations of graphic and screenshot files. If it finds - them, it will copy them into the repo. - - :param repodir: repo directory to scan - """ - allowed_extensions = ('png', 'jpg', 'jpeg') - graphicnames = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner') - screenshotdirs = ('phoneScreenshots', 'sevenInchScreenshots', - 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots') - - sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z][A-Z-.@]*')) - sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')) + sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*')) + sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*')) for d in sorted(sourcedirs): if not os.path.isdir(d): continue for root, dirs, files in os.walk(d): segments = root.split('/') - destdir = os.path.join('repo', segments[1], segments[-1]) # repo/packageName/locale + packageName = segments[1] + if packageName not in apps: + logging.debug(packageName + ' does not have app metadata, skipping l18n scan.') + continue + locale = segments[-1] + destdir = os.path.join('repo', packageName, locale) for f in files: + if f == 'full_description.txt': + _set_localized_text_entry(apps[packageName], locale, 'Description', + os.path.join(root, f)) + continue + elif f == 'short_description.txt': + _set_localized_text_entry(apps[packageName], locale, 'Summary', + os.path.join(root, f)) + continue + elif f == 'title.txt': + _set_localized_text_entry(apps[packageName], locale, 'Name', + os.path.join(root, f)) + continue + elif f == 'video.txt': + _set_localized_text_entry(apps[packageName], locale, 'Video', + os.path.join(root, f)) + continue + elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt': + _set_localized_text_entry(apps[packageName], segments[-2], 'WhatsNew', + os.path.join(root, f)) + continue + base, extension = common.get_extension(f) - if base in graphicnames and extension in allowed_extensions: + if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir) shutil.copy(os.path.join(root, f), destdir) for d in dirs: - if d in screenshotdirs: + if d in SCREENSHOT_DIRS: for f in glob.glob(os.path.join(root, d, '*.*')): _, extension = common.get_extension(f) - if extension in allowed_extensions: + if extension in ALLOWED_EXTENSIONS: screenshotdestdir = os.path.join(destdir, d) os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) logging.debug('copying ' + f + ' ' + screenshotdestdir) @@ -622,18 +762,14 @@ def insert_graphics(repodir, apps): logging.warning('Found "%s" graphic without metadata for app "%s"!' % (filename, packageName)) continue - if 'localized' not in apps[packageName]: - apps[packageName]['localized'] = collections.OrderedDict() - if locale not in apps[packageName]['localized']: - apps[packageName]['localized'][locale] = collections.OrderedDict() - graphics = apps[packageName]['localized'][locale] + graphics = _get_localized_dict(apps[packageName], locale) - if extension not in allowed_extensions: + if extension not in ALLOWED_EXTENSIONS: logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f) - elif base in graphicnames: + elif base in GRAPHIC_NAMES: # there can only be zero or one of these per locale graphics[base] = filename - elif screenshotdir in screenshotdirs: + elif screenshotdir in SCREENSHOT_DIRS: # there can any number of these per locale logging.debug('adding ' + base + ':' + f) if screenshotdir not in graphics: @@ -1363,8 +1499,9 @@ def main(): if newmetadata: apps = metadata.read_metadata() + copy_triple_t_store_metadata(apps) insert_obbs(repodirs[0], apps, apks) - insert_graphics(repodirs[0], apps) + insert_localized_app_metadata(apps) # Scan the archive repo for apks as well if len(repodirs) > 1: diff --git a/tests/metadata/dump/org.adaway.yaml b/tests/metadata/dump/org.adaway.yaml index 7f6d1589..3e5b2394 100644 --- a/tests/metadata/dump/org.adaway.yaml +++ b/tests/metadata/dump/org.adaway.yaml @@ -2,6 +2,7 @@ AntiFeatures: [] ArchivePolicy: null AuthorEmail: null AuthorName: null +AuthorWebSite: null AutoName: AdAway AutoUpdateMode: Version v%v Binaries: null diff --git a/tests/metadata/dump/org.smssecure.smssecure.yaml b/tests/metadata/dump/org.smssecure.smssecure.yaml index 0fdd3372..06b70b8d 100644 --- a/tests/metadata/dump/org.smssecure.smssecure.yaml +++ b/tests/metadata/dump/org.smssecure.smssecure.yaml @@ -2,6 +2,7 @@ AntiFeatures: [] ArchivePolicy: null AuthorEmail: null AuthorName: null +AuthorWebSite: null AutoName: SMSSecure AutoUpdateMode: Version v%v Binaries: null diff --git a/tests/metadata/dump/org.videolan.vlc.yaml b/tests/metadata/dump/org.videolan.vlc.yaml index d905830c..e173fcba 100644 --- a/tests/metadata/dump/org.videolan.vlc.yaml +++ b/tests/metadata/dump/org.videolan.vlc.yaml @@ -2,6 +2,7 @@ AntiFeatures: [] ArchivePolicy: 9 versions AuthorEmail: null AuthorName: null +AuthorWebSite: null AutoName: VLC AutoUpdateMode: None Binaries: null diff --git a/tests/metadata/info.guardianproject.urzip.yml b/tests/metadata/info.guardianproject.urzip.yml index c81f9486..a1650816 100644 --- a/tests/metadata/info.guardianproject.urzip.yml +++ b/tests/metadata/info.guardianproject.urzip.yml @@ -7,6 +7,7 @@ Categories: - 1 - 2.0 CurrentVersionCode: 2147483647 +AuthorWebSite: https://guardianproject.info Description: | It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始,一键触摸即可显示 APK 的指纹,并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接,让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。 diff --git a/tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt b/tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt new file mode 100644 index 00000000..29d6383b --- /dev/null +++ b/tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt @@ -0,0 +1 @@ +100 diff --git a/tests/metadata/info.guardianproject.urzip/en-US/full_description.txt b/tests/metadata/info.guardianproject.urzip/en-US/full_description.txt new file mode 100644 index 00000000..387b61d2 --- /dev/null +++ b/tests/metadata/info.guardianproject.urzip/en-US/full_description.txt @@ -0,0 +1 @@ +full description diff --git a/tests/metadata/info.guardianproject.urzip/en-US/short_description.txt b/tests/metadata/info.guardianproject.urzip/en-US/short_description.txt new file mode 100644 index 00000000..306eb438 --- /dev/null +++ b/tests/metadata/info.guardianproject.urzip/en-US/short_description.txt @@ -0,0 +1 @@ +short description diff --git a/tests/metadata/info.guardianproject.urzip/en-US/title.txt b/tests/metadata/info.guardianproject.urzip/en-US/title.txt new file mode 100644 index 00000000..787215d4 --- /dev/null +++ b/tests/metadata/info.guardianproject.urzip/en-US/title.txt @@ -0,0 +1 @@ +title diff --git a/tests/metadata/info.guardianproject.urzip/en-US/video.txt b/tests/metadata/info.guardianproject.urzip/en-US/video.txt new file mode 100644 index 00000000..2b68b523 --- /dev/null +++ b/tests/metadata/info.guardianproject.urzip/en-US/video.txt @@ -0,0 +1 @@ +video diff --git a/tests/update.TestCase b/tests/update.TestCase index df85b277..b9154b06 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -2,12 +2,16 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 +import git import inspect import logging import optparse import os +import shutil import sys +import tempfile import unittest +import yaml from binascii import unhexlify localmodule = os.path.realpath( @@ -17,6 +21,7 @@ if localmodule not in sys.path: sys.path.insert(0, localmodule) import fdroidserver.common +import fdroidserver.metadata import fdroidserver.update from fdroidserver.common import FDroidPopen @@ -24,6 +29,95 @@ from fdroidserver.common import FDroidPopen class UpdateTest(unittest.TestCase): '''fdroid update''' + def testInsertStoreMetadata(self): + config = dict() + fdroidserver.common.fill_config_defaults(config) + config['accepted_formats'] = ('txt', 'yml') + fdroidserver.update.config = config + fdroidserver.update.options = fdroidserver.common.options + os.chdir(os.path.join(localmodule, 'tests')) + + apps = dict() + for packageName in ('info.guardianproject.urzip', 'org.videolan.vlc', 'obb.mainpatch.current'): + apps[packageName] = dict() + apps[packageName]['id'] = packageName + apps[packageName]['CurrentVersionCode'] = 0xcafebeef + apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100 + fdroidserver.update.insert_localized_app_metadata(apps) + + self.assertEqual(3, len(apps)) + for packageName, app in apps.items(): + self.assertTrue('localized' in app) + self.assertTrue('en-US' in app['localized']) + self.assertEqual(1, len(app['localized'])) + if packageName == 'info.guardianproject.urzip': + self.assertEqual(5, len(app['localized']['en-US'])) + self.assertEqual('full description\n', app['localized']['en-US']['Description']) + self.assertEqual('title\n', app['localized']['en-US']['Name']) + self.assertEqual('short description\n', app['localized']['en-US']['Summary']) + self.assertEqual('video\n', app['localized']['en-US']['Video']) + self.assertEqual('100\n', app['localized']['en-US']['WhatsNew']) + elif packageName == 'org.videolan.vlc': + self.assertEqual('icon.png', app['localized']['en-US']['icon']) + self.assertEqual(9, len(app['localized']['en-US']['phoneScreenshots'])) + self.assertEqual(15, len(app['localized']['en-US']['sevenInchScreenshots'])) + elif packageName == 'obb.mainpatch.current': + self.assertEqual('icon.png', app['localized']['en-US']['icon']) + self.assertEqual('featureGraphic.png', app['localized']['en-US']['featureGraphic']) + self.assertEqual(1, len(app['localized']['en-US']['phoneScreenshots'])) + self.assertEqual(1, len(app['localized']['en-US']['sevenInchScreenshots'])) + + def test_insert_triple_t_metadata(self): + importer = os.path.join(localmodule, 'tests', 'tmp', 'importer') + packageName = 'org.fdroid.ci.test.app' + if not os.path.isdir(importer): + logging.warning('skipping test_insert_triple_t_metadata, import.TestCase must run first!') + return + tmpdir = os.path.join(localmodule, '.testfiles') + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + tmptestsdir = tempfile.mkdtemp(prefix='test_insert_triple_t_metadata-', dir=tmpdir) + packageDir = os.path.join(tmptestsdir, 'build', packageName) + shutil.copytree(importer, packageDir) + + # always use the same commit so these tests work when ci-test-app.git is updated + repo = git.Repo(packageDir) + for remote in repo.remotes: + remote.fetch() + repo.git.reset('--hard', 'b9e5d1a0d8d6fc31d4674b2f0514fef10762ed4f') + repo.git.clean('-fdx') + + os.mkdir(os.path.join(tmptestsdir, 'metadata')) + metadata = dict() + metadata['Description'] = 'This is just a test app' + with open(os.path.join(tmptestsdir, 'metadata', packageName + '.yml'), 'w') as fp: + yaml.dump(metadata, fp) + + config = dict() + fdroidserver.common.fill_config_defaults(config) + config['accepted_formats'] = ('yml') + fdroidserver.common.config = config + fdroidserver.update.config = config + fdroidserver.update.options = fdroidserver.common.options + os.chdir(tmptestsdir) + + apps = fdroidserver.metadata.read_metadata(xref=True) + fdroidserver.update.copy_triple_t_store_metadata(apps) + + # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata() + correctlocales = [ + 'ar', 'ast_ES', 'az', 'ca', 'ca_ES', 'cs-CZ', 'cs_CZ', 'da', + 'da-DK', 'de', 'de-DE', 'el', 'en-US', 'es', 'es-ES', 'es_ES', 'et', + 'fi', 'fr', 'fr-FR', 'he_IL', 'hi-IN', 'hi_IN', 'hu', 'id', 'it', + 'it-IT', 'it_IT', 'iw-IL', 'ja', 'ja-JP', 'kn_IN', 'ko', 'ko-KR', + 'ko_KR', 'lt', 'nb', 'nb_NO', 'nl', 'nl-NL', 'no', 'pl', 'pl-PL', + 'pl_PL', 'pt', 'pt-BR', 'pt-PT', 'pt_BR', 'ro', 'ro_RO', 'ru-RU', + 'ru_RU', 'sv-SE', 'sv_SE', 'te', 'tr', 'tr-TR', 'uk', 'uk_UA', 'vi', + 'vi_VN', 'zh-CN', 'zh_CN', 'zh_TW', + ] + locales = sorted(list(apps['org.fdroid.ci.test.app']['localized'].keys())) + self.assertEqual(correctlocales, locales) + def javagetsig(self, apkfile): getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig') if not os.path.exists(getsig_dir + "/getsig.class"): @@ -84,7 +178,7 @@ class UpdateTest(unittest.TestCase): self.assertIsNone(pysig, "python sig should be None: " + str(sig)) def testScanApksAndObbs(self): - os.chdir(os.path.dirname(__file__)) + os.chdir(os.path.join(localmodule, 'tests')) if os.path.basename(os.getcwd()) != 'tests': raise Exception('This test must be run in the "tests/" subdir') @@ -131,7 +225,7 @@ class UpdateTest(unittest.TestCase): self.assertIsNone(apk.get('obbPatchFile')) def test_scan_invalid_apk(self): - os.chdir(os.path.dirname(__file__)) + os.chdir(os.path.join(localmodule, 'tests')) if os.path.basename(os.getcwd()) != 'tests': raise Exception('This test must be run in the "tests/" subdir')