Merge branch 'full-app-store-metadata-scraping' into 'master'

Full app store metadata scraping

Closes #204 and #143

See merge request !253

fdroidserver/common.py

@@ -93,8 +93,12 @@ default_config = {
     'keystore': 'keystore.jks',
     'smartcardoptions': [],
     'char_limits': {
+        'Author': 256,
+        'Name': 30,
         'Summary': 80,
         'Description': 4000,
+        'Video': 256,
+        'WhatsNew': 500,
     },
     'keyaliases': {},
     'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",

fdroidserver/metadata.py

@@ -70,6 +70,7 @@ app_fields = set([
     'License',
     'Author Name',
     'Author Email',
+    'Author Web Site',
     'Web Site',
     'Source Code',
     'Issue Tracker',
@@ -119,6 +120,7 @@ class App(dict):
         self.License = 'Unknown'
         self.AuthorName = None
         self.AuthorEmail = None
+        self.AuthorWebSite = None
         self.WebSite = ''
         self.SourceCode = ''
         self.IssueTracker = ''
@@ -1199,6 +1201,7 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
     w_field_always('License')
     w_field_nonempty('Author Name')
     w_field_nonempty('Author Email')
+    w_field_nonempty('Author Web Site')
     w_field_always('Web Site')
     w_field_always('Source Code')
     w_field_always('Issue Tracker')

fdroidserver/update.py

@@ -66,6 +66,11 @@ all_screen_densities = ['0'] + screen_densities
 UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
 UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
 
+ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
+GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
+SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
+                   'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
+
 
 def dpi_to_px(density):
     return (int(density) * 48) / 160
@@ -551,54 +556,189 @@ def insert_obbs(repodir, apps, apks):
                 break
 
 
-def insert_graphics(repodir, apps):
-    """Scans for screenshot PNG files in statically defined screenshots
-    directory and adds them to the app metadata.  The screenshots and
-    graphic must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
+def _get_localized_dict(app, locale):
+    '''get the dict to add localized store metadata to'''
+    if 'localized' not in app:
+        app['localized'] = collections.OrderedDict()
+    if locale not in app['localized']:
+        app['localized'][locale] = collections.OrderedDict()
+    return app['localized'][locale]
+
+
+def _set_localized_text_entry(app, locale, key, f):
+    limit = config['char_limits'][key]
+    localized = _get_localized_dict(app, locale)
+    with open(f) as fp:
+        text = fp.read()[:limit]
+        if len(text) > 0:
+            localized[key] = text
+
+
+def _set_author_entry(app, key, f):
+    limit = config['char_limits']['Author']
+    with open(f) as fp:
+        text = fp.read()[:limit]
+        if len(text) > 0:
+            app[key] = text
+
+
+def copy_triple_t_store_metadata(apps):
+    """Include store metadata from the app's source repo
+
+    The Triple-T Gradle Play Publisher is a plugin that has a standard
+    file layout for all of the metadata and graphics that the Google
+    Play Store accepts.  Since F-Droid has the git repo, it can just
+    pluck those files directly.  This method reads any text files into
+    the app dict, then copies any graphics into the fdroid repo
+    directory structure.
+
+    This needs to be run before insert_localized_app_metadata() so that
+    the graphics files that are copied into the fdroid repo get
+    properly indexed.
+
+    https://github.com/Triple-T/gradle-play-publisher#upload-images
+    https://github.com/Triple-T/gradle-play-publisher#play-store-metadata
+
+    """
+
+    if not os.path.isdir('build'):
+        return  # nothing to do
+
+    for packageName, app in apps.items():
+        for d in glob.glob(os.path.join('build', packageName, '*', 'src', '*', 'play')):
+            logging.debug('Triple-T Gradle Play Publisher: ' + d)
+            for root, dirs, files in os.walk(d):
+                segments = root.split('/')
+                locale = segments[-2]
+                for f in files:
+                    if f == 'fulldescription':
+                        _set_localized_text_entry(app, locale, 'Description',
+                                                  os.path.join(root, f))
+                        continue
+                    elif f == 'shortdescription':
+                        _set_localized_text_entry(app, locale, 'Summary',
+                                                  os.path.join(root, f))
+                        continue
+                    elif f == 'title':
+                        _set_localized_text_entry(app, locale, 'Name',
+                                                  os.path.join(root, f))
+                        continue
+                    elif f == 'video':
+                        _set_localized_text_entry(app, locale, 'Video',
+                                                  os.path.join(root, f))
+                        continue
+                    elif f == 'whatsnew':
+                        _set_localized_text_entry(app, segments[-1], 'WhatsNew',
+                                                  os.path.join(root, f))
+                        continue
+                    elif f == 'contactEmail':
+                        _set_author_entry(app, 'AuthorEmail', os.path.join(root, f))
+                        continue
+                    elif f == 'contactPhone':
+                        _set_author_entry(app, 'AuthorPhone', os.path.join(root, f))
+                        continue
+                    elif f == 'contactWebsite':
+                        _set_author_entry(app, 'AuthorWebSite', os.path.join(root, f))
+                        continue
+
+                    base, extension = common.get_extension(f)
+                    dirname = os.path.basename(root)
+                    if dirname in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
+                        if segments[-2] == 'listing':
+                            locale = segments[-3]
+                        else:
+                            locale = segments[-2]
+                        destdir = os.path.join('repo', packageName, locale)
+                        os.makedirs(destdir, mode=0o755, exist_ok=True)
+                        sourcefile = os.path.join(root, f)
+                        destfile = os.path.join(destdir, dirname + '.' + extension)
+                        logging.debug('copying ' + sourcefile + ' ' + destfile)
+                        shutil.copy(sourcefile, destfile)
+
+
+def insert_localized_app_metadata(apps):
+    """scans standard locations for graphics and localized text
+
+    Scans for localized description files, store graphics, and
+    screenshot PNG files in statically defined screenshots directory
+    and adds them to the app metadata.  The screenshots and graphic
+    must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
     and must be in the following layout:
 
     repo/packageName/locale/featureGraphic.png
     repo/packageName/locale/phoneScreenshots/1.png
     repo/packageName/locale/phoneScreenshots/2.png
 
+    The changelog files must be text files named with the versionCode
+    ending with ".txt" and must be in the following layout:
+    https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#changelogs-whats-new
+
+    repo/packageName/locale/changelogs/12345.txt
+
+    This will scan the each app's source repo then the metadata/ dir
+    for these standard locations of changelog files.  If it finds
+    them, they will be added to the dict of all packages, with the
+    versions in the metadata/ folder taking precendence over the what
+    is in the app's source repo.
+
     Where "packageName" is the app's packageName and "locale" is the locale
     of the graphics, e.g. what language they are in, using the IETF RFC5646
-    format (en-US, fr-CA, es-MX, etc).  This is following this pattern:
+    format (en-US, fr-CA, es-MX, etc).
+
+    This will also scan the app's git for a fastlane folder, and the
+    metadata/ folder and the apps' source repos for standard locations
+    of graphic and screenshot files.  If it finds them, it will copy
+    them into the repo.  The fastlane files follow this pattern:
     https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
 
-    This will also scan the metadata/ folder and the apps' source repos
-    for standard locations of graphic and screenshot files.  If it finds
-    them, it will copy them into the repo.
-
-    :param repodir: repo directory to scan
-
     """
 
-    allowed_extensions = ('png', 'jpg', 'jpeg')
-    graphicnames = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
-    screenshotdirs = ('phoneScreenshots', 'sevenInchScreenshots',
-                      'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
-
-    sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z][A-Z-.@]*'))
-    sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*'))
+    sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
+    sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 
     for d in sorted(sourcedirs):
         if not os.path.isdir(d):
             continue
         for root, dirs, files in os.walk(d):
             segments = root.split('/')
-            destdir = os.path.join('repo', segments[1], segments[-1])  # repo/packageName/locale
+            packageName = segments[1]
+            if packageName not in apps:
+                logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
+                continue
+            locale = segments[-1]
+            destdir = os.path.join('repo', packageName, locale)
             for f in files:
+                if f == 'full_description.txt':
+                    _set_localized_text_entry(apps[packageName], locale, 'Description',
+                                              os.path.join(root, f))
+                    continue
+                elif f == 'short_description.txt':
+                    _set_localized_text_entry(apps[packageName], locale, 'Summary',
+                                              os.path.join(root, f))
+                    continue
+                elif f == 'title.txt':
+                    _set_localized_text_entry(apps[packageName], locale, 'Name',
+                                              os.path.join(root, f))
+                    continue
+                elif f == 'video.txt':
+                    _set_localized_text_entry(apps[packageName], locale, 'Video',
+                                              os.path.join(root, f))
+                    continue
+                elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
+                    _set_localized_text_entry(apps[packageName], segments[-2], 'WhatsNew',
+                                              os.path.join(root, f))
+                    continue
+
                 base, extension = common.get_extension(f)
-                if base in graphicnames and extension in allowed_extensions:
+                if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
                     os.makedirs(destdir, mode=0o755, exist_ok=True)
                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
                     shutil.copy(os.path.join(root, f), destdir)
             for d in dirs:
-                if d in screenshotdirs:
+                if d in SCREENSHOT_DIRS:
                     for f in glob.glob(os.path.join(root, d, '*.*')):
                         _, extension = common.get_extension(f)
-                        if extension in allowed_extensions:
+                        if extension in ALLOWED_EXTENSIONS:
                             screenshotdestdir = os.path.join(destdir, d)
                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
@@ -622,18 +762,14 @@ def insert_graphics(repodir, apps):
                 logging.warning('Found "%s" graphic without metadata for app "%s"!'
                                 % (filename, packageName))
                 continue
-            if 'localized' not in apps[packageName]:
-                apps[packageName]['localized'] = collections.OrderedDict()
-            if locale not in apps[packageName]['localized']:
-                apps[packageName]['localized'][locale] = collections.OrderedDict()
-            graphics = apps[packageName]['localized'][locale]
+            graphics = _get_localized_dict(apps[packageName], locale)
 
-            if extension not in allowed_extensions:
+            if extension not in ALLOWED_EXTENSIONS:
                 logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f)
-            elif base in graphicnames:
+            elif base in GRAPHIC_NAMES:
                 # there can only be zero or one of these per locale
                 graphics[base] = filename
-            elif screenshotdir in screenshotdirs:
+            elif screenshotdir in SCREENSHOT_DIRS:
                 # there can any number of these per locale
                 logging.debug('adding ' + base + ':' + f)
                 if screenshotdir not in graphics:
@@ -1363,8 +1499,9 @@ def main():
     if newmetadata:
         apps = metadata.read_metadata()
 
+    copy_triple_t_store_metadata(apps)
     insert_obbs(repodirs[0], apps, apks)
-    insert_graphics(repodirs[0], apps)
+    insert_localized_app_metadata(apps)
 
     # Scan the archive repo for apks as well
     if len(repodirs) > 1:

tests/metadata/dump/org.adaway.yaml

@@ -2,6 +2,7 @@ AntiFeatures: []
 ArchivePolicy: null
 AuthorEmail: null
 AuthorName: null
+AuthorWebSite: null
 AutoName: AdAway
 AutoUpdateMode: Version v%v
 Binaries: null

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -2,6 +2,7 @@ AntiFeatures: []
 ArchivePolicy: null
 AuthorEmail: null
 AuthorName: null
+AuthorWebSite: null
 AutoName: SMSSecure
 AutoUpdateMode: Version v%v
 Binaries: null

tests/metadata/dump/org.videolan.vlc.yaml

@@ -2,6 +2,7 @@ AntiFeatures: []
 ArchivePolicy: 9 versions
 AuthorEmail: null
 AuthorName: null
+AuthorWebSite: null
 AutoName: VLC
 AutoUpdateMode: None
 Binaries: null

tests/metadata/info.guardianproject.urzip.yml

@@ -7,6 +7,7 @@ Categories:
 - 1
 - 2.0
 CurrentVersionCode: 2147483647
+AuthorWebSite: https://guardianproject.info
 Description: |
   It’s Urzip 是一个获得已安装 APK 相关信息的实用工具。它从您的设备上已安装的所有应用开始，一键触摸即可显示 APK 的指纹，并且提供到达 virustotal.com 和 androidobservatory.org 的快捷链接，让您方便地了解特定 APK 的档案。它还可以让您导出签名证书和生成 ApkSignaturePin Pin 文件供 TrustedIntents 库使用。
 

tests/metadata/info.guardianproject.urzip/en-US/changelogs/100.txt

@@ -0,0 +1 @@
+100

tests/metadata/info.guardianproject.urzip/en-US/full_description.txt

@@ -0,0 +1 @@
+full description

tests/metadata/info.guardianproject.urzip/en-US/short_description.txt

@@ -0,0 +1 @@
+short description

tests/metadata/info.guardianproject.urzip/en-US/title.txt

@@ -0,0 +1 @@
+title

tests/metadata/info.guardianproject.urzip/en-US/video.txt

@@ -0,0 +1 @@
+video

tests/update.TestCase

@@ -2,12 +2,16 @@
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
+import git
 import inspect
 import logging
 import optparse
 import os
+import shutil
 import sys
+import tempfile
 import unittest
+import yaml
 from binascii import unhexlify
 
 localmodule = os.path.realpath(
@@ -17,6 +21,7 @@ if localmodule not in sys.path:
     sys.path.insert(0, localmodule)
 
 import fdroidserver.common
+import fdroidserver.metadata
 import fdroidserver.update
 from fdroidserver.common import FDroidPopen
 
@@ -24,6 +29,95 @@ from fdroidserver.common import FDroidPopen
 class UpdateTest(unittest.TestCase):
     '''fdroid update'''
 
+    def testInsertStoreMetadata(self):
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['accepted_formats'] = ('txt', 'yml')
+        fdroidserver.update.config = config
+        fdroidserver.update.options = fdroidserver.common.options
+        os.chdir(os.path.join(localmodule, 'tests'))
+
+        apps = dict()
+        for packageName in ('info.guardianproject.urzip', 'org.videolan.vlc', 'obb.mainpatch.current'):
+            apps[packageName] = dict()
+            apps[packageName]['id'] = packageName
+            apps[packageName]['CurrentVersionCode'] = 0xcafebeef
+        apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100
+        fdroidserver.update.insert_localized_app_metadata(apps)
+
+        self.assertEqual(3, len(apps))
+        for packageName, app in apps.items():
+            self.assertTrue('localized' in app)
+            self.assertTrue('en-US' in app['localized'])
+            self.assertEqual(1, len(app['localized']))
+            if packageName == 'info.guardianproject.urzip':
+                self.assertEqual(5, len(app['localized']['en-US']))
+                self.assertEqual('full description\n', app['localized']['en-US']['Description'])
+                self.assertEqual('title\n', app['localized']['en-US']['Name'])
+                self.assertEqual('short description\n', app['localized']['en-US']['Summary'])
+                self.assertEqual('video\n', app['localized']['en-US']['Video'])
+                self.assertEqual('100\n', app['localized']['en-US']['WhatsNew'])
+            elif packageName == 'org.videolan.vlc':
+                self.assertEqual('icon.png', app['localized']['en-US']['icon'])
+                self.assertEqual(9, len(app['localized']['en-US']['phoneScreenshots']))
+                self.assertEqual(15, len(app['localized']['en-US']['sevenInchScreenshots']))
+            elif packageName == 'obb.mainpatch.current':
+                self.assertEqual('icon.png', app['localized']['en-US']['icon'])
+                self.assertEqual('featureGraphic.png', app['localized']['en-US']['featureGraphic'])
+                self.assertEqual(1, len(app['localized']['en-US']['phoneScreenshots']))
+                self.assertEqual(1, len(app['localized']['en-US']['sevenInchScreenshots']))
+
+    def test_insert_triple_t_metadata(self):
+        importer = os.path.join(localmodule, 'tests', 'tmp', 'importer')
+        packageName = 'org.fdroid.ci.test.app'
+        if not os.path.isdir(importer):
+            logging.warning('skipping test_insert_triple_t_metadata, import.TestCase must run first!')
+            return
+        tmpdir = os.path.join(localmodule, '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        tmptestsdir = tempfile.mkdtemp(prefix='test_insert_triple_t_metadata-', dir=tmpdir)
+        packageDir = os.path.join(tmptestsdir, 'build', packageName)
+        shutil.copytree(importer, packageDir)
+
+        # always use the same commit so these tests work when ci-test-app.git is updated
+        repo = git.Repo(packageDir)
+        for remote in repo.remotes:
+            remote.fetch()
+        repo.git.reset('--hard', 'b9e5d1a0d8d6fc31d4674b2f0514fef10762ed4f')
+        repo.git.clean('-fdx')
+
+        os.mkdir(os.path.join(tmptestsdir, 'metadata'))
+        metadata = dict()
+        metadata['Description'] = 'This is just a test app'
+        with open(os.path.join(tmptestsdir, 'metadata', packageName + '.yml'), 'w') as fp:
+            yaml.dump(metadata, fp)
+
+        config = dict()
+        fdroidserver.common.fill_config_defaults(config)
+        config['accepted_formats'] = ('yml')
+        fdroidserver.common.config = config
+        fdroidserver.update.config = config
+        fdroidserver.update.options = fdroidserver.common.options
+        os.chdir(tmptestsdir)
+
+        apps = fdroidserver.metadata.read_metadata(xref=True)
+        fdroidserver.update.copy_triple_t_store_metadata(apps)
+
+        # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata()
+        correctlocales = [
+            'ar', 'ast_ES', 'az', 'ca', 'ca_ES', 'cs-CZ', 'cs_CZ', 'da',
+            'da-DK', 'de', 'de-DE', 'el', 'en-US', 'es', 'es-ES', 'es_ES', 'et',
+            'fi', 'fr', 'fr-FR', 'he_IL', 'hi-IN', 'hi_IN', 'hu', 'id', 'it',
+            'it-IT', 'it_IT', 'iw-IL', 'ja', 'ja-JP', 'kn_IN', 'ko', 'ko-KR',
+            'ko_KR', 'lt', 'nb', 'nb_NO', 'nl', 'nl-NL', 'no', 'pl', 'pl-PL',
+            'pl_PL', 'pt', 'pt-BR', 'pt-PT', 'pt_BR', 'ro', 'ro_RO', 'ru-RU',
+            'ru_RU', 'sv-SE', 'sv_SE', 'te', 'tr', 'tr-TR', 'uk', 'uk_UA', 'vi',
+            'vi_VN', 'zh-CN', 'zh_CN', 'zh_TW',
+        ]
+        locales = sorted(list(apps['org.fdroid.ci.test.app']['localized'].keys()))
+        self.assertEqual(correctlocales, locales)
+
     def javagetsig(self, apkfile):
         getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
         if not os.path.exists(getsig_dir + "/getsig.class"):
@@ -84,7 +178,7 @@ class UpdateTest(unittest.TestCase):
         self.assertIsNone(pysig, "python sig should be None: " + str(sig))
 
     def testScanApksAndObbs(self):
-        os.chdir(os.path.dirname(__file__))
+        os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':
             raise Exception('This test must be run in the "tests/" subdir')
 
@@ -131,7 +225,7 @@ class UpdateTest(unittest.TestCase):
                 self.assertIsNone(apk.get('obbPatchFile'))
 
     def test_scan_invalid_apk(self):
-        os.chdir(os.path.dirname(__file__))
+        os.chdir(os.path.join(localmodule, 'tests'))
         if os.path.basename(os.getcwd()) != 'tests':
             raise Exception('This test must be run in the "tests/" subdir')