Merge branch 'indexing-fixes' into 'master'

indexing fixes

Closes #303

See merge request !270

.gitignore

@@ -39,5 +39,6 @@ makebuildserver.config.py
 /tests/archive/index-v1.jar
 /tests/repo/index.jar
 /tests/repo/index-v1.jar
+/tests/repo/info.guardianproject.urzip/
 /tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
 /unsigned/

fdroidserver/build.py

@@ -544,7 +544,7 @@ def get_apk_metadata_androguard(apkfile):
 def get_metadata_from_apk(app, build, apkfile):
     """get the required metadata from the built APK"""
 
-    if common.set_command_in_config('aapt'):
+    if common.SdkToolsPopen(['aapt', 'version'], output=False):
         vercode, version, foundid, nativecode = get_apk_metadata_aapt(apkfile)
     else:
         vercode, version, foundid, nativecode = get_apk_metadata_androguard(apkfile)

fdroidserver/common.py

@@ -1727,7 +1727,7 @@ def isApkAndDebuggable(apkfile, config):
     if get_file_extension(apkfile) != 'apk':
         return False
 
-    if set_command_in_config('aapt'):
+    if SdkToolsPopen(['aapt', 'version'], output=False):
         return get_apk_debuggable_aapt(apkfile)
     else:
         return get_apk_debuggable_androguard(apkfile)

fdroidserver/index.py

@@ -169,7 +169,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
 
     appslist = []
     output['apps'] = appslist
-    for appid, appdict in apps.items():
+    for packageName, appdict in apps.items():
         d = collections.OrderedDict()
         appslist.append(d)
         for k, v in sorted(appdict.items()):
@@ -190,7 +190,7 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
             elif k == 'CurrentVersion':  # TODO make SuggestedVersionName the canonical name
                 k = 'suggestedVersionName'
             elif k == 'AutoName':
-                if 'Name' not in apps[appid]:
+                if 'Name' not in apps[packageName]:
                     d['name'] = v
                 continue
             else:
@@ -201,6 +201,9 @@ def make_v1(apps, packages, repodir, repodict, requestsdict):
     output['packages'] = output_packages
     for package in packages:
         packageName = package['packageName']
+        if packageName not in apps:
+            logging.info('Ignoring package without metadata: ' + package['apkName'])
+            continue
         if packageName in output_packages:
             packagelist = output_packages[packageName]
         else:

fdroidserver/metadata.py

@@ -21,7 +21,7 @@ import json
 import os
 import re
 import glob
-import cgi
+import html
 import logging
 import textwrap
 import io
@@ -492,10 +492,10 @@ class DescriptionFormatter:
         self.laststate = self.state
         self.state = self.stNONE
 
-    def formatted(self, txt, html):
+    def formatted(self, txt, htmlbody):
         res = ''
-        if html:
-            txt = cgi.escape(txt)
+        if htmlbody:
+            txt = html.escape(txt, quote=False)
         while True:
             index = txt.find("''")
             if index == -1:
@@ -503,7 +503,7 @@ class DescriptionFormatter:
             res += txt[:index]
             txt = txt[index:]
             if txt.startswith("'''"):
-                if html:
+                if htmlbody:
                     if self.bold:
                         res += '</b>'
                     else:
@@ -511,7 +511,7 @@ class DescriptionFormatter:
                 self.bold = not self.bold
                 txt = txt[3:]
             else:
-                if html:
+                if htmlbody:
                     if self.ital:
                         res += '</i>'
                     else:
@@ -538,7 +538,7 @@ class DescriptionFormatter:
                     url, urltext = self.linkResolver(url)
                 else:
                     urltext = url
-                res_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
+                res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
                 res_plain += urltext
                 txt = txt[index + 2:]
             else:
@@ -554,7 +554,7 @@ class DescriptionFormatter:
                     url = url[:index2]
                     if url == urltxt:
                         warn_or_exception("Url title is just the URL - use [url]")
-                res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
+                res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
                 res_plain += urltxt
                 if urltxt != url:
                     res_plain += ' (' + url + ')'

fdroidserver/update.py

@@ -676,14 +676,14 @@ def insert_localized_app_metadata(apps):
     must be PNG or JPEG files ending with ".png", ".jpg", or ".jpeg"
     and must be in the following layout:
     # TODO replace these docs with link to All_About_Descriptions_Graphics_and_Screenshots
-    # TODO mention that the 'localized' section is not in metadata.yml, so key names are like Java vars: camelCase with first letter lowercase.
+
     repo/packageName/locale/featureGraphic.png
     repo/packageName/locale/phoneScreenshots/1.png
     repo/packageName/locale/phoneScreenshots/2.png
 
     The changelog files must be text files named with the versionCode
     ending with ".txt" and must be in the following layout:
-    https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#changelogs-whats-new
+    https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#changelogs-whats-new
 
     repo/packageName/locale/changelogs/12345.txt
 
@@ -701,7 +701,7 @@ def insert_localized_app_metadata(apps):
     metadata/ folder and the apps' source repos for standard locations
     of graphic and screenshot files.  If it finds them, it will copy
     them into the repo.  The fastlane files follow this pattern:
-    https://github.com/fastlane/fastlane/blob/1.109.0/supply/README.md#images-and-screenshots
+    https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
 
     """
 
@@ -718,7 +718,6 @@ def insert_localized_app_metadata(apps):
                 logging.debug(packageName + ' does not have app metadata, skipping l18n scan.')
                 continue
             locale = segments[-1]
-            destdir = os.path.join('repo', packageName, locale)
             for f in files:
                 if f == 'full_description.txt':
                     _set_localized_text_entry(apps[packageName], locale, 'description',
@@ -737,11 +736,15 @@ def insert_localized_app_metadata(apps):
                                               os.path.join(root, f))
                     continue
                 elif f == str(apps[packageName]['CurrentVersionCode']) + '.txt':
-                    _set_localized_text_entry(apps[packageName], segments[-2], 'whatsNew',
+                    locale = segments[-2]
+                    _set_localized_text_entry(apps[packageName], locale, 'whatsNew',
                                               os.path.join(root, f))
                     continue
 
                 base, extension = common.get_extension(f)
+                if locale == 'images':
+                    locale = segments[-2]
+                destdir = os.path.join('repo', packageName, locale)
                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
                     os.makedirs(destdir, mode=0o755, exist_ok=True)
                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
@@ -783,7 +786,7 @@ def insert_localized_app_metadata(apps):
                 graphics[base] = filename
             elif screenshotdir in SCREENSHOT_DIRS:
                 # there can any number of these per locale
-                logging.debug('adding ' + base + ':' + f)
+                logging.debug('adding to ' + screenshotdir + ': ' + f)
                 if screenshotdir not in graphics:
                     graphics[screenshotdir] = []
                 graphics[screenshotdir].append(filename)
@@ -849,12 +852,13 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             repo_file['versionName'] = shasum
             # the static ID is the SHA256 unless it is set in the metadata
             repo_file['packageName'] = shasum
-            n = name_utf8.split('_')
+
+            n = name_utf8.rsplit('_', maxsplit=1)
             if len(n) == 2:
                 packageName = n[0]
                 versionCode = n[1].split('.')[0]
                 if re.match('^-?[0-9]+$', versionCode) \
-                   and common.is_valid_package_name(name_utf8.split('_')[0]):
+                   and common.is_valid_package_name(n[0]):
                     repo_file['packageName'] = packageName
                     repo_file['versionCode'] = int(versionCode)
             srcfilename = name + b'_src.tar.gz'
@@ -1124,11 +1128,9 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk):
             apk['antiFeatures'].add('KnownVuln')
 
         try:
-            if common.set_command_in_config('aapt'):
-                logging.warning("Using AAPT for metadata")
+            if SdkToolsPopen(['aapt', 'version'], output=False):
                 scan_apk_aapt(apk, apkfile)
             else:
-                logging.warning("Using androguard for metadata")
                 scan_apk_androguard(apk, apkfile)
         except BuildException:
             return True, None, False

tests/androguard_test.py

@@ -4,12 +4,8 @@ import inspect
 import logging
 import optparse
 import os
-import shutil
 import sys
-import tempfile
 import unittest
-import yaml
-from binascii import unhexlify
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
@@ -53,9 +49,10 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options.clean = True
         fdroidserver.update.options.delete_unknown = True
 
-        self.assertTrue(fdroidserver.common.set_command_in_config('aapt'))
+        self.assertTrue(fdroidserver.common.SdkToolsPopen('aapt'))
         try:
             from androguard.core.bytecodes.apk import APK
+            dir(APK)
         except ImportError:
             raise Exception("androguard not installed!")
 
@@ -89,4 +86,4 @@ if __name__ == "__main__":
 
     newSuite = unittest.TestSuite()
     newSuite.addTest(unittest.makeSuite(UpdateTest))
-    unittest.main()
+    unittest.main()

tests/update.TestCase

@@ -37,6 +37,8 @@ class UpdateTest(unittest.TestCase):
         fdroidserver.update.options = fdroidserver.common.options
         os.chdir(os.path.join(localmodule, 'tests'))
 
+        shutil.rmtree(os.path.join('repo', 'info.guardianproject.urzip'), ignore_errors=True)
+
         apps = dict()
         for packageName in ('info.guardianproject.urzip', 'org.videolan.vlc', 'obb.mainpatch.current'):
             apps[packageName] = dict()
@@ -45,17 +47,23 @@ class UpdateTest(unittest.TestCase):
         apps['info.guardianproject.urzip']['CurrentVersionCode'] = 100
         fdroidserver.update.insert_localized_app_metadata(apps)
 
+        appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US')
+        self.assertTrue(os.path.isfile(os.path.join(appdir, 'icon.png')))
+        self.assertTrue(os.path.isfile(os.path.join(appdir, 'featureGraphic.png')))
+
         self.assertEqual(3, len(apps))
         for packageName, app in apps.items():
             self.assertTrue('localized' in app)
             self.assertTrue('en-US' in app['localized'])
             self.assertEqual(1, len(app['localized']))
             if packageName == 'info.guardianproject.urzip':
-                self.assertEqual(5, len(app['localized']['en-US']))
+                self.assertEqual(7, len(app['localized']['en-US']))
                 self.assertEqual('full description\n', app['localized']['en-US']['description'])
                 self.assertEqual('title\n', app['localized']['en-US']['name'])
                 self.assertEqual('short description\n', app['localized']['en-US']['summary'])
                 self.assertEqual('video\n', app['localized']['en-US']['video'])
+                self.assertEqual('icon.png', app['localized']['en-US']['icon'])
+                self.assertEqual('featureGraphic.png', app['localized']['en-US']['featureGraphic'])
                 self.assertEqual('100\n', app['localized']['en-US']['whatsNew'])
             elif packageName == 'org.videolan.vlc':
                 self.assertEqual('icon.png', app['localized']['en-US']['icon'])
@@ -244,6 +252,7 @@ class UpdateTest(unittest.TestCase):
 
         fdroidserver.update.options = type('', (), {})()
         fdroidserver.update.options.clean = True
+        fdroidserver.update.options.rename_apks = False
         fdroidserver.update.options.delete_unknown = True
 
         for icon_dir in fdroidserver.update.get_all_icon_dirs('repo'):