Merge branch 'pile-o-2.0-bugfixes' into 'master'

Pile o' 2.0 bugfixes

Closes #719

See merge request fdroid/fdroidserver!753

buildserver/provision-apt-get-install

@@ -115,6 +115,7 @@ packages="
  scons
  sqlite3
  subversion
+ sudo
  swig
  unzip
  xsltproc

completion/bash-completion

@@ -60,20 +60,14 @@ __apk_vercode() {
 }
 
 __vercode() {
-	local p v
+	if [ $prev = ":" ]; then
-	echo $cur | IFS=':' read p v
+	    appid="${COMP_WORDS[COMP_CWORD-2]}"
-
+	elif [ $cur = ":" ]; then
-	COMPREPLY=( $( compgen -P "${p}:" -W "$( while read line; do
+	    appid=$prev
-		if [[ "$line" == "Build Version:"* ]]
+	    cur=""
-		then
+	fi
-			line="${line#*,}"
+	versionCodes=`sed -En 's,^ +versionCode: +([0-9]+) *$,\1,p' metadata/${appid}.yml`
-			printf "${line%%,*} "
+	COMPREPLY=( $( compgen -W "$versionCodes" -- $cur ) )
-		elif [[ "$line" == "Build:"* ]]
-		then
-			line="${line#*,}"
-			printf "${line%%,*} "
-		fi
-		done < "metadata/${p}.yml" )" -- $cur ) )
 }
 
 __complete_options() {
@@ -91,11 +85,16 @@ __complete_build() {
 	opts="-v -q -l -s -t -f -a -w"
 
 	lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --no-tarball --force --all --wiki --no-refresh"
+	case "${prev}" in
+		:)
+			__vercode
+			return 0;;
+	esac
 	case "${cur}" in
 		-*)
 			__complete_options
 			return 0;;
-		*:*)
+		:)
 			__vercode
 			return 0;;
 		*)

fdroidserver/__main__.py

@@ -137,9 +137,9 @@ def main():
 
     system_langcode, system_encoding = locale.getdefaultlocale()
     if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
-        logging.warn(_("Encoding is set to '{enc}' fdroid might run "
+        logging.warning(_("Encoding is set to '{enc}' fdroid might run "
-                       "into encoding issues. Please set it to 'UTF-8' "
+                          "into encoding issues. Please set it to 'UTF-8' "
-                       "for best results.".format(enc=system_encoding)))
+                          "for best results.".format(enc=system_encoding)))
 
     try:
         mod.main()

fdroidserver/build.py

@@ -278,7 +278,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
             ftp.get(toolsversion_log, os.path.join(log_dir, toolsversion_log))
             logging.debug('retrieved %s', toolsversion_log)
         except Exception as e:
-            logging.warn('could not get %s from builder vm: %s' % (toolsversion_log, e))
+            logging.warning('could not get %s from builder vm: %s' % (toolsversion_log, e))
 
         # Retrieve the built files...
         logging.info("Retrieving build output...")

fdroidserver/import.py

@@ -131,9 +131,9 @@ def main():
         if not package:
             raise FDroidException(_("Couldn't find package ID"))
         if not versionName:
-            logging.warn(_("Couldn't find latest version name"))
+            logging.warning(_('Could not find latest version name'))
         if not versionCode:
-            logging.warn(_("Couldn't find latest version code"))
+            logging.warning(_('Could not find latest version code'))
     else:
         raise FDroidException(_("No gradle project could be found. Specify --subdir?"))
 

fdroidserver/init.py

@@ -130,7 +130,7 @@ def main():
         if 'sdk_path' in test_config:
             common.write_to_config(test_config, 'sdk_path', options.android_home)
     else:
-        logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
+        logging.warning('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
         logging.info('Try running `fdroid init` in an empty directory.')
         raise FDroidException('Repository already exists.')
 
@@ -209,8 +209,8 @@ def main():
                     opensc_so = files[0]
                 else:
                     opensc_so = '/usr/lib/opensc-pkcs11.so'
-                    logging.warn('No OpenSC PKCS#11 module found, '
+                    logging.warning('No OpenSC PKCS#11 module found, '
-                                 + 'install OpenSC then edit "opensc-fdroid.cfg"!')
+                                    + 'install OpenSC then edit "opensc-fdroid.cfg"!')
             with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f:
                 opensc_fdroid = f.read()
             opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid,

fdroidserver/install.py

@@ -109,8 +109,8 @@ def main():
                 continue
 
             if fail == "INSTALL_FAILED_ALREADY_EXISTS":
-                logging.warn(_("'{apkfilename}' is already installed on {dev}.")
+                logging.warning(_('"{apkfilename}" is already installed on {dev}.')
-                             .format(apkfilename=apk, dev=dev))
+                                .format(apkfilename=apk, dev=dev))
             else:
                 raise FDroidException(_("Failed to install '{apkfilename}' on {dev}: {error}")
                                       .format(apkfilename=apk, dev=dev, error=fail))

fdroidserver/scanner.py

@@ -159,7 +159,7 @@ def scan_source(build_dir, build=metadata.Build()):
     def warnproblem(what, path_in_build_dir):
         if toignore(path_in_build_dir):
             return
-        logging.warn('Found %s at %s' % (what, path_in_build_dir))
+        logging.warning('Found %s at %s' % (what, path_in_build_dir))
         if json_per_build is not None:
             json_per_build['warnings'].append([what, path_in_build_dir])
 
@@ -356,8 +356,8 @@ def main():
                 json_per_appid['current-source-state'] = json_per_build
                 count = scan_source(build_dir)
                 if count > 0:
-                    logging.warn(_('Scanner found {count} problems in {appid}:')
+                    logging.warning(_('Scanner found {count} problems in {appid}:')
-                                 .format(count=count, appid=appid))
+                                    .format(count=count, appid=appid))
                     probcount += count
                 app.builds = []
 
@@ -378,19 +378,19 @@ def main():
 
                 count = scan_source(build_dir, build)
                 if count > 0:
-                    logging.warn(_('Scanner found {count} problems in {appid}:{versionCode}:')
+                    logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:')
-                                 .format(count=count, appid=appid, versionCode=build.versionCode))
+                                    .format(count=count, appid=appid, versionCode=build.versionCode))
                     probcount += count
 
         except BuildException as be:
-            logging.warn("Could not scan app %s due to BuildException: %s" % (
+            logging.warning('Could not scan app %s due to BuildException: %s' % (
                 appid, be))
             probcount += 1
         except VCSException as vcse:
-            logging.warn("VCS error while scanning app %s: %s" % (appid, vcse))
+            logging.warning('VCS error while scanning app %s: %s' % (appid, vcse))
             probcount += 1
         except Exception:
-            logging.warn("Could not scan app %s due to unknown error: %s" % (
+            logging.warning('Could not scan app %s due to unknown error: %s' % (
                 appid, traceback.format_exc()))
             probcount += 1
 

fdroidserver/server.py

@@ -207,7 +207,7 @@ def update_awsbucket_libcloud(repo_section):
                         s3url = 's3://' + awsbucket + '/' + obj.name
                         logging.info(' deleting ' + s3url)
                         if not driver.delete_object(obj):
-                            logging.warn('Could not delete ' + s3url)
+                            logging.warning('Could not delete ' + s3url)
                         upload = True
 
             if upload:
@@ -229,7 +229,7 @@ def update_awsbucket_libcloud(repo_section):
         object_name, obj = objs.popitem()
         s3url = 's3://' + awsbucket + '/' + object_name
         if object_name.startswith(upload_dir):
-            logging.warn(' deleting ' + s3url)
+            logging.warning(' deleting ' + s3url)
             driver.delete_object(obj)
         else:
             logging.info(' skipping ' + s3url)
@@ -773,8 +773,9 @@ def main():
             and not config.get('binary_transparency_remote') \
             and not config.get('virustotal_apikey') \
             and local_copy_dir is None:
-        logging.warn(_('No option set! Edit your config.py to set at least one of these:')
+        logging.warning(_('No option set! Edit your config.py to set at least one of these:')
-                     + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, virustotal_apikey, androidobservatory, or binary_transparency_remote')
+                        + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, '
+                        + 'virustotal_apikey, androidobservatory, or binary_transparency_remote')
         sys.exit(1)
 
     repo_sections = ['repo']

fdroidserver/signatures.py

@@ -76,7 +76,9 @@ def extract(options):
                         if tmp_apk and os.path.exists(tmp_apk):
                             os.remove(tmp_apk)
                 else:
-                    logging.warn(_('refuse downloading via insecure HTTP connection (use HTTPS or specify --no-https-check): {apkfilename}').format(apkfilename=apk))
+                    logging.warning(_('refuse downloading via insecure HTTP connection '
+                                      '(use HTTPS or specify --no-https-check): {apkfilename}')
+                                    .format(apkfilename=apk))
         except FDroidException as e:
             logging.warning(_("Failed fetching signatures for '{apkfilename}': {error}")
                             .format(apkfilename=apk, error=e))

fdroidserver/update.py

@@ -366,7 +366,7 @@ def update_wiki(apps, sortedids, apks):
                 else:
                     logging.debug("Page " + page.name + " is unchanged")
             else:
-                logging.warn("Deleting page " + page.name)
+                logging.warning('Deleting page ' + page.name)
                 page.delete('No longer published')
         for pagename, text in genp.items():
             logging.debug("Checking " + pagename)
@@ -2162,15 +2162,15 @@ def main():
                 msg = _("{apkfilename} ({appid}) has no metadata!") \
                     .format(apkfilename=apk['apkName'], appid=apk['packageName'])
                 if options.delete_unknown:
-                    logging.warn(msg + '\n\t' + _("deleting: repo/{apkfilename}")
+                    logging.warning(msg + '\n\t' + _("deleting: repo/{apkfilename}")
-                                 .format(apkfilename=apk['apkName']))
+                                    .format(apkfilename=apk['apkName']))
                     rmf = os.path.join(repodirs[0], apk['apkName'])
                     if not os.path.exists(rmf):
                         logging.error(_("Could not find {path} to remove it").format(path=rmf))
                     else:
                         os.remove(rmf)
                 else:
-                    logging.warn(msg + '\n\t' + _("Use `fdroid update -c` to create it."))
+                    logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
 
     copy_triple_t_store_metadata(apps)
     insert_obbs(repodirs[0], apps, apks)

fdroidserver/vmtools.py

@@ -117,7 +117,7 @@ def get_build_vm(srvdir, provider=None):
             logging.debug('build vm provider \'virtualbox\' selected')
             return VirtualboxBuildVm(abssrvdir)
         else:
-            logging.warn('build vm provider not supported: \'%s\'', provider)
+            logging.warning('build vm provider not supported: \'%s\'', provider)
 
     # try guessing provider from installed software
     kvm_installed = shutil.which('kvm') is not None
@@ -451,8 +451,8 @@ class LibvirtBuildVm(FDroidBuildVm):
                 os.remove('box.img')
 
         else:
-            logging.warn("could not connect to storage-pool 'default', "
+            logging.warning("could not connect to storage-pool 'default', "
-                         "skip packaging buildserver box")
+                            "skip packaging buildserver box")
 
     def box_add(self, boxname, boxfile, force=True):
         boximg = '%s_vagrant_box_image_0.img' % (boxname)

gradlew-fdroid

@@ -168,6 +168,7 @@ for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do
         if [[ $line == 'distributionUrl='* ]]; then
             wrapper_ver=${line#*/gradle-}
             wrapper_ver=${wrapper_ver%-*.zip}
+            wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds
             break 2
         fi
     done < $f
@@ -186,9 +187,11 @@ for f in {.,..}/build.gradle{,.kts}; do
         if [[ -z "$plugin_pver" && $line == *'com.android.tools.build:gradle:'* ]]; then
             plugin_pver=${line#*[\'\"]com.android.tools.build:gradle:}
             plugin_pver=${plugin_pver%[\'\"]*}
+            plugin_pver=$(printf $plugin_pver | tr -d '\r') # strip Windows linefeeds
         elif [[ -z "$wrapper_ver" && $line == *'gradleVersion = '* ]]; then
             wrapper_ver=${line#*gradleVersion*=*[\'\"]}
             wrapper_ver=${wrapper_ver%[\'\"]*}
+            wrapper_ver=$(printf $wrapper_ver | tr -d '\r') # strip Windows linefeeds
         fi
     done < $f
 done

tests/common.TestCase

@@ -1315,6 +1315,11 @@ class CommonTest(unittest.TestCase):
                         self.assertEqual(ret, ('ACRA', None, 'srclib/ACRA'))
 
     def test_run_yamllint_wellformed(self):
+        try:
+            import yamllint.config
+            yamllint.config  # make pyflakes ignore this
+        except ImportError:
+            self.skipTest('yamllint not installed')
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('wellformed.yml', 'w') as f:
                 f.write(textwrap.dedent('''\
@@ -1328,6 +1333,11 @@ class CommonTest(unittest.TestCase):
             self.assertEqual(result, '')
 
     def test_run_yamllint_malformed(self):
+        try:
+            import yamllint.config
+            yamllint.config  # make pyflakes ignore this
+        except ImportError:
+            self.skipTest('yamllint not installed')
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with open('malformed.yml', 'w') as f:
                 f.write(textwrap.dedent('''\

tests/scanner.TestCase

@@ -53,6 +53,33 @@ class ScannerTest(unittest.TestCase):
             self.assertEqual(should, fatal_problems,
                              "%s should have %d errors!" % (d, should))
 
+    def test_get_gradle_compile_commands(self):
+        test_files = [
+            ('source-files/fdroid/fdroidclient/build.gradle', 'yes', 17),
+            ('source-files/com.nextcloud.client/build.gradle', 'generic', 24),
+            ('source-files/com.kunzisoft.testcase/build.gradle', 'libre', 4),
+            ('source-files/cn.wildfirechat.chat/chat/build.gradle', 'yes', 33),
+            ('source-files/org.tasks/app/build.gradle.kts', 'generic', 39),
+            ('source-files/at.bitfire.davdroid/build.gradle', 'standard', 16),
+            ('source-files/se.manyver/android/app/build.gradle', 'indie', 29),
+            ('source-files/osmandapp/osmand/build.gradle', 'free', 5),
+            ('source-files/eu.siacs.conversations/build.gradle', 'free', 23),
+            ('source-files/org.mozilla.rocket/app/build.gradle', 'focus', 42),
+        ]
+
+        for f, flavor, count in test_files:
+            i = 0
+            build = fdroidserver.metadata.Build()
+            build.gradle = [flavor]
+            regexs = fdroidserver.scanner.get_gradle_compile_commands(build)
+            with open(f) as fp:
+                for line in fp.readlines():
+                    for regex in regexs:
+                        m = regex.match(line)
+                        if m:
+                            i += 1
+            self.assertEqual(count, i)
+
     def test_build_local_scanner(self):
         """`fdroid build` calls scanner functions, test them here"""
         testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)

tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed Jan 30 10:59:12 CET 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip

tests/test-gradlew-fdroid

@@ -19,8 +19,13 @@ exit_value=0
 basedir=$(cd $(dirname $0)/..; pwd)
 export https_proxy=127.7.7.7:7 # fake proxy to block downloading
 
+# force test file to have Windows linefeeds
+sed -i -e $'s/\r$//' -e  $'s/$/\r/' \
+    $basedir/tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
+
 run_test osmandapp/osmand 2.2.1
 run_test com.integreight.onesheeld 3.3
 run_test se.manyver/android 5.5
+run_test yuriykulikov/AlarmClock 5.1.1
 
 exit $exit_value